Срез 0 [4.3]: эмулятор ИС Сканирование (src/scan_emulator), unit-тесты, отметка задачи
process_once(root, fixtures, delay): новые <uuid>_<имя> -> ответ из фикстуры (точное имя, затем префикс, file_uid подставляется) либо INVALID_DOCUMENT, атомарная запись ответа, затем переименование в .done. Цикл python -m src.scan_emulator с корректной остановкой по SIGINT/SIGTERM. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Andrey Epifancev
parent
97ce49eed2
commit
b489f6e0b3
2
Makefile
2
Makefile
@ -45,6 +45,6 @@ check-full: check test-integration
|
||||
up:
|
||||
docker compose -f $(COMPOSE_FILE) --project-name $(COMPOSE_PROJECT) up -d --wait postgres nats minio
|
||||
|
||||
# Эмулятор ИС Сканирование: модуль появится в задаче 9
|
||||
# Эмулятор ИС Сканирование (параметры — SCAN_HOTFOLDER_PATH, SCAN_FIXTURES_DIR, SCAN_EMULATOR_DELAY из api/.env)
|
||||
emulator:
|
||||
cd api && $(BIN)/python -m src.scan_emulator
|
||||
|
||||
220
api/src/scan_emulator/__init__.py
Normal file
220
api/src/scan_emulator/__init__.py
Normal file
@ -0,0 +1,220 @@
|
||||
"""Эмулятор ИС Сканирование (Content AI) для разработки и тестов (D-6, R-11, spec scan-emulator).
|
||||
|
||||
Воспроизводит протокол горячих папок: для нового файла `<uuid>_<name>.<ext>` во входящем каталоге через задержку кладёт
|
||||
`<output>/<uuid>/<uuid>.json` с эталонным ответом из `SCAN_FIXTURES_DIR` и переименовывает входящий файл в `….done`.
|
||||
Эмулятор читает собственные переменные окружения (`EmulatorSettings`), а не настройки шлюза: в его контейнере нет
|
||||
переменных MinIO/NATS. С каталогами работает через `LocalHotFolder`, тем самым тестируя и его.
|
||||
"""
|
||||
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from dotenv import dotenv_values
|
||||
from loguru import logger
|
||||
|
||||
from src.gateway.hotfolder import LocalHotFolder
|
||||
|
||||
__all__ = ["EmulatorSettings", "process_once", "run"]
|
||||
|
||||
# `<uuid>_<name>`: канонический UUID (36 символов) и непустое имя после `_`
|
||||
_INBOX_NAME = re.compile(
|
||||
r"^(?P<uuid>[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})_(?P<name>.+)$"
|
||||
)
|
||||
_DONE_SUFFIX = ".done"
|
||||
_ERROR_CODE = "INVALID_DOCUMENT"
|
||||
|
||||
|
||||
def _read_float(env: Mapping[str, str], name: str, default: float, *, positive: bool) -> float:
|
||||
raw = env.get(name)
|
||||
if raw is None or raw.strip() == "":
|
||||
return default
|
||||
try:
|
||||
value = float(raw)
|
||||
except ValueError:
|
||||
raise ValueError(f"{name}: ожидается число, получено {raw!r}") from None
|
||||
if not math.isfinite(value) or value < 0 or (positive and value == 0):
|
||||
raise ValueError(
|
||||
f"{name}: ожидается {'положительное' if positive else 'неотрицательное'} число, получено {raw!r}"
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EmulatorSettings:
|
||||
"""Настройки эмулятора из переменных окружения (имена совпадают с `.env.example`)."""
|
||||
|
||||
hotfolder_path: Path
|
||||
input_dir: str = "Input"
|
||||
output_dir: str = "Output"
|
||||
fixtures_dir: Path = Path("../../fixtures/ocr")
|
||||
delay: float = 2.0
|
||||
poll_interval: float = 1.0
|
||||
|
||||
@classmethod
|
||||
def from_env(cls, environ: Mapping[str, str] | None = None) -> "EmulatorSettings":
|
||||
"""Разбор окружения. Без аргумента: `os.environ` поверх файла `ENV_FILE` (по умолчанию `.env`, если он есть).
|
||||
|
||||
`SCAN_HOTFOLDER_PATH` обязателен (`ValueError`); `SCAN_INPUT_DIR`=Input, `SCAN_OUTPUT_DIR`=Output,
|
||||
`SCAN_FIXTURES_DIR`=../../fixtures/ocr, `SCAN_EMULATOR_DELAY`=2 (с, >= 0),
|
||||
`SCAN_EMULATOR_POLL_SECONDS`=1 (с, > 0).
|
||||
"""
|
||||
if environ is None:
|
||||
env_file = os.environ.get("ENV_FILE", ".env")
|
||||
file_values = {k: v for k, v in dotenv_values(env_file).items() if v is not None}
|
||||
env: Mapping[str, str] = {**file_values, **os.environ}
|
||||
else:
|
||||
env = environ
|
||||
path = env.get("SCAN_HOTFOLDER_PATH", "").strip()
|
||||
if not path:
|
||||
raise ValueError("SCAN_HOTFOLDER_PATH не задан: укажите корень горячей папки (каталог с Input/ и Output/)")
|
||||
return cls(
|
||||
hotfolder_path=Path(path),
|
||||
input_dir=env.get("SCAN_INPUT_DIR") or "Input",
|
||||
output_dir=env.get("SCAN_OUTPUT_DIR") or "Output",
|
||||
fixtures_dir=Path(env.get("SCAN_FIXTURES_DIR") or "../../fixtures/ocr"),
|
||||
delay=_read_float(env, "SCAN_EMULATOR_DELAY", 2.0, positive=False),
|
||||
poll_interval=_read_float(env, "SCAN_EMULATOR_POLL_SECONDS", 1.0, positive=True),
|
||||
)
|
||||
|
||||
|
||||
def _error_answer(job_id: str, message: str) -> dict[str, Any]:
|
||||
return {"file_uid": job_id, "error": {"code": _ERROR_CODE, "message": message}, "data_ocr": []}
|
||||
|
||||
|
||||
def _find_fixture(fixtures: Path, stem: str) -> Path | None:
|
||||
"""`<stem>.json`, затем `<stem>*.json` (первый по алфавиту). Имя сравнивается буквально, без glob."""
|
||||
try:
|
||||
names = sorted(p.name for p in fixtures.iterdir() if p.name.lower().endswith(".json") and p.is_file())
|
||||
except OSError:
|
||||
return None
|
||||
if f"{stem}.json" in names:
|
||||
return fixtures / f"{stem}.json"
|
||||
for name in names:
|
||||
if name.startswith(stem):
|
||||
return fixtures / name
|
||||
return None
|
||||
|
||||
|
||||
def _build_answer(job_id: str, name: str, fixtures: Path) -> dict[str, Any]:
|
||||
"""Ответ ИС на файл `name` (без префикса uuid): фикстура с подставленным `file_uid` либо ошибка Content AI."""
|
||||
fixture = _find_fixture(fixtures, Path(name).stem)
|
||||
if fixture is None:
|
||||
return _error_answer(job_id, f"{name}: нет эталонного ответа в фикстурах")
|
||||
try:
|
||||
data = json.loads(fixture.read_text(encoding="utf-8"))
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError("корень не объект JSON")
|
||||
except (OSError, ValueError) as exc: # json.JSONDecodeError и UnicodeDecodeError — подклассы ValueError
|
||||
logger.error("Эмулятор: фикстура {} повреждена ({}); для {} отдаём {}", fixture, exc, name, _ERROR_CODE)
|
||||
return _error_answer(job_id, f"{name}: эталонный ответ {fixture.name} повреждён и не разобран")
|
||||
return {"file_uid": job_id, **{key: value for key, value in data.items() if key != "file_uid"}}
|
||||
|
||||
|
||||
def _write_answer(hotfolder: LocalHotFolder, job_id: str, answer: dict[str, Any]) -> None:
|
||||
"""Атомарная запись ответа: `.part` в том же каталоге, затем `os.replace`."""
|
||||
target = hotfolder.answer_path(job_id)
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
part = target.parent / f".{target.name}.part"
|
||||
try:
|
||||
part.write_text(json.dumps(answer, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
os.replace(part, target)
|
||||
except BaseException:
|
||||
part.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
|
||||
def process_once(
|
||||
root: Path | str,
|
||||
fixtures: Path | str,
|
||||
delay: float,
|
||||
*,
|
||||
input_dir: str = "Input",
|
||||
output_dir: str = "Output",
|
||||
) -> list[str]:
|
||||
"""Один проход по входящему каталогу; возвращает uuid обработанных файлов.
|
||||
|
||||
Берёт новые `<uuid>_<name>.<ext>` (не dot-файлы, не `*.done`, не каталоги), при наличии работы один раз за проход
|
||||
ждёт `delay` секунд, для каждого файла пишет `<output>/<uuid>/<uuid>.json` и только затем переименовывает входящий
|
||||
файл в `….done`. Имена не по шаблону `<uuid>_` игнорируются (одно предупреждение за проход). Нет входящего
|
||||
каталога — пустой результат.
|
||||
"""
|
||||
hotfolder = LocalHotFolder(root, input_dir, output_dir)
|
||||
fixtures_dir = Path(fixtures)
|
||||
try:
|
||||
entries = sorted(hotfolder.input_dir.iterdir(), key=lambda p: p.name)
|
||||
except FileNotFoundError:
|
||||
return []
|
||||
pending: list[tuple[Path, str, str]] = []
|
||||
foreign: list[str] = []
|
||||
for entry in entries:
|
||||
if entry.name.startswith(".") or entry.name.endswith(_DONE_SUFFIX) or not entry.is_file():
|
||||
continue
|
||||
match = _INBOX_NAME.match(entry.name)
|
||||
if match is None:
|
||||
foreign.append(entry.name)
|
||||
continue
|
||||
pending.append((entry, match["uuid"], match["name"]))
|
||||
if foreign:
|
||||
logger.warning("Эмулятор: файлы не по шаблону <uuid>_<имя> пропущены: {}", ", ".join(foreign))
|
||||
if not pending:
|
||||
return []
|
||||
if delay > 0:
|
||||
time.sleep(delay)
|
||||
processed: list[str] = []
|
||||
for entry, job_id, name in pending:
|
||||
try:
|
||||
_write_answer(hotfolder, job_id, _build_answer(job_id, name, fixtures_dir))
|
||||
os.replace(entry, entry.with_name(entry.name + _DONE_SUFFIX))
|
||||
except FileNotFoundError:
|
||||
logger.warning("Эмулятор: файл {} исчез во время обработки", entry.name)
|
||||
continue
|
||||
processed.append(job_id)
|
||||
logger.info("Эмулятор: {} -> {}", entry.name, hotfolder.answer_path(job_id))
|
||||
return processed
|
||||
|
||||
|
||||
def _count_fixtures(fixtures: Path) -> int:
|
||||
return sum(1 for p in fixtures.iterdir() if p.name.lower().endswith(".json") and p.is_file())
|
||||
|
||||
|
||||
def run(settings: EmulatorSettings, stop: threading.Event, *, max_iterations: int | None = None) -> None:
|
||||
"""Цикл эмулятора: `process_once` каждые `poll_interval` секунд, пока не взведён `stop` (или не выполнено
|
||||
`max_iterations` проходов — для тестов). Сбой прохода логируется и не останавливает цикл."""
|
||||
hotfolder = LocalHotFolder(settings.hotfolder_path, settings.input_dir, settings.output_dir)
|
||||
logger.info(
|
||||
"Эмулятор ИС Сканирование: корень {}, входящий каталог {}, исходящий каталог {}, задержка {} с, опрос {} с",
|
||||
hotfolder.root,
|
||||
hotfolder.input_dir,
|
||||
hotfolder.output_dir,
|
||||
settings.delay,
|
||||
settings.poll_interval,
|
||||
)
|
||||
if settings.fixtures_dir.is_dir():
|
||||
logger.info("Фикстуры: {}, фикстур: {}", settings.fixtures_dir, _count_fixtures(settings.fixtures_dir))
|
||||
else:
|
||||
logger.warning(
|
||||
"Каталог фикстур {} не найден: на все файлы будет отвечать {}", settings.fixtures_dir, _ERROR_CODE
|
||||
)
|
||||
iterations = 0
|
||||
while not stop.is_set() and (max_iterations is None or iterations < max_iterations):
|
||||
try:
|
||||
process_once(
|
||||
settings.hotfolder_path,
|
||||
settings.fixtures_dir,
|
||||
settings.delay,
|
||||
input_dir=settings.input_dir,
|
||||
output_dir=settings.output_dir,
|
||||
)
|
||||
except Exception: # noqa: BLE001 — эмулятор живёт, пока его не остановят
|
||||
logger.exception("Эмулятор: сбой прохода по входящему каталогу")
|
||||
iterations += 1
|
||||
stop.wait(settings.poll_interval)
|
||||
logger.info("Эмулятор остановлен")
|
||||
46
api/src/scan_emulator/__main__.py
Normal file
46
api/src/scan_emulator/__main__.py
Normal file
@ -0,0 +1,46 @@
|
||||
"""Точка входа эмулятора: `python -m src.scan_emulator` (make emulator)."""
|
||||
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
from types import FrameType
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from src.scan_emulator import EmulatorSettings, run
|
||||
|
||||
|
||||
def main(stop: threading.Event | None = None) -> int:
|
||||
"""Запуск цикла эмулятора; код возврата процесса. SIGINT/SIGTERM — штатная остановка (0).
|
||||
|
||||
`stop` — для тестов: событие остановки вместо сигналов. Нет `SCAN_HOTFOLDER_PATH` или корня горячей папки — ошибка
|
||||
и ненулевой код (эмулятор не создаёт корень сам: его создаёт и монтирует окружение).
|
||||
"""
|
||||
try:
|
||||
settings = EmulatorSettings.from_env()
|
||||
except ValueError as exc:
|
||||
logger.error("Эмулятор не запущен: {}", exc)
|
||||
return 2
|
||||
if not settings.hotfolder_path.is_dir():
|
||||
logger.error(
|
||||
"Эмулятор не запущен: корень горячей папки {} не существует или не каталог", settings.hotfolder_path
|
||||
)
|
||||
return 2
|
||||
|
||||
event = stop if stop is not None else threading.Event()
|
||||
|
||||
def _on_signal(signum: int, _frame: FrameType | None) -> None:
|
||||
logger.info("Получен сигнал {}, останавливаюсь", signal.Signals(signum).name)
|
||||
event.set()
|
||||
|
||||
previous = {sig: signal.signal(sig, _on_signal) for sig in (signal.SIGINT, signal.SIGTERM)}
|
||||
try:
|
||||
run(settings, event)
|
||||
finally:
|
||||
for sig, handler in previous.items():
|
||||
signal.signal(sig, handler)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
393
api/tests/unit/test_emulator.py
Normal file
393
api/tests/unit/test_emulator.py
Normal file
@ -0,0 +1,393 @@
|
||||
"""Эмулятор ИС Сканирование (spec scan-emulator, D-6, R-11, slice-0-skeleton).
|
||||
|
||||
Сценарии spec «Точное имя», «Префикс», «Нет пары» — на копиях реальных `../fixtures/ocr/*.json` (маркер `fixtures`);
|
||||
остальное — на синтетических фикстурах в `tmp_path`, без внешних репозиториев и без Docker.
|
||||
"""
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import threading
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from loguru import logger
|
||||
|
||||
from src.gateway.hotfolder import LocalHotFolder, inbox_name
|
||||
from src.scan_emulator import EmulatorSettings, process_once, run
|
||||
from src.scan_emulator.__main__ import main
|
||||
|
||||
API_DIR = Path(__file__).resolve().parents[2]
|
||||
# `<корень scan-gateway>/../fixtures/ocr` (workspace и worktree `.wt/<срез>/`), как у теста идентичности контракта
|
||||
REAL_FIXTURES = API_DIR.parent.parent / "fixtures" / "ocr"
|
||||
|
||||
|
||||
def _uid() -> str:
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
def _hotfolder(tmp_path: Path) -> LocalHotFolder:
|
||||
root = tmp_path / "hotfolder"
|
||||
root.mkdir()
|
||||
return LocalHotFolder(root)
|
||||
|
||||
|
||||
def _fixtures_dir(tmp_path: Path, files: dict[str, Any] | None = None) -> Path:
|
||||
path = tmp_path / "fixtures"
|
||||
path.mkdir(exist_ok=True)
|
||||
for name, content in (files or {}).items():
|
||||
text = content if isinstance(content, str) else json.dumps(content, ensure_ascii=False)
|
||||
(path / name).write_text(text, encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def _drop(hf: LocalHotFolder, name: str, data: bytes = b"%PDF") -> str:
|
||||
"""Положить `<uuid>_<name>` во входящий каталог, вернуть uuid."""
|
||||
job_id = _uid()
|
||||
hf.put(inbox_name(job_id, name), data)
|
||||
return job_id
|
||||
|
||||
|
||||
def _answer(hf: LocalHotFolder, job_id: str) -> dict[str, Any]:
|
||||
result: dict[str, Any] = json.loads(hf.answer_path(job_id).read_text(encoding="utf-8"))
|
||||
return result
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def log_messages() -> Iterator[list[str]]:
|
||||
messages: list[str] = []
|
||||
handler_id = logger.add(lambda message: messages.append(str(message)), level="DEBUG", format="{level}|{message}")
|
||||
yield messages
|
||||
logger.remove(handler_id)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def real_fixtures(tmp_path: Path) -> Path:
|
||||
if not REAL_FIXTURES.is_dir():
|
||||
pytest.skip("fixtures недоступен: положите репозиторий fixtures рядом с workspace")
|
||||
target = tmp_path / "real-fixtures"
|
||||
target.mkdir()
|
||||
for source in REAL_FIXTURES.glob("*.json"):
|
||||
shutil.copy(source, target / source.name)
|
||||
return target
|
||||
|
||||
|
||||
@pytest.mark.fixtures
|
||||
class TestRealFixtures:
|
||||
@pytest.mark.parametrize("name", ["aep_5.pdf", "aep_5.PDF"])
|
||||
def test_exact_name(self, tmp_path: Path, real_fixtures: Path, name: str) -> None:
|
||||
hf = _hotfolder(tmp_path)
|
||||
job_id = _drop(hf, name)
|
||||
assert process_once(hf.root, real_fixtures, 0) == [job_id]
|
||||
expected = json.loads((real_fixtures / "aep_5.json").read_text(encoding="utf-8"))
|
||||
expected["file_uid"] = job_id
|
||||
assert _answer(hf, job_id) == expected
|
||||
assert (hf.input_dir / f"{job_id}_{name}.done").is_file()
|
||||
assert not (hf.input_dir / f"{job_id}_{name}").exists()
|
||||
|
||||
def test_prefix(self, tmp_path: Path, real_fixtures: Path) -> None:
|
||||
hf = _hotfolder(tmp_path)
|
||||
job_id = _drop(hf, "aep_8.pdf")
|
||||
process_once(hf.root, real_fixtures, 0)
|
||||
expected = json.loads((real_fixtures / "aep_8_form4.json").read_text(encoding="utf-8"))
|
||||
expected["file_uid"] = job_id
|
||||
assert _answer(hf, job_id) == expected
|
||||
assert (hf.input_dir / f"{job_id}_aep_8.pdf.done").is_file()
|
||||
|
||||
def test_no_pair(self, tmp_path: Path, real_fixtures: Path) -> None:
|
||||
hf = _hotfolder(tmp_path)
|
||||
job_id = _drop(hf, "unknown.pdf")
|
||||
process_once(hf.root, real_fixtures, 0)
|
||||
assert _answer(hf, job_id) == {
|
||||
"file_uid": job_id,
|
||||
"error": {"code": "INVALID_DOCUMENT", "message": "unknown.pdf: нет эталонного ответа в фикстурах"},
|
||||
"data_ocr": [],
|
||||
}
|
||||
assert (hf.input_dir / f"{job_id}_unknown.pdf.done").is_file()
|
||||
|
||||
|
||||
class TestFixtureLookup:
|
||||
def test_exact_beats_prefix(self, tmp_path: Path) -> None:
|
||||
hf = _hotfolder(tmp_path)
|
||||
fixtures = _fixtures_dir(tmp_path, {"a.json": {"file_uid": "x", "v": "exact"}, "a0.json": {"v": "prefix"}})
|
||||
job_id = _drop(hf, "a.pdf")
|
||||
process_once(hf.root, fixtures, 0)
|
||||
assert _answer(hf, job_id) == {"file_uid": job_id, "v": "exact"}
|
||||
|
||||
def test_prefix_first_alphabetical(self, tmp_path: Path) -> None:
|
||||
hf = _hotfolder(tmp_path)
|
||||
fixtures = _fixtures_dir(tmp_path, {"a_zz.json": {"v": "zz"}, "a_b.json": {"v": "b"}, "a_c.json": {"v": "c"}})
|
||||
job_id = _drop(hf, "a.pdf")
|
||||
# `a_*` не совпадает с префиксом `a` по имени без расширения? совпадает: `a*.json`
|
||||
process_once(hf.root, fixtures, 0)
|
||||
assert _answer(hf, job_id)["v"] == "b"
|
||||
|
||||
def test_file_uid_added_when_absent_and_replaced_when_present(self, tmp_path: Path) -> None:
|
||||
hf = _hotfolder(tmp_path)
|
||||
fixtures = _fixtures_dir(tmp_path, {"a.json": {"data_ocr": [1]}, "b.json": {"file_uid": "old", "data_ocr": []}})
|
||||
first, second = _drop(hf, "a.pdf"), _drop(hf, "b.pdf")
|
||||
process_once(hf.root, fixtures, 0)
|
||||
assert _answer(hf, first) == {"file_uid": first, "data_ocr": [1]}
|
||||
assert _answer(hf, second) == {"file_uid": second, "data_ocr": []}
|
||||
|
||||
def test_name_without_extension(self, tmp_path: Path) -> None:
|
||||
hf = _hotfolder(tmp_path)
|
||||
fixtures = _fixtures_dir(tmp_path, {"scan.json": {"v": 1}})
|
||||
job_id = _drop(hf, "scan")
|
||||
process_once(hf.root, fixtures, 0)
|
||||
assert _answer(hf, job_id) == {"file_uid": job_id, "v": 1}
|
||||
|
||||
def test_name_with_glob_characters_is_literal(self, tmp_path: Path) -> None:
|
||||
hf = _hotfolder(tmp_path)
|
||||
fixtures = _fixtures_dir(tmp_path, {"a.json": {"v": 1}})
|
||||
job_id = _drop(hf, "[a]*.pdf")
|
||||
process_once(hf.root, fixtures, 0)
|
||||
assert _answer(hf, job_id)["error"]["code"] == "INVALID_DOCUMENT"
|
||||
|
||||
def test_non_ascii_not_escaped(self, tmp_path: Path) -> None:
|
||||
hf = _hotfolder(tmp_path)
|
||||
fixtures = _fixtures_dir(tmp_path, {"a.json": {"text": "Баланс"}})
|
||||
job_id = _drop(hf, "a.pdf")
|
||||
process_once(hf.root, fixtures, 0)
|
||||
assert "Баланс" in hf.answer_path(job_id).read_text(encoding="utf-8")
|
||||
|
||||
def test_missing_fixtures_dir_gives_error_answer(self, tmp_path: Path) -> None:
|
||||
hf = _hotfolder(tmp_path)
|
||||
job_id = _drop(hf, "a.pdf")
|
||||
process_once(hf.root, tmp_path / "нет-такого", 0)
|
||||
assert _answer(hf, job_id)["error"]["code"] == "INVALID_DOCUMENT"
|
||||
|
||||
@pytest.mark.parametrize("content", ["{не json", "[1, 2]", ""])
|
||||
def test_broken_fixture_gives_error_answer(self, tmp_path: Path, log_messages: list[str], content: str) -> None:
|
||||
hf = _hotfolder(tmp_path)
|
||||
fixtures = _fixtures_dir(tmp_path, {"a.json": content})
|
||||
job_id = _drop(hf, "a.pdf")
|
||||
assert process_once(hf.root, fixtures, 0) == [job_id]
|
||||
answer = _answer(hf, job_id)
|
||||
assert answer["file_uid"] == job_id
|
||||
assert answer["data_ocr"] == []
|
||||
assert answer["error"]["code"] == "INVALID_DOCUMENT"
|
||||
assert "a.pdf" in answer["error"]["message"]
|
||||
assert "a.json" in answer["error"]["message"]
|
||||
assert any(m.startswith("ERROR|") and "a.json" in m for m in log_messages)
|
||||
assert (hf.input_dir / f"{job_id}_a.pdf.done").is_file()
|
||||
|
||||
|
||||
class TestProcessOnce:
|
||||
def test_returns_processed_uuids(self, tmp_path: Path) -> None:
|
||||
hf = _hotfolder(tmp_path)
|
||||
fixtures = _fixtures_dir(tmp_path, {"a.json": {"v": 1}})
|
||||
ids = {_drop(hf, "a.pdf"), _drop(hf, "a.pdf")}
|
||||
assert set(process_once(hf.root, fixtures, 0)) == ids
|
||||
|
||||
def test_nothing_to_do(self, tmp_path: Path) -> None:
|
||||
hf = _hotfolder(tmp_path)
|
||||
assert process_once(hf.root, _fixtures_dir(tmp_path), 0) == []
|
||||
|
||||
def test_missing_input_dir_is_not_an_error(self, tmp_path: Path) -> None:
|
||||
hf = _hotfolder(tmp_path)
|
||||
assert not hf.input_dir.exists()
|
||||
assert process_once(hf.root, _fixtures_dir(tmp_path), 0) == []
|
||||
|
||||
def test_second_pass_does_not_reprocess(self, tmp_path: Path) -> None:
|
||||
hf = _hotfolder(tmp_path)
|
||||
fixtures = _fixtures_dir(tmp_path, {"a.json": {"v": 1}})
|
||||
job_id = _drop(hf, "a.pdf")
|
||||
assert process_once(hf.root, fixtures, 0) == [job_id]
|
||||
mtime = hf.answer_path(job_id).stat().st_mtime_ns
|
||||
assert process_once(hf.root, fixtures, 0) == []
|
||||
assert hf.answer_path(job_id).stat().st_mtime_ns == mtime
|
||||
|
||||
def test_ignores_done_dotfiles_and_non_uuid(self, tmp_path: Path, log_messages: list[str]) -> None:
|
||||
hf = _hotfolder(tmp_path)
|
||||
fixtures = _fixtures_dir(tmp_path, {"a.json": {"v": 1}})
|
||||
hf.input_dir.mkdir()
|
||||
job_id = _uid()
|
||||
ignored = [
|
||||
f"{job_id}_a.pdf.done",
|
||||
f".{job_id}_a.pdf.part",
|
||||
"a.pdf",
|
||||
"not-a-uuid_a.pdf",
|
||||
f"{job_id}a.pdf",
|
||||
f"{job_id}_",
|
||||
f"{job_id[:-1]}_a.pdf",
|
||||
]
|
||||
for name in ignored:
|
||||
(hf.input_dir / name).write_bytes(b"x")
|
||||
(hf.input_dir / f"{_uid()}_dir.pdf").mkdir()
|
||||
assert process_once(hf.root, fixtures, 0) == []
|
||||
assert sorted(p.name for p in hf.input_dir.iterdir() if p.is_file()) == sorted(ignored)
|
||||
assert not hf.output_dir.exists()
|
||||
warnings = [m for m in log_messages if m.startswith("WARNING|")]
|
||||
assert len(warnings) == 1
|
||||
assert "not-a-uuid_a.pdf" in warnings[0]
|
||||
|
||||
def test_delay_once_per_pass(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
hf = _hotfolder(tmp_path)
|
||||
fixtures = _fixtures_dir(tmp_path, {"a.json": {"v": 1}})
|
||||
sleeps: list[float] = []
|
||||
monkeypatch.setattr("src.scan_emulator.time.sleep", sleeps.append)
|
||||
assert process_once(hf.root, fixtures, 2) == []
|
||||
assert sleeps == [] # нет работы — нет задержки
|
||||
_drop(hf, "a.pdf")
|
||||
_drop(hf, "a.pdf")
|
||||
process_once(hf.root, fixtures, 2)
|
||||
assert sleeps == [2]
|
||||
|
||||
def test_answer_exists_before_done_and_is_atomic(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
hf = _hotfolder(tmp_path)
|
||||
fixtures = _fixtures_dir(tmp_path, {"a.json": {"v": 1}})
|
||||
job_id = _drop(hf, "a.pdf")
|
||||
seen: dict[str, bool] = {}
|
||||
original_replace = __import__("os").replace
|
||||
|
||||
def spy(src: Path, dst: Path) -> None:
|
||||
if str(dst).endswith(".done"):
|
||||
seen["answer_before_done"] = hf.answer_path(job_id).is_file()
|
||||
original_replace(src, dst)
|
||||
|
||||
monkeypatch.setattr("src.scan_emulator.os.replace", spy)
|
||||
process_once(hf.root, fixtures, 0)
|
||||
assert seen == {"answer_before_done": True}
|
||||
assert [p.name for p in hf.answer_path(job_id).parent.iterdir()] == [f"{job_id}.json"]
|
||||
|
||||
def test_custom_dir_names(self, tmp_path: Path) -> None:
|
||||
root = tmp_path / "hf"
|
||||
root.mkdir()
|
||||
hf = LocalHotFolder(root, "In", "Out")
|
||||
fixtures = _fixtures_dir(tmp_path, {"a.json": {"v": 1}})
|
||||
job_id = _drop(hf, "a.pdf")
|
||||
assert process_once(root, fixtures, 0, input_dir="In", output_dir="Out") == [job_id]
|
||||
assert hf.answer_path(job_id).is_file()
|
||||
|
||||
def test_interop_put_emulator_poll(self, tmp_path: Path) -> None:
|
||||
"""Сквозной «put -> эмулятор -> poll» без Docker (D-9)."""
|
||||
hf = _hotfolder(tmp_path)
|
||||
fixtures = _fixtures_dir(tmp_path, {"report.json": {"file_uid": "x", "data_ocr": [{"k": "v"}]}})
|
||||
job_id = _uid()
|
||||
assert hf.poll(job_id) is None
|
||||
hf.put(inbox_name(job_id, "report.pdf"), b"%PDF-1.4")
|
||||
assert hf.poll(job_id) is None
|
||||
process_once(hf.root, fixtures, 0)
|
||||
raw = hf.poll(job_id)
|
||||
assert raw is not None
|
||||
assert json.loads(raw) == {"file_uid": job_id, "data_ocr": [{"k": "v"}]}
|
||||
|
||||
|
||||
class TestSettings:
|
||||
def test_defaults(self, tmp_path: Path) -> None:
|
||||
settings = EmulatorSettings.from_env({"SCAN_HOTFOLDER_PATH": str(tmp_path)})
|
||||
assert settings.hotfolder_path == tmp_path
|
||||
assert (settings.input_dir, settings.output_dir) == ("Input", "Output")
|
||||
assert settings.delay == 2
|
||||
assert settings.poll_interval == 1
|
||||
assert settings.fixtures_dir == Path("../../fixtures/ocr")
|
||||
|
||||
def test_overrides(self, tmp_path: Path) -> None:
|
||||
settings = EmulatorSettings.from_env(
|
||||
{
|
||||
"SCAN_HOTFOLDER_PATH": str(tmp_path),
|
||||
"SCAN_INPUT_DIR": "In",
|
||||
"SCAN_OUTPUT_DIR": "Out",
|
||||
"SCAN_FIXTURES_DIR": "fx",
|
||||
"SCAN_EMULATOR_DELAY": "0.5",
|
||||
"SCAN_EMULATOR_POLL_SECONDS": "0.1",
|
||||
}
|
||||
)
|
||||
assert (settings.input_dir, settings.output_dir) == ("In", "Out")
|
||||
assert settings.fixtures_dir == Path("fx")
|
||||
assert (settings.delay, settings.poll_interval) == (0.5, 0.1)
|
||||
|
||||
def test_path_is_required(self) -> None:
|
||||
with pytest.raises(ValueError, match="SCAN_HOTFOLDER_PATH"):
|
||||
EmulatorSettings.from_env({})
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("name", "value"),
|
||||
[("SCAN_EMULATOR_DELAY", "-1"), ("SCAN_EMULATOR_DELAY", "abc"), ("SCAN_EMULATOR_POLL_SECONDS", "0")],
|
||||
)
|
||||
def test_invalid_numbers(self, tmp_path: Path, name: str, value: str) -> None:
|
||||
with pytest.raises(ValueError, match=name):
|
||||
EmulatorSettings.from_env({"SCAN_HOTFOLDER_PATH": str(tmp_path), name: value})
|
||||
|
||||
|
||||
def _settings(tmp_path: Path, fixtures: Path, *, poll: float = 0.01) -> EmulatorSettings:
|
||||
return EmulatorSettings(
|
||||
hotfolder_path=tmp_path / "hotfolder",
|
||||
input_dir="Input",
|
||||
output_dir="Output",
|
||||
fixtures_dir=fixtures,
|
||||
delay=0,
|
||||
poll_interval=poll,
|
||||
)
|
||||
|
||||
|
||||
class TestRun:
|
||||
def test_startup_log_and_processing(self, tmp_path: Path, log_messages: list[str]) -> None:
|
||||
hf = _hotfolder(tmp_path)
|
||||
fixtures = _fixtures_dir(tmp_path, {"a.json": {"v": 1}, "b.json": {"v": 2}})
|
||||
(fixtures / "readme.txt").write_text("не фикстура")
|
||||
job_id = _drop(hf, "a.pdf")
|
||||
run(_settings(tmp_path, fixtures), threading.Event(), max_iterations=2)
|
||||
assert hf.answer_path(job_id).is_file()
|
||||
startup = "\n".join(log_messages)
|
||||
assert str(hf.root) in startup
|
||||
assert str(hf.input_dir) in startup
|
||||
assert str(hf.output_dir) in startup
|
||||
assert str(fixtures) in startup
|
||||
assert any("фикстур: 2" in m for m in log_messages)
|
||||
|
||||
def test_missing_fixtures_dir_warns_and_continues(self, tmp_path: Path, log_messages: list[str]) -> None:
|
||||
hf = _hotfolder(tmp_path)
|
||||
missing = tmp_path / "нет"
|
||||
job_id = _drop(hf, "a.pdf")
|
||||
run(_settings(tmp_path, missing), threading.Event(), max_iterations=1)
|
||||
assert any(m.startswith("WARNING|") and str(missing) in m for m in log_messages)
|
||||
assert _answer(hf, job_id)["error"]["code"] == "INVALID_DOCUMENT"
|
||||
|
||||
def test_stop_event_ends_loop_without_waiting(self, tmp_path: Path) -> None:
|
||||
_hotfolder(tmp_path)
|
||||
stop = threading.Event()
|
||||
stop.set()
|
||||
run(_settings(tmp_path, _fixtures_dir(tmp_path), poll=60), stop) # вернётся сразу, а не через 60 с
|
||||
|
||||
def test_loop_survives_pass_error(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, log_messages: list[str]
|
||||
) -> None:
|
||||
_hotfolder(tmp_path)
|
||||
calls: list[int] = []
|
||||
|
||||
def boom(*args: object, **kwargs: object) -> list[str]:
|
||||
calls.append(1)
|
||||
raise OSError("сбой")
|
||||
|
||||
monkeypatch.setattr("src.scan_emulator.process_once", boom)
|
||||
run(_settings(tmp_path, _fixtures_dir(tmp_path)), threading.Event(), max_iterations=2)
|
||||
assert len(calls) == 2
|
||||
assert any(m.startswith("ERROR|") for m in log_messages)
|
||||
|
||||
|
||||
class TestMain:
|
||||
def test_no_hotfolder_path_exits_nonzero(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
monkeypatch.delenv("SCAN_HOTFOLDER_PATH", raising=False)
|
||||
monkeypatch.setenv("ENV_FILE", str(tmp_path / "нет.env"))
|
||||
assert main() != 0
|
||||
|
||||
def test_hotfolder_root_missing_exits_nonzero(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
monkeypatch.setenv("SCAN_HOTFOLDER_PATH", str(tmp_path / "нет-корня"))
|
||||
assert main() != 0
|
||||
|
||||
def test_clean_exit_on_stop(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
hf = _hotfolder(tmp_path)
|
||||
fixtures = _fixtures_dir(tmp_path, {"a.json": {"v": 1}})
|
||||
monkeypatch.setenv("SCAN_HOTFOLDER_PATH", str(hf.root))
|
||||
monkeypatch.setenv("SCAN_FIXTURES_DIR", str(fixtures))
|
||||
monkeypatch.setenv("SCAN_EMULATOR_DELAY", "0")
|
||||
monkeypatch.setenv("SCAN_EMULATOR_POLL_SECONDS", "0.01")
|
||||
job_id = _drop(hf, "a.pdf")
|
||||
stop = threading.Event()
|
||||
threading.Timer(0.3, stop.set).start()
|
||||
assert main(stop) == 0
|
||||
assert hf.answer_path(job_id).is_file()
|
||||
@ -52,7 +52,7 @@
|
||||
|
||||
### Task 9: [4.3] `src/scan_emulator/__init__.py` + `__main__.py`
|
||||
|
||||
- [ ] `src/scan_emulator/__init__.py` + `__main__.py`: функция `process_once(root, fixtures, delay)` и цикл; поиск фикстуры по имени/префиксу, подстановка `file_uid`, ошибка `INVALID_DOCUMENT`, переименование в `.done`; `tests/unit/test_emulator.py` — сценарии spec «Точное имя», «Префикс», «Нет пары» на `tmp_path` с копиями `../fixtures/ocr/*.json` (маркер `fixtures`, skip без фикстур); проверка: зелёный
|
||||
- [x] `src/scan_emulator/__init__.py` + `__main__.py`: функция `process_once(root, fixtures, delay)` и цикл; поиск фикстуры по имени/префиксу, подстановка `file_uid`, ошибка `INVALID_DOCUMENT`, переименование в `.done`; `tests/unit/test_emulator.py` — сценарии spec «Точное имя», «Префикс», «Нет пары» на `tmp_path` с копиями `../fixtures/ocr/*.json` (маркер `fixtures`, skip без фикстур); проверка: зелёный
|
||||
|
||||
### Task 10: [5.1] `src/normalizer/__init__.py`
|
||||
|
||||
|
||||
@ -37,7 +37,7 @@
|
||||
на `tmp_path`; проверка: put пишет `<uuid>_<name>`, poll возвращает `None` до появления ответа и байты после
|
||||
- [x] 4.2 `src/gateway/smb.py`: `SmbHotFolder` поверх `smbclient` (UNC `\\host\path\Input`), те же методы; unit-тест с
|
||||
подменённым `smbclient` (вызовы `open_file`, `listdir`); проверка: тест зелёный, в режиме `local` модуль не импортируется
|
||||
- [ ] 4.3 `src/scan_emulator/__init__.py` + `__main__.py`: функция `process_once(root, fixtures, delay)` и цикл; поиск
|
||||
- [x] 4.3 `src/scan_emulator/__init__.py` + `__main__.py`: функция `process_once(root, fixtures, delay)` и цикл; поиск
|
||||
фикстуры по имени/префиксу, подстановка `file_uid`, ошибка `INVALID_DOCUMENT`, переименование в `.done`;
|
||||
`tests/unit/test_emulator.py` — сценарии spec «Точное имя», «Префикс», «Нет пары» на `tmp_path` с копиями
|
||||
`../fixtures/ocr/*.json` (маркер `fixtures`, skip без фикстур); проверка: зелёный
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user