977 lines
48 KiB
Python
977 lines
48 KiB
Python
"""Воркер шлюза (`src.core.worker`, задача 5.3): сценарии spec image-recognition-gateway, D-2, D-3, D-7, R-14..R-20.
|
||
|
||
Обвязка (фейки S3/NATS, шпион горячей папки, эмулятор ИС, журнал событий) — `tests/worker_helpers.py`, фикстура
|
||
`make_env` — `tests/unit/conftest.py`. Параллелизм и остановка при очереди на слот — `test_worker_concurrency.py`.
|
||
"""
|
||
|
||
import asyncio
|
||
import json
|
||
import shutil
|
||
import uuid
|
||
from collections.abc import Callable
|
||
from datetime import UTC, datetime, timedelta
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import pytest
|
||
from jsonschema import Draft202012Validator, FormatChecker
|
||
|
||
from src.contract import (
|
||
ErrorCode,
|
||
MessageError,
|
||
Metrics,
|
||
ResultStatus,
|
||
StatementDocument,
|
||
)
|
||
from src.core.clients import JSON_CONTENT_TYPE
|
||
from src.core.worker import MAX_ERROR_MESSAGE_CHARS, MAX_WARNING_CHARS, MAX_WARNINGS
|
||
from src.gateway.hotfolder import AnswerFile
|
||
from src.gateway.merge import dump_merged, merge_answers
|
||
from src.normalizer import NOT_IMPLEMENTED_MESSAGE, NOT_OBJECT_MESSAGE, NormalizeOutcome, OcrNormalizer
|
||
from src.normalizer.rules import load_rules
|
||
from tests.worker_helpers import (
|
||
API_DIR,
|
||
BUCKET,
|
||
CONTRACTS_DIR,
|
||
PDF_BYTES,
|
||
SUCCESS_ANSWER,
|
||
Env,
|
||
EnvFactory,
|
||
emulator,
|
||
eventually,
|
||
merged_raw,
|
||
)
|
||
|
||
# `<корень scan-gateway>/../fixtures/ocr` (workspace и worktree `.wt/<срез>/`), как в тестах эмулятора
|
||
REAL_FIXTURES = API_DIR.parent.parent / "fixtures" / "ocr"
|
||
RESULT_VALIDATOR = Draft202012Validator(
|
||
json.loads((CONTRACTS_DIR / "recognition-result.schema.json").read_text(encoding="utf-8")),
|
||
format_checker=FormatChecker(),
|
||
)
|
||
STATEMENT_EXAMPLE = json.loads((CONTRACTS_DIR / "examples" / "result-statement-minimal.json").read_text("utf-8"))
|
||
REAL_MESSAGE = "Не удалось определить тип документа."
|
||
INVALID_DOCUMENT_ANSWER = {
|
||
"file_uid": "x",
|
||
"error": {"code": "INVALID_DOCUMENT", "message": REAL_MESSAGE},
|
||
"data_ocr": {"table_structure": []},
|
||
}
|
||
|
||
|
||
def _json(obj: object) -> bytes:
|
||
return json.dumps(obj, ensure_ascii=False).encode("utf-8")
|
||
|
||
|
||
def _raw(env: Env, job_id: str) -> Any:
|
||
"""Склеенный сырой ответ из S3: всегда валидный JSON с типом application/json."""
|
||
key = (BUCKET, f"raw/{job_id}.json")
|
||
assert env.storage.content_types[key] == JSON_CONTENT_TYPE
|
||
return json.loads(env.storage.objects[key])
|
||
|
||
|
||
def _real_fixture(name: str) -> bytes:
|
||
if not REAL_FIXTURES.is_dir():
|
||
pytest.skip("fixtures недоступен: положите репозиторий fixtures рядом с workspace")
|
||
return (REAL_FIXTURES / name).read_bytes()
|
||
|
||
|
||
# Логи воркера перехватываются (`log_lines` из conftest): вывод pytest чистый
|
||
pytestmark = pytest.mark.usefixtures("log_lines")
|
||
|
||
|
||
class RaisingNormalizer:
|
||
def normalize(self, raw: object) -> NormalizeOutcome:
|
||
raise RuntimeError("нормализатор сломан")
|
||
|
||
|
||
class StubNormalizer:
|
||
"""Возвращает заданный исход (done / needs_check / error) — в Срезе 0 успешный путь достижим только так."""
|
||
|
||
def __init__(self, outcome: NormalizeOutcome) -> None:
|
||
self.outcome = outcome
|
||
self.calls: list[object] = []
|
||
|
||
def normalize(self, raw: object) -> NormalizeOutcome:
|
||
self.calls.append(raw)
|
||
return self.outcome
|
||
|
||
|
||
def assert_error(body: dict[str, Any], code: ErrorCode, *, raw: bool) -> None:
|
||
assert body["status"] == "error"
|
||
assert body["error"]["code"] == code.value
|
||
assert body["error"]["message"]
|
||
if raw:
|
||
assert body["raw_s3_key"] == f"raw/{body['job_id']}.json"
|
||
else:
|
||
assert "raw_s3_key" not in body
|
||
assert "result_s3" not in body
|
||
|
||
|
||
# --- подтверждение запроса до обработки ---------------------------------------------------------------------------
|
||
async def test_ack_is_published_before_s3_read_and_before_file_appears(make_env: EnvFactory) -> None:
|
||
env = await make_env()
|
||
request = env.request()
|
||
async with emulator(env):
|
||
await env.run(request)
|
||
job_id = request["job_id"]
|
||
name = f"{job_id}.pdf"
|
||
kinds = [event for event in env.events if event[1] in (f"ack:{job_id}:1", request["s3"]["key"], name)]
|
||
# очистка вызывается всегда после записи (шпион её журналирует; удаляет `HotFolder.cleanup` только при SCAN_CLEANUP)
|
||
assert kinds == [
|
||
("publish", f"ack:{job_id}:1"),
|
||
("s3_get", request["s3"]["key"]),
|
||
("put", name),
|
||
("cleanup", name),
|
||
]
|
||
|
||
|
||
async def test_file_in_inbox_is_byte_for_byte_s3_object(make_env: EnvFactory) -> None:
|
||
env = await make_env()
|
||
request = env.request(original_name="aep_5.pdf")
|
||
await env.deliver(request)
|
||
path = env.local.input_dir / f"{request['job_id']}.pdf"
|
||
await eventually(path.exists)
|
||
assert path.read_bytes() == PDF_BYTES
|
||
await env.worker.stop()
|
||
|
||
|
||
async def test_repeated_request_gets_second_ack_and_one_file(make_env: EnvFactory) -> None:
|
||
env = await make_env()
|
||
request = env.request(original_name="aep_5.pdf")
|
||
await env.deliver(request)
|
||
job_id = request["job_id"]
|
||
await eventually(lambda: len(env.hotfolder.puts) == 1)
|
||
await env.deliver({**request, "attempt": 2}) # ядро не дождалось ack и повторило запрос
|
||
await eventually(lambda: len(env.acks(job_id)) == 2)
|
||
await asyncio.sleep(0.05)
|
||
assert env.acks(job_id) == [f"ack:{job_id}:1", f"ack:{job_id}:2"]
|
||
assert env.inbox() == [f"{job_id}.pdf"]
|
||
assert env.hotfolder.puts == [f"{job_id}.pdf"]
|
||
assert len(env.storage.get_calls) == 1
|
||
|
||
|
||
async def test_repeated_request_after_result_gets_ack_without_reprocessing(make_env: EnvFactory) -> None:
|
||
env = await make_env()
|
||
request = env.request()
|
||
async with emulator(env):
|
||
await env.run(request)
|
||
await env.deliver(request)
|
||
await asyncio.wait_for(env.worker.wait_idle(), 5)
|
||
job_id = request["job_id"]
|
||
assert env.acks(job_id) == [f"ack:{job_id}:1", f"ack:{job_id}:1"]
|
||
assert len(env.results(job_id)) == 1
|
||
assert env.hotfolder.puts == [f"{job_id}.pdf"]
|
||
|
||
|
||
async def test_done_cache_is_bounded_lru(make_env: EnvFactory) -> None:
|
||
env = await make_env(DONE_CACHE_SIZE=2)
|
||
requests = [env.request() for _ in range(3)]
|
||
async with emulator(env):
|
||
for request in requests:
|
||
await env.run(request)
|
||
# первый job_id вытеснен из LRU размера 2 — его повтор обрабатывается заново (в проде — 1000 заданий назад)
|
||
await env.deliver(requests[0])
|
||
await asyncio.wait_for(env.worker.wait_idle(), 5)
|
||
# третий ещё в кэше — повтор только с ack
|
||
await env.deliver(requests[2])
|
||
await asyncio.wait_for(env.worker.wait_idle(), 5)
|
||
assert len(env.results(requests[0]["job_id"])) == 2
|
||
assert len(env.results(requests[2]["job_id"])) == 1
|
||
assert len(env.acks(requests[2]["job_id"])) == 2
|
||
|
||
|
||
async def test_on_request_returns_without_waiting_for_publication(make_env: EnvFactory) -> None:
|
||
env = await make_env()
|
||
env.client.block = asyncio.Event()
|
||
request = env.request()
|
||
await asyncio.wait_for(env.deliver(request), 0.2) # публикация ack висит, а колбэк уже вернулся (T11.4)
|
||
assert env.client.published == []
|
||
env.client.block.set()
|
||
async with emulator(env):
|
||
await asyncio.wait_for(env.worker.wait_idle(), 5)
|
||
assert len(env.results(request["job_id"])) == 1
|
||
|
||
|
||
# --- результат по контракту ---------------------------------------------------------------------------------------
|
||
async def test_success_answer_gives_not_implemented_with_raw_key(make_env: EnvFactory) -> None:
|
||
env = await make_env()
|
||
request = env.request(attempt=3)
|
||
async with emulator(env):
|
||
body = await env.run(request)
|
||
job_id = request["job_id"]
|
||
assert_error(body, ErrorCode.NOT_IMPLEMENTED, raw=True)
|
||
assert body["error"]["message"] == NOT_IMPLEMENTED_MESSAGE
|
||
assert body["file_id"] == request["file_id"]
|
||
assert [mid for mid, _ in env.published()] == [f"ack:{job_id}:3", f"result:{job_id}:3"]
|
||
assert "metrics" not in body # ошибочных файлов нет — warnings опущено
|
||
answer = json.loads(env.local.answer_path(job_id).read_bytes())
|
||
assert answer["error"] == {"code": "", "message": ""} # успех в форме реального ответа ИС
|
||
assert _raw(env, job_id) == merged_raw(job_id, (f"{job_id}.json", answer, None))
|
||
# сырой ответ сохранён раньше публикации результата
|
||
assert env.events.index(("s3_put", f"raw/{job_id}.json")) < env.events.index(("publish", f"result:{job_id}:3"))
|
||
assert env.inbox() == [f"{job_id}.pdf.done"]
|
||
|
||
|
||
@pytest.mark.fixtures
|
||
async def test_real_fixture_error_is_passed_through(make_env: EnvFactory, tmp_path: Path) -> None:
|
||
if not REAL_FIXTURES.is_dir():
|
||
pytest.skip("fixtures недоступен: положите репозиторий fixtures рядом с workspace")
|
||
real = tmp_path / "real-fixtures"
|
||
real.mkdir()
|
||
for source in REAL_FIXTURES.glob("*.json"):
|
||
shutil.copy(source, real / source.name)
|
||
(real / "aep_5.pdf").write_bytes(PDF_BYTES) # пара ответа `aep_5.json` по sha256 содержимого запроса
|
||
expected = json.loads((real / "aep_5.json").read_text(encoding="utf-8"))["error"]
|
||
env = await make_env()
|
||
request = env.request(original_name="aep_5.PDF")
|
||
async with emulator(env, real):
|
||
body = await env.run(request)
|
||
assert_error(body, ErrorCode.INVALID_DOCUMENT, raw=True)
|
||
assert body["error"]["message"] == expected["message"]
|
||
job_id = request["job_id"]
|
||
assert body["metrics"] == {"warnings": [f"{job_id}.json: INVALID_DOCUMENT: {expected['message']}"]}
|
||
assert _raw(env, job_id)["files"][0]["response"]["error"] == expected
|
||
|
||
|
||
async def test_unknown_name_gets_emulator_invalid_document(make_env: EnvFactory) -> None:
|
||
env = await make_env()
|
||
request = env.request(original_name="unknown.pdf", data=b"%PDF unknown content") # пары по содержимому нет
|
||
async with emulator(env):
|
||
body = await env.run(request)
|
||
assert_error(body, ErrorCode.INVALID_DOCUMENT, raw=True)
|
||
assert "нет пары по содержимому" in body["error"]["message"]
|
||
|
||
|
||
@pytest.mark.parametrize("status", [ResultStatus.DONE, ResultStatus.NEEDS_CHECK])
|
||
async def test_successful_outcome_writes_result_file(make_env: EnvFactory, status: ResultStatus) -> None:
|
||
document = StatementDocument.model_validate(STATEMENT_EXAMPLE["documents"][0])
|
||
normalizer = StubNormalizer(NormalizeOutcome(status=status, documents=[document]))
|
||
env = await make_env(normalizer=normalizer)
|
||
request = env.request()
|
||
async with emulator(env):
|
||
body = await env.run(request)
|
||
job_id, file_id = request["job_id"], request["file_id"]
|
||
key = f"results/{file_id}/{job_id}/result.json"
|
||
assert body["status"] == status.value
|
||
assert body["result_s3"] == {"bucket": BUCKET, "key": key}
|
||
assert body["result_schema_version"] == "1.0"
|
||
assert body["raw_s3_key"] == f"raw/{job_id}.json"
|
||
assert "error" not in body
|
||
result = json.loads(env.storage.objects[(BUCKET, key)].decode("utf-8"))
|
||
RESULT_VALIDATOR.validate(result)
|
||
assert result["status"] == status.value
|
||
assert result["job_id"] == job_id
|
||
assert result["file_id"] == file_id
|
||
assert result["service"]["name"] == "scan-gateway"
|
||
assert result["raw_s3_key"] == f"raw/{job_id}.json"
|
||
assert result["documents"] == STATEMENT_EXAMPLE["documents"]
|
||
assert env.storage.content_types[(BUCKET, key)] == JSON_CONTENT_TYPE
|
||
# нормализатор получает склеенный ответ — тот же объект, что записан в raw/
|
||
assert normalizer.calls == [merged_raw(job_id, (f"{job_id}.json", {**SUCCESS_ANSWER, "file_uid": job_id}, None))]
|
||
assert normalizer.calls == [_raw(env, job_id)]
|
||
assert env.events.index(("s3_put", key)) < env.events.index(("publish", f"result:{job_id}:1"))
|
||
|
||
|
||
async def test_result_file_write_failure_is_unknown_error_with_raw_key(make_env: EnvFactory) -> None:
|
||
document = StatementDocument.model_validate(STATEMENT_EXAMPLE["documents"][0])
|
||
env = await make_env(normalizer=StubNormalizer(NormalizeOutcome(status=ResultStatus.DONE, documents=[document])))
|
||
env.storage.fail_put_prefixes = ("results/",)
|
||
request = env.request()
|
||
async with emulator(env):
|
||
body = await env.run(request)
|
||
assert_error(body, ErrorCode.UNKNOWN_ERROR, raw=True)
|
||
assert "результат" in body["error"]["message"]
|
||
|
||
|
||
# --- сырой ответ и нормализатор -----------------------------------------------------------------------------------
|
||
async def test_normalizer_exception_is_unknown_error_with_raw_saved(make_env: EnvFactory) -> None:
|
||
env = await make_env(normalizer=RaisingNormalizer())
|
||
request = env.request()
|
||
async with emulator(env):
|
||
body = await env.run(request)
|
||
assert_error(body, ErrorCode.UNKNOWN_ERROR, raw=True)
|
||
assert "нормализатор" in body["error"]["message"]
|
||
assert (BUCKET, f"raw/{request['job_id']}.json") in env.storage.objects
|
||
|
||
|
||
async def test_raw_write_failure_is_unknown_error_without_raw_key(make_env: EnvFactory) -> None:
|
||
normalizer = StubNormalizer(
|
||
NormalizeOutcome(status=ResultStatus.ERROR, error=MessageError(code="UNREADABLE", message="x"))
|
||
)
|
||
env = await make_env(normalizer=normalizer)
|
||
env.storage.fail_put_prefixes = ("raw/",)
|
||
request = env.request()
|
||
async with emulator(env):
|
||
body = await env.run(request)
|
||
assert_error(body, ErrorCode.UNKNOWN_ERROR, raw=False)
|
||
assert "сырой ответ" in body["error"]["message"]
|
||
assert normalizer.calls == [] # без сохранённого сырого ответа нормализатор не вызывается (D-4)
|
||
|
||
|
||
async def test_answer_not_json_is_parse_error_and_invalid_format(make_env: EnvFactory) -> None:
|
||
"""Сценарий «Все файлы не JSON»: файл — `parse_error` в склейке, задание заканчивается `INVALID_FORMAT`."""
|
||
env = await make_env()
|
||
request = env.request()
|
||
job_id = request["job_id"]
|
||
await env.deliver(request)
|
||
await eventually(lambda: len(env.hotfolder.puts) == 1)
|
||
env.answer(job_id, b"\xff\xfe<html>not json")
|
||
await asyncio.wait_for(env.worker.wait_idle(), 5)
|
||
[(_, body)] = env.results(job_id)
|
||
assert_error(body, ErrorCode.INVALID_FORMAT, raw=True)
|
||
[entry] = _raw(env, job_id)["files"]
|
||
assert entry["name"] == f"{job_id}.json"
|
||
assert entry["response"] is None
|
||
assert entry["parse_error"]
|
||
[warning] = body["metrics"]["warnings"]
|
||
assert warning.startswith(f"{job_id}.json: INVALID_FORMAT: ")
|
||
assert entry["parse_error"] in warning
|
||
|
||
|
||
async def test_invalid_json_after_settle_is_not_reread(make_env: EnvFactory) -> None:
|
||
"""R-24: набор уже настоялся — недописанный JSON не перечитывается, а становится `parse_error` файла."""
|
||
env = await make_env()
|
||
full = _json({**SUCCESS_ANSWER, "file_uid": "x"})
|
||
env.hotfolder.poll_script = [None, [AnswerFile("x.json", full[:10])], [AnswerFile("x.json", full)]]
|
||
body = await env.run(env.request())
|
||
assert_error(body, ErrorCode.INVALID_FORMAT, raw=True)
|
||
assert env.hotfolder.polls == 2
|
||
[entry] = _raw(env, body["job_id"])["files"]
|
||
assert entry["response"] is None
|
||
assert entry["parse_error"]
|
||
|
||
|
||
async def test_merged_raw_contains_every_file_by_name(make_env: EnvFactory) -> None:
|
||
"""Сценарий «Склеенный сырой ответ»: оба файла в `raw/`, по возрастанию имени, нормализатор получает склейку."""
|
||
normalizer = StubNormalizer(
|
||
NormalizeOutcome(status=ResultStatus.ERROR, error=MessageError(code="NOT_IMPLEMENTED", message="x"))
|
||
)
|
||
env = await make_env(normalizer=normalizer)
|
||
request = env.request()
|
||
job_id = request["job_id"]
|
||
first = {**SUCCESS_ANSWER, "file_uid": job_id}
|
||
second = {**INVALID_DOCUMENT_ANSWER, "file_uid": job_id}
|
||
env.hotfolder.poll_script = [
|
||
[AnswerFile(f"{job_id}.json", _json(first)), AnswerFile(f"{job_id}_ofr.json", _json(second))]
|
||
]
|
||
await env.run(request)
|
||
expected = merged_raw(job_id, (f"{job_id}.json", first, None), (f"{job_id}_ofr.json", second, None))
|
||
assert _raw(env, job_id) == expected
|
||
assert normalizer.calls == [expected]
|
||
assert env.events.index(("s3_put", f"raw/{job_id}.json")) < env.events.index(("publish", f"result:{job_id}:1"))
|
||
|
||
|
||
async def test_one_erroneous_file_does_not_fail_the_job(make_env: EnvFactory) -> None:
|
||
"""Сценарий «Ошибка одного из документов»: `NOT_IMPLEMENTED`, ровно одна строка warnings про `_ofr`."""
|
||
env = await make_env()
|
||
request = env.request()
|
||
job_id = request["job_id"]
|
||
env.hotfolder.poll_script = [
|
||
[
|
||
AnswerFile(f"{job_id}.json", _json(SUCCESS_ANSWER)),
|
||
AnswerFile(f"{job_id}_ofr.json", _json(INVALID_DOCUMENT_ANSWER)),
|
||
]
|
||
]
|
||
body = await env.run(request)
|
||
assert_error(body, ErrorCode.NOT_IMPLEMENTED, raw=True)
|
||
assert body["error"]["message"] == NOT_IMPLEMENTED_MESSAGE
|
||
assert body["metrics"] == {"warnings": [f"{job_id}_ofr.json: INVALID_DOCUMENT: {REAL_MESSAGE}"]}
|
||
|
||
|
||
async def test_parse_error_of_one_file_goes_to_warnings(make_env: EnvFactory) -> None:
|
||
env = await make_env()
|
||
request = env.request()
|
||
job_id = request["job_id"]
|
||
env.hotfolder.poll_script = [
|
||
[
|
||
AnswerFile(f"{job_id}.json", _json(SUCCESS_ANSWER)),
|
||
AnswerFile(f"{job_id}_bad.json", b"{not json"),
|
||
AnswerFile(f"{job_id}_list.json", b"[1, 2]"),
|
||
]
|
||
]
|
||
body = await env.run(request)
|
||
assert_error(body, ErrorCode.NOT_IMPLEMENTED, raw=True)
|
||
bad, listed = body["metrics"]["warnings"]
|
||
assert bad.startswith(f"{job_id}_bad.json: INVALID_FORMAT: ")
|
||
assert listed == f"{job_id}_list.json: INVALID_FORMAT: {NOT_OBJECT_MESSAGE}"
|
||
files = _raw(env, job_id)["files"]
|
||
assert [f["name"] for f in files] == [f"{job_id}.json", f"{job_id}_bad.json", f"{job_id}_list.json"]
|
||
assert files[1]["response"] is None and files[1]["parse_error"]
|
||
assert files[2] == {"name": f"{job_id}_list.json", "response": [1, 2], "parse_error": None}
|
||
|
||
|
||
async def test_all_files_erroneous_take_code_of_first(make_env: EnvFactory) -> None:
|
||
"""Сценарий «Все документы ошибочны»: код первого по имени, две строки warnings, не-JSON — INVALID_FORMAT."""
|
||
env = await make_env()
|
||
request = env.request()
|
||
job_id = request["job_id"]
|
||
env.hotfolder.poll_script = [
|
||
[AnswerFile(f"{job_id}.json", _json(INVALID_DOCUMENT_ANSWER)), AnswerFile(f"{job_id}_ofr.json", b"garbage")]
|
||
]
|
||
body = await env.run(request)
|
||
assert_error(body, ErrorCode.INVALID_DOCUMENT, raw=True)
|
||
assert body["error"]["message"] == REAL_MESSAGE
|
||
first, second = body["metrics"]["warnings"]
|
||
assert first == f"{job_id}.json: INVALID_DOCUMENT: {REAL_MESSAGE}"
|
||
assert second.startswith(f"{job_id}_ofr.json: INVALID_FORMAT: ")
|
||
|
||
|
||
async def test_unknown_code_is_unknown_error_with_original_in_warnings(make_env: EnvFactory) -> None:
|
||
env = await make_env()
|
||
request = env.request()
|
||
job_id = request["job_id"]
|
||
answer = {"file_uid": job_id, "error": {"code": "SOME_NEW_CODE", "message": "новое"}, "data_ocr": {}}
|
||
env.hotfolder.poll_script = [[AnswerFile(f"{job_id}.json", _json(answer))]]
|
||
body = await env.run(request)
|
||
assert_error(body, ErrorCode.UNKNOWN_ERROR, raw=True)
|
||
assert "SOME_NEW_CODE" in body["error"]["message"]
|
||
assert body["metrics"] == {"warnings": [f"{job_id}.json: SOME_NEW_CODE: новое"]}
|
||
|
||
|
||
async def _unserialisable_depth() -> bytes:
|
||
"""JSON вложенности, который разбирается, но после обёртки склейки не сериализуется (`RecursionError`).
|
||
|
||
Глубина подбирается в потоке тем же способом, каким воркер вызывает склейку и сериализацию (`to_thread`): окно
|
||
~2 уровня зависит от глубины стека потока и версии интерпретатора.
|
||
"""
|
||
for depth in range(900, 1000):
|
||
data = ("[" * depth + "]" * depth).encode()
|
||
merged = await asyncio.to_thread(merge_answers, "probe", [AnswerFile("p.json", data)])
|
||
if merged["files"][0]["parse_error"] is not None:
|
||
continue
|
||
try:
|
||
await asyncio.to_thread(dump_merged, merged)
|
||
except RecursionError:
|
||
return data
|
||
raise AssertionError("не найдена глубина: разбирается, но не сериализуется")
|
||
|
||
|
||
async def test_unserialisable_answer_is_unknown_error_without_raw_and_worker_survives(make_env: EnvFactory) -> None:
|
||
"""M-2 (T3.1): `RecursionError` при сериализации склейки — как сбой записи сырого ответа (`UNKNOWN_ERROR`, без
|
||
`raw_s3_key`), а не непредвиденная ошибка; воркер продолжает работать."""
|
||
env = await make_env()
|
||
request = env.request()
|
||
job_id = request["job_id"]
|
||
env.hotfolder.poll_script = [[AnswerFile(f"{job_id}.json", await _unserialisable_depth())]]
|
||
body = await env.run(request)
|
||
assert_error(body, ErrorCode.UNKNOWN_ERROR, raw=False)
|
||
assert "сырой ответ ИС не сохранён" in body["error"]["message"]
|
||
assert (BUCKET, f"raw/{job_id}.json") not in env.storage.objects
|
||
async with emulator(env): # воркер жив: следующее задание обрабатывается штатно
|
||
assert_error(await env.run(env.request()), ErrorCode.NOT_IMPLEMENTED, raw=True)
|
||
|
||
|
||
@pytest.mark.fixtures
|
||
async def test_real_aep_6_success_is_not_implemented_without_warnings(make_env: EnvFactory) -> None:
|
||
"""Регрессия: реальный успешный ответ `aep_6.json` (`error = {"code": "", "message": ""}`) — не ошибка ИС."""
|
||
data = _real_fixture("aep_6.json")
|
||
env = await make_env()
|
||
request = env.request(original_name="aep_6.pdf")
|
||
job_id = request["job_id"]
|
||
env.hotfolder.poll_script = [[AnswerFile(f"{job_id}.json", data)]]
|
||
body = await env.run(request)
|
||
assert_error(body, ErrorCode.NOT_IMPLEMENTED, raw=True)
|
||
assert body["error"]["message"] == NOT_IMPLEMENTED_MESSAGE
|
||
assert "metrics" not in body
|
||
assert _raw(env, job_id) == merged_raw(job_id, (f"{job_id}.json", json.loads(data), None))
|
||
|
||
|
||
@pytest.mark.fixtures
|
||
async def test_real_aep_3_with_ofr_is_not_implemented_with_one_warning(make_env: EnvFactory) -> None:
|
||
main, ofr = _real_fixture("aep_3.json"), _real_fixture("aep_3_ofr.json")
|
||
env = await make_env()
|
||
request = env.request(original_name="aep_3.pdf")
|
||
job_id = request["job_id"]
|
||
env.hotfolder.poll_script = [[AnswerFile(f"{job_id}.json", main), AnswerFile(f"{job_id}_ofr.json", ofr)]]
|
||
body = await env.run(request)
|
||
assert_error(body, ErrorCode.NOT_IMPLEMENTED, raw=True)
|
||
assert body["metrics"] == {"warnings": [f"{job_id}_ofr.json: INVALID_DOCUMENT: {REAL_MESSAGE}"]}
|
||
assert _raw(env, job_id) == merged_raw(
|
||
job_id, (f"{job_id}.json", json.loads(main), None), (f"{job_id}_ofr.json", json.loads(ofr), None)
|
||
)
|
||
|
||
|
||
@pytest.mark.parametrize("status", [ResultStatus.DONE, ResultStatus.NEEDS_CHECK])
|
||
async def test_outcome_warnings_join_metrics_in_envelope_and_result(make_env: EnvFactory, status: ResultStatus) -> None:
|
||
document = StatementDocument.model_validate(STATEMENT_EXAMPLE["documents"][0])
|
||
outcome = NormalizeOutcome(
|
||
status=status, documents=[document], metrics=Metrics(pages=2, warnings=["свои"]), warnings=["a.json: X: y"]
|
||
)
|
||
env = await make_env(normalizer=StubNormalizer(outcome))
|
||
request = env.request()
|
||
async with emulator(env):
|
||
body = await env.run(request)
|
||
assert body["metrics"] == {"pages": 2, "warnings": ["свои", "a.json: X: y"]}
|
||
result = json.loads(env.storage.objects[(BUCKET, f"results/{request['file_id']}/{request['job_id']}/result.json")])
|
||
RESULT_VALIDATOR.validate(result)
|
||
assert result["metrics"] == body["metrics"]
|
||
|
||
|
||
async def test_error_outcome_without_warnings_keeps_its_metrics(make_env: EnvFactory) -> None:
|
||
outcome = NormalizeOutcome(
|
||
status=ResultStatus.ERROR, error=MessageError(code="UNREADABLE", message="x"), metrics=Metrics(pages=1)
|
||
)
|
||
env = await make_env(normalizer=StubNormalizer(outcome))
|
||
async with emulator(env):
|
||
body = await env.run(env.request())
|
||
assert body["metrics"] == {"pages": 1}
|
||
|
||
|
||
async def test_huge_warnings_are_bounded(make_env: EnvFactory) -> None:
|
||
"""R-30: число строк warnings и длина каждой ограничены — конверт остаётся меньше лимита NATS."""
|
||
lines = [f"f{i}.json: INVALID_DOCUMENT: " + "я" * 50_000 for i in range(MAX_WARNINGS * 3)]
|
||
outcome = NormalizeOutcome(
|
||
status=ResultStatus.ERROR, error=MessageError(code="INVALID_DOCUMENT", message="m"), warnings=lines
|
||
)
|
||
env = await make_env(normalizer=StubNormalizer(outcome))
|
||
async with emulator(env):
|
||
body = await env.run(env.request())
|
||
warnings = body["metrics"]["warnings"]
|
||
assert len(warnings) == MAX_WARNINGS
|
||
assert all(len(line) <= MAX_WARNING_CHARS for line in warnings)
|
||
assert warnings[0].startswith("f0.json: INVALID_DOCUMENT: я")
|
||
assert warnings[-2].startswith(f"f{MAX_WARNINGS - 2}.json")
|
||
assert str(MAX_WARNINGS * 3 - (MAX_WARNINGS - 1)) in warnings[-1]
|
||
assert f"raw/{body['job_id']}.json" in warnings[-1]
|
||
assert all(len(payload) < 900_000 for _, payload, _ in env.client.published)
|
||
|
||
|
||
async def test_answer_ready_only_after_settle(make_env: EnvFactory) -> None:
|
||
"""Сценарий «Ответ из одного файла» с настоящим settle: ответ принимается после стабилизации набора."""
|
||
env = await make_env(SCAN_SETTLE_SECONDS=0.05)
|
||
async with emulator(env):
|
||
body = await env.run(env.request())
|
||
assert_error(body, ErrorCode.NOT_IMPLEMENTED, raw=True)
|
||
assert env.hotfolder.polls >= 2 # первый опрос, увидевший набор, его ещё не отдаёт
|
||
|
||
|
||
async def test_set_not_settled_before_deadline_is_timeout(make_env: EnvFactory) -> None:
|
||
"""Сценарий «Набор не успел настояться до дедлайна»: `*.json` есть, но не настоялся — `TIMEOUT`, без raw."""
|
||
env = await make_env(SCAN_SETTLE_SECONDS=30, SCAN_TIMEOUT_SECONDS=0.2)
|
||
request = env.request()
|
||
async with emulator(env):
|
||
body = await env.run(request)
|
||
assert_error(body, ErrorCode.TIMEOUT, raw=False)
|
||
assert env.local.answer_path(request["job_id"]).exists()
|
||
assert (BUCKET, f"raw/{request['job_id']}.json") not in env.storage.objects
|
||
|
||
|
||
# --- таймаут ------------------------------------------------------------------------------------------------------
|
||
async def test_timeout_by_scan_timeout_keeps_file_and_never_resends(make_env: EnvFactory) -> None:
|
||
env = await make_env(SCAN_TIMEOUT_SECONDS=0.2)
|
||
request = env.request(original_name="aep_5.pdf", deadline_s=60)
|
||
started = asyncio.get_running_loop().time()
|
||
body = await env.run(request)
|
||
elapsed = asyncio.get_running_loop().time() - started
|
||
assert_error(body, ErrorCode.TIMEOUT, raw=False)
|
||
assert 0.15 < elapsed < 2
|
||
assert env.inbox() == [f"{request['job_id']}.pdf"] # входящий файл остаётся
|
||
assert env.hotfolder.puts == [f"{request['job_id']}.pdf"] # повторной отправки нет
|
||
assert env.hotfolder.polls >= 5
|
||
|
||
|
||
async def test_timeout_by_reply_deadline(make_env: EnvFactory) -> None:
|
||
env = await make_env(SCAN_TIMEOUT_SECONDS=300)
|
||
request = env.request(deadline_s=0.2)
|
||
started = asyncio.get_running_loop().time()
|
||
body = await env.run(request)
|
||
assert_error(body, ErrorCode.TIMEOUT, raw=False)
|
||
assert asyncio.get_running_loop().time() - started < 2
|
||
assert len(env.hotfolder.puts) == 1
|
||
|
||
|
||
async def test_deadline_passed_before_sending_gives_timeout_without_file(make_env: EnvFactory) -> None:
|
||
env = await make_env()
|
||
request = env.request(deadline_s=-1)
|
||
body = await env.run(request)
|
||
assert_error(body, ErrorCode.TIMEOUT, raw=False)
|
||
assert env.hotfolder.puts == []
|
||
assert env.storage.get_calls == []
|
||
assert env.acks(request["job_id"]) == [f"ack:{request['job_id']}:1"]
|
||
|
||
|
||
async def test_deadline_uses_injected_clock(make_env: EnvFactory) -> None:
|
||
"""Дедлайн считается по инжектируемым часам (R-20): часы «убежали» на час — таймаут на первом же опросе."""
|
||
shift = {"value": timedelta(0)}
|
||
|
||
def clock() -> datetime:
|
||
return datetime.now(UTC) + shift["value"]
|
||
|
||
env = await make_env(clock=clock, SCAN_TIMEOUT_SECONDS=60)
|
||
request = env.request(deadline_s=120)
|
||
await env.deliver(request)
|
||
await eventually(lambda: len(env.hotfolder.puts) == 1)
|
||
shift["value"] = timedelta(hours=1)
|
||
await asyncio.wait_for(env.worker.wait_idle(), 2)
|
||
[(_, body)] = env.results(request["job_id"])
|
||
assert_error(body, ErrorCode.TIMEOUT, raw=False)
|
||
|
||
|
||
# --- ошибки транспорта --------------------------------------------------------------------------------------------
|
||
async def test_missing_hotfolder_root_is_recognition_failed(make_env: EnvFactory) -> None:
|
||
env = await make_env(root_exists=False)
|
||
request = env.request()
|
||
body = await env.run(request)
|
||
assert_error(body, ErrorCode.RECOGNITION_FAILED, raw=False)
|
||
assert "горячая папка недоступна" in body["error"]["message"]
|
||
assert not env.root.exists() # корень не создаётся (R-5)
|
||
assert env.hotfolder.healthy() is False
|
||
|
||
|
||
async def test_unusable_original_name_is_recognition_failed(make_env: EnvFactory) -> None:
|
||
env = await make_env()
|
||
body = await env.run(env.request(original_name=".."))
|
||
assert_error(body, ErrorCode.RECOGNITION_FAILED, raw=False)
|
||
assert "расширени" in body["error"]["message"]
|
||
assert env.hotfolder.puts == []
|
||
|
||
|
||
async def test_transient_poll_errors_do_not_fail_the_job(make_env: EnvFactory) -> None:
|
||
env = await make_env()
|
||
env.hotfolder.poll_errors = [OSError(5, "STATUS_SHARING_VIOLATION"), ConnectionResetError("smb")]
|
||
async with emulator(env):
|
||
body = await env.run(env.request())
|
||
assert_error(body, ErrorCode.NOT_IMPLEMENTED, raw=True)
|
||
|
||
|
||
async def test_poll_failing_until_deadline_is_recognition_failed(make_env: EnvFactory) -> None:
|
||
env = await make_env(SCAN_TIMEOUT_SECONDS=0.1)
|
||
env.hotfolder.poll_always = OSError(13, "STATUS_ACCESS_DENIED")
|
||
body = await env.run(env.request())
|
||
assert_error(body, ErrorCode.RECOGNITION_FAILED, raw=False)
|
||
assert "горячая папка недоступна" in body["error"]["message"]
|
||
|
||
|
||
async def test_poll_failing_sometimes_then_silence_is_timeout(make_env: EnvFactory) -> None:
|
||
env = await make_env(SCAN_TIMEOUT_SECONDS=0.1)
|
||
env.hotfolder.poll_errors = [OSError(13, "STATUS_ACCESS_DENIED")]
|
||
body = await env.run(env.request())
|
||
assert_error(body, ErrorCode.TIMEOUT, raw=False)
|
||
|
||
|
||
async def test_poll_error_alternating_with_empty_polls_is_timeout_with_error_description(
|
||
make_env: EnvFactory,
|
||
) -> None:
|
||
"""R-39 (design #80): при settle > 0 устойчивая ошибка чтения чередуется с пустыми опросами (готовность сбрасывает
|
||
состояние), `failures != polls` — код остаётся `TIMEOUT`, но текст несёт описание последней ошибки опроса."""
|
||
env = await make_env(SCAN_TIMEOUT_SECONDS=0.2)
|
||
env.hotfolder.poll_script = [None, PermissionError(13, "STATUS_ACCESS_DENIED"), *([None] * 2)] * 50
|
||
body = await env.run(env.request())
|
||
assert_error(body, ErrorCode.TIMEOUT, raw=False)
|
||
assert "ИС Сканирование не ответила" in body["error"]["message"]
|
||
assert "STATUS_ACCESS_DENIED" in body["error"]["message"]
|
||
|
||
|
||
async def test_missing_s3_object_is_file_read_error(make_env: EnvFactory) -> None:
|
||
env = await make_env()
|
||
request = env.request(data=None)
|
||
body = await env.run(request)
|
||
assert_error(body, ErrorCode.FILE_READ_ERROR, raw=False)
|
||
assert request["s3"]["key"] in body["error"]["message"]
|
||
assert env.hotfolder.puts == []
|
||
assert env.acks(request["job_id"]) == [f"ack:{request['job_id']}:1"]
|
||
|
||
|
||
async def test_foreign_bucket_is_file_read_error_without_reading(make_env: EnvFactory) -> None:
|
||
env = await make_env()
|
||
request = env.request(bucket="other-bucket")
|
||
body = await env.run(request)
|
||
assert_error(body, ErrorCode.FILE_READ_ERROR, raw=False)
|
||
assert "other-bucket" in body["error"]["message"]
|
||
assert env.storage.get_calls == []
|
||
assert env.acks(request["job_id"]) == [f"ack:{request['job_id']}:1"]
|
||
|
||
|
||
async def test_exception_outside_steps_is_unknown_error(make_env: EnvFactory) -> None:
|
||
class BadOutcomeNormalizer:
|
||
def normalize(self, raw: object) -> NormalizeOutcome:
|
||
return "не исход" # type: ignore[return-value]
|
||
|
||
env = await make_env(normalizer=BadOutcomeNormalizer())
|
||
async with emulator(env):
|
||
body = await env.run(env.request())
|
||
assert_error(body, ErrorCode.UNKNOWN_ERROR, raw=True)
|
||
|
||
|
||
# --- SCHEMA_MISMATCH без ack (R-16) -------------------------------------------------------------------------------
|
||
def _mutations() -> list[tuple[str, Callable[[dict[str, Any]], dict[str, Any]]]]:
|
||
return [
|
||
("major_2", lambda r: {**r, "schema_version": "2.0"}),
|
||
("pipeline_tabular", lambda r: {**r, "pipeline": "tabular", "file_type": "XLSX"}),
|
||
("no_s3", lambda r: {k: v for k, v in r.items() if k != "s3"}),
|
||
("bad_file_type", lambda r: {**r, "file_type": "DOCX"}),
|
||
("naive_deadline", lambda r: {**r, "reply_deadline": "2026-10-03T12:45:00"}),
|
||
("attempt_zero", lambda r: {**r, "attempt": 0}),
|
||
("empty_name", lambda r: {**r, "original_name": ""}),
|
||
]
|
||
|
||
|
||
@pytest.mark.parametrize(("case", "mutate"), _mutations(), ids=[case for case, _ in _mutations()])
|
||
async def test_invalid_request_is_schema_mismatch_without_ack(
|
||
make_env: EnvFactory, case: str, mutate: Callable[[dict[str, Any]], dict[str, Any]]
|
||
) -> None:
|
||
env = await make_env()
|
||
request = env.request()
|
||
body = mutate(request)
|
||
await env.deliver(body)
|
||
await asyncio.wait_for(env.worker.wait_idle(), 5)
|
||
job_id = request["job_id"]
|
||
published = env.published()
|
||
assert env.acks(job_id) == []
|
||
attempt = body["attempt"] if body["attempt"] >= 1 else 1
|
||
assert [mid for mid, _ in published] == [f"result:{job_id}:{attempt}"]
|
||
assert_error(published[0][1], ErrorCode.SCHEMA_MISMATCH, raw=False)
|
||
assert published[0][1]["file_id"] == request["file_id"]
|
||
assert env.storage.get_calls == []
|
||
assert env.hotfolder.puts == []
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
"data",
|
||
[
|
||
b"not json at all",
|
||
b"\xff\xfe\x00broken utf-8",
|
||
b"[1, 2, 3]",
|
||
b'"string"',
|
||
b"[" * 100_000, # глубокая вложенность: RecursionError парсера не роняет воркер
|
||
json.dumps({"schema_version": "1.0", "job_id": "not-a-uuid", "file_id": str(uuid.uuid4())}).encode(),
|
||
json.dumps({"schema_version": "1.0", "job_id": str(uuid.uuid4())}).encode(),
|
||
json.dumps({"job_id": 5, "file_id": None}).encode(),
|
||
],
|
||
ids=["text", "bad_utf8", "array", "string", "deep", "bad_job_id", "no_file_id", "wrong_types"],
|
||
)
|
||
async def test_unparsable_request_without_ids_is_dropped(
|
||
make_env: EnvFactory, data: bytes, log_lines: list[str]
|
||
) -> None:
|
||
env = await make_env()
|
||
await env.deliver(data)
|
||
await asyncio.wait_for(env.worker.wait_idle(), 5)
|
||
assert env.client.published == []
|
||
assert env.hotfolder.puts == []
|
||
assert any("SCHEMA_MISMATCH" in line for line in log_lines)
|
||
|
||
|
||
@pytest.mark.parametrize(("attempt", "expected"), [(3, 3), (True, 1), ("2", 1), (-1, 1), (2**40, 1)])
|
||
async def test_schema_mismatch_attempt_from_body(make_env: EnvFactory, attempt: Any, expected: int) -> None:
|
||
env = await make_env()
|
||
request = env.request()
|
||
await env.deliver({**request, "schema_version": "2.0", "attempt": attempt})
|
||
await asyncio.wait_for(env.worker.wait_idle(), 5)
|
||
assert [mid for mid, _ in env.published()] == [f"result:{request['job_id']}:{expected}"]
|
||
|
||
|
||
async def test_schema_mismatch_does_not_register_job_id(make_env: EnvFactory) -> None:
|
||
"""Исправленный запрос с тем же job_id после SCHEMA_MISMATCH обрабатывается (это не повтор)."""
|
||
env = await make_env()
|
||
request = env.request()
|
||
await env.deliver({**request, "schema_version": "2.0"})
|
||
async with emulator(env):
|
||
await env.deliver(request)
|
||
await asyncio.wait_for(env.worker.wait_idle(), 5)
|
||
assert len(env.hotfolder.puts) == 1
|
||
|
||
|
||
async def test_schema_mismatch_for_active_or_done_job_id_is_not_published(
|
||
make_env: EnvFactory, log_lines: list[str]
|
||
) -> None:
|
||
"""T12.1: битый запрос с job_id задания в работе или уже обработанного не публикует `result:<job>:<attempt>`.
|
||
|
||
Иначе ядро (дедуп по `result_msg_id`) отбросило бы настоящий итог с тем же Nats-Msg-Id. UUID в теле может прийти
|
||
в верхнем регистре — ключ дедупликации нормализуется.
|
||
"""
|
||
env = await make_env()
|
||
request = env.request()
|
||
job_id = request["job_id"]
|
||
broken = {**request, "schema_version": "2.0"}
|
||
await env.deliver(request)
|
||
await eventually(lambda: len(env.hotfolder.puts) == 1) # задание ждёт ответа ИС
|
||
await env.deliver(broken)
|
||
await env.deliver({**broken, "job_id": job_id.upper()})
|
||
async with emulator(env):
|
||
await asyncio.wait_for(env.worker.wait_idle(), 5)
|
||
await env.deliver(broken) # задание уже обработано
|
||
await env.deliver({**broken, "job_id": job_id.upper()})
|
||
await asyncio.wait_for(env.worker.wait_idle(), 5)
|
||
published = env.published()
|
||
assert [mid for mid, _ in published] == [f"ack:{job_id}:1", f"result:{job_id}:1"]
|
||
assert_error(published[1][1], ErrorCode.NOT_IMPLEMENTED, raw=True)
|
||
assert sum("SCHEMA_MISMATCH для задания в работе/обработанного" in line for line in log_lines) == 4
|
||
|
||
|
||
# --- публикация, размер и кодировка -------------------------------------------------------------------------------
|
||
async def test_publish_failure_is_logged_and_worker_keeps_working(make_env: EnvFactory, log_lines: list[str]) -> None:
|
||
env = await make_env()
|
||
env.client.fail = True
|
||
first = env.request()
|
||
async with emulator(env):
|
||
await env.deliver(first)
|
||
await asyncio.wait_for(env.worker.wait_idle(), 5)
|
||
assert env.client.published == []
|
||
assert any(line.startswith("ERROR|") and first["job_id"] in line for line in log_lines)
|
||
env.client.fail = False
|
||
second = env.request()
|
||
body = await env.run(second)
|
||
assert_error(body, ErrorCode.NOT_IMPLEMENTED, raw=True)
|
||
assert len(env.hotfolder.puts) == 2
|
||
|
||
|
||
async def test_huge_error_message_is_truncated(make_env: EnvFactory) -> None:
|
||
huge = "я" * 2_000_000
|
||
normalizer = StubNormalizer(
|
||
NormalizeOutcome(status=ResultStatus.ERROR, error=MessageError(code="INVALID_DOCUMENT", message=huge))
|
||
)
|
||
env = await make_env(normalizer=normalizer)
|
||
async with emulator(env):
|
||
body = await env.run(env.request())
|
||
assert_error(body, ErrorCode.INVALID_DOCUMENT, raw=True)
|
||
message = body["error"]["message"]
|
||
assert len(message) == MAX_ERROR_MESSAGE_CHARS
|
||
assert message.endswith("…")
|
||
assert all(len(payload) < 900_000 for _, payload, _ in env.client.published)
|
||
|
||
|
||
async def test_lone_surrogate_in_is_error_message_does_not_break_publishing(make_env: EnvFactory) -> None:
|
||
env = await make_env()
|
||
request = env.request()
|
||
job_id = request["job_id"]
|
||
await env.deliver(request)
|
||
await eventually(lambda: len(env.hotfolder.puts) == 1)
|
||
env.answer(job_id, b'{"file_uid": "x", "error": {"code": "INVALID_DOCUMENT", "message": "bad \\ud800 tail"}}')
|
||
await asyncio.wait_for(env.worker.wait_idle(), 5)
|
||
[(_, body)] = env.results(job_id)
|
||
assert_error(body, ErrorCode.INVALID_DOCUMENT, raw=True)
|
||
assert body["error"]["message"].startswith("bad ")
|
||
assert body["error"]["message"].endswith(" tail")
|
||
[warning] = body["metrics"]["warnings"]
|
||
assert warning.startswith(f"{job_id}.json: INVALID_DOCUMENT: bad ")
|
||
assert warning.endswith(" tail")
|
||
# сырой ответ записан валидным JSON, суррогат сохранён экранированным
|
||
assert _raw(env, job_id)["files"][0]["response"]["error"]["message"] == "bad \ud800 tail"
|
||
|
||
|
||
async def test_lone_surrogate_in_original_name_is_schema_mismatch(make_env: EnvFactory) -> None:
|
||
"""Одиночный суррогат в `original_name` схема запроса не пропускает (pydantic `string_unicode`): SCHEMA_MISMATCH
|
||
без ack и без файла в горячей папке; сам конверт публикуется."""
|
||
env = await make_env()
|
||
request = env.request(original_name="скан\udc80.pdf")
|
||
data = json.dumps(request, ensure_ascii=True).encode("ascii") # в JSON суррогат приходит экранированным
|
||
await env.deliver(data)
|
||
await asyncio.wait_for(env.worker.wait_idle(), 5)
|
||
job_id = request["job_id"]
|
||
[(_, body)] = env.results(job_id)
|
||
assert_error(body, ErrorCode.SCHEMA_MISMATCH, raw=False)
|
||
assert env.acks(job_id) == []
|
||
assert env.hotfolder.puts == []
|
||
|
||
|
||
# --- остановка и прямой вызов handle (параллелизм — test_worker_concurrency.py) ---------------------------------------
|
||
async def test_stop_cancels_jobs_and_rejects_new_requests(make_env: EnvFactory) -> None:
|
||
env = await make_env()
|
||
request = env.request()
|
||
await env.deliver(request)
|
||
await eventually(lambda: len(env.hotfolder.puts) == 1)
|
||
await asyncio.wait_for(env.worker.stop(), 1)
|
||
assert env.worker.active_jobs == 0
|
||
assert env.results(request["job_id"]) == []
|
||
await env.deliver(env.request())
|
||
await asyncio.wait_for(env.worker.wait_idle(), 1)
|
||
assert len(env.client.published) == 1 # только ack первого запроса
|
||
await env.worker.stop() # повторная остановка безопасна
|
||
|
||
|
||
async def test_service_info_and_handle_returns_published_message(make_env: EnvFactory) -> None:
|
||
from src.contract import RecognitionRequest # noqa: PLC0415
|
||
|
||
env = await make_env()
|
||
request = RecognitionRequest.model_validate(env.request(data=None))
|
||
message = await env.worker.handle(request)
|
||
assert message.service.name == "scan-gateway"
|
||
assert message.service.version == env.settings.VERSION
|
||
assert message.error is not None and message.error.code is ErrorCode.FILE_READ_ERROR
|
||
[(_, body)] = env.results(str(request.job_id))
|
||
assert body == message.to_wire()
|
||
|
||
|
||
# --- Срез 3a: имя <job_id>.<ext>, очистка под флагом, OcrNormalizer (D-1, D-2, решения №2, №3, №26) -----------------
|
||
class TestSlice3a:
|
||
async def test_no_extension_is_recognition_failed(self, make_env: EnvFactory) -> None:
|
||
env = await make_env()
|
||
body = env.request(original_name="scan")
|
||
result = await env.run(body)
|
||
assert result["status"] == "error" and result["error"]["code"] == "RECOGNITION_FAILED"
|
||
assert "расширени" in result["error"]["message"]
|
||
assert env.inbox() == []
|
||
|
||
async def test_cleanup_after_answer_when_enabled(self, make_env: EnvFactory) -> None:
|
||
env = await make_env(SCAN_CLEANUP=True)
|
||
body = env.request()
|
||
async with emulator(env):
|
||
result = await env.run(body)
|
||
assert result["status"] == "error" and result["error"]["code"] == "NOT_IMPLEMENTED" # NullNormalizer
|
||
assert env.inbox() == [] # и <job>.pdf, и <job>.pdf.done удалены
|
||
assert not env.local.answer_dir(body["job_id"]).exists()
|
||
assert ("cleanup", f"{body['job_id']}.pdf") in env.events
|
||
|
||
async def test_no_cleanup_when_disabled(self, make_env: EnvFactory) -> None:
|
||
env = await make_env(SCAN_CLEANUP=False)
|
||
body = env.request()
|
||
async with emulator(env):
|
||
await env.run(body)
|
||
assert env.inbox() == [f"{body['job_id']}.pdf.done"]
|
||
assert env.local.answer_dir(body["job_id"]).is_dir()
|
||
|
||
async def test_no_cleanup_on_timeout(self, make_env: EnvFactory) -> None:
|
||
env = await make_env(SCAN_CLEANUP=True)
|
||
body = env.request(deadline_s=0.2)
|
||
result = await env.run(body)
|
||
assert result["error"]["code"] == "TIMEOUT"
|
||
assert env.inbox() == [f"{body['job_id']}.pdf"]
|
||
assert ("cleanup", f"{body['job_id']}.pdf") not in env.events
|
||
|
||
async def test_cleanup_error_does_not_change_result(
|
||
self, make_env: EnvFactory, monkeypatch: pytest.MonkeyPatch, log_lines: list[str]
|
||
) -> None:
|
||
env = await make_env(SCAN_CLEANUP=True)
|
||
cleanup_calls: list[tuple[str, str]] = []
|
||
|
||
def deny(job_id: str, name: str) -> bool:
|
||
cleanup_calls.append((job_id, name))
|
||
raise PermissionError("нет прав")
|
||
|
||
monkeypatch.setattr(env.hotfolder.inner, "cleanup", deny)
|
||
body = env.request()
|
||
async with emulator(env):
|
||
result = await env.run(body)
|
||
job_id = body["job_id"]
|
||
# очистка действительно вызывалась (а не пропущена), и её ошибка не изменила исход задания
|
||
assert cleanup_calls == [(job_id, f"{job_id}.pdf")]
|
||
assert ("cleanup", f"{job_id}.pdf") in env.events
|
||
assert result["status"] == "error" and result["error"]["code"] == "NOT_IMPLEMENTED"
|
||
assert "warnings" not in result.get("metrics", {}) # ошибка очистки в конверт не попадает
|
||
# ошибка очистки — ровно одна строка лога уровня WARNING
|
||
warnings = [line for line in log_lines if line.startswith("WARNING|")]
|
||
assert len(warnings) == 1
|
||
assert "Очистка горячей папки не удалась" in warnings[0] and job_id in warnings[0] and "нет прав" in warnings[0]
|
||
|
||
async def test_existing_input_waits_for_answer(self, make_env: EnvFactory) -> None:
|
||
"""Входящий файл уже лежит (повтор задания), ответа нет — TIMEOUT; файл перезаписан объектом S3, как в старом
|
||
АРМ (change hotfolder-direct-write, решение владельца 09.10.2026)."""
|
||
env = await make_env()
|
||
body = env.request(deadline_s=0.3)
|
||
env.local.put(f"{body['job_id']}.pdf", b"already there")
|
||
result = await env.run(body)
|
||
assert result["error"]["code"] == "TIMEOUT"
|
||
assert (env.local.input_dir / f"{body['job_id']}.pdf").read_bytes() == PDF_BYTES
|
||
assert env.inbox() == [f"{body['job_id']}.pdf"]
|
||
|
||
async def test_ocr_normalizer_produces_documents(self, make_env: EnvFactory) -> None:
|
||
env = await make_env(normalizer=OcrNormalizer(load_rules(API_DIR / "rules")))
|
||
body = env.request()
|
||
async with emulator(env):
|
||
result = await env.run(body)
|
||
assert result["status"] == "needs_check" # SUCCESS_ANSWER: confidence < 100 в шапке
|
||
key = result["result_s3"]["key"]
|
||
stored = json.loads(env.storage.objects[(BUCKET, key)])
|
||
assert stored["documents"][0]["document_type"] == "BALANCE"
|
||
assert stored["documents"][0]["lines"][0]["line_code"] == "1600"
|
||
assert result["raw_s3_key"] == f"raw/{body['job_id']}.json"
|