2a7ceca334
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012vJkmBGKKqMoEMzvsXG3y2
392 lines
19 KiB
Python
392 lines
19 KiB
Python
"""Нормализатор ответа ИС Сканирование (spec image-recognition-gateway «Результат по контракту», D-5, R-12, R-25).
|
||
|
||
Вход `NullNormalizer` — склеенный ответ `{"job_id", "files": [{"name", "response", "parse_error"}]}` (D-4).
|
||
Успех файла — `error.code == ""` (реальный успешный ответ несёт `error = {"code": "", "message": ""}`, а не `null`);
|
||
ошибочный файл — непустой `error.code`, `parse_error` или корень ответа не объект. Сводка: все файлы ошибочны — код
|
||
первого по имени; есть успешный — `NOT_IMPLEMENTED`; ошибочные файлы — строки `warnings`.
|
||
Реальные ответы `../fixtures/ocr/*.json` — маркер `fixtures`.
|
||
Срез 3a: настоящая нормализация — `tests/unit/normalizer/test_ocr_normalizer.py`.
|
||
"""
|
||
|
||
import copy
|
||
import json
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import pytest
|
||
from pydantic import ValidationError
|
||
|
||
from src.contract import ErrorCode, MessageError, Metrics, ResultStatus, StatementDocument
|
||
from src.gateway.hotfolder import AnswerFile
|
||
from src.gateway.merge import merge_answers
|
||
from src.normalizer import NOT_IMPLEMENTED_MESSAGE, NOT_OBJECT_MESSAGE, NormalizeOutcome, Normalizer, NullNormalizer
|
||
|
||
API_DIR = Path(__file__).resolve().parents[2]
|
||
# `<корень scan-gateway>/../fixtures/ocr` (workspace и worktree `.wt/<срез>/`), как у test_emulator.py
|
||
REAL_FIXTURES = API_DIR.parent.parent / "fixtures" / "ocr"
|
||
JOB = "3f1c2b7a-0000-4000-8000-000000000001"
|
||
REAL_MESSAGE = "Не удалось определить тип документа."
|
||
SUCCESS_ERROR = {"code": "", "message": ""}
|
||
|
||
|
||
def _real(name: str) -> dict[str, Any]:
|
||
if not REAL_FIXTURES.is_dir():
|
||
pytest.skip("fixtures недоступен: положите репозиторий fixtures рядом с workspace")
|
||
loaded: dict[str, Any] = json.loads((REAL_FIXTURES / name).read_text(encoding="utf-8"))
|
||
return loaded
|
||
|
||
|
||
def _real_set(*names: str) -> dict[str, Any]:
|
||
"""Склейка реальных ответов под именами каталога ответа: `aep_N.json` — `<job>.json`, `aep_N_<x>.json` —
|
||
`<job>_<x>.json`."""
|
||
if not REAL_FIXTURES.is_dir():
|
||
pytest.skip("fixtures недоступен: положите репозиторий fixtures рядом с workspace")
|
||
files = []
|
||
for name in names:
|
||
suffix = name.removesuffix(".json").partition("_")[2].partition("_")[2]
|
||
target = f"{JOB}_{suffix}.json" if suffix else f"{JOB}.json"
|
||
files.append(AnswerFile(target, (REAL_FIXTURES / name).read_bytes()))
|
||
return dict(merge_answers(JOB, files))
|
||
|
||
|
||
def _file(name: str, response: object = None, parse_error: str | None = None) -> dict[str, Any]:
|
||
return {"name": name, "response": response, "parse_error": parse_error}
|
||
|
||
|
||
def _merged(*files: dict[str, Any]) -> dict[str, Any]:
|
||
return {"job_id": JOB, "files": list(files)}
|
||
|
||
|
||
def _one(response: object) -> dict[str, Any]:
|
||
return _merged(_file(f"{JOB}.json", response))
|
||
|
||
|
||
def _success(**extra: Any) -> dict[str, Any]:
|
||
return {"file_uid": JOB, "error": dict(SUCCESS_ERROR), "data_ocr": {"table_structure": []}, **extra}
|
||
|
||
|
||
def _failure(code: object, message: object = "сообщение ИС") -> dict[str, Any]:
|
||
return {"file_uid": JOB, "error": {"code": code, "message": message}, "data_ocr": {"table_structure": []}}
|
||
|
||
|
||
def _assert_error(outcome: NormalizeOutcome, code: ErrorCode, message: str | None = None) -> None:
|
||
assert outcome.status is ResultStatus.ERROR
|
||
assert outcome.error is not None
|
||
assert outcome.error.code is code
|
||
if message is not None:
|
||
assert outcome.error.message == message
|
||
assert outcome.documents == []
|
||
assert outcome.metrics is None
|
||
|
||
|
||
def _assert_not_implemented(outcome: NormalizeOutcome, warnings: list[str]) -> None:
|
||
_assert_error(outcome, ErrorCode.NOT_IMPLEMENTED, NOT_IMPLEMENTED_MESSAGE)
|
||
assert outcome.warnings == warnings
|
||
|
||
|
||
@pytest.mark.fixtures
|
||
class TestRealFixtures:
|
||
"""Регрессия: реальные успешные ответы (`error = {"code": "", "message": ""}`) — не ошибка."""
|
||
|
||
@pytest.mark.parametrize("name", ["aep_6.json", "aep_3.json"])
|
||
def test_real_success_is_not_an_error(self, name: str) -> None:
|
||
assert _real(name)["error"] == SUCCESS_ERROR
|
||
_assert_not_implemented(NullNormalizer().normalize(_real_set(name)), [])
|
||
|
||
def test_aep_3_with_ofr_gives_not_implemented_and_one_warning(self) -> None:
|
||
outcome = NullNormalizer().normalize(_real_set("aep_3.json", "aep_3_ofr.json"))
|
||
_assert_not_implemented(outcome, [f"{JOB}_ofr.json: INVALID_DOCUMENT: {REAL_MESSAGE}"])
|
||
|
||
@pytest.mark.parametrize("name", ["aep_3_ofr.json", "aep_5.json", "aep_8_form4.json"])
|
||
def test_real_error_alone_is_invalid_document(self, name: str) -> None:
|
||
merged = _real_set(name)
|
||
[entry] = merged["files"]
|
||
outcome = NullNormalizer().normalize(merged)
|
||
_assert_error(outcome, ErrorCode.INVALID_DOCUMENT, REAL_MESSAGE)
|
||
assert outcome.warnings == [f"{entry['name']}: INVALID_DOCUMENT: {REAL_MESSAGE}"]
|
||
|
||
|
||
class TestFileSuccess:
|
||
"""Успех файла — `error.code == ""`; отсутствующий/`null` `error` и `error` без кода ошибкой не считаются."""
|
||
|
||
def test_success_shape_of_real_response(self) -> None:
|
||
_assert_not_implemented(NullNormalizer().normalize(_one(_success())), [])
|
||
|
||
@pytest.mark.parametrize(
|
||
"error",
|
||
[None, {"code": None, "message": "m"}, {"message": "Нет кода"}, {}, "Что-то сломалось", ["a"], 42, 0, ""],
|
||
ids=["null", "code_null", "no_code", "empty_object", "string", "list", "int", "zero", "empty_string"],
|
||
)
|
||
def test_no_error_code_is_success(self, error: object) -> None:
|
||
_assert_not_implemented(NullNormalizer().normalize(_one(_success(error=error))), [])
|
||
|
||
def test_error_key_absent(self) -> None:
|
||
response = _success()
|
||
del response["error"]
|
||
_assert_not_implemented(NullNormalizer().normalize(_one(response)), [])
|
||
|
||
def test_minimal_object(self) -> None:
|
||
_assert_not_implemented(NullNormalizer().normalize(_one({})), [])
|
||
|
||
|
||
class TestFileError:
|
||
def test_known_code_and_message_verbatim(self) -> None:
|
||
outcome = NullNormalizer().normalize(_one(_failure("FILE_READ_ERROR", "Файл повреждён: стр. 3 «пусто»")))
|
||
_assert_error(outcome, ErrorCode.FILE_READ_ERROR, "Файл повреждён: стр. 3 «пусто»")
|
||
assert outcome.warnings == [f"{JOB}.json: FILE_READ_ERROR: Файл повреждён: стр. 3 «пусто»"]
|
||
|
||
@pytest.mark.parametrize("code", [c.value for c in ErrorCode])
|
||
def test_every_contract_code_passes(self, code: str) -> None:
|
||
_assert_error(NullNormalizer().normalize(_one(_failure(code))), ErrorCode(code), "сообщение ИС")
|
||
|
||
def test_unknown_code_is_unknown_error_keeping_original(self) -> None:
|
||
outcome = NullNormalizer().normalize(_one(_failure("SOME_NEW_CODE", "Превышена квота")))
|
||
_assert_error(outcome, ErrorCode.UNKNOWN_ERROR)
|
||
assert outcome.error is not None
|
||
assert "SOME_NEW_CODE" in outcome.error.message
|
||
assert "Превышена квота" in outcome.error.message
|
||
assert outcome.warnings == [f"{JOB}.json: SOME_NEW_CODE: Превышена квота"]
|
||
|
||
def test_lowercase_code_is_unknown(self) -> None:
|
||
outcome = NullNormalizer().normalize(_one(_failure("invalid_document", "m")))
|
||
_assert_error(outcome, ErrorCode.UNKNOWN_ERROR)
|
||
assert outcome.error is not None
|
||
assert "invalid_document" in outcome.error.message
|
||
assert outcome.warnings == [f"{JOB}.json: invalid_document: m"]
|
||
|
||
@pytest.mark.parametrize("code", [5, ["INVALID_DOCUMENT"], {"a": 1}, True])
|
||
def test_code_not_a_string_is_unknown_error(self, code: object) -> None:
|
||
outcome = NullNormalizer().normalize(_one(_failure(code, "m-text")))
|
||
_assert_error(outcome, ErrorCode.UNKNOWN_ERROR)
|
||
assert outcome.error is not None
|
||
assert repr(code) in outcome.error.message
|
||
assert "m-text" in outcome.error.message
|
||
assert outcome.warnings == [f"{JOB}.json: {code!r}: m-text"]
|
||
|
||
@pytest.mark.parametrize("message", [None, ""])
|
||
def test_missing_or_empty_message_gets_default(self, message: object) -> None:
|
||
response = _failure("INVALID_DOCUMENT", message)
|
||
outcome = NullNormalizer().normalize(_one(response))
|
||
_assert_error(outcome, ErrorCode.INVALID_DOCUMENT)
|
||
assert outcome.error is not None
|
||
assert "INVALID_DOCUMENT" in outcome.error.message
|
||
assert outcome.warnings == [f"{JOB}.json: INVALID_DOCUMENT: {outcome.error.message}"]
|
||
|
||
def test_message_absent_gets_default(self) -> None:
|
||
outcome = NullNormalizer().normalize(_one({"error": {"code": "UNREADABLE"}}))
|
||
_assert_error(outcome, ErrorCode.UNREADABLE)
|
||
assert outcome.error is not None
|
||
assert outcome.error.message
|
||
|
||
def test_message_not_a_string_is_not_lost(self) -> None:
|
||
outcome = NullNormalizer().normalize(_one(_failure("UNREADABLE", {"detail": "x"})))
|
||
_assert_error(outcome, ErrorCode.UNREADABLE)
|
||
assert outcome.error is not None
|
||
assert "{'detail': 'x'}" in outcome.error.message
|
||
|
||
def test_unknown_code_and_no_message(self) -> None:
|
||
outcome = NullNormalizer().normalize(_one({"error": {"code": "WEIRD"}}))
|
||
_assert_error(outcome, ErrorCode.UNKNOWN_ERROR)
|
||
assert outcome.error is not None
|
||
assert "WEIRD" in outcome.error.message
|
||
|
||
def test_error_wins_over_data(self) -> None:
|
||
response = _success(error={"code": "RECOGNITION_FAILED", "message": "сбой"})
|
||
_assert_error(NullNormalizer().normalize(_one(response)), ErrorCode.RECOGNITION_FAILED, "сбой")
|
||
|
||
def test_parse_error_is_invalid_format(self) -> None:
|
||
"""Сценарий «Все файлы не JSON»."""
|
||
merged = _merged(_file(f"{JOB}.json", None, "JSONDecodeError: Expecting value: line 1 column 1 (char 0)"))
|
||
outcome = NullNormalizer().normalize(merged)
|
||
_assert_error(outcome, ErrorCode.INVALID_FORMAT)
|
||
assert outcome.error is not None
|
||
assert "Expecting value" in outcome.error.message
|
||
[warning] = outcome.warnings
|
||
assert warning.startswith(f"{JOB}.json: INVALID_FORMAT: ")
|
||
assert "JSONDecodeError: Expecting value: line 1 column 1 (char 0)" in warning
|
||
|
||
@pytest.mark.parametrize("response", [[], [{"error": None}], "строка", "", None, 0, 42, 1.5, True])
|
||
def test_root_not_object_is_invalid_format(self, response: object) -> None:
|
||
"""Сценарий «Корень ответа не объект»."""
|
||
outcome = NullNormalizer().normalize(_one(response))
|
||
_assert_error(outcome, ErrorCode.INVALID_FORMAT, NOT_OBJECT_MESSAGE)
|
||
assert outcome.warnings == [f"{JOB}.json: INVALID_FORMAT: {NOT_OBJECT_MESSAGE}"]
|
||
|
||
|
||
class TestSummary:
|
||
def test_all_files_erroneous_takes_first_by_name(self) -> None:
|
||
"""Сценарий «Все документы ошибочны»: первый — INVALID_DOCUMENT, второй — не JSON."""
|
||
merged = _merged(
|
||
_file(f"{JOB}.json", _failure("INVALID_DOCUMENT", "нет типа")),
|
||
_file(f"{JOB}_ofr.json", None, "JSONDecodeError: Expecting value"),
|
||
)
|
||
outcome = NullNormalizer().normalize(merged)
|
||
_assert_error(outcome, ErrorCode.INVALID_DOCUMENT, "нет типа")
|
||
assert len(outcome.warnings) == 2
|
||
assert outcome.warnings[0] == f"{JOB}.json: INVALID_DOCUMENT: нет типа"
|
||
assert outcome.warnings[1].startswith(f"{JOB}_ofr.json: INVALID_FORMAT: ")
|
||
|
||
def test_first_erroneous_parse_error_gives_invalid_format(self) -> None:
|
||
merged = _merged(
|
||
_file(f"{JOB}.json", None, "JSONDecodeError: x"),
|
||
_file(f"{JOB}_ofr.json", _failure("INVALID_DOCUMENT")),
|
||
)
|
||
_assert_error(NullNormalizer().normalize(merged), ErrorCode.INVALID_FORMAT)
|
||
|
||
def test_one_success_gives_not_implemented_with_warnings_for_the_rest(self) -> None:
|
||
"""Сценарий «Ошибка одного из документов»."""
|
||
merged = _merged(
|
||
_file(f"{JOB}.json", _success()),
|
||
_file(f"{JOB}_ofr.json", _failure("INVALID_DOCUMENT", "нет типа")),
|
||
_file(f"{JOB}_x.json", [1]),
|
||
)
|
||
_assert_not_implemented(
|
||
NullNormalizer().normalize(merged),
|
||
[f"{JOB}_ofr.json: INVALID_DOCUMENT: нет типа", f"{JOB}_x.json: INVALID_FORMAT: {NOT_OBJECT_MESSAGE}"],
|
||
)
|
||
|
||
def test_success_not_first_still_not_implemented(self) -> None:
|
||
merged = _merged(_file("a.json", _failure("UNREADABLE", "u")), _file("b.json", _success()))
|
||
_assert_not_implemented(NullNormalizer().normalize(merged), ["a.json: UNREADABLE: u"])
|
||
|
||
def test_several_successes_no_warnings(self) -> None:
|
||
merged = _merged(_file("a.json", _success()), _file("b.json", _success(error=None)))
|
||
_assert_not_implemented(NullNormalizer().normalize(merged), [])
|
||
|
||
def test_input_is_not_mutated(self) -> None:
|
||
merged = _merged(_file("a.json", _failure("WEIRD", "m")), _file("b.json", None, "e"))
|
||
before = copy.deepcopy(merged)
|
||
NullNormalizer().normalize(merged)
|
||
assert merged == before
|
||
|
||
|
||
class TestMergedStructure:
|
||
@pytest.mark.parametrize("raw", [[], "строка", None, 0, True])
|
||
def test_not_an_object_is_invalid_format(self, raw: object) -> None:
|
||
_assert_error(NullNormalizer().normalize(raw), ErrorCode.INVALID_FORMAT)
|
||
|
||
@pytest.mark.parametrize(
|
||
"raw", [{}, {"job_id": JOB}, {"job_id": JOB, "files": None}, {"job_id": JOB, "files": {"a.json": {}}}]
|
||
)
|
||
def test_files_not_a_list_is_invalid_format(self, raw: object) -> None:
|
||
_assert_error(NullNormalizer().normalize(raw), ErrorCode.INVALID_FORMAT)
|
||
|
||
def test_no_files_is_invalid_format(self) -> None:
|
||
_assert_error(NullNormalizer().normalize(_merged()), ErrorCode.INVALID_FORMAT)
|
||
|
||
@pytest.mark.parametrize("entry", [None, "a.json", [], {"response": _success()}])
|
||
def test_malformed_entry_is_erroneous_file(self, entry: object) -> None:
|
||
merged = {"job_id": JOB, "files": [entry, _file("b.json", _success())]}
|
||
outcome = NullNormalizer().normalize(merged)
|
||
_assert_error(outcome, ErrorCode.NOT_IMPLEMENTED)
|
||
[warning] = outcome.warnings
|
||
assert warning.startswith("files[0]: INVALID_FORMAT: ")
|
||
|
||
def test_real_merge_output_is_accepted(self) -> None:
|
||
merged = merge_answers(JOB, [AnswerFile(f"{JOB}.json", json.dumps(_success()).encode())])
|
||
_assert_not_implemented(NullNormalizer().normalize(merged), [])
|
||
|
||
|
||
class TestNormalizeOutcome:
|
||
def test_error_outcome(self) -> None:
|
||
outcome = NormalizeOutcome(status="error", error=MessageError(code=ErrorCode.UNREADABLE, message="m"))
|
||
assert outcome.status is ResultStatus.ERROR
|
||
assert outcome.documents == []
|
||
assert outcome.metrics is None
|
||
assert outcome.warnings == []
|
||
|
||
def test_warnings_are_kept(self) -> None:
|
||
outcome = NormalizeOutcome(
|
||
status="error", error=MessageError(code=ErrorCode.UNREADABLE, message="m"), warnings=["a: B: c"]
|
||
)
|
||
assert outcome.warnings == ["a: B: c"]
|
||
|
||
def test_error_requires_error(self) -> None:
|
||
with pytest.raises(ValidationError):
|
||
NormalizeOutcome(status="error")
|
||
|
||
@pytest.mark.parametrize("status", ["done", "needs_check"])
|
||
def test_success_forbids_error(self, status: str) -> None:
|
||
with pytest.raises(ValidationError):
|
||
NormalizeOutcome(
|
||
status=status,
|
||
documents=[_statement()],
|
||
error=MessageError(code=ErrorCode.UNREADABLE, message="m"),
|
||
)
|
||
|
||
@pytest.mark.parametrize("status", ["done", "needs_check"])
|
||
def test_success_requires_documents(self, status: str) -> None:
|
||
with pytest.raises(ValidationError):
|
||
NormalizeOutcome(status=status)
|
||
|
||
@pytest.mark.parametrize("status", ["done", "needs_check"])
|
||
def test_success_outcome(self, status: str) -> None:
|
||
metrics = Metrics(pages=2, rows=10)
|
||
outcome = NormalizeOutcome(status=status, documents=[_statement()], metrics=metrics)
|
||
assert outcome.status is ResultStatus(status)
|
||
assert outcome.error is None
|
||
assert outcome.metrics == metrics
|
||
assert len(outcome.documents) == 1
|
||
|
||
def test_error_forbids_documents(self) -> None:
|
||
with pytest.raises(ValidationError):
|
||
NormalizeOutcome(
|
||
status="error",
|
||
documents=[_statement()],
|
||
error=MessageError(code=ErrorCode.UNREADABLE, message="m"),
|
||
)
|
||
|
||
def test_error_may_carry_metrics(self) -> None:
|
||
outcome = NormalizeOutcome(
|
||
status="error",
|
||
error=MessageError(code=ErrorCode.UNREADABLE, message="m"),
|
||
metrics=Metrics(pages=1),
|
||
)
|
||
assert outcome.metrics == Metrics(pages=1)
|
||
|
||
@pytest.mark.parametrize("status", ["processing", "ok", "", None])
|
||
def test_status_restricted(self, status: object) -> None:
|
||
with pytest.raises(ValidationError):
|
||
NormalizeOutcome(status=status, error=MessageError(code=ErrorCode.UNREADABLE, message="m"))
|
||
|
||
def test_frozen(self) -> None:
|
||
outcome = NormalizeOutcome(status="error", error=MessageError(code=ErrorCode.UNREADABLE, message="m"))
|
||
with pytest.raises(ValidationError):
|
||
outcome.status = ResultStatus.DONE
|
||
|
||
def test_unknown_field_forbidden(self) -> None:
|
||
with pytest.raises(ValidationError):
|
||
NormalizeOutcome(
|
||
status="error",
|
||
error=MessageError(code=ErrorCode.UNREADABLE, message="m"),
|
||
extra_field=1,
|
||
)
|
||
|
||
|
||
class TestProtocol:
|
||
def test_null_normalizer_is_normalizer(self) -> None:
|
||
assert isinstance(NullNormalizer(), Normalizer)
|
||
|
||
def test_object_without_normalize_is_not(self) -> None:
|
||
assert not isinstance(object(), Normalizer)
|
||
|
||
def test_custom_implementation_is_accepted(self) -> None:
|
||
class Custom:
|
||
def normalize(self, raw: object) -> NormalizeOutcome:
|
||
return NormalizeOutcome(status="done", documents=[_statement()])
|
||
|
||
normalizer: Normalizer = Custom()
|
||
assert normalizer.normalize({}).status is ResultStatus.DONE
|
||
|
||
|
||
def _statement() -> StatementDocument:
|
||
return StatementDocument.model_validate(
|
||
{
|
||
"document_type": "BALANCE",
|
||
"locator": "pages:1-2",
|
||
"header": {},
|
||
"columns": [],
|
||
"lines": [],
|
||
}
|
||
)
|