"""Нормализатор ответа ИС Сканирование (spec image-recognition-gateway «Результат по контракту», D-5, R-12, R-25). Вход `NullNormalizer` — склеенный ответ `{"job_id", "files": [{"name", "response", "parse_error"}]}` (D-4). Успех файла — `error.code == ""` (реальный успешный ответ несёт `error = {"code": "", "message": ""}`, а не `null`); ошибочный файл — непустой `error.code`, `parse_error` или корень ответа не объект. Сводка: все файлы ошибочны — код первого по имени; есть успешный — `NOT_IMPLEMENTED`; ошибочные файлы — строки `warnings`. Реальные ответы `../fixtures/ocr/*.json` — маркер `fixtures`. Срез 3a: настоящая нормализация — `tests/unit/normalizer/test_ocr_normalizer.py`. """ import copy import json from pathlib import Path from typing import Any import pytest from pydantic import ValidationError from src.contract import ErrorCode, MessageError, Metrics, ResultStatus, StatementDocument from src.gateway.hotfolder import AnswerFile from src.gateway.merge import merge_answers from src.normalizer import NOT_IMPLEMENTED_MESSAGE, NOT_OBJECT_MESSAGE, NormalizeOutcome, Normalizer, NullNormalizer API_DIR = Path(__file__).resolve().parents[2] # `<корень scan-gateway>/../fixtures/ocr` (workspace и worktree `.wt/<срез>/`), как у test_emulator.py REAL_FIXTURES = API_DIR.parent.parent / "fixtures" / "ocr" JOB = "3f1c2b7a-0000-4000-8000-000000000001" REAL_MESSAGE = "Не удалось определить тип документа." SUCCESS_ERROR = {"code": "", "message": ""} def _real(name: str) -> dict[str, Any]: if not REAL_FIXTURES.is_dir(): pytest.skip("fixtures недоступен: положите репозиторий fixtures рядом с workspace") loaded: dict[str, Any] = json.loads((REAL_FIXTURES / name).read_text(encoding="utf-8")) return loaded def _real_set(*names: str) -> dict[str, Any]: """Склейка реальных ответов под именами каталога ответа: `aep_N.json` — `.json`, `aep_N_.json` — `_.json`.""" if not REAL_FIXTURES.is_dir(): pytest.skip("fixtures недоступен: положите репозиторий fixtures рядом с workspace") files = [] for name in names: suffix = name.removesuffix(".json").partition("_")[2].partition("_")[2] target = f"{JOB}_{suffix}.json" if suffix else f"{JOB}.json" files.append(AnswerFile(target, (REAL_FIXTURES / name).read_bytes())) return dict(merge_answers(JOB, files)) def _file(name: str, response: object = None, parse_error: str | None = None) -> dict[str, Any]: return {"name": name, "response": response, "parse_error": parse_error} def _merged(*files: dict[str, Any]) -> dict[str, Any]: return {"job_id": JOB, "files": list(files)} def _one(response: object) -> dict[str, Any]: return _merged(_file(f"{JOB}.json", response)) def _success(**extra: Any) -> dict[str, Any]: return {"file_uid": JOB, "error": dict(SUCCESS_ERROR), "data_ocr": {"table_structure": []}, **extra} def _failure(code: object, message: object = "сообщение ИС") -> dict[str, Any]: return {"file_uid": JOB, "error": {"code": code, "message": message}, "data_ocr": {"table_structure": []}} def _assert_error(outcome: NormalizeOutcome, code: ErrorCode, message: str | None = None) -> None: assert outcome.status is ResultStatus.ERROR assert outcome.error is not None assert outcome.error.code is code if message is not None: assert outcome.error.message == message assert outcome.documents == [] assert outcome.metrics is None def _assert_not_implemented(outcome: NormalizeOutcome, warnings: list[str]) -> None: _assert_error(outcome, ErrorCode.NOT_IMPLEMENTED, NOT_IMPLEMENTED_MESSAGE) assert outcome.warnings == warnings @pytest.mark.fixtures class TestRealFixtures: """Регрессия: реальные успешные ответы (`error = {"code": "", "message": ""}`) — не ошибка.""" @pytest.mark.parametrize("name", ["aep_6.json", "aep_3.json"]) def test_real_success_is_not_an_error(self, name: str) -> None: assert _real(name)["error"] == SUCCESS_ERROR _assert_not_implemented(NullNormalizer().normalize(_real_set(name)), []) def test_aep_3_with_ofr_gives_not_implemented_and_one_warning(self) -> None: outcome = NullNormalizer().normalize(_real_set("aep_3.json", "aep_3_ofr.json")) _assert_not_implemented(outcome, [f"{JOB}_ofr.json: INVALID_DOCUMENT: {REAL_MESSAGE}"]) @pytest.mark.parametrize("name", ["aep_3_ofr.json", "aep_5.json", "aep_8_form4.json"]) def test_real_error_alone_is_invalid_document(self, name: str) -> None: merged = _real_set(name) [entry] = merged["files"] outcome = NullNormalizer().normalize(merged) _assert_error(outcome, ErrorCode.INVALID_DOCUMENT, REAL_MESSAGE) assert outcome.warnings == [f"{entry['name']}: INVALID_DOCUMENT: {REAL_MESSAGE}"] class TestFileSuccess: """Успех файла — `error.code == ""`; отсутствующий/`null` `error` и `error` без кода ошибкой не считаются.""" def test_success_shape_of_real_response(self) -> None: _assert_not_implemented(NullNormalizer().normalize(_one(_success())), []) @pytest.mark.parametrize( "error", [None, {"code": None, "message": "m"}, {"message": "Нет кода"}, {}, "Что-то сломалось", ["a"], 42, 0, ""], ids=["null", "code_null", "no_code", "empty_object", "string", "list", "int", "zero", "empty_string"], ) def test_no_error_code_is_success(self, error: object) -> None: _assert_not_implemented(NullNormalizer().normalize(_one(_success(error=error))), []) def test_error_key_absent(self) -> None: response = _success() del response["error"] _assert_not_implemented(NullNormalizer().normalize(_one(response)), []) def test_minimal_object(self) -> None: _assert_not_implemented(NullNormalizer().normalize(_one({})), []) class TestFileError: def test_known_code_and_message_verbatim(self) -> None: outcome = NullNormalizer().normalize(_one(_failure("FILE_READ_ERROR", "Файл повреждён: стр. 3 «пусто»"))) _assert_error(outcome, ErrorCode.FILE_READ_ERROR, "Файл повреждён: стр. 3 «пусто»") assert outcome.warnings == [f"{JOB}.json: FILE_READ_ERROR: Файл повреждён: стр. 3 «пусто»"] @pytest.mark.parametrize("code", [c.value for c in ErrorCode]) def test_every_contract_code_passes(self, code: str) -> None: _assert_error(NullNormalizer().normalize(_one(_failure(code))), ErrorCode(code), "сообщение ИС") def test_unknown_code_is_unknown_error_keeping_original(self) -> None: outcome = NullNormalizer().normalize(_one(_failure("SOME_NEW_CODE", "Превышена квота"))) _assert_error(outcome, ErrorCode.UNKNOWN_ERROR) assert outcome.error is not None assert "SOME_NEW_CODE" in outcome.error.message assert "Превышена квота" in outcome.error.message assert outcome.warnings == [f"{JOB}.json: SOME_NEW_CODE: Превышена квота"] def test_lowercase_code_is_unknown(self) -> None: outcome = NullNormalizer().normalize(_one(_failure("invalid_document", "m"))) _assert_error(outcome, ErrorCode.UNKNOWN_ERROR) assert outcome.error is not None assert "invalid_document" in outcome.error.message assert outcome.warnings == [f"{JOB}.json: invalid_document: m"] @pytest.mark.parametrize("code", [5, ["INVALID_DOCUMENT"], {"a": 1}, True]) def test_code_not_a_string_is_unknown_error(self, code: object) -> None: outcome = NullNormalizer().normalize(_one(_failure(code, "m-text"))) _assert_error(outcome, ErrorCode.UNKNOWN_ERROR) assert outcome.error is not None assert repr(code) in outcome.error.message assert "m-text" in outcome.error.message assert outcome.warnings == [f"{JOB}.json: {code!r}: m-text"] @pytest.mark.parametrize("message", [None, ""]) def test_missing_or_empty_message_gets_default(self, message: object) -> None: response = _failure("INVALID_DOCUMENT", message) outcome = NullNormalizer().normalize(_one(response)) _assert_error(outcome, ErrorCode.INVALID_DOCUMENT) assert outcome.error is not None assert "INVALID_DOCUMENT" in outcome.error.message assert outcome.warnings == [f"{JOB}.json: INVALID_DOCUMENT: {outcome.error.message}"] def test_message_absent_gets_default(self) -> None: outcome = NullNormalizer().normalize(_one({"error": {"code": "UNREADABLE"}})) _assert_error(outcome, ErrorCode.UNREADABLE) assert outcome.error is not None assert outcome.error.message def test_message_not_a_string_is_not_lost(self) -> None: outcome = NullNormalizer().normalize(_one(_failure("UNREADABLE", {"detail": "x"}))) _assert_error(outcome, ErrorCode.UNREADABLE) assert outcome.error is not None assert "{'detail': 'x'}" in outcome.error.message def test_unknown_code_and_no_message(self) -> None: outcome = NullNormalizer().normalize(_one({"error": {"code": "WEIRD"}})) _assert_error(outcome, ErrorCode.UNKNOWN_ERROR) assert outcome.error is not None assert "WEIRD" in outcome.error.message def test_error_wins_over_data(self) -> None: response = _success(error={"code": "RECOGNITION_FAILED", "message": "сбой"}) _assert_error(NullNormalizer().normalize(_one(response)), ErrorCode.RECOGNITION_FAILED, "сбой") def test_parse_error_is_invalid_format(self) -> None: """Сценарий «Все файлы не JSON».""" merged = _merged(_file(f"{JOB}.json", None, "JSONDecodeError: Expecting value: line 1 column 1 (char 0)")) outcome = NullNormalizer().normalize(merged) _assert_error(outcome, ErrorCode.INVALID_FORMAT) assert outcome.error is not None assert "Expecting value" in outcome.error.message [warning] = outcome.warnings assert warning.startswith(f"{JOB}.json: INVALID_FORMAT: ") assert "JSONDecodeError: Expecting value: line 1 column 1 (char 0)" in warning @pytest.mark.parametrize("response", [[], [{"error": None}], "строка", "", None, 0, 42, 1.5, True]) def test_root_not_object_is_invalid_format(self, response: object) -> None: """Сценарий «Корень ответа не объект».""" outcome = NullNormalizer().normalize(_one(response)) _assert_error(outcome, ErrorCode.INVALID_FORMAT, NOT_OBJECT_MESSAGE) assert outcome.warnings == [f"{JOB}.json: INVALID_FORMAT: {NOT_OBJECT_MESSAGE}"] class TestSummary: def test_all_files_erroneous_takes_first_by_name(self) -> None: """Сценарий «Все документы ошибочны»: первый — INVALID_DOCUMENT, второй — не JSON.""" merged = _merged( _file(f"{JOB}.json", _failure("INVALID_DOCUMENT", "нет типа")), _file(f"{JOB}_ofr.json", None, "JSONDecodeError: Expecting value"), ) outcome = NullNormalizer().normalize(merged) _assert_error(outcome, ErrorCode.INVALID_DOCUMENT, "нет типа") assert len(outcome.warnings) == 2 assert outcome.warnings[0] == f"{JOB}.json: INVALID_DOCUMENT: нет типа" assert outcome.warnings[1].startswith(f"{JOB}_ofr.json: INVALID_FORMAT: ") def test_first_erroneous_parse_error_gives_invalid_format(self) -> None: merged = _merged( _file(f"{JOB}.json", None, "JSONDecodeError: x"), _file(f"{JOB}_ofr.json", _failure("INVALID_DOCUMENT")), ) _assert_error(NullNormalizer().normalize(merged), ErrorCode.INVALID_FORMAT) def test_one_success_gives_not_implemented_with_warnings_for_the_rest(self) -> None: """Сценарий «Ошибка одного из документов».""" merged = _merged( _file(f"{JOB}.json", _success()), _file(f"{JOB}_ofr.json", _failure("INVALID_DOCUMENT", "нет типа")), _file(f"{JOB}_x.json", [1]), ) _assert_not_implemented( NullNormalizer().normalize(merged), [f"{JOB}_ofr.json: INVALID_DOCUMENT: нет типа", f"{JOB}_x.json: INVALID_FORMAT: {NOT_OBJECT_MESSAGE}"], ) def test_success_not_first_still_not_implemented(self) -> None: merged = _merged(_file("a.json", _failure("UNREADABLE", "u")), _file("b.json", _success())) _assert_not_implemented(NullNormalizer().normalize(merged), ["a.json: UNREADABLE: u"]) def test_several_successes_no_warnings(self) -> None: merged = _merged(_file("a.json", _success()), _file("b.json", _success(error=None))) _assert_not_implemented(NullNormalizer().normalize(merged), []) def test_input_is_not_mutated(self) -> None: merged = _merged(_file("a.json", _failure("WEIRD", "m")), _file("b.json", None, "e")) before = copy.deepcopy(merged) NullNormalizer().normalize(merged) assert merged == before class TestMergedStructure: @pytest.mark.parametrize("raw", [[], "строка", None, 0, True]) def test_not_an_object_is_invalid_format(self, raw: object) -> None: _assert_error(NullNormalizer().normalize(raw), ErrorCode.INVALID_FORMAT) @pytest.mark.parametrize( "raw", [{}, {"job_id": JOB}, {"job_id": JOB, "files": None}, {"job_id": JOB, "files": {"a.json": {}}}] ) def test_files_not_a_list_is_invalid_format(self, raw: object) -> None: _assert_error(NullNormalizer().normalize(raw), ErrorCode.INVALID_FORMAT) def test_no_files_is_invalid_format(self) -> None: _assert_error(NullNormalizer().normalize(_merged()), ErrorCode.INVALID_FORMAT) @pytest.mark.parametrize("entry", [None, "a.json", [], {"response": _success()}]) def test_malformed_entry_is_erroneous_file(self, entry: object) -> None: merged = {"job_id": JOB, "files": [entry, _file("b.json", _success())]} outcome = NullNormalizer().normalize(merged) _assert_error(outcome, ErrorCode.NOT_IMPLEMENTED) [warning] = outcome.warnings assert warning.startswith("files[0]: INVALID_FORMAT: ") def test_real_merge_output_is_accepted(self) -> None: merged = merge_answers(JOB, [AnswerFile(f"{JOB}.json", json.dumps(_success()).encode())]) _assert_not_implemented(NullNormalizer().normalize(merged), []) class TestNormalizeOutcome: def test_error_outcome(self) -> None: outcome = NormalizeOutcome(status="error", error=MessageError(code=ErrorCode.UNREADABLE, message="m")) assert outcome.status is ResultStatus.ERROR assert outcome.documents == [] assert outcome.metrics is None assert outcome.warnings == [] def test_warnings_are_kept(self) -> None: outcome = NormalizeOutcome( status="error", error=MessageError(code=ErrorCode.UNREADABLE, message="m"), warnings=["a: B: c"] ) assert outcome.warnings == ["a: B: c"] def test_error_requires_error(self) -> None: with pytest.raises(ValidationError): NormalizeOutcome(status="error") @pytest.mark.parametrize("status", ["done", "needs_check"]) def test_success_forbids_error(self, status: str) -> None: with pytest.raises(ValidationError): NormalizeOutcome( status=status, documents=[_statement()], error=MessageError(code=ErrorCode.UNREADABLE, message="m"), ) @pytest.mark.parametrize("status", ["done", "needs_check"]) def test_success_requires_documents(self, status: str) -> None: with pytest.raises(ValidationError): NormalizeOutcome(status=status) @pytest.mark.parametrize("status", ["done", "needs_check"]) def test_success_outcome(self, status: str) -> None: metrics = Metrics(pages=2, rows=10) outcome = NormalizeOutcome(status=status, documents=[_statement()], metrics=metrics) assert outcome.status is ResultStatus(status) assert outcome.error is None assert outcome.metrics == metrics assert len(outcome.documents) == 1 def test_error_forbids_documents(self) -> None: with pytest.raises(ValidationError): NormalizeOutcome( status="error", documents=[_statement()], error=MessageError(code=ErrorCode.UNREADABLE, message="m"), ) def test_error_may_carry_metrics(self) -> None: outcome = NormalizeOutcome( status="error", error=MessageError(code=ErrorCode.UNREADABLE, message="m"), metrics=Metrics(pages=1), ) assert outcome.metrics == Metrics(pages=1) @pytest.mark.parametrize("status", ["processing", "ok", "", None]) def test_status_restricted(self, status: object) -> None: with pytest.raises(ValidationError): NormalizeOutcome(status=status, error=MessageError(code=ErrorCode.UNREADABLE, message="m")) def test_frozen(self) -> None: outcome = NormalizeOutcome(status="error", error=MessageError(code=ErrorCode.UNREADABLE, message="m")) with pytest.raises(ValidationError): outcome.status = ResultStatus.DONE def test_unknown_field_forbidden(self) -> None: with pytest.raises(ValidationError): NormalizeOutcome( status="error", error=MessageError(code=ErrorCode.UNREADABLE, message="m"), extra_field=1, ) class TestProtocol: def test_null_normalizer_is_normalizer(self) -> None: assert isinstance(NullNormalizer(), Normalizer) def test_object_without_normalize_is_not(self) -> None: assert not isinstance(object(), Normalizer) def test_custom_implementation_is_accepted(self) -> None: class Custom: def normalize(self, raw: object) -> NormalizeOutcome: return NormalizeOutcome(status="done", documents=[_statement()]) normalizer: Normalizer = Custom() assert normalizer.normalize({}).status is ResultStatus.DONE def _statement() -> StatementDocument: return StatementDocument.model_validate( { "document_type": "BALANCE", "locator": "pages:1-2", "header": {}, "columns": [], "lines": [], } )