diff --git a/api/src/normalizer/__init__.py b/api/src/normalizer/__init__.py index 2b2f180..5cfcc13 100644 --- a/api/src/normalizer/__init__.py +++ b/api/src/normalizer/__init__.py @@ -5,6 +5,9 @@ (`src.gateway.merge.merge_answers`). Результат — `NormalizeOutcome`, из которого воркер собирает конверт (`warnings` -> `metrics.warnings`) и, для done/needs_check, файл `results/{file_id}/{job_id}/result.json`. +Срез 3a: настоящая нормализация — `OcrNormalizer` (`src/normalizer/ocr/`, тесты — +`tests/unit/normalizer/test_ocr_normalizer.py`); `NullNormalizer` остаётся для тестов воркера Среза 0. + В Срезе 0 единственная реализация — `NullNormalizer`: сводит ошибки по файлам (успех файла — `error.code == ""`), при хотя бы одном успешном файле отвечает `NOT_IMPLEMENTED`. Настоящий разбор `data_ocr` (§11 `05-domain-rules.md`) появится в Срезе 3 новой реализацией протокола `Normalizer`. @@ -17,7 +20,15 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator from src.contract import ErrorCode, MessageError, Metrics, ResultStatus, StatementDocument -__all__ = ["NOT_IMPLEMENTED_MESSAGE", "NOT_OBJECT_MESSAGE", "NormalizeOutcome", "Normalizer", "NullNormalizer"] +__all__ = [ + "NOT_IMPLEMENTED_MESSAGE", + "NOT_OBJECT_MESSAGE", + "NO_TABLES_MESSAGE", + "NormalizeOutcome", + "Normalizer", + "NullNormalizer", + "OcrNormalizer", +] NOT_IMPLEMENTED_MESSAGE = "нормализация ответа ИС Сканирование реализуется в Срезе 3" NOT_OBJECT_MESSAGE = "ответ ИС не является JSON-объектом" @@ -164,3 +175,7 @@ def _error_outcome(code: ErrorCode, message: str, warnings: list[str] | None = N return NormalizeOutcome( status=ResultStatus.ERROR, error=MessageError(code=code, message=message), warnings=warnings or [] ) + + +# в конце модуля: `OcrNormalizer` берёт `NormalizeOutcome` и помощники ошибок из этого модуля лениво (цикл импортов) +from src.normalizer.ocr import NO_TABLES_MESSAGE, OcrNormalizer # noqa: E402 diff --git a/api/src/normalizer/ocr/__init__.py b/api/src/normalizer/ocr/__init__.py index 776b1fa..f949e12 100644 --- a/api/src/normalizer/ocr/__init__.py +++ b/api/src/normalizer/ocr/__init__.py @@ -1,3 +1,98 @@ """Нормализатор OCR: конвейер чистых функций `text` → `dates` → `amounts` → `layout` → `rows` → `header` → `document` (D-4, §11.1). Проверенные на стенде функции старого ядра (`../reference-old-arm/api/src/core/PDFWorker/`) перенесены дословно; DB-обвязка не переносится.""" + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any + +import src.normalizer as normalizer_package # цикл: `src.normalizer` импортирует этот модуль в конце своего `__init__` +from src.contract import ErrorCode, Metrics, ResultStatus, StatementDocument +from src.normalizer.ocr.document import build_document +from src.normalizer.ocr.header import parse_file_header +from src.normalizer.rules import OcrRules + +if TYPE_CHECKING: + from src.normalizer import NormalizeOutcome + +__all__ = ["NO_TABLES_MESSAGE", "OcrNormalizer"] + +NO_TABLES_MESSAGE = "ни одной таблицы не разобрано" +NO_CODE_COLUMN_MESSAGE = "нет шапки и столбца кода" +FULL_CONFIDENCE = 100.0 + + +class OcrNormalizer: + """Нормализатор ответа ИС Сканирование: ошибочные файлы — как в Срезе 0, из успешных собираются документы по + одному на таблицу; статус — `done` только когда всё распознано (D-6).""" + + def __init__(self, rules: OcrRules) -> None: + self._rules = rules + + def normalize(self, raw: object) -> "NormalizeOutcome": + # имена пакета берутся при вызове: при импорте этого модуля `src.normalizer` ещё не дочитан до конца + outcome_cls = normalizer_package.NormalizeOutcome + error_outcome = normalizer_package._error_outcome + file_error = normalizer_package._file_error + + if not isinstance(raw, dict) or not isinstance(raw.get("files"), list): + return error_outcome(ErrorCode.INVALID_FORMAT, normalizer_package.NOT_MERGED_MESSAGE) + files: list[object] = raw["files"] + if not files: + return error_outcome(ErrorCode.INVALID_FORMAT, normalizer_package.NO_FILES_MESSAGE) + warnings: list[str] = [] + documents: list[StatementDocument] = [] + fully_recognized = True + succeeded = 0 + first_error = None + for index, entry in enumerate(files): + error = file_error(index, entry) + if error is not None: + warnings.append(error.warning) + first_error = first_error or error + continue + succeeded += 1 + assert isinstance(entry, dict) # noqa: S101 — проверено _file_error + file_documents, file_warnings, file_recognized = self._normalize_file(entry) + documents.extend(file_documents) + warnings.extend(file_warnings) + fully_recognized = fully_recognized and file_recognized + if not succeeded and first_error is not None: + return error_outcome(first_error.code, first_error.summary, warnings) + if not documents: + return error_outcome(ErrorCode.RECOGNITION_FAILED, NO_TABLES_MESSAGE, warnings) + status = ResultStatus.DONE if fully_recognized else ResultStatus.NEEDS_CHECK + rows = sum(len(document.lines) for document in documents) + return outcome_cls(status=status, documents=documents, metrics=Metrics(rows=rows), warnings=warnings) + + def _normalize_file(self, entry: dict[str, Any]) -> tuple[list[StatementDocument], list[str], bool]: + """Документы одного успешного файла, предупреждения и признак «всё распознано».""" + name = str(entry["name"]) + response = entry["response"] + data_ocr = response.get("data_ocr") if isinstance(response, dict) else None + if not isinstance(data_ocr, Mapping): + return [], [f"{name}: data_ocr отсутствует или не объект"], False + header = parse_file_header(data_ocr, self._rules.lookalikes) + warnings: list[str] = [] + if header.inn is None or header.inn_confidence != FULL_CONFIDENCE: + warnings.append(f"{name}: ИНН не распознан") + if header.report_date is None: + warnings.append(f"{name}: дата отчёта не разобрана") + tables = data_ocr.get("table_structure") + documents: list[StatementDocument] = [] + recognized = True + for index, table in enumerate(tables if isinstance(tables, list) else []): + prefix = f"{name}: table:{index}: " + try: + built = build_document(table if isinstance(table, Mapping) else {}, index, name, header, self._rules) + except Exception as exc: # noqa: BLE001 — §11.1 п. 9: ошибка одной таблицы не роняет файл + warnings.append(f"{prefix}BACKEND_PARSING_ERROR: {type(exc).__name__}: {exc}") + recognized = False + continue + if built is None: + warnings.append(prefix + NO_CODE_COLUMN_MESSAGE) + recognized = False + continue + documents.append(built.document) + warnings.extend(prefix + warning for warning in built.warnings) + recognized = recognized and built.fully_recognized + return documents, warnings, recognized and not warnings diff --git a/api/tests/unit/normalizer/test_ocr_normalizer.py b/api/tests/unit/normalizer/test_ocr_normalizer.py new file mode 100644 index 0000000..c3f0015 --- /dev/null +++ b/api/tests/unit/normalizer/test_ocr_normalizer.py @@ -0,0 +1,188 @@ +"""`OcrNormalizer` (spec «Статус результата нормализации», «Результат по контракту», D-6).""" + +import copy +import json +from pathlib import Path +from typing import Any + +import pytest + +import src.normalizer.ocr as ocr_module +from src.contract import ErrorCode, ResultStatus +from src.gateway.hotfolder import AnswerFile +from src.gateway.merge import merge_answers +from src.normalizer import NO_TABLES_MESSAGE, OcrNormalizer +from src.normalizer.rules import load_rules +from tests.worker_helpers import SUCCESS_ANSWER + +API_DIR = Path(__file__).resolve().parents[3] +REAL_FIXTURES = API_DIR.parent.parent / "fixtures" / "ocr" +JOB = "3f1c2b7a-0000-4000-8000-000000000001" +RULES = load_rules(API_DIR / "rules") + + +def _merged(*files: tuple[str, Any]) -> dict[str, Any]: + answers = [AnswerFile(name, json.dumps(body, ensure_ascii=False).encode()) for name, body in files] + return dict(merge_answers(JOB, answers)) + + +def _real_set(*names: str) -> dict[str, Any]: + if not REAL_FIXTURES.is_dir(): + pytest.skip("fixtures недоступен: положите репозиторий fixtures рядом с workspace") + files = [] + for name in names: + suffix = name.removesuffix(".json").partition("_")[2].partition("_")[2] + target = f"{JOB}_{suffix}.json" if suffix else f"{JOB}.json" + files.append(AnswerFile(target, (REAL_FIXTURES / name).read_bytes())) + return dict(merge_answers(JOB, files)) + + +CLEAN: dict[str, Any] = { + "file_uid": JOB, + "error": {"code": "", "message": ""}, + "data_ocr": { + "label": {"form_name": "Бухгалтерская отчетность", "confidence": 100}, + "organization": {"form_name": "ООО Ромашка", "confidence": 100}, + "inn": {"form_name": "7700000000", "confidence": 100}, + "units": {"form_name": "тыс. руб.", "confidence": 100}, + "date": {"form_name": "на 31 декабря 2024 г.", "confidence": 100}, + "table_structure": [ + { + "label": {"form_name": "Бухгалтерский баланс", "confidence": 100}, + "rows": [ + { + "row_index": 0, + "cells": [ + {"column_index": 0, "value": "Наименование показателя", "confidence": 100}, + {"column_index": 1, "value": "Код", "confidence": 100}, + {"column_index": 2, "value": "На 31 декабря 2024 г.", "confidence": 100}, + ], + }, + { + "row_index": 1, + "cells": [ + {"column_index": 0, "value": "Основные средства", "confidence": 100}, + {"column_index": 1, "value": "1150", "confidence": 100}, + {"column_index": 2, "value": "1 234", "confidence": 100}, + ], + }, + ], + } + ], + }, +} + + +class TestStatus: + def test_done_when_everything_recognized(self) -> None: + outcome = OcrNormalizer(RULES).normalize(_merged((f"{JOB}.json", CLEAN))) + assert outcome.status is ResultStatus.DONE and outcome.warnings == [] + assert len(outcome.documents) == 1 and outcome.documents[0].document_type == "BALANCE" + assert outcome.metrics is not None and outcome.metrics.rows == 1 + + def test_needs_check_with_table_warning(self) -> None: + noisy = copy.deepcopy(CLEAN) + noisy["data_ocr"]["table_structure"][0]["rows"][1]["cells"][2]["confidence"] = 90 + outcome = OcrNormalizer(RULES).normalize(_merged((f"{JOB}.json", noisy))) + assert outcome.status is ResultStatus.NEEDS_CHECK + + def test_no_tables_is_recognition_failed(self) -> None: + empty = copy.deepcopy(CLEAN) + empty["data_ocr"]["table_structure"] = [] + outcome = OcrNormalizer(RULES).normalize(_merged((f"{JOB}.json", empty))) + assert outcome.status is ResultStatus.ERROR + assert outcome.error is not None and outcome.error.code is ErrorCode.RECOGNITION_FAILED + assert outcome.error.message == NO_TABLES_MESSAGE + + def test_skipped_table_is_warned(self) -> None: + bad = copy.deepcopy(CLEAN) + empty_table = {"label": {"form_name": "x"}, "rows": [{"row_index": 0, "cells": []}]} + bad["data_ocr"]["table_structure"].append(empty_table) + outcome = OcrNormalizer(RULES).normalize(_merged((f"{JOB}.json", bad))) + assert outcome.status is ResultStatus.NEEDS_CHECK + assert f"{JOB}.json: table:1: нет шапки и столбца кода" in outcome.warnings + + def test_exception_in_one_table_becomes_warning(self, monkeypatch: pytest.MonkeyPatch) -> None: + real = ocr_module.build_document + + def boom(table: Any, index: int, *args: Any, **kwargs: Any) -> Any: + if index == 1: + raise RuntimeError("сломалось") + return real(table, index, *args, **kwargs) + + monkeypatch.setattr(ocr_module, "build_document", boom) + two = copy.deepcopy(CLEAN) + two["data_ocr"]["table_structure"].append(copy.deepcopy(two["data_ocr"]["table_structure"][0])) + outcome = OcrNormalizer(RULES).normalize(_merged((f"{JOB}.json", two))) + assert outcome.status is ResultStatus.NEEDS_CHECK and len(outcome.documents) == 1 + assert f"{JOB}.json: table:1: BACKEND_PARSING_ERROR: RuntimeError: сломалось" in outcome.warnings + + def test_file_level_warnings(self) -> None: + noisy = copy.deepcopy(CLEAN) + noisy["data_ocr"]["inn"] = {"form_name": "+", "confidence": 0} + noisy["data_ocr"]["date"] = {"form_name": "Общество", "confidence": 100} + outcome = OcrNormalizer(RULES).normalize(_merged((f"{JOB}.json", noisy))) + assert f"{JOB}.json: ИНН не распознан" in outcome.warnings + assert f"{JOB}.json: дата отчёта не разобрана" in outcome.warnings + + +class TestErrorFilesStillHandled: + def test_all_files_failed_keeps_first_code(self) -> None: + failed = {"file_uid": JOB, "error": {"code": "INVALID_DOCUMENT", "message": "нет"}, "data_ocr": {}} + outcome = OcrNormalizer(RULES).normalize(_merged((f"{JOB}.json", failed))) + assert outcome.status is ResultStatus.ERROR and outcome.error is not None + assert outcome.error.code is ErrorCode.INVALID_DOCUMENT + assert outcome.warnings == [f"{JOB}.json: INVALID_DOCUMENT: нет"] + + def test_mixed_files(self) -> None: + failed = {"file_uid": JOB, "error": {"code": "INVALID_DOCUMENT", "message": "нет"}, "data_ocr": {}} + outcome = OcrNormalizer(RULES).normalize(_merged((f"{JOB}.json", CLEAN), (f"{JOB}_ofr.json", failed))) + assert outcome.status is ResultStatus.DONE # ошибочный файл — предупреждение, не needs_check + assert outcome.warnings == [f"{JOB}_ofr.json: INVALID_DOCUMENT: нет"] + + def test_not_merged_structure(self) -> None: + outcome = OcrNormalizer(RULES).normalize({"job_id": JOB}) + assert outcome.status is ResultStatus.ERROR and outcome.error is not None + assert outcome.error.code is ErrorCode.INVALID_FORMAT + + def test_success_answer_helper_from_worker_tests(self) -> None: + outcome = OcrNormalizer(RULES).normalize(_merged((f"{JOB}.json", SUCCESS_ANSWER))) + assert outcome.status is ResultStatus.NEEDS_CHECK # confidence < 100 у шапки + + +@pytest.mark.fixtures +class TestRealFixtures: + def test_aep6(self) -> None: + outcome = OcrNormalizer(RULES).normalize(_real_set("aep_6.json")) + assert outcome.status is ResultStatus.NEEDS_CHECK + types = [d.document_type for d in outcome.documents] + assert types == ["BALANCE", "INCOME"] + balance, income = outcome.documents + assert [(c.column_index, c.label, str(c.period_date)) for c in balance.columns] == [ + (0, "На 30 июня 2025 г", "2025-06-30"), + (1, "На 31 декабря 2024 г.", "2024-12-31"), + (2, "На 31 декабря 2023 г.", "2023-12-31"), + ] + codes = [line.line_code for line in balance.lines] + assert "1105" in codes and "1110" in codes + assert all(line.line_code for line in balance.lines) + assert [str(c.period_date) for c in income.columns] == ["2025-06-30", "2024-06-30"] + revenue = next(line for line in income.lines if line.line_code == "2110") + assert str(revenue.values[0].amount) == "13070" + assert balance.header.units == "тыс. руб." and balance.header.inn is None and balance.header.report_date is None + + def test_aep3(self) -> None: + outcome = OcrNormalizer(RULES).normalize(_real_set("aep_3.json")) + assert outcome.status is ResultStatus.NEEDS_CHECK + balance = next(d for d in outcome.documents if d.document_type == "BALANCE") + assert all(c.period_date is None for c in balance.columns) + assert balance.header.inn == "3121200351" + assert balance.header.inn_confidence is not None and balance.header.inn_confidence < 100 + assert str(balance.header.report_date) == "2024-12-31" + assert any(line.line_code == "1120" for line in balance.lines) + assert any("шапка синтезирована" in w for w in outcome.warnings) + + def test_aep3_with_failed_ofr(self) -> None: + outcome = OcrNormalizer(RULES).normalize(_real_set("aep_3.json", "aep_3_ofr.json")) + assert outcome.status is ResultStatus.NEEDS_CHECK + assert any(w.startswith(f"{JOB}_ofr.json: INVALID_DOCUMENT:") for w in outcome.warnings) diff --git a/api/tests/unit/test_normalizer.py b/api/tests/unit/test_normalizer.py index 7a751b6..8c58e00 100644 --- a/api/tests/unit/test_normalizer.py +++ b/api/tests/unit/test_normalizer.py @@ -5,6 +5,7 @@ ошибочный файл — непустой `error.code`, `parse_error` или корень ответа не объект. Сводка: все файлы ошибочны — код первого по имени; есть успешный — `NOT_IMPLEMENTED`; ошибочные файлы — строки `warnings`. Реальные ответы `../fixtures/ocr/*.json` — маркер `fixtures`. +Срез 3a: настоящая нормализация — `tests/unit/normalizer/test_ocr_normalizer.py`. """ import copy