From 6882493d987c16e9f707c509aa7d3c3d95481ca9 Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Wed, 7 Oct 2026 00:25:48 +0400 Subject: [PATCH] =?UTF-8?q?feat(normalizer):=20=D1=88=D0=B0=D0=BF=D0=BA?= =?UTF-8?q?=D0=B0=20=D0=B4=D0=BE=D0=BA=D1=83=D0=BC=D0=B5=D0=BD=D1=82=D0=B0?= =?UTF-8?q?=20=D0=B8=D0=B7=20data=5Focr=20=E2=80=94=20=D0=98=D0=9D=D0=9D,?= =?UTF-8?q?=20=D0=B5=D0=B4=D0=B8=D0=BD=D0=B8=D1=86=D1=8B,=20=D0=B4=D0=B0?= =?UTF-8?q?=D1=82=D0=B0=20=D0=BE=D1=82=D1=87=D1=91=D1=82=D0=B0=20(=C2=A711?= =?UTF-8?q?.1=20=D0=BF.=208)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Haiku 4.5 Claude-Session: https://claude.ai/code/session_012vJkmBGKKqMoEMzvsXG3y2 --- api/src/normalizer/ocr/header.py | 62 ++++++++++++++++++++ api/tests/unit/normalizer/test_header.py | 75 ++++++++++++++++++++++++ 2 files changed, 137 insertions(+) create mode 100644 api/src/normalizer/ocr/header.py create mode 100644 api/tests/unit/normalizer/test_header.py diff --git a/api/src/normalizer/ocr/header.py b/api/src/normalizer/ocr/header.py new file mode 100644 index 0000000..7bbfb6b --- /dev/null +++ b/api/src/normalizer/ocr/header.py @@ -0,0 +1,62 @@ +"""Шапка документа из полей `data_ocr` (§11.1 п. 8): ИНН — только цифры, единицы — очищенный текст, дата отчёта — +`parse_date` после замены двойников; «распознано» — `confidence == 100` и сырой текст без мусора.""" + +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import date + +from src.normalizer.ocr.dates import parse_date +from src.normalizer.ocr.text import clean_value, collapse_whitespace, digits_only, fix_lookalikes + +__all__ = ["FileHeader", "parse_file_header"] + +FULL_CONFIDENCE = 100.0 +CAPPED_CONFIDENCE = 99.0 + + +@dataclass(frozen=True) +class FileHeader: + organization_name: str | None + organization_confidence: float | None + inn: str | None + inn_confidence: float | None + units: str | None + report_date_text: str | None + report_date: date | None + report_date_recognized: bool + + +def _field(data_ocr: Mapping[str, object], name: str) -> tuple[str, float | None]: + raw = data_ocr.get(name) + if not isinstance(raw, Mapping): + return "", None + text = raw.get("form_name") + confidence = raw.get("confidence") + text_str = str(text) if isinstance(text, str) else "" + conf_float = float(confidence) if isinstance(confidence, (int, float)) else None + return (text_str, conf_float) + + +def parse_file_header(data_ocr: Mapping[str, object], lookalikes: Mapping[str, str]) -> FileHeader: + organization, organization_confidence = _field(data_ocr, "organization") + inn_raw, inn_confidence = _field(data_ocr, "inn") + units_raw, _ = _field(data_ocr, "units") + date_raw, date_confidence = _field(data_ocr, "date") + + organization_name = collapse_whitespace(organization) or None + inn = digits_only(inn_raw) or None + if inn is None: + inn_confidence = None + elif inn_confidence is not None and inn_raw.strip() != inn: + inn_confidence = min(inn_confidence, CAPPED_CONFIDENCE) + report_date = parse_date(fix_lookalikes(date_raw, lookalikes)) if date_raw.strip() else None + return FileHeader( + organization_name=organization_name, + organization_confidence=organization_confidence if organization_name is not None else None, + inn=inn, + inn_confidence=inn_confidence, + units=clean_value(units_raw) or None, + report_date_text=date_raw.strip() or None, + report_date=report_date, + report_date_recognized=date_confidence == FULL_CONFIDENCE and report_date is not None, + ) diff --git a/api/tests/unit/normalizer/test_header.py b/api/tests/unit/normalizer/test_header.py new file mode 100644 index 0000000..0551a44 --- /dev/null +++ b/api/tests/unit/normalizer/test_header.py @@ -0,0 +1,75 @@ +"""Шапка документа из `data_ocr` (§11.1 п. 8) на реальных `aep_3.json` (мусор) и `aep_6.json` (чисто).""" + +import json +from datetime import date +from pathlib import Path + +import pytest + +from src.normalizer.ocr.header import FileHeader, parse_file_header + +API_DIR = Path(__file__).resolve().parents[3] +REAL_FIXTURES = API_DIR.parent.parent / "fixtures" / "ocr" +LOOKALIKES = {"О": "0", "о": "0", "З": "3", "l": "1", "I": "1", "|": "1", "б": "6", "S": "5", "Ч": "4"} + + +def _real(name: str) -> dict: + if not REAL_FIXTURES.is_dir(): + pytest.skip("fixtures недоступен: положите репозиторий fixtures рядом с workspace") + return dict(json.loads((REAL_FIXTURES / name).read_text(encoding="utf-8"))["data_ocr"]) + + +def _field(text: str, confidence: float) -> dict: + return {"form_name": text, "confidence": confidence} + + +def test_clean_header() -> None: + header = parse_file_header( + { + "organization": _field("ООО Ромашка", 97.5), + "inn": _field("7700000000", 100), + "units": _field("тыс. руб.", 100), + "date": _field("на 31 декабря 2024 г.", 100), + }, + LOOKALIKES, + ) + assert header == FileHeader( + "ООО Ромашка", + 97.5, + "7700000000", + 100.0, + "тыс. руб.", + "на 31 декабря 2024 г.", + date(2024, 12, 31), + True, + ) + + +def test_noisy_inn_caps_confidence() -> None: + header = parse_file_header({"inn": _field("3|1 21 2| о! О' 0 0; 3: 51", 100)}, LOOKALIKES) + assert header.inn == "3121200351" + assert header.inn_confidence == 99.0 + assert header.organization_name is None and header.organization_confidence is None + + +def test_missing_fields() -> None: + header = parse_file_header({}, LOOKALIKES) + assert header == FileHeader(None, None, None, None, None, None, None, False) + + +@pytest.mark.fixtures +def test_aep6_header() -> None: + header = parse_file_header(_real("aep_6.json"), LOOKALIKES) + assert header.inn is None # текст «+» + assert header.units == "тыс. руб." + assert header.report_date is None # в поле date — название организации + assert header.report_date_text is not None and header.report_date_recognized is False + assert header.organization_name is not None and "Дорстроймонтаж" in header.organization_name + + +@pytest.mark.fixtures +def test_aep3_header() -> None: + header = parse_file_header(_real("aep_3.json"), LOOKALIKES) + assert header.inn == "3121200351" and header.inn_confidence is not None and header.inn_confidence < 100 + assert header.report_date == date(2024, 12, 31) + assert header.report_date_recognized is False # confidence 75