diff --git a/api/src/normalizer/ocr/header.py b/api/src/normalizer/ocr/header.py new file mode 100644 index 0000000..7bbfb6b --- /dev/null +++ b/api/src/normalizer/ocr/header.py @@ -0,0 +1,62 @@ +"""Шапка документа из полей `data_ocr` (§11.1 п. 8): ИНН — только цифры, единицы — очищенный текст, дата отчёта — +`parse_date` после замены двойников; «распознано» — `confidence == 100` и сырой текст без мусора.""" + +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import date + +from src.normalizer.ocr.dates import parse_date +from src.normalizer.ocr.text import clean_value, collapse_whitespace, digits_only, fix_lookalikes + +__all__ = ["FileHeader", "parse_file_header"] + +FULL_CONFIDENCE = 100.0 +CAPPED_CONFIDENCE = 99.0 + + +@dataclass(frozen=True) +class FileHeader: + organization_name: str | None + organization_confidence: float | None + inn: str | None + inn_confidence: float | None + units: str | None + report_date_text: str | None + report_date: date | None + report_date_recognized: bool + + +def _field(data_ocr: Mapping[str, object], name: str) -> tuple[str, float | None]: + raw = data_ocr.get(name) + if not isinstance(raw, Mapping): + return "", None + text = raw.get("form_name") + confidence = raw.get("confidence") + text_str = str(text) if isinstance(text, str) else "" + conf_float = float(confidence) if isinstance(confidence, (int, float)) else None + return (text_str, conf_float) + + +def parse_file_header(data_ocr: Mapping[str, object], lookalikes: Mapping[str, str]) -> FileHeader: + organization, organization_confidence = _field(data_ocr, "organization") + inn_raw, inn_confidence = _field(data_ocr, "inn") + units_raw, _ = _field(data_ocr, "units") + date_raw, date_confidence = _field(data_ocr, "date") + + organization_name = collapse_whitespace(organization) or None + inn = digits_only(inn_raw) or None + if inn is None: + inn_confidence = None + elif inn_confidence is not None and inn_raw.strip() != inn: + inn_confidence = min(inn_confidence, CAPPED_CONFIDENCE) + report_date = parse_date(fix_lookalikes(date_raw, lookalikes)) if date_raw.strip() else None + return FileHeader( + organization_name=organization_name, + organization_confidence=organization_confidence if organization_name is not None else None, + inn=inn, + inn_confidence=inn_confidence, + units=clean_value(units_raw) or None, + report_date_text=date_raw.strip() or None, + report_date=report_date, + report_date_recognized=date_confidence == FULL_CONFIDENCE and report_date is not None, + ) diff --git a/api/tests/unit/normalizer/test_header.py b/api/tests/unit/normalizer/test_header.py new file mode 100644 index 0000000..0551a44 --- /dev/null +++ b/api/tests/unit/normalizer/test_header.py @@ -0,0 +1,75 @@ +"""Шапка документа из `data_ocr` (§11.1 п. 8) на реальных `aep_3.json` (мусор) и `aep_6.json` (чисто).""" + +import json +from datetime import date +from pathlib import Path + +import pytest + +from src.normalizer.ocr.header import FileHeader, parse_file_header + +API_DIR = Path(__file__).resolve().parents[3] +REAL_FIXTURES = API_DIR.parent.parent / "fixtures" / "ocr" +LOOKALIKES = {"О": "0", "о": "0", "З": "3", "l": "1", "I": "1", "|": "1", "б": "6", "S": "5", "Ч": "4"} + + +def _real(name: str) -> dict: + if not REAL_FIXTURES.is_dir(): + pytest.skip("fixtures недоступен: положите репозиторий fixtures рядом с workspace") + return dict(json.loads((REAL_FIXTURES / name).read_text(encoding="utf-8"))["data_ocr"]) + + +def _field(text: str, confidence: float) -> dict: + return {"form_name": text, "confidence": confidence} + + +def test_clean_header() -> None: + header = parse_file_header( + { + "organization": _field("ООО Ромашка", 97.5), + "inn": _field("7700000000", 100), + "units": _field("тыс. руб.", 100), + "date": _field("на 31 декабря 2024 г.", 100), + }, + LOOKALIKES, + ) + assert header == FileHeader( + "ООО Ромашка", + 97.5, + "7700000000", + 100.0, + "тыс. руб.", + "на 31 декабря 2024 г.", + date(2024, 12, 31), + True, + ) + + +def test_noisy_inn_caps_confidence() -> None: + header = parse_file_header({"inn": _field("3|1 21 2| о! О' 0 0; 3: 51", 100)}, LOOKALIKES) + assert header.inn == "3121200351" + assert header.inn_confidence == 99.0 + assert header.organization_name is None and header.organization_confidence is None + + +def test_missing_fields() -> None: + header = parse_file_header({}, LOOKALIKES) + assert header == FileHeader(None, None, None, None, None, None, None, False) + + +@pytest.mark.fixtures +def test_aep6_header() -> None: + header = parse_file_header(_real("aep_6.json"), LOOKALIKES) + assert header.inn is None # текст «+» + assert header.units == "тыс. руб." + assert header.report_date is None # в поле date — название организации + assert header.report_date_text is not None and header.report_date_recognized is False + assert header.organization_name is not None and "Дорстроймонтаж" in header.organization_name + + +@pytest.mark.fixtures +def test_aep3_header() -> None: + header = parse_file_header(_real("aep_3.json"), LOOKALIKES) + assert header.inn == "3121200351" and header.inn_confidence is not None and header.inn_confidence < 100 + assert header.report_date == date(2024, 12, 31) + assert header.report_date_recognized is False # confidence 75