From 8ebf572a22e3a92f807fc2a2912f926ff7f9cc84 Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Wed, 7 Oct 2026 00:37:30 +0400 Subject: [PATCH] =?UTF-8?q?fix(normalizer):=20values=20=D0=BF=D0=BE=20?= =?UTF-8?q?=D1=87=D0=B8=D1=81=D0=BB=D1=83=20=D0=BA=D0=BE=D0=BB=D0=BE=D0=BD?= =?UTF-8?q?=D0=BE=D0=BA,=20=D0=BF=D1=80=D0=B5=D0=B4=D1=83=D0=BF=D1=80?= =?UTF-8?q?=D0=B5=D0=B6=D0=B4=D0=B5=D0=BD=D0=B8=D1=8F=20=D1=80=D0=B5=D0=B4?= =?UTF-8?q?=D0=B0=D0=BA=D1=86=D0=B8=D0=B9=20=D0=B8=20=D0=BA=D0=BE=D0=BB?= =?UTF-8?q?=D0=BE=D0=BD=D0=BE=D0=BA=20(T8.1=E2=80=93T8.6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012vJkmBGKKqMoEMzvsXG3y2 --- api/src/normalizer/ocr/document.py | 32 ++-- api/tests/unit/normalizer/test_document.py | 212 ++++++++++++++++++++- 2 files changed, 230 insertions(+), 14 deletions(-) diff --git a/api/src/normalizer/ocr/document.py b/api/src/normalizer/ocr/document.py index 1a2ab5c..7d3cc6b 100644 --- a/api/src/normalizer/ocr/document.py +++ b/api/src/normalizer/ocr/document.py @@ -71,9 +71,7 @@ def _resolve_type(label: str, parsed: ParsedTable, rules: OcrRules, build: _Buil return "OTHER" -def _columns( - parsed: ParsedTable, lookalikes: Mapping[str, str], build: _Build -) -> tuple[list[StatementColumn], list[int]]: +def _columns(parsed: ParsedTable, lookalikes: Mapping[str, str], build: _Build) -> list[StatementColumn]: """Колонки значений первой шапки: `label` — текст ячейки шапки, `period_date` — `parse_date`.""" header = parsed.rows[0] columns: list[StatementColumn] = [] @@ -86,7 +84,9 @@ def _columns( columns.append(StatementColumn(column_index=position, label=label, period_date=period)) if parsed.synthetic_header: build.warn("шапка синтезирована: колонки без дат") - return columns, list(header.layout.value_idxs) + if not columns: + build.warn("нет колонок значений") + return columns def _latest_period(columns: list[StatementColumn]) -> date | None: @@ -131,23 +131,21 @@ def _value(cell: Cell | None, column_index: int, build: _Build) -> StatementValu ) -def _line( - row: ParsedRow, value_idxs: list[int], edition: Edition | None, rules: OcrRules, build: _Build -) -> StatementLine: +def _line(row: ParsedRow, column_count: int, edition: Edition | None, rules: OcrRules, build: _Build) -> StatementLine: row_cells = row.cells code_cell = row_cells[row.layout.code_idx] code = clean_value(code_cell.value) code_clean = code == code_cell.value.strip() label_idx = row.layout.label_idx name_cell = row_cells.get(label_idx) if label_idx is not None else None - known_here = edition is not None and code in edition.lines - if known_here and edition is not None: + if edition is not None and code in edition.lines: name = edition.lines[code] # spec: уверенность кода и словарного имени — confidence ячейки, если текст чистый и код в справочнике code_confidence = code_cell.confidence if code_clean else min(code_cell.confidence, CAPPED_CONFIDENCE) name_confidence = code_confidence else: - elsewhere = rules.editions_with(code) + # пустая редакция уже даёт одно «редакция не заполнена» — по строке не предупреждаем + elsewhere = rules.editions_with(code) if edition is not None and edition.lines else [] if elsewhere: build.warn(f"код {code} из редакции {', '.join(elsewhere)}") name = collapse_whitespace(name_cell.value) if name_cell is not None else "" @@ -156,7 +154,13 @@ def _line( name_confidence = name_cell.confidence if name_cell is not None else 0.0 if code_confidence < FULL_CONFIDENCE or name_confidence < FULL_CONFIDENCE: build.fully_recognized = False - values = [_value(row_cells.get(source_idx), position, build) for position, source_idx in enumerate(value_idxs)] + # values — по позиции колонки документа (R-3): лишние индексы раскладки строки отбрасываются, недостающие — + # «ячейки нет» + value_idxs = row.layout.value_idxs + values = [ + _value(row_cells.get(value_idxs[position]) if position < len(value_idxs) else None, position, build) + for position in range(column_count) + ] return StatementLine( line_code=code, line_code_confidence=code_confidence, @@ -197,8 +201,10 @@ def build_document( build = _Build(warnings=[]) label = _label_text(table) document_type = _resolve_type(label, parsed, rules, build) - columns, value_idxs = _columns(parsed, rules.lookalikes, build) + columns = _columns(parsed, rules.lookalikes, build) data_rows = [row for row in parsed.rows[1:] if row.kind is not RowKind.HEADER] + if any(len(row.layout.value_idxs) != len(columns) for row in data_rows): + build.warn("повторная шапка изменила число колонок значений") # одно предупреждение на таблицу codes = [clean_value(cell_text(row.cells, row.layout.code_idx)) for row in data_rows] edition = _choose_edition(header, columns, codes, rules, build) lines: list[StatementLine] = [] @@ -209,7 +215,7 @@ def build_document( build.warn(f"код {code} повторяется: оставлена первая строка") continue seen.add(code) - lines.append(_line(row, list(row.layout.value_idxs) or value_idxs, edition, rules, build)) + lines.append(_line(row, len(columns), edition, rules, build)) statement_header = _statement_header(label, header, build) document = StatementDocument( document_type=document_type, diff --git a/api/tests/unit/normalizer/test_document.py b/api/tests/unit/normalizer/test_document.py index 9793886..263f2e0 100644 --- a/api/tests/unit/normalizer/test_document.py +++ b/api/tests/unit/normalizer/test_document.py @@ -1,12 +1,15 @@ """Сборка `statementDocument` из таблицы OCR (spec «Нормализация ответа ИС», D-6, D-7).""" +from dataclasses import replace from datetime import date from decimal import Decimal from pathlib import Path +import pytest + from src.normalizer.ocr.document import BuiltDocument, build_document from src.normalizer.ocr.header import FileHeader -from src.normalizer.rules import load_rules +from src.normalizer.rules import Edition, OcrRules, load_rules API_DIR = Path(__file__).resolve().parents[3] RULES = load_rules(API_DIR / "rules") @@ -212,3 +215,210 @@ def test_missing_label_omits_form_name() -> None: def test_table_rows_not_list_is_none() -> None: assert build_document({"label": {"form_name": "x"}, "rows": "oops"}, 0, "job.json", HEADER_2024, RULES) is None + + +REPEATED_HEADER = "повторная шапка изменила число колонок значений" + + +def _head(*dates: str) -> dict: + cells = [(0, "Наименование показателя", 100), (1, "Код", 100)] + cells += [(2 + i, text, 100) for i, text in enumerate(dates)] + return _row(0, *cells) + + +class TestRepeatedHeader: + """R-3 (design #21): `columns` — по первой шапке, `values` каждой строки — по позиции этой колонки.""" + + def test_wider_first_header_pads_missing_values(self) -> None: + table = _table( + "Бухгалтерский баланс", + _head("На 31 декабря 2024 г.", "На 31 декабря 2023 г.", "На 31 декабря 2022 г."), + _row(1, (0, "ОС", 100), (1, "1150", 100), (2, "1", 100), (3, "2", 100), (4, "3", 100)), + _row( + 2, + (0, "Наименование показателя", 100), + (1, "Код", 100), + (2, "На 31 декабря 2024 г.", 100), + (3, "На 31 декабря 2023 г.", 100), + ), + _row(3, (0, "НМА", 100), (1, "1110", 100), (2, "4", 100), (3, "5", 100)), + ) + built = _build(table) + assert [line.line_code for line in built.document.lines] == ["1150", "1110"] + assert all(len(line.values) == 3 for line in built.document.lines) + last = built.document.lines[1].values[2] + assert (last.amount, last.string_value, last.confidence, last.is_recognized) == (None, None, 0, False) + assert built.document.lines[1].values[1].amount == Decimal("5") + assert built.warnings.count(REPEATED_HEADER) == 1 and built.fully_recognized is False + + def test_narrower_first_header_drops_extra_indexes(self) -> None: + table = _table( + "Бухгалтерский баланс", + _head("На 31 декабря 2024 г.", "На 31 декабря 2023 г."), + _row(1, (0, "ОС", 100), (1, "1150", 100), (2, "1", 100), (3, "2", 100)), + _row( + 2, + (0, "Наименование показателя", 100), + (1, "Код", 100), + (2, "На 31 декабря 2024 г.", 100), + (3, "На 31 декабря 2023 г.", 100), + (4, "На 31 декабря 2022 г.", 100), + ), + _row(3, (0, "НМА", 100), (1, "1110", 100), (2, "4", 100), (3, "5", 100), (4, "6", 100)), + ) + built = _build(table) + assert [c.column_index for c in built.document.columns] == [0, 1] + for line in built.document.lines: + assert [v.column_index for v in line.values] == [0, 1] # нет «сироты» с column_index = 2 + assert built.document.lines[1].values[1].amount == Decimal("5") + assert built.warnings.count(REPEATED_HEADER) == 1 and built.fully_recognized is False + + def test_same_layout_after_repeated_header_is_silent(self) -> None: + table = _table( + "Бухгалтерский баланс", + *CLEAN_BALANCE["rows"][:2], + _row( + 5, + (0, "Наименование показателя", 100), + (1, "Код", 100), + (2, "На 31 декабря 2024 г.", 100), + (3, "На 31 декабря 2023 г.", 100), + ), + _row(6, (0, "Итого по разделу I", 100), (1, "1100", 100), (2, "1", 100), (3, "2", 100)), + ) + built = _build(table) + assert [line.line_code for line in built.document.lines] == ["1150", "1100"] + assert built.warnings == [] and built.fully_recognized is True + + +def test_latin_o_in_code_drops_row() -> None: + """R-4 (design #22): лукалайки применяются только к датам — «11O0» не целое число, строка отбрасывается.""" + table = _table( + "Бухгалтерский баланс", + CLEAN_BALANCE["rows"][0], + _row(1, (0, "ОС", 100), (1, "1150", 100), (2, "1", 100), (3, "2", 100)), + _row(2, (0, "Итого", 100), (1, "11O0", 100), (2, "1", 100), (3, "2", 100)), + ) + assert [line.line_code for line in _build(table).document.lines] == ["1150"] + + +class TestEditionWarnings: + def test_empty_edition_gives_single_warning(self) -> None: + """R-5 (design #23): пустая fsbu4 — одно «не заполнена», без «код N из редакции 66n» по строкам.""" + header = FileHeader("ООО", 100.0, "7700000000", 100.0, "тыс. руб.", None, None, False) + table = _table( + "Бухгалтерский баланс", + _head("На 30 июня 2025 г"), + _row(1, (0, "ОС", 100), (1, "1150", 100), (2, "1", 100)), + _row(2, (0, "НМА", 100), (1, "1110", 100), (2, "2", 100)), + ) + built = _build(table, header) + assert built.edition_id == "fsbu4" + assert built.warnings == ["редакция fsbu4 не заполнена"] + assert not any("из редакции" in w for w in built.warnings) + + def test_code_from_other_nonempty_edition_is_reported(self) -> None: + rules = OcrRules( + editions=[ + Edition(id="old", valid_until=date(2024, 12, 31), lines={"1150": "Основные средства"}), + Edition(id="new", valid_from=date(2025, 1, 1), lines={"1110": "Нематериальные активы"}), + ], + lookalikes=RULES.lookalikes, + labels=RULES.labels, + ) + table = _table( + "Бухгалтерский баланс", + CLEAN_BALANCE["rows"][0], + _row(1, (0, "ОС", 100), (1, "1150", 100), (2, "1", 100), (3, "2", 100)), + _row(2, (0, "НМА", 100), (1, "1110", 100), (2, "3", 100), (3, "4", 100)), + ) + built = build_document(table, 0, "job.json", HEADER_2024, rules) + assert built is not None and built.edition_id == "old" + assert built.warnings == ["код 1110 из редакции new"] + assert built.document.lines[1].line_name == "НМА" and built.document.lines[1].line_code_confidence == 99 + + +def test_table_without_value_columns() -> None: + """R-6 (design #25): нет колонок значений — предупреждение и needs_check; values пустые.""" + table = _table( + "Бухгалтерский баланс", + _row(0, (0, "Наименование показателя", 100), (1, "Код", 100)), + _row(1, (0, "ОС", 100), (1, "1150", 100)), + ) + built = _build(table) + assert built.document.columns == [] and built.document.lines[0].values == [] + assert "нет колонок значений" in built.warnings and built.fully_recognized is False + assert REPEATED_HEADER not in built.warnings + + +class TestHeaderConditions: + """D-6: каждое отклонение шапки файла лишает документ статуса «всё распознано».""" + + @pytest.mark.parametrize( + "changes", + [ + {"inn_confidence": 90.0}, + {"inn": None, "inn_confidence": None}, + {"organization_confidence": 90.0}, + {"organization_name": None, "organization_confidence": None}, + {"report_date": None, "report_date_recognized": False}, + {"report_date_recognized": False}, + ], + ids=["inn-low", "inn-none", "org-low", "org-none", "no-date", "date-not-recognized"], + ) + def test_header_deviation_is_not_fully_recognized(self, changes: dict) -> None: + assert _build(CLEAN_BALANCE, replace(HEADER_2024, **changes)).fully_recognized is False + + def test_baseline_is_fully_recognized(self) -> None: + assert _build(CLEAN_BALANCE, HEADER_2024).fully_recognized is True + + +def test_column_without_date_warning() -> None: + table = _table( + "Бухгалтерский баланс", + _head("На отчетную дату"), + _row(1, (0, "ОС", 100), (1, "1150", 100), (2, "1", 100)), + ) + built = _build(table) + assert "колонка 0 без даты: На отчетную дату" in built.warnings + assert built.document.columns[0].period_date is None and built.fully_recognized is False + + +def test_edition_not_determined_warning() -> None: + header = FileHeader(None, None, None, None, None, None, None, False) + table = _table( + "Приложение", + _row(0, (0, "Наименование показателя", 100), (1, "Код", 100), (2, "На 1 января", 100)), + _row(1, (0, "a", 100), (1, "5100", 100), (2, "1", 100)), + _row(2, (0, "b", 100), (1, "5200", 100), (2, "2", 100)), + ) + built = _build(table, header) + assert built.edition_id is None and "редакция не определена" in built.warnings + assert built.document.lines[0].line_name == "a" and built.fully_recognized is False + + +class TestKnownCodeConfidence: + """R-2, ветка «код в выбранной редакции»: confidence ячейки, если текст чистый, иначе не выше 99.""" + + def test_clean_code_keeps_cell_confidence(self) -> None: + table = _table( + "Бухгалтерский баланс", + CLEAN_BALANCE["rows"][0], + _row(1, (0, "ОС", 80), (1, "1150", 80), (2, "1", 100), (3, "2", 100)), + ) + line = _build(table).document.lines[0] + assert line.line_code_confidence == 80 and line.line_name_confidence == 80 + assert line.line_name == "Основные средства" + + def test_dirty_code_is_capped_at_99(self) -> None: + """«+1150» проходит `int()`, `clean_value` убирает «+» — текст не равен сырому, уверенность ≤ 99.""" + table = _table( + "Бухгалтерский баланс", + CLEAN_BALANCE["rows"][0], + _row(1, (0, "ОС", 100), (1, "+1150", 100), (2, "1", 100), (3, "2", 100)), + ) + built = _build(table) + line = built.document.lines[0] + assert line.line_code == "1150" and line.line_name == "Основные средства" + assert line.line_code_confidence == 99 and line.line_name_confidence == 99 + assert built.fully_recognized is False