8ebf572a22
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012vJkmBGKKqMoEMzvsXG3y2
425 lines
21 KiB
Python
425 lines
21 KiB
Python
"""Сборка `statementDocument` из таблицы OCR (spec «Нормализация ответа ИС», D-6, D-7)."""
|
||
|
||
from dataclasses import replace
|
||
from datetime import date
|
||
from decimal import Decimal
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
|
||
from src.normalizer.ocr.document import BuiltDocument, build_document
|
||
from src.normalizer.ocr.header import FileHeader
|
||
from src.normalizer.rules import Edition, OcrRules, load_rules
|
||
|
||
API_DIR = Path(__file__).resolve().parents[3]
|
||
RULES = load_rules(API_DIR / "rules")
|
||
HEADER_2024 = FileHeader(
|
||
"ООО Ромашка", 100.0, "7700000000", 100.0, "тыс. руб.", "на 31 декабря 2024 г.", date(2024, 12, 31), True
|
||
)
|
||
|
||
|
||
def _row(index: int, *cells: tuple[int, str, float]) -> dict:
|
||
return {"row_index": index, "cells": [{"column_index": i, "value": v, "confidence": c} for i, v, c in cells]}
|
||
|
||
|
||
def _table(label: str, *rows: dict) -> dict:
|
||
return {"label": {"form_name": label, "confidence": 100}, "rows": list(rows)}
|
||
|
||
|
||
CLEAN_BALANCE = _table(
|
||
"Бухгалтерский баланс",
|
||
_row(
|
||
0,
|
||
(0, "Наименование показателя", 100),
|
||
(1, "Код", 100),
|
||
(2, "На 31 декабря 2024 г.", 100),
|
||
(3, "На 31 декабря 2023 г.", 100),
|
||
),
|
||
_row(1, (0, "Основные средства", 100), (1, "1150", 100), (2, "1 234", 100), (3, "(56)", 100)),
|
||
_row(2, (0, "Итого по разделу I", 100), (1, "1100", 100), (2, "1 234", 100), (3, "-", 100)),
|
||
)
|
||
|
||
|
||
def _build(table: dict, header: FileHeader = HEADER_2024, index: int = 0) -> BuiltDocument:
|
||
built = build_document(table, index, "job.json", header, RULES)
|
||
assert built is not None
|
||
return built
|
||
|
||
|
||
class TestCleanTable:
|
||
def test_document_fields(self) -> None:
|
||
built = _build(CLEAN_BALANCE)
|
||
doc = built.document
|
||
assert doc.document_type == "BALANCE"
|
||
assert doc.locator == "file:job.json;table:0"
|
||
assert doc.header.form_name == "Бухгалтерский баланс"
|
||
assert doc.header.inn == "7700000000" and doc.header.inn_confidence == 100
|
||
assert doc.header.report_date == date(2024, 12, 31) and doc.header.report_date_text == "на 31 декабря 2024 г."
|
||
assert doc.header.units == "тыс. руб." and doc.header.okud is None
|
||
assert [(c.column_index, c.label, c.period_date) for c in doc.columns] == [
|
||
(0, "На 31 декабря 2024 г.", date(2024, 12, 31)),
|
||
(1, "На 31 декабря 2023 г.", date(2023, 12, 31)),
|
||
]
|
||
assert doc.checks is None
|
||
|
||
def test_lines_and_values(self) -> None:
|
||
doc = _build(CLEAN_BALANCE).document
|
||
assert [line.line_code for line in doc.lines] == ["1150", "1100"]
|
||
first = doc.lines[0]
|
||
assert (
|
||
first.line_name == "Основные средства"
|
||
and first.line_code_confidence == 100
|
||
and first.line_name_confidence == 100
|
||
)
|
||
values = first.values
|
||
assert [v.column_index for v in values] == [0, 1]
|
||
assert (
|
||
values[0].amount == Decimal("1234") and values[0].string_value is None and values[0].is_recognized is True
|
||
)
|
||
assert values[1].amount == Decimal("-56") and values[1].is_recognized is True
|
||
blank = doc.lines[1].values[1]
|
||
assert blank.amount is None and blank.string_value is None and blank.is_recognized is True
|
||
|
||
def test_fully_recognized_and_edition(self) -> None:
|
||
built = _build(CLEAN_BALANCE)
|
||
assert built.fully_recognized is True and built.warnings == [] and built.edition_id == "66n"
|
||
|
||
|
||
class TestNoise:
|
||
def test_unknown_code_uses_ocr_name_and_caps_confidence(self) -> None:
|
||
table = _table(
|
||
"Бухгалтерский баланс",
|
||
CLEAN_BALANCE["rows"][0],
|
||
_row(1, (0, "Расшифровка", 100), (1, "11501", 100), (2, "1", 100), (3, "2", 100)),
|
||
_row(2, (0, "ОС", 100), (1, "1150", 90), (2, "1", 100), (3, "2", 100)),
|
||
)
|
||
built = _build(table)
|
||
doc = built.document
|
||
# кода нет в справочнике — имя из OCR, уверенность кода не выше 99 (spec: «иначе min(confidence, 99)»)
|
||
assert doc.lines[0].line_code == "11501" and doc.lines[0].line_name == "Расшифровка"
|
||
assert doc.lines[0].line_code_confidence == 99 and doc.lines[0].line_name_confidence == 100
|
||
assert doc.lines[1].line_code == "1150" and doc.lines[1].line_code_confidence == 90 # уже < 100, не поднимается
|
||
assert doc.lines[1].line_name == "Основные средства"
|
||
assert built.fully_recognized is False
|
||
|
||
def test_non_integer_code_row_is_dropped(self) -> None:
|
||
"""Перенос `_classify_row`: «1150 :» не целое число → строка считается повторным заголовком и пропускается."""
|
||
table = _table(
|
||
"Бухгалтерский баланс",
|
||
CLEAN_BALANCE["rows"][0],
|
||
_row(1, (0, "НМА", 100), (1, "1110", 100), (2, "1", 100), (3, "2", 100)),
|
||
_row(2, (0, "ОС", 100), (1, "1150 :", 100), (2, "1", 100), (3, "2", 100)),
|
||
)
|
||
assert [line.line_code for line in _build(table).document.lines] == ["1110"]
|
||
|
||
def test_duplicate_code_keeps_first(self) -> None:
|
||
"""Решение №15: DBML уникален по (meta_id, line_code) — повтор кода даёт одну строку и предупреждение."""
|
||
table = _table(
|
||
"Бухгалтерский баланс",
|
||
CLEAN_BALANCE["rows"][0],
|
||
_row(1, (0, "Основные средства", 100), (1, "1150", 100), (2, "1", 100), (3, "2", 100)),
|
||
_row(2, (0, "Основные средства", 100), (1, "1150", 100), (2, "7", 100), (3, "8", 100)),
|
||
)
|
||
built = _build(table)
|
||
assert [line.line_code for line in built.document.lines] == ["1150"]
|
||
assert built.document.lines[0].values[0].amount == Decimal("1")
|
||
assert "код 1150 повторяется: оставлена первая строка" in built.warnings and built.fully_recognized is False
|
||
|
||
def test_missing_cells_keep_column_count(self) -> None:
|
||
"""Review Focus 2: у строки нет ячеек под колонками — values всё равно по числу колонок."""
|
||
table = _table("Бухгалтерский баланс", CLEAN_BALANCE["rows"][0], _row(1, (0, "ОС", 100), (1, "1150", 100)))
|
||
values = _build(table).document.lines[0].values
|
||
assert [(v.column_index, v.amount, v.string_value, v.confidence, v.is_recognized) for v in values] == [
|
||
(0, None, None, 0, False),
|
||
(1, None, None, 0, False),
|
||
]
|
||
|
||
def test_unverified_amount_is_not_recognized(self) -> None:
|
||
table = _table(
|
||
"Бухгалтерский баланс",
|
||
CLEAN_BALANCE["rows"][0],
|
||
_row(1, (0, "ОС", 100), (1, "1150", 100), (2, "(7744536}", 100), (3, "x", 100)),
|
||
)
|
||
values = _build(table).document.lines[0].values
|
||
assert (
|
||
values[0].amount == Decimal("7744536")
|
||
and values[0].is_recognized is False
|
||
and values[0].string_value is None
|
||
)
|
||
assert values[1].amount is None and values[1].is_recognized is False and values[1].string_value == "x"
|
||
|
||
def test_synthetic_header_columns_without_dates(self) -> None:
|
||
table = _table(
|
||
"Бухгалтерский баланс",
|
||
_row(0, (1, "1150", 100), (0, "ОС", 100), (2, "1", 100)),
|
||
_row(1, (1, "1110", 100), (0, "НМА", 100), (2, "2", 100)),
|
||
)
|
||
built = _build(table)
|
||
assert [(c.column_index, c.label, c.period_date) for c in built.document.columns] == [(0, "", None)]
|
||
assert "шапка синтезирована" in " ".join(built.warnings) and built.fully_recognized is False
|
||
|
||
|
||
class TestTypeAndEdition:
|
||
def test_type_by_codes_when_label_unknown(self) -> None:
|
||
built = _build(_table("Бvxгалтерский баланс", *CLEAN_BALANCE["rows"]))
|
||
assert built.document.document_type == "BALANCE" and "тип определён по кодам" in built.warnings
|
||
|
||
def test_label_wins_over_codes_with_warning(self) -> None:
|
||
built = _build(_table("Отчет о финансовых результатах", *CLEAN_BALANCE["rows"]))
|
||
assert built.document.document_type == "INCOME"
|
||
assert any(w.startswith("тип по кодам (BALANCE) не совпал с label") for w in built.warnings)
|
||
|
||
def test_other_when_nothing_known(self) -> None:
|
||
table = _table(
|
||
"Приложение",
|
||
CLEAN_BALANCE["rows"][0],
|
||
_row(1, (0, "a", 100), (1, "5100", 100), (2, "1", 100), (3, "2", 100)),
|
||
_row(2, (0, "b", 100), (1, "5200", 100), (2, "1", 100), (3, "2", 100)),
|
||
)
|
||
built = _build(table)
|
||
assert built.document.document_type == "OTHER" and "тип не определён: OTHER" in built.warnings
|
||
|
||
def test_edition_2025_is_empty(self) -> None:
|
||
header = FileHeader("ООО", 100.0, "7700000000", 100.0, "тыс. руб.", None, None, False)
|
||
table = _table(
|
||
"Бухгалтерский баланс",
|
||
_row(0, (0, "Наименование показателя", 100), (1, "Код", 100), (2, "На 30 июня 2025 г.", 100)),
|
||
_row(1, (0, "Основные средства", 100), (1, "1150", 100), (2, "1", 100)),
|
||
)
|
||
built = _build(table, header)
|
||
assert built.edition_id == "fsbu4" and "редакция fsbu4 не заполнена" in built.warnings
|
||
assert built.document.lines[0].line_name == "Основные средства" # из OCR
|
||
assert built.document.lines[0].line_name_confidence == 100 # confidence ячейки наименования
|
||
assert built.fully_recognized is False
|
||
|
||
def test_edition_by_codes_when_no_dates(self) -> None:
|
||
header = FileHeader(None, None, None, None, None, None, None, False)
|
||
table = _table("Бухгалтерский баланс", _row(0, (1, "1150", 100), (0, "ОС", 100), (2, "1", 100)))
|
||
built = _build(table, header)
|
||
assert built.edition_id == "66n" and "редакция определена по кодам: 66n" in built.warnings
|
||
|
||
def test_unparsed_table_is_none(self) -> None:
|
||
assert (
|
||
build_document(_table("x", _row(0, (0, "a", 100), (1, "b", 100))), 0, "job.json", HEADER_2024, RULES)
|
||
is None
|
||
)
|
||
|
||
|
||
def test_missing_label_omits_form_name() -> None:
|
||
"""Контракт: `form_name` без `null` — при отсутствии label поле не задаётся, сборка не падает."""
|
||
table = {"rows": CLEAN_BALANCE["rows"]}
|
||
built = _build(table)
|
||
assert built.document.header.form_name is None
|
||
assert "form_name" not in built.document.to_wire()["header"]
|
||
|
||
|
||
def test_table_rows_not_list_is_none() -> None:
|
||
assert build_document({"label": {"form_name": "x"}, "rows": "oops"}, 0, "job.json", HEADER_2024, RULES) is None
|
||
|
||
|
||
REPEATED_HEADER = "повторная шапка изменила число колонок значений"
|
||
|
||
|
||
def _head(*dates: str) -> dict:
|
||
cells = [(0, "Наименование показателя", 100), (1, "Код", 100)]
|
||
cells += [(2 + i, text, 100) for i, text in enumerate(dates)]
|
||
return _row(0, *cells)
|
||
|
||
|
||
class TestRepeatedHeader:
|
||
"""R-3 (design #21): `columns` — по первой шапке, `values` каждой строки — по позиции этой колонки."""
|
||
|
||
def test_wider_first_header_pads_missing_values(self) -> None:
|
||
table = _table(
|
||
"Бухгалтерский баланс",
|
||
_head("На 31 декабря 2024 г.", "На 31 декабря 2023 г.", "На 31 декабря 2022 г."),
|
||
_row(1, (0, "ОС", 100), (1, "1150", 100), (2, "1", 100), (3, "2", 100), (4, "3", 100)),
|
||
_row(
|
||
2,
|
||
(0, "Наименование показателя", 100),
|
||
(1, "Код", 100),
|
||
(2, "На 31 декабря 2024 г.", 100),
|
||
(3, "На 31 декабря 2023 г.", 100),
|
||
),
|
||
_row(3, (0, "НМА", 100), (1, "1110", 100), (2, "4", 100), (3, "5", 100)),
|
||
)
|
||
built = _build(table)
|
||
assert [line.line_code for line in built.document.lines] == ["1150", "1110"]
|
||
assert all(len(line.values) == 3 for line in built.document.lines)
|
||
last = built.document.lines[1].values[2]
|
||
assert (last.amount, last.string_value, last.confidence, last.is_recognized) == (None, None, 0, False)
|
||
assert built.document.lines[1].values[1].amount == Decimal("5")
|
||
assert built.warnings.count(REPEATED_HEADER) == 1 and built.fully_recognized is False
|
||
|
||
def test_narrower_first_header_drops_extra_indexes(self) -> None:
|
||
table = _table(
|
||
"Бухгалтерский баланс",
|
||
_head("На 31 декабря 2024 г.", "На 31 декабря 2023 г."),
|
||
_row(1, (0, "ОС", 100), (1, "1150", 100), (2, "1", 100), (3, "2", 100)),
|
||
_row(
|
||
2,
|
||
(0, "Наименование показателя", 100),
|
||
(1, "Код", 100),
|
||
(2, "На 31 декабря 2024 г.", 100),
|
||
(3, "На 31 декабря 2023 г.", 100),
|
||
(4, "На 31 декабря 2022 г.", 100),
|
||
),
|
||
_row(3, (0, "НМА", 100), (1, "1110", 100), (2, "4", 100), (3, "5", 100), (4, "6", 100)),
|
||
)
|
||
built = _build(table)
|
||
assert [c.column_index for c in built.document.columns] == [0, 1]
|
||
for line in built.document.lines:
|
||
assert [v.column_index for v in line.values] == [0, 1] # нет «сироты» с column_index = 2
|
||
assert built.document.lines[1].values[1].amount == Decimal("5")
|
||
assert built.warnings.count(REPEATED_HEADER) == 1 and built.fully_recognized is False
|
||
|
||
def test_same_layout_after_repeated_header_is_silent(self) -> None:
|
||
table = _table(
|
||
"Бухгалтерский баланс",
|
||
*CLEAN_BALANCE["rows"][:2],
|
||
_row(
|
||
5,
|
||
(0, "Наименование показателя", 100),
|
||
(1, "Код", 100),
|
||
(2, "На 31 декабря 2024 г.", 100),
|
||
(3, "На 31 декабря 2023 г.", 100),
|
||
),
|
||
_row(6, (0, "Итого по разделу I", 100), (1, "1100", 100), (2, "1", 100), (3, "2", 100)),
|
||
)
|
||
built = _build(table)
|
||
assert [line.line_code for line in built.document.lines] == ["1150", "1100"]
|
||
assert built.warnings == [] and built.fully_recognized is True
|
||
|
||
|
||
def test_latin_o_in_code_drops_row() -> None:
|
||
"""R-4 (design #22): лукалайки применяются только к датам — «11O0» не целое число, строка отбрасывается."""
|
||
table = _table(
|
||
"Бухгалтерский баланс",
|
||
CLEAN_BALANCE["rows"][0],
|
||
_row(1, (0, "ОС", 100), (1, "1150", 100), (2, "1", 100), (3, "2", 100)),
|
||
_row(2, (0, "Итого", 100), (1, "11O0", 100), (2, "1", 100), (3, "2", 100)),
|
||
)
|
||
assert [line.line_code for line in _build(table).document.lines] == ["1150"]
|
||
|
||
|
||
class TestEditionWarnings:
|
||
def test_empty_edition_gives_single_warning(self) -> None:
|
||
"""R-5 (design #23): пустая fsbu4 — одно «не заполнена», без «код N из редакции 66n» по строкам."""
|
||
header = FileHeader("ООО", 100.0, "7700000000", 100.0, "тыс. руб.", None, None, False)
|
||
table = _table(
|
||
"Бухгалтерский баланс",
|
||
_head("На 30 июня 2025 г"),
|
||
_row(1, (0, "ОС", 100), (1, "1150", 100), (2, "1", 100)),
|
||
_row(2, (0, "НМА", 100), (1, "1110", 100), (2, "2", 100)),
|
||
)
|
||
built = _build(table, header)
|
||
assert built.edition_id == "fsbu4"
|
||
assert built.warnings == ["редакция fsbu4 не заполнена"]
|
||
assert not any("из редакции" in w for w in built.warnings)
|
||
|
||
def test_code_from_other_nonempty_edition_is_reported(self) -> None:
|
||
rules = OcrRules(
|
||
editions=[
|
||
Edition(id="old", valid_until=date(2024, 12, 31), lines={"1150": "Основные средства"}),
|
||
Edition(id="new", valid_from=date(2025, 1, 1), lines={"1110": "Нематериальные активы"}),
|
||
],
|
||
lookalikes=RULES.lookalikes,
|
||
labels=RULES.labels,
|
||
)
|
||
table = _table(
|
||
"Бухгалтерский баланс",
|
||
CLEAN_BALANCE["rows"][0],
|
||
_row(1, (0, "ОС", 100), (1, "1150", 100), (2, "1", 100), (3, "2", 100)),
|
||
_row(2, (0, "НМА", 100), (1, "1110", 100), (2, "3", 100), (3, "4", 100)),
|
||
)
|
||
built = build_document(table, 0, "job.json", HEADER_2024, rules)
|
||
assert built is not None and built.edition_id == "old"
|
||
assert built.warnings == ["код 1110 из редакции new"]
|
||
assert built.document.lines[1].line_name == "НМА" and built.document.lines[1].line_code_confidence == 99
|
||
|
||
|
||
def test_table_without_value_columns() -> None:
|
||
"""R-6 (design #25): нет колонок значений — предупреждение и needs_check; values пустые."""
|
||
table = _table(
|
||
"Бухгалтерский баланс",
|
||
_row(0, (0, "Наименование показателя", 100), (1, "Код", 100)),
|
||
_row(1, (0, "ОС", 100), (1, "1150", 100)),
|
||
)
|
||
built = _build(table)
|
||
assert built.document.columns == [] and built.document.lines[0].values == []
|
||
assert "нет колонок значений" in built.warnings and built.fully_recognized is False
|
||
assert REPEATED_HEADER not in built.warnings
|
||
|
||
|
||
class TestHeaderConditions:
|
||
"""D-6: каждое отклонение шапки файла лишает документ статуса «всё распознано»."""
|
||
|
||
@pytest.mark.parametrize(
|
||
"changes",
|
||
[
|
||
{"inn_confidence": 90.0},
|
||
{"inn": None, "inn_confidence": None},
|
||
{"organization_confidence": 90.0},
|
||
{"organization_name": None, "organization_confidence": None},
|
||
{"report_date": None, "report_date_recognized": False},
|
||
{"report_date_recognized": False},
|
||
],
|
||
ids=["inn-low", "inn-none", "org-low", "org-none", "no-date", "date-not-recognized"],
|
||
)
|
||
def test_header_deviation_is_not_fully_recognized(self, changes: dict) -> None:
|
||
assert _build(CLEAN_BALANCE, replace(HEADER_2024, **changes)).fully_recognized is False
|
||
|
||
def test_baseline_is_fully_recognized(self) -> None:
|
||
assert _build(CLEAN_BALANCE, HEADER_2024).fully_recognized is True
|
||
|
||
|
||
def test_column_without_date_warning() -> None:
|
||
table = _table(
|
||
"Бухгалтерский баланс",
|
||
_head("На отчетную дату"),
|
||
_row(1, (0, "ОС", 100), (1, "1150", 100), (2, "1", 100)),
|
||
)
|
||
built = _build(table)
|
||
assert "колонка 0 без даты: На отчетную дату" in built.warnings
|
||
assert built.document.columns[0].period_date is None and built.fully_recognized is False
|
||
|
||
|
||
def test_edition_not_determined_warning() -> None:
|
||
header = FileHeader(None, None, None, None, None, None, None, False)
|
||
table = _table(
|
||
"Приложение",
|
||
_row(0, (0, "Наименование показателя", 100), (1, "Код", 100), (2, "На 1 января", 100)),
|
||
_row(1, (0, "a", 100), (1, "5100", 100), (2, "1", 100)),
|
||
_row(2, (0, "b", 100), (1, "5200", 100), (2, "2", 100)),
|
||
)
|
||
built = _build(table, header)
|
||
assert built.edition_id is None and "редакция не определена" in built.warnings
|
||
assert built.document.lines[0].line_name == "a" and built.fully_recognized is False
|
||
|
||
|
||
class TestKnownCodeConfidence:
|
||
"""R-2, ветка «код в выбранной редакции»: confidence ячейки, если текст чистый, иначе не выше 99."""
|
||
|
||
def test_clean_code_keeps_cell_confidence(self) -> None:
|
||
table = _table(
|
||
"Бухгалтерский баланс",
|
||
CLEAN_BALANCE["rows"][0],
|
||
_row(1, (0, "ОС", 80), (1, "1150", 80), (2, "1", 100), (3, "2", 100)),
|
||
)
|
||
line = _build(table).document.lines[0]
|
||
assert line.line_code_confidence == 80 and line.line_name_confidence == 80
|
||
assert line.line_name == "Основные средства"
|
||
|
||
def test_dirty_code_is_capped_at_99(self) -> None:
|
||
"""«+1150» проходит `int()`, `clean_value` убирает «+» — текст не равен сырому, уверенность ≤ 99."""
|
||
table = _table(
|
||
"Бухгалтерский баланс",
|
||
CLEAN_BALANCE["rows"][0],
|
||
_row(1, (0, "ОС", 100), (1, "+1150", 100), (2, "1", 100), (3, "2", 100)),
|
||
)
|
||
built = _build(table)
|
||
line = built.document.lines[0]
|
||
assert line.line_code == "1150" and line.line_name == "Основные средства"
|
||
assert line.line_code_confidence == 99 and line.line_name_confidence == 99
|
||
assert built.fully_recognized is False
|