9b0f01ef1c
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W
354 lines
19 KiB
Python
354 lines
19 KiB
Python
"""Строки таблицы OCR (§11.1 п. 3–5): шапка настоящая/повторная/синтетическая, классы строк, тип по кодам."""
|
||
|
||
import json
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
|
||
from src.normalizer.ocr.layout import ColumnLayout, cell_text, cells_by_index
|
||
from src.normalizer.ocr.rows import (
|
||
ParsedTable,
|
||
RowKind,
|
||
classify_row,
|
||
is_numbering_row,
|
||
parse_table_rows,
|
||
table_type_by_codes,
|
||
table_type_by_unanimous_codes,
|
||
)
|
||
|
||
API_DIR = Path(__file__).resolve().parents[3]
|
||
REAL_FIXTURES = API_DIR.parent.parent / "fixtures" / "ocr"
|
||
KNOWN = frozenset({"1105", "1110", "1120", "1130", "1150", "1600", "2110", "2120", "2100"})
|
||
|
||
|
||
def _row(index: int, *cells: tuple[int, str, float]) -> dict:
|
||
return {"row_index": index, "cells": [{"column_index": i, "value": v, "confidence": c} for i, v, c in cells]}
|
||
|
||
|
||
HEADER = _row(
|
||
0,
|
||
(0, "Наименование показателя", 100),
|
||
(1, "Код", 100),
|
||
(2, "На 30 июня\n2025 г", 100),
|
||
(4, "На 31 декабря\n2024 г.", 100),
|
||
)
|
||
|
||
|
||
def _real_table(name: str, index: int) -> list[dict]:
|
||
if not REAL_FIXTURES.is_dir():
|
||
pytest.skip("fixtures недоступен: положите репозиторий fixtures рядом с workspace")
|
||
data = json.loads((REAL_FIXTURES / name).read_text(encoding="utf-8"))
|
||
return list(data["data_ocr"]["table_structure"][index]["rows"])
|
||
|
||
|
||
class TestClassifyRow:
|
||
layout = ColumnLayout(label_idx=0, code_idx=1, value_idxs=(2,))
|
||
|
||
@pytest.mark.parametrize(
|
||
("label", "code", "expected"),
|
||
[
|
||
("Выручка", "2110", RowKind.ROW),
|
||
("Итого по разделу I", "1100", RowKind.TOTAL),
|
||
("Баланс", "1600", RowKind.BALANCE),
|
||
("Наименование", "Код", RowKind.HEADER),
|
||
("в том числе:", "", None),
|
||
],
|
||
)
|
||
def test_kinds(self, label: str, code: str, expected: RowKind | None) -> None:
|
||
cells = cells_by_index(_row(1, (0, label, 100), (1, code, 100), (2, "1", 100)))
|
||
assert classify_row(cells, self.layout) is expected
|
||
|
||
|
||
class TestParseTableRows:
|
||
def test_header_then_rows(self) -> None:
|
||
rows = [
|
||
HEADER,
|
||
_row(1, (0, "Выручка", 100), (1, "2110", 100), (2, "13 070", 100), (4, "", 100)),
|
||
_row(2, (0, "в том числе:", 93), (1, "", 100), (2, "13 070", 100), (4, "", 100)),
|
||
]
|
||
parsed = parse_table_rows(rows, KNOWN)
|
||
assert isinstance(parsed, ParsedTable) and not parsed.synthetic_header
|
||
assert [r.kind for r in parsed.rows] == [RowKind.HEADER, RowKind.ROW]
|
||
assert parsed.rows[0].layout == ColumnLayout(label_idx=0, code_idx=1, value_idxs=(2, 4))
|
||
assert parsed.rows[1].ordinal == 1
|
||
|
||
def test_numbering_row_is_not_data(self) -> None:
|
||
"""AURORA-1517: строка «1 2 3 4 5» под шапкой — часть заголовка, не строка данных с кодом «3»."""
|
||
rows = [
|
||
HEADER,
|
||
_row(1, (0, "1", 100), (1, "2", 100), (2, "3", 100), (4, "4", 100)),
|
||
_row(2, (0, "Выручка", 100), (1, "2110", 100), (2, "13 070", 100), (4, "", 100)),
|
||
]
|
||
parsed = parse_table_rows(rows, KNOWN)
|
||
assert parsed is not None
|
||
assert [r.kind for r in parsed.rows] == [RowKind.HEADER, RowKind.ROW]
|
||
assert parsed.rows[1].ordinal == 2
|
||
assert parsed.skipped_non_numeric_code == 0 and parsed.skipped_no_code_with_amounts == 0
|
||
|
||
def test_numbering_row_with_empty_cells(self) -> None:
|
||
rows = [
|
||
HEADER,
|
||
_row(1, (0, "1", 100), (1, "2", 100), (2, "3", 100), (3, "", 100), (4, "5", 100)),
|
||
_row(2, (0, "Выручка", 100), (1, "2110", 100), (2, "13 070", 100)),
|
||
]
|
||
parsed = parse_table_rows(rows, KNOWN)
|
||
assert parsed is not None and [r.kind for r in parsed.rows] == [RowKind.HEADER, RowKind.ROW]
|
||
|
||
@pytest.mark.parametrize(
|
||
"values",
|
||
[
|
||
("1", "2"), # меньше трёх ячеек
|
||
("5", "3", "4"), # не по возрастанию
|
||
("1", "2", "2"),
|
||
("1105", "2", "3"), # не короткие
|
||
("Выручка", "2110", "5"),
|
||
],
|
||
)
|
||
def test_not_numbering_row(self, values: tuple[str, ...]) -> None:
|
||
assert not is_numbering_row(cells_by_index(_row(1, *((i, v, 100) for i, v in enumerate(values)))))
|
||
|
||
def test_second_header_overrides_layout(self) -> None:
|
||
"""Review Focus 1: после повторной шапки с другим набором колонок строки идут по новой раскладке."""
|
||
rows = [
|
||
HEADER,
|
||
_row(1, (0, "Выручка", 100), (1, "2110", 100), (2, "1", 100), (4, "2", 100)),
|
||
_row(2, (0, "Наименование показателя", 100), (1, "Код", 100), (3, "За 2024 г.", 100)),
|
||
_row(3, (0, "Себестоимость", 100), (1, "2120", 100), (3, "(5)", 100)),
|
||
]
|
||
parsed = parse_table_rows(rows, KNOWN)
|
||
assert parsed is not None
|
||
kinds = [r.kind for r in parsed.rows]
|
||
assert kinds == [RowKind.HEADER, RowKind.ROW, RowKind.ROW] # повторная шапка не становится строкой
|
||
assert parsed.rows[1].layout.value_idxs == (2, 4)
|
||
assert parsed.rows[2].layout.value_idxs == (3,)
|
||
|
||
def test_rows_before_header_use_inferred_layout(self) -> None:
|
||
rows = [
|
||
_row(0, (1, "1110", 100), (0, "Нематериальные активы", 100), (2, "5", 100)),
|
||
_row(1, (0, "Наименование показателя", 100), (1, "Код", 100), (2, "2024", 100)),
|
||
_row(2, (0, "Основные средства", 100), (1, "1150", 100), (2, "7", 100)),
|
||
]
|
||
parsed = parse_table_rows(rows, KNOWN)
|
||
assert parsed is not None and not parsed.synthetic_header
|
||
assert [r.ordinal for r in parsed.rows] == [-1, 0, 2] # шапка встаёт раньше первой строки данных
|
||
assert parsed.rows[0].kind is RowKind.HEADER
|
||
|
||
def test_synthetic_header_when_none(self) -> None:
|
||
rows = [
|
||
_row(0, (1, "1110", 100), (0, "Нематериальные активы", 100), (2, "5", 100)),
|
||
_row(1, (1, "1150", 100), (0, "Основные средства", 100), (2, "7", 100)),
|
||
]
|
||
parsed = parse_table_rows(rows, KNOWN)
|
||
assert parsed is not None and parsed.synthetic_header
|
||
header = parsed.rows[0]
|
||
assert header.kind is RowKind.HEADER and header.ordinal == -1
|
||
assert header.cells[1].value == "Код" and header.cells[0].value == "Наименование показателя"
|
||
assert header.layout.value_idxs == (2,) and header.cells.get(2) is None # тексты периодов пустые
|
||
|
||
def test_no_header_no_codes_is_none(self) -> None:
|
||
assert parse_table_rows([_row(0, (0, "текст", 100), (1, "мусор", 100))], KNOWN) is None
|
||
assert parse_table_rows([], KNOWN) is None
|
||
|
||
@pytest.mark.fixtures
|
||
def test_aep3_balance_without_header(self) -> None:
|
||
parsed = parse_table_rows(_real_table("aep_3.json", 0), KNOWN)
|
||
assert parsed is not None and parsed.synthetic_header
|
||
assert parsed.rows[0].layout.code_idx == 1
|
||
assert any(r.cells[1].value.strip() == "1120" for r in parsed.rows[1:])
|
||
|
||
@pytest.mark.fixtures
|
||
def test_aep6_balance_with_header(self) -> None:
|
||
parsed = parse_table_rows(_real_table("aep_6.json", 0), KNOWN)
|
||
assert parsed is not None and not parsed.synthetic_header
|
||
assert parsed.rows[0].layout == ColumnLayout(label_idx=0, code_idx=1, value_idxs=(2, 4, 5))
|
||
assert table_type_by_codes(parsed) == "BALANCE"
|
||
|
||
|
||
class TestSkippedRows:
|
||
"""Решение №29: отброшенные строки считаются, не меняя сам разбор (строки данных те же, что и раньше)."""
|
||
|
||
def test_clean_table_counts_nothing(self) -> None:
|
||
rows = [HEADER, _row(1, (0, "Выручка", 100), (1, "2110", 100), (2, "5", 100), (4, "6", 100))]
|
||
parsed = parse_table_rows(rows, KNOWN)
|
||
assert parsed is not None
|
||
assert (parsed.skipped_non_numeric_code, parsed.skipped_no_code_with_amounts) == (0, 0)
|
||
|
||
def test_non_numeric_code_is_counted(self) -> None:
|
||
rows = [
|
||
HEADER,
|
||
_row(1, (0, "ОС", 100), (1, "1150 :", 100), (2, "5", 100), (4, "6", 100)),
|
||
_row(2, (0, "НМА", 100), (1, "1l10", 100), (2, "5", 100), (4, "6", 100)), # латинская l вместо единицы
|
||
_row(3, (0, "Выручка", 100), (1, "2110", 100), (2, "5", 100), (4, "6", 100)),
|
||
]
|
||
parsed = parse_table_rows(rows, KNOWN)
|
||
assert parsed is not None
|
||
assert [r.ordinal for r in parsed.rows] == [0, 3] # разбор прежний: строки с кодом-мусором отброшены
|
||
assert (parsed.skipped_non_numeric_code, parsed.skipped_no_code_with_amounts) == (2, 0)
|
||
|
||
def test_empty_code_counted_only_with_amount_in_value_columns(self) -> None:
|
||
rows = [
|
||
HEADER,
|
||
_row(1, (0, "БАЛАНС", 100), (1, "", 100), (2, "5 553 383", 100), (4, "", 100)), # сумма — считается
|
||
_row(2, (0, "Итого", 100), (4, "(77)", 100)), # ячейки кода нет вовсе, сумма в колонке 4 — считается
|
||
_row(3, (0, "в том числе:", 93), (1, "", 100), (2, "", 100), (4, "-", 100)), # прочерки — не сумма
|
||
_row(4, (0, "Раздел I", 100), (1, "", 100), (2, "текст", 100)), # текст — не сумма
|
||
_row(5, (0, "Справочно", 100), (1, "", 100), (3, "9", 100)), # сумма вне колонок значений — не считается
|
||
_row(6, (0, "Выручка", 100), (1, "2110", 100), (2, "1", 100)),
|
||
]
|
||
parsed = parse_table_rows(rows, KNOWN)
|
||
assert parsed is not None and [r.ordinal for r in parsed.rows] == [0, 6]
|
||
assert (parsed.skipped_non_numeric_code, parsed.skipped_no_code_with_amounts) == (0, 2)
|
||
|
||
def test_repeated_header_is_not_counted_and_later_rows_use_new_layout(self) -> None:
|
||
rows = [
|
||
HEADER,
|
||
_row(1, (0, "Выручка", 100), (1, "2110", 100), (2, "1", 100)),
|
||
_row(2, (0, "Наименование показателя", 100), (1, "Код", 100), (3, "За 2024 г.", 100)),
|
||
_row(3, (0, "Мусор", 100), (1, "21x0", 100), (3, "7", 100)),
|
||
_row(4, (0, "БАЛАНС", 100), (1, "", 100), (3, "8", 100)), # сумма в колонке новой раскладки (3)
|
||
_row(5, (0, "Прочее", 100), (1, "", 100), (2, "8", 100)), # колонка 2 в новой раскладке — не значение
|
||
]
|
||
parsed = parse_table_rows(rows, KNOWN)
|
||
assert parsed is not None and [r.ordinal for r in parsed.rows] == [0, 1]
|
||
assert (parsed.skipped_non_numeric_code, parsed.skipped_no_code_with_amounts) == (1, 1)
|
||
|
||
def test_leading_rows_before_header_are_counted(self) -> None:
|
||
rows = [
|
||
_row(0, (1, "1110", 100), (0, "Нематериальные активы", 100), (2, "5", 100)),
|
||
_row(1, (1, "11x0", 100), (0, "Мусор", 100), (2, "5", 100)),
|
||
_row(2, (1, "", 100), (0, "БАЛАНС", 100), (2, "9", 100)),
|
||
_row(3, (1, "", 100), (0, "в том числе:", 100), (2, "", 100)),
|
||
_row(4, (0, "Наименование показателя", 100), (1, "Код", 100), (2, "2024", 100)),
|
||
_row(5, (0, "Основные средства", 100), (1, "1150", 100), (2, "7", 100)),
|
||
]
|
||
parsed = parse_table_rows(rows, KNOWN)
|
||
assert parsed is not None and not parsed.synthetic_header
|
||
assert [r.ordinal for r in parsed.rows] == [-1, 0, 5] # строка шапки — не в счёте
|
||
assert (parsed.skipped_non_numeric_code, parsed.skipped_no_code_with_amounts) == (1, 1)
|
||
|
||
def test_synthetic_header_table_counts_leading_rows(self) -> None:
|
||
rows = [
|
||
_row(0, (1, "1110", 100), (0, "Нематериальные активы", 100), (2, "5", 100)),
|
||
_row(1, (1, "1150 :", 100), (0, "ОС", 100), (2, "5", 100)),
|
||
_row(2, (1, "", 100), (0, "БАЛАНС", 100), (2, "9", 100)),
|
||
]
|
||
parsed = parse_table_rows(rows, KNOWN)
|
||
assert parsed is not None and parsed.synthetic_header
|
||
assert (parsed.skipped_non_numeric_code, parsed.skipped_no_code_with_amounts) == (1, 1)
|
||
|
||
@pytest.mark.fixtures
|
||
@pytest.mark.parametrize(
|
||
("name", "index", "non_numeric", "no_code_amounts"),
|
||
[("aep_3.json", 0, 8, 2), ("aep_3.json", 1, 2, 2), ("aep_6.json", 0, 2, 19), ("aep_6.json", 1, 1, 2)],
|
||
)
|
||
def test_real_tables(self, name: str, index: int, non_numeric: int, no_code_amounts: int) -> None:
|
||
"""aep_3: «БАЛАНС» и «Выручка» с ПУСТЫМ кодом и суммами теряются; aep_6: «в том числе:» с суммами (OQ-141)."""
|
||
parsed = parse_table_rows(_real_table(name, index), KNOWN)
|
||
assert parsed is not None
|
||
assert (parsed.skipped_non_numeric_code, parsed.skipped_no_code_with_amounts) == (non_numeric, no_code_amounts)
|
||
|
||
|
||
class TestMergedCodeRows:
|
||
"""OQ-156 (§11.1 п. 4): ИС склеила соседние строки формы — код `1500\\n1700`, суммы через перевод строки."""
|
||
|
||
MERGED = _row(
|
||
7,
|
||
(0, "Итого по разделу V\n\nБАЛАНС", 97),
|
||
(1, "1500\n1700", 99),
|
||
(2, "917 775\n2 371 333", 100),
|
||
(4, "783 941\n2 096 754", 90),
|
||
)
|
||
|
||
@staticmethod
|
||
def _data(parsed: ParsedTable | None) -> list[tuple[str, str, str, str]]:
|
||
assert parsed is not None
|
||
return [
|
||
(cell_text(r.cells, 0), cell_text(r.cells, 1), cell_text(r.cells, 2), cell_text(r.cells, 4))
|
||
for r in parsed.rows
|
||
if r.kind is not RowKind.HEADER
|
||
]
|
||
|
||
def test_split_into_rows_by_parts(self) -> None:
|
||
parsed = parse_table_rows([HEADER, self.MERGED], KNOWN)
|
||
assert self._data(parsed) == [
|
||
("Итого по разделу V", "1500", "917 775", "783 941"),
|
||
("БАЛАНС", "1700", "2 371 333", "2 096 754"),
|
||
]
|
||
assert parsed is not None
|
||
assert [r.ordinal for r in parsed.rows[1:]] == [7, 7]
|
||
assert parsed.rows[1].kind is RowKind.TOTAL
|
||
assert (parsed.rows[2].cells[1].confidence, parsed.rows[2].cells[4].confidence) == (99, 90)
|
||
assert (parsed.split_merged_codes, parsed.skipped_non_numeric_code) == (1, 0)
|
||
|
||
def test_blank_value_cell_stays_blank_in_every_part(self) -> None:
|
||
row = _row(1, (0, "А\nБ", 100), (1, "1150\n1160", 100), (2, "5\n6", 100), (4, "", 100))
|
||
assert self._data(parse_table_rows([HEADER, row], KNOWN)) == [("А", "1150", "5", ""), ("Б", "1160", "6", "")]
|
||
|
||
def test_label_parts_mismatch_keeps_whole_label(self) -> None:
|
||
row = _row(1, (0, "Итого", 100), (1, "1150\n1160", 100), (2, "5\n6", 100))
|
||
assert [d[0] for d in self._data(parse_table_rows([HEADER, row], KNOWN))] == ["Итого", "Итого"]
|
||
|
||
@pytest.mark.parametrize(
|
||
"row",
|
||
[
|
||
_row(1, (0, "А\nБ", 100), (1, "1150\n1160", 100), (2, "5", 100)), # одна сумма на два кода
|
||
_row(1, (0, "А\nБ", 100), (1, "1150\n116O", 100), (2, "5\n6", 100)), # вторая часть — не целое
|
||
_row(1, (0, "А\nБ", 100), (1, "1150\n1160", 100), (2, "5\n6", 100), (4, "1\n2\n3", 100)), # 3 части
|
||
],
|
||
)
|
||
def test_parts_mismatch_is_skipped_as_before(self, row: dict) -> None:
|
||
parsed = parse_table_rows([HEADER, row], KNOWN)
|
||
assert self._data(parsed) == []
|
||
assert parsed is not None and (parsed.split_merged_codes, parsed.skipped_non_numeric_code) == (0, 1)
|
||
|
||
def test_rows_before_header_are_split_too(self) -> None:
|
||
rows = [
|
||
_row(0, (1, "1110", 100), (0, "НМА", 100), (2, "5", 100)),
|
||
_row(1, (1, "1150\n1160", 100), (0, "ОС\nДоходные вложения", 100), (2, "7\n8", 100)),
|
||
]
|
||
parsed = parse_table_rows(rows, KNOWN)
|
||
assert parsed is not None and parsed.synthetic_header
|
||
assert [r.cells[1].value for r in parsed.rows[1:]] == ["1110", "1150", "1160"]
|
||
assert parsed.split_merged_codes == 1
|
||
|
||
@pytest.mark.fixtures
|
||
@pytest.mark.parametrize("name", ["Бухгалтерская отчетность 1 кв.25.json", "БО 1 кв 2025.json"])
|
||
def test_real_balance_keeps_1500_and_1700(self, name: str) -> None:
|
||
parsed = parse_table_rows(_real_table(name, 0), KNOWN)
|
||
assert parsed is not None
|
||
codes = [r.cells[r.layout.code_idx].value for r in parsed.rows if r.kind is not RowKind.HEADER]
|
||
assert "1500" in codes and "1700" in codes
|
||
assert parsed.split_merged_codes == 1
|
||
|
||
|
||
class TestTableType:
|
||
def test_by_first_digit(self) -> None:
|
||
rows = [
|
||
HEADER,
|
||
_row(1, (0, "Выручка", 100), (1, "2110", 100), (2, "1", 100)),
|
||
_row(2, (0, "Себестоимость", 100), (1, "2120", 100), (2, "1", 100)),
|
||
_row(3, (0, "ОС", 100), (1, "1150", 100), (2, "1", 100)),
|
||
]
|
||
assert table_type_by_codes(parse_table_rows(rows, KNOWN)) == "INCOME"
|
||
|
||
def test_tie_or_empty_is_none(self) -> None:
|
||
rows = [
|
||
HEADER,
|
||
_row(1, (0, "a", 100), (1, "2110", 100), (2, "1", 100)),
|
||
_row(2, (0, "b", 100), (1, "1150", 100), (2, "1", 100)),
|
||
]
|
||
assert table_type_by_codes(parse_table_rows(rows, KNOWN)) is None
|
||
assert table_type_by_codes(ParsedTable(rows=[], synthetic_header=False)) is None
|
||
|
||
def test_unanimous_requires_single_group(self) -> None:
|
||
"""#30: тип «по единодушным кодам» — только когда все коды данных одной группы."""
|
||
income_only = [
|
||
HEADER,
|
||
_row(1, (0, "Выручка", 100), (1, "2110", 100), (2, "1", 100)),
|
||
_row(2, (0, "Себестоимость", 100), (1, "2120", 100), (2, "1", 100)),
|
||
]
|
||
assert table_type_by_unanimous_codes(parse_table_rows(income_only, KNOWN)) == "INCOME"
|
||
mixed = [*income_only, _row(3, (0, "ОС", 100), (1, "1150", 100), (2, "1", 100))]
|
||
assert table_type_by_unanimous_codes(parse_table_rows(mixed, KNOWN)) is None
|
||
assert table_type_by_unanimous_codes(ParsedTable(rows=[], synthetic_header=False)) is None
|