arm-tabular-parser/api/tests/unit/parser/test_detect.py

"""Детекторы по содержимому (design D-2, spec «Листы ОСВ находятся по содержимому», «Заголовок таблицы и колонки
сумм определяются по тексту»): вид листа, блок заголовка, шесть колонок сумм, колонки текста и ИНН, extra, повтор
заголовка в данных. Сетки собираются `gridbuilder`; позиции в них произвольные — детектор их не знает заранее.
"""

from decimal import Decimal

import pytest
from gridbuilder import L, M, at, sheet_from_rows

from src.parser.detect import (
    ColumnMap,
    SheetKind,
    TableHeader,
    classify_sheet,
    find_table_header,
    first_data_row,
    is_header_repeat,
)
from src.parser.errors import InvalidFormatError
from src.parser.grid import Cell, Sheet
from src.parser.rules import Rules, load_rules

TITLE = "Оборотно-сальдовая ведомость по счету 60 за 1 квартал 2025 г."
OPENING = "Сальдо на начало периода"
TURNOVER = "Обороты за период"
CLOSING = "Сальдо на конец периода"


@pytest.fixture(scope="module")
def rules() -> Rules:
    return load_rules()


def _doc_head() -> list[list[object]]:
    """Шапка варианта 1 (строки 1–5); заголовок таблицы — с строки 6."""
    return [["АО «Ромашка»"], [TITLE], [], ["Выводимые данные: БУ (данные бухгалтерского учета)"], []]


def _rb_like() -> Sheet:
    """Вариант 1, заголовок в две строки (как АО РБ): суммы E/H, J/L, O/Q, подзаголовки merged на две строки."""
    return sheet_from_rows(
        [
            *_doc_head(),
            [M("Счет", 4), M(OPENING, 5), M(TURNOVER, 5), M(CLOSING, 3)],  # 6
            [M("Контрагенты", 4, 2), M("Дебет", 3, 2), M("Кредит", 2, 2), M("Дебет", 2, 2)]
            + [M("Кредит", 3, 2), M("Дебет", 2, 2), M("Кредит", 1, 2)],  # 7
            [],  # 8 — под merged-подзаголовками
            [L(0), *at({1: "60", 8: 300, 10: 1000, 12: 900, 17: 200})],  # 9
            [L(1), *at({1: "60.01", 8: 300, 10: 1000, 12: 900, 17: 200})],
            [L(2), *at({1: "ООО Поставщик", 8: 300, 10: 1000, 12: 900, 17: 200})],
            at({1: "Итого", 8: 300, 10: 1000, 12: 900, 17: 200}),
        ]
    )


def _schastl_like() -> Sheet:
    """Вариант 1, три строки (как Счастливочная): Счет / Подразделение / Контрагенты + ИНН в E, суммы G/J, K/N, Q/S."""
    return sheet_from_rows(
        [
            *_doc_head(),
            [M("Счет", 6), M(OPENING, 4), M(TURNOVER, 6), M(CLOSING, 3)],  # 6
            [M("Подразделение", 6), M("Дебет", 3, 2), M("Кредит", 1, 2), M("Дебет", 3, 2)]
            + [M("Кредит", 3, 2), M("Дебет", 2, 2), M("Кредит", 1, 2)],  # 7
            [M("Контрагенты", 4), M("ИНН", 2)],  # 8
            [L(0), *at({1: "62", 7: 500, 11: 100, 14: 50, 17: 550})],  # 9
            [L(1), *at({1: "62.01", 7: 500, 11: 100, 14: 50, 17: 550})],
            [L(2), *at({1: "БП Аренда", 7: 500, 11: 100, 14: 50, 17: 550})],
            [L(3), *at({1: "МАЛЕЧКИНСКАЯ ПТИЦЕФАБРИКА ООО", 5: "3523023093", 7: 500, 11: 100, 14: 50, 17: 550})],
            at({1: "Итого", 7: 500, 11: 100, 14: 50, 17: 550, 20: 777, 21: Decimal("0.5")}),  # мусор правее S
        ]
    )


# --- classify_sheet ---


def test_classify_osv_sheet_by_title_anywhere_in_first_rows(rules: Rules) -> None:
    sheet = sheet_from_rows([[], [None, None, "ОБОРОТНО-САЛЬДОВАЯ\nведомость по счёту 60"]])

    assert classify_sheet(sheet, rules) is SheetKind.OSV


@pytest.mark.parametrize("marker", ["Бухгалтерский баланс", "Отчет о финансовых результатах", "Форма по ОКУД 0710001"])
def test_classify_statement_sheet(rules: Rules, marker: str) -> None:
    sheet = sheet_from_rows([["ООО Ромашка"], [marker]])

    assert classify_sheet(sheet, rules) is SheetKind.STATEMENT


def test_classify_other_sheet(rules: Rules) -> None:
    sheet = sheet_from_rows([["Справка"], ["Наименование", "Сумма"], ["Аренда", 100]])

    assert classify_sheet(sheet, rules) is SheetKind.OTHER


def test_osv_marker_wins_over_statement_marker(rules: Rules) -> None:
    sheet = sheet_from_rows([["Бухгалтерский баланс"], [TITLE]])

    assert classify_sheet(sheet, rules) is SheetKind.OSV


def test_osv_marker_far_below_the_head_is_not_searched(rules: Rules) -> None:
    sheet = sheet_from_rows([*([["строка"]] * 40), [TITLE]])

    assert classify_sheet(sheet, rules) is SheetKind.OTHER


# --- find_table_header: варианты заголовка ---


def test_two_row_header_rb_like(rules: Rules) -> None:
    sheet = _rb_like()

    header = find_table_header(sheet, rules)

    assert header.columns == ColumnMap(
        opening_debit=5,
        opening_credit=8,
        turnover_debit=10,
        turnover_credit=12,
        closing_debit=15,
        closing_credit=17,
        collapsed=frozenset(),
    )
    assert header.columns.as_diagnostics() == {
        "opening_debit": "E",
        "opening_credit": "H",
        "turnover_debit": "J",
        "turnover_credit": "L",
        "closing_debit": "O",
        "closing_credit": "Q",
    }
    assert (header.first_row, header.last_row) == (6, 8)
    assert header.text_col == 1
    assert header.inn_col is None
    assert [header.right_edge] == [17]
    assert header.extra_cols == {}
    assert header.group_label_rows == [6, 7]
    assert [first_data_row(sheet, header)] == [9]


def test_three_row_header_with_inn_schastl_like(rules: Rules) -> None:
    sheet = _schastl_like()

    header = find_table_header(sheet, rules)

    assert header.columns.as_diagnostics() == {
        "opening_debit": "G",
        "opening_credit": "J",
        "turnover_debit": "K",
        "turnover_credit": "N",
        "closing_debit": "Q",
        "closing_credit": "S",
    }
    assert [header.inn_col] == [5]
    assert (header.first_row, header.last_row) == (6, 8)
    assert header.group_label_rows == [6, 7, 8]
    assert [first_data_row(sheet, header)] == [9]


def test_garbage_right_of_table_is_neither_sum_nor_extra(rules: Rules) -> None:
    header = find_table_header(_schastl_like(), rules)

    assert [header.right_edge] == [19]
    assert header.extra_cols == {}
    sums = {
        header.columns.opening_debit,
        header.columns.opening_credit,
        header.columns.turnover_debit,
        header.columns.turnover_credit,
        header.columns.closing_debit,
        header.columns.closing_credit,
    }
    assert not sums & {20, 21}


def test_four_row_header(rules: Rules) -> None:
    sheet = sheet_from_rows(
        [
            *_doc_head(),
            ["Счет", None, M(OPENING, 2), M(TURNOVER, 2), M(CLOSING, 2)],  # 6
            ["Подразделение", None, "Дебет", "Кредит", "Дебет", "Кредит", "Дебет", "Кредит"],
            ["Контрагенты", "ИНН"],
            ["Договоры"],  # 9
            [L(0), *at({1: "62", 3: 10, 8: 10})],
            [L(3), *at({1: "Договор 1", 3: 10, 8: 10})],
        ]
    )

    header = find_table_header(sheet, rules)

    assert (header.first_row, header.last_row) == (6, 9)
    assert header.group_label_rows == [6, 7, 8, 9]
    assert [header.inn_col] == [2]
    assert [header.columns.opening_debit] == [3]
    assert [header.columns.closing_credit] == [8]
    assert [first_data_row(sheet, header)] == [10]


def test_label_row_of_other_grouping_belongs_to_the_block(rules: Rules) -> None:
    # «Основные средства» — группировка OTHER из словаря osv_groups: строка блока, а не первая строка данных
    sheet = sheet_from_rows(
        [
            [TITLE],
            ["Счет", M(OPENING, 2), M(TURNOVER, 2), M(CLOSING, 2)],
            ["Подразделение", "Дебет", "Кредит", "Дебет", "Кредит", "Дебет", "Кредит"],
            ["Основные средства"],
            [L(0), *at({1: "01", 2: 10, 6: 10})],
        ]
    )

    header = find_table_header(sheet, rules)

    assert header.group_label_rows == [2, 3, 4]
    assert [first_data_row(sheet, header)] == [5]


def test_counterparty_named_like_a_grouping_right_after_header_is_data(rules: Rules) -> None:
    # «Контрагенты-Сервис ООО» матчит метку «Контрагент…», но у неё суммы — это строка данных, а не часть блока
    sheet = sheet_from_rows(
        [
            [TITLE],
            ["Счет", M(OPENING, 2), M(TURNOVER, 2), M(CLOSING, 2)],
            ["Контрагенты", "Дебет", "Кредит", "Дебет", "Кредит", "Дебет", "Кредит"],
            [L(1), *at({1: "Контрагенты-Сервис ООО", 2: 10, 6: 10})],
        ]
    )

    header = find_table_header(sheet, rules)

    assert [header.last_row] == [3]
    assert [first_data_row(sheet, header)] == [4]


def test_variant_2_subkonto_header(rules: Rules) -> None:
    sheet = sheet_from_rows(
        [
            ["Оборотно-сальдовая ведомость по счету 60"],
            ["Период: 1 квартал 2026 г."],
            ["Детализация по субсчетам, субконто: Контрагенты, Договоры"],
            ["Выводимые данные: сумма"],
            ["Субконто", M("Сальдо на начало периода", 2), M("Оборот за период", 2), M(CLOSING, 2)],  # 5
            [None, "Дебет", "Кредит", "Дебет", "Кредит", "Дебет", "Кредит"],
            [L(0), *at({1: "60", 2: 10, 7: 10})],
        ]
    )

    header = find_table_header(sheet, rules)

    assert (header.first_row, header.last_row) == (5, 6)
    assert header.group_label_rows == [5]
    assert [header.columns.turnover_debit] == [4]
    assert header.text_col == 1


def test_one_row_header_with_sides_in_group_cells(rules: Rules) -> None:
    sheet = sheet_from_rows(
        [
            [TITLE],
            [
                "Счет",
                "Нач. сальдо Дт",
                "Нач. сальдо Кт",
                "Оборот Дт",
                "Оборот Кт",
                "Кон. сальдо Д-т",
                "Кон. сальдо К-т",
            ],
            [L(0), *at({1: "60", 2: 10, 7: 10})],
        ]
    )

    header = find_table_header(sheet, rules)

    assert header.columns == ColumnMap(2, 3, 4, 5, 6, 7, frozenset())
    assert (header.first_row, header.last_row) == (2, 2)
    assert [first_data_row(sheet, header)] == [3]


def test_unmerged_group_titles_take_subheaders_up_to_the_next_group(rules: Rules) -> None:
    sheet = sheet_from_rows(
        [
            [TITLE],
            ["Счет", OPENING, None, TURNOVER, None, CLOSING],
            [None, "Дт", "Кт", "Дт", "Кт", "Дт", "Кт"],
            [L(0), *at({1: "60", 2: 10, 7: 10})],
        ]
    )

    header = find_table_header(sheet, rules)

    assert header.columns == ColumnMap(2, 3, 4, 5, 6, 7, frozenset())
    assert [header.right_edge] == [7]


def test_table_text_column_is_not_necessarily_the_first(rules: Rules) -> None:
    sheet = sheet_from_rows(
        [
            [TITLE],
            [None, "Счет", M(OPENING, 2), M(TURNOVER, 2), M(CLOSING, 2)],
            [None, "Контрагенты", "Дебет", "Кредит", "Дебет", "Кредит", "Дебет", "Кредит"],
            [L(0), *at({2: "60", 3: 10, 8: 10})],
            [L(1), *at({2: "ООО Поставщик", 3: 10, 8: 10})],
        ]
    )

    header = find_table_header(sheet, rules)

    assert [header.text_col] == [2]
    assert [header.columns.opening_debit] == [3]
    assert header.group_label_rows == [2, 3]


# --- Без «начала», свёрнутое сальдо ---


def test_header_without_opening_group(rules: Rules) -> None:
    sheet = sheet_from_rows(
        [
            [TITLE],
            ["Счет", M(TURNOVER, 2), M(CLOSING, 2)],
            ["Контрагенты", "Дебет", "Кредит", "Дебет", "Кредит"],
            [L(0), *at({1: "60", 2: 10, 5: 10})],
        ]
    )

    header = find_table_header(sheet, rules)

    assert header.columns == ColumnMap(None, None, 2, 3, 4, 5, frozenset())
    assert header.columns.as_diagnostics() == {
        "turnover_debit": "B",
        "turnover_credit": "C",
        "closing_debit": "D",
        "closing_credit": "E",
    }


def test_collapsed_closing_balance_is_one_column_in_debit_slot(rules: Rules) -> None:
    sheet = sheet_from_rows(
        [
            [TITLE],
            ["Счет", M(OPENING, 2), M(TURNOVER, 2), M(CLOSING, 1, 2)],
            ["Контрагенты", "Дебет", "Кредит", "Дебет", "Кредит", None],
            [L(0), *at({1: "60", 2: 10, 4: 5, 6: 15})],
            [L(1), *at({1: "ООО Должник", 2: 10, 4: 5, 6: -15})],
        ]
    )

    header = find_table_header(sheet, rules)

    assert [header.columns.closing_debit] == [6]
    assert header.columns.closing_credit is None
    assert header.columns.collapsed == frozenset({"closing"})
    assert [header.right_edge] == [6]
    assert header.columns.as_diagnostics()["closing_debit"] == "F"


def test_collapsed_group_with_empty_subheaders_under_wide_title(rules: Rules) -> None:
    sheet = sheet_from_rows(
        [
            [TITLE],
            ["Счет", M(OPENING, 2), M(TURNOVER, 2), M(CLOSING, 2)],
            ["Контрагенты", None, None, "Дебет", "Кредит", "Дебет", "Кредит"],
            [L(0), *at({1: "60", 2: 10, 4: 5, 6: 15})],
        ]
    )

    header = find_table_header(sheet, rules)

    assert header.columns.collapsed == frozenset({"opening"})
    assert (header.columns.opening_debit, header.columns.opening_credit) == (2, None)


def test_collapsed_turnover_is_invalid_format(rules: Rules) -> None:
    sheet = sheet_from_rows(
        [
            [TITLE],
            ["Счет", M(OPENING, 2), M(TURNOVER, 1, 2), M(CLOSING, 2)],
            ["Контрагенты", "Дебет", "Кредит", None, "Дебет", "Кредит"],
            [L(0), *at({1: "60", 2: 10, 6: 10})],
        ]
    )

    with pytest.raises(InvalidFormatError, match="обороты"):
        find_table_header(sheet, rules)


def test_group_with_debit_but_no_credit_is_invalid_format(rules: Rules) -> None:
    sheet = sheet_from_rows(
        [
            [TITLE],
            ["Счет", M(OPENING, 2), M(TURNOVER, 2), M(CLOSING, 2)],
            ["Контрагенты", "Дебет", "Кредит", "Дебет", "Кредит", "Дебет", "Итог"],
            [L(0), *at({1: "60", 2: 10, 6: 10})],
        ]
    )

    with pytest.raises(InvalidFormatError, match=r"сальдо на конец.*Кредит"):
        find_table_header(sheet, rules)


# --- Ошибки: нет заголовка ---


def test_no_table_header_is_invalid_format_naming_the_groups(rules: Rules) -> None:
    sheet = sheet_from_rows([[TITLE], ["Наименование", "Сумма"], ["Аренда", 100]])

    with pytest.raises(InvalidFormatError, match="сальдо на начало") as caught:
        find_table_header(sheet, rules)

    assert "обороты" in caught.value.message
    assert "сальдо на конец" in caught.value.message


def test_single_group_is_not_a_header_and_missing_groups_are_named(rules: Rules) -> None:
    sheet = sheet_from_rows([[TITLE], ["Счет", M(OPENING, 2)], [None, "Дебет", "Кредит"], ["60", 10]])

    with pytest.raises(InvalidFormatError) as caught:
        find_table_header(sheet, rules)

    assert "обороты" in caught.value.message
    assert "сальдо на конец" in caught.value.message
    assert "сальдо на начало" not in caught.value.message


def test_header_without_text_column_left_of_sums_is_invalid_format(rules: Rules) -> None:
    sheet = sheet_from_rows([[TITLE], [M(TURNOVER, 2), M(CLOSING, 2)], ["Дебет", "Кредит", "Дебет", "Кредит"], [1, 2]])

    with pytest.raises(InvalidFormatError, match="текст"):
        find_table_header(sheet, rules)


# --- ИНН и extra ---


def test_kpp_between_inn_and_sums_goes_to_extra(rules: Rules) -> None:
    sheet = sheet_from_rows(
        [
            [TITLE],
            ["Счет", None, None, M(OPENING, 2), M(TURNOVER, 2), M(CLOSING, 2)],  # 2
            ["Контрагенты", "ИНН", "КПП", "Дебет", "Кредит", "Дебет", "Кредит", "Дебет", "Кредит"],
            [L(0), *at({1: "60", 4: 10, 9: 10})],
            [L(1), *at({1: "ООО Поставщик", 2: "7701234567", 3: "770101001", 4: 10, 9: 10, 11: 99})],
        ]
    )

    header = find_table_header(sheet, rules)

    assert [header.inn_col] == [2]
    assert header.extra_cols == {3: "КПП"}
    assert [header.columns.opening_debit] == [4]
    assert [header.right_edge] == [9]


def test_unnamed_column_with_data_is_extra_under_its_letter(rules: Rules) -> None:
    sheet = sheet_from_rows(
        [
            [TITLE],
            ["Счет", None, M(OPENING, 2), M(TURNOVER, 2), M(CLOSING, 2)],
            ["Контрагенты", None, "Дебет", "Кредит", "Дебет", "Кредит", "Дебет", "Кредит"],
            [L(0), *at({1: "60", 2: "прим.", 3: 10, 8: 10})],
        ]
    )

    header = find_table_header(sheet, rules)

    assert header.extra_cols == {2: "B"}


def test_inn_column_found_by_digits_without_header(rules: Rules) -> None:
    rows: list[list[object]] = [
        [TITLE],
        ["Счет", None, None, M(OPENING, 2), M(TURNOVER, 2), M(CLOSING, 2)],
        ["Контрагенты", None, None, "Дебет", "Кредит", "Дебет", "Кредит", "Дебет", "Кредит"],
        [L(0), *at({1: "60", 4: 10, 9: 10})],
    ]
    inns: list[object] = ["7701234567", Decimal("3523023093"), "770123456789", "7701234567", "не указан"]
    for index, inn in enumerate(inns):
        rows.append([L(1), *at({1: f"Контрагент {index}", 2: inn, 3: f"код {index}", 4: 1, 9: 1})])

    header = find_table_header(sheet_from_rows(rows), rules)

    assert [header.inn_col] == [2]  # 4 из 5 непустых — 10/12 цифр (80 %)
    assert header.extra_cols == {3: "C"}


def test_mostly_non_inn_digits_are_not_an_inn_column(rules: Rules) -> None:
    rows: list[list[object]] = [
        [TITLE],
        ["Счет", None, M(OPENING, 2), M(TURNOVER, 2), M(CLOSING, 2)],
        ["Контрагенты", "Код", "Дебет", "Кредит", "Дебет", "Кредит", "Дебет", "Кредит"],
    ]
    for index, code in enumerate(["7701234567", "12", "345", "7701234567", "000123"]):
        rows.append([L(1), *at({1: f"Контрагент {index}", 2: code, 3: 1, 8: 1})])

    header = find_table_header(sheet_from_rows(rows), rules)

    assert header.inn_col is None
    assert header.extra_cols == {2: "Код"}


# --- Пустые строки и повтор заголовка (Review Focus 4, 5) ---


def test_whitespace_rows_after_header_are_empty(rules: Rules) -> None:
    sheet = sheet_from_rows(
        [
            [TITLE],
            ["Счет", M(OPENING, 2), M(TURNOVER, 2), M(CLOSING, 2)],
            ["Контрагенты", "Дебет", "Кредит", "Дебет", "Кредит", "Дебет", "Кредит"],
            [" ", "\u00a0", None, "  "],
            [],
            [L(0), *at({1: "60", 2: 10, 7: 10})],
        ]
    )

    header = find_table_header(sheet, rules)

    assert [header.last_row] == [3]
    assert [first_data_row(sheet, header)] == [6]


def test_first_data_row_of_table_without_data_is_past_the_sheet(rules: Rules) -> None:
    sheet = sheet_from_rows(
        [[TITLE], ["Счет", M(OPENING, 2), M(TURNOVER, 2), M(CLOSING, 2)], ["Контрагенты", *(["Дебет", "Кредит"] * 3)]]
    )

    header = find_table_header(sheet, rules)

    assert first_data_row(sheet, header) == sheet.n_rows + 1


def _with_repeated_header() -> Sheet:
    group_row = ["Счет", M(OPENING, 2), M(TURNOVER, 2), M(CLOSING, 2)]
    side_row = ["Контрагенты", "Дебет", "Кредит", "Дебет", "Кредит", "Дебет", "Кредит"]
    return sheet_from_rows(
        [
            [TITLE],
            group_row,  # 2
            side_row,  # 3
            [L(0), *at({1: "60", 2: 30, 7: 30})],  # 4
            [L(1), *at({1: "ООО Первый", 2: 10, 7: 10})],  # 5
            [],  # 6
            group_row,  # 7 — повтор на «странице» печати
            side_row,  # 8
            [L(1), *at({1: "ООО Второй", 2: 20, 7: 20})],  # 9
            ["Сальдо на начало периода", "Обороты за период", None, None, 5],  # 10 — повтор без merged
            [L(1), *at({1: "Контрагенты", 2: 1, 7: 1})],  # 11 — контрагент с таким именем, но с суммами
        ]
    )


def test_header_repeat_inside_data_is_recognised(rules: Rules) -> None:
    sheet = _with_repeated_header()
    header = find_table_header(sheet, rules)

    assert is_header_repeat(sheet, 7, header, rules)
    assert is_header_repeat(sheet, 8, header, rules)
    assert is_header_repeat(sheet, 10, header, rules)


@pytest.mark.parametrize("row", [4, 5, 6, 9, 11])
def test_data_and_empty_rows_are_not_header_repeats(rules: Rules, row: int) -> None:
    sheet = _with_repeated_header()
    header = find_table_header(sheet, rules)

    assert not is_header_repeat(sheet, row, header, rules)


def test_header_is_found_from_the_top(rules: Rules) -> None:
    sheet = _with_repeated_header()

    header: TableHeader = find_table_header(sheet, rules)

    assert [header.first_row] == [2]


def test_row_with_only_cells_right_of_table_is_not_a_header_repeat(rules: Rules) -> None:
    # В блоке rb-подобного заголовка есть пустая строка (под merged-подзаголовками); строка, где есть только «хвост»
    # правее таблицы, после отсечения по right_edge тоже пуста — но это не повтор заголовка
    base = _rb_like()
    sheet = Sheet(
        name=base.name,
        cells={**base.cells, (base.n_rows + 1, 19): Cell(Decimal("375490642.46"), "number")},
        merged=base.merged,
        row_outline=base.row_outline,
        n_rows=base.n_rows + 1,
        n_cols=19,
    )
    header = find_table_header(sheet, rules)

    assert not is_header_repeat(sheet, sheet.n_rows, header, rules)