diff --git a/api/src/parser/statements/__init__.py b/api/src/parser/statements/__init__.py new file mode 100644 index 0000000..5ac937c --- /dev/null +++ b/api/src/parser/statements/__init__.py @@ -0,0 +1,4 @@ +"""Конвейер форм ББ/ОФР (Срез 2b): columns → detect → header → lines → checks → merge/statement. + +Как и в ОСВ, структура ищется по содержимому: синонимы и соотношения — `rules/statement_*.yaml`. +""" diff --git a/api/src/parser/statements/columns.py b/api/src/parser/statements/columns.py new file mode 100644 index 0000000..6d1c757 --- /dev/null +++ b/api/src/parser/statements/columns.py @@ -0,0 +1,253 @@ +"""Заголовок таблицы формы, колонка кодов, колонка наименований, колонки-периоды (design D-4). + +Колонка кодов — та, где не меньше `line_codes.min_share` непустых значений блока — коды строк форм; строка заголовка +— с подписью «Код» и ячейками-периодами правее; заголовок в две строки склеивается; строка нумерации «1 2 3 4 5» +пропускается. Ячейки заголовка без периода («Изменение, %») в колонки не попадают. +""" + +import re +from dataclasses import dataclass +from datetime import date + +from src.parser.grid import Cell, Sheet +from src.parser.rules import Rules + +__all__ = [ + "LayoutNotFoundError", + "PeriodColumn", + "TableLayout", + "cell_label", + "find_code_column", + "find_layout", + "is_numbering_row", + "line_code", +] + +_CODE = re.compile(r"^(\d{4,5})(\.0+)?$") +_SHORT_TEXT = 3 # значения колонки «Пояснения»: 1–2 символа/цифры +_MIN_NUMBERING_CELLS = 3 +NO_CODE_COLUMN = "колонка кодов не найдена" + + +class LayoutNotFoundError(Exception): + """Таблица формы не найдена: `reason` — что именно (колонка кодов).""" + + def __init__(self, reason: str) -> None: + super().__init__(reason) + self.reason = reason + + +@dataclass(frozen=True) +class PeriodColumn: + col: int + label: str # как в заголовке (переносы → пробел); "" у продолжения без заголовка + period_date: date | None + + +@dataclass(frozen=True) +class TableLayout: + header_row: int | None # None — заголовка таблицы нет (лист-продолжение) + header_last_row: int | None # вторая строка заголовка или строка нумерации + code_col: int + name_col: int | None + periods: tuple[PeriodColumn, ...] + skipped_labels: tuple[str, ...] # заголовки правее кодов без периода + first_data_row: int + + +def cell_label(sheet: Sheet, row: int, col: int) -> str: + return " ".join(sheet.text_at(row, col).split()) + + +def line_code(cell: Cell, rules: Rules) -> str | None: + """Код строки формы: 4–5 цифр, первые четыре — в диапазонах словаря; число `1110.0` читается как «1110».""" + found = _CODE.match(cell.text.replace(" ", "")) + if found is None: + return None + code = found.group(1) + head = int(code[:4]) + ranges = rules.statement_headers.code_ranges.values() + return code if any(low <= head <= high for low, high in ranges) else None + + +def find_code_column(sheet: Sheet, rules: Rules, first_row: int, last_row: int) -> int | None: + """Колонка с наибольшим числом кодов, если кодов ≥ `min_codes` и их доля среди непустых ≥ `min_share`.""" + counts: dict[int, list[int]] = {} + for row in range(first_row, last_row + 1): + for col, cell in sheet.row_cells(row): + tally = counts.setdefault(col, [0, 0]) + tally[1] += 1 + if line_code(cell, rules) is not None: + tally[0] += 1 + headers = rules.statement_headers + best: int | None = None + for col in sorted(counts): + codes, nonempty = counts[col] + if codes < headers.min_codes or codes / nonempty < headers.code_min_share: + continue + if best is None or codes > counts[best][0]: + best = col + return best + + +def is_numbering_row(sheet: Sheet, row: int, rules: Rules) -> bool: + cells = sheet.row_cells(row) + pattern = rules.statement_headers.column_numbering + return len(cells) >= _MIN_NUMBERING_CELLS and all(pattern.search(cell.text) for _, cell in cells) + + +def _first_code_row(sheet: Sheet, rules: Rules, code_col: int, first_row: int, last_row: int) -> int: + for row in range(first_row, last_row + 1): + if line_code(sheet.cell(row, code_col), rules) is not None: + return row + return last_row + 1 + + +def _is_period_label(label: str, rules: Rules, *, allow_bare_year: bool) -> bool: + if rules.statement_periods.period_end(label) is None: + return False + patterns = ( + rules.statement_periods.on_date, + rules.statement_periods.numeric_date, + rules.statement_periods.for_months, + rules.statement_periods.for_range, + ) + return ( + allow_bare_year + or not rules.statement_periods.for_year.search(label) + or any(pattern.search(label) for pattern in patterns) + ) + + +def _find_header_row(sheet: Sheet, rules: Rules, code_col: int, first_row: int, first_code_row: int) -> int | None: + """Последняя над первым кодом строка с подписью «Код» в колонке кодов; без подписи — последняя строка с двумя и + более ячейками-периодами правее кодов либо с одной, но не голым годом (титульный блок НПЗ содержит «2025»).""" + captioned = [ + row + for row in range(first_row, first_code_row) + if rules.statement_headers.is_code_caption(sheet.text_at(row, code_col)) + ] + if captioned: + return captioned[-1] + for row in range(first_code_row - 1, first_row - 1, -1): + labels = [cell_label(sheet, row, col) for col, _ in sheet.row_cells(row) if col > code_col] + strong = [label for label in labels if _is_period_label(label, rules, allow_bare_year=False)] + weak = [label for label in labels if _is_period_label(label, rules, allow_bare_year=True)] + if strong or len(weak) >= 2: # noqa: PLR2004 - две даты подряд — заголовок, одна «2025» — титул + return row + return None + + +def _header_last_row(sheet: Sheet, rules: Rules, header_row: int, code_col: int, first_code_row: int) -> int: + """Вторая строка заголовка (продолжение подписей без кодов) и строка нумерации колонок.""" + last = header_row + following = header_row + 1 + is_following_numbering = is_numbering_row(sheet, following, rules) + is_following_empty = sheet.is_row_empty(following) + if following < first_code_row and not is_following_numbering and not is_following_empty: + # Проверяем, является ли следующая строка продолжением заголовка: + # 1. Все непустые ячейки находятся правее кодов, ИЛИ + # 2. В колонке кодов пусто, нет чисел, и есть ячейки правее кодов + all_cells_right = all(col > code_col for col, _ in sheet.row_cells(following)) + code_col_empty = sheet.text_at(following, code_col) == "" + no_numbers = not any(cell.kind == "number" for _, cell in sheet.row_cells(following)) + has_cells_right = any(col > code_col for col, _ in sheet.row_cells(following)) + + if all_cells_right or (code_col_empty and no_numbers and has_cells_right): + last = following + if last + 1 < first_code_row and is_numbering_row(sheet, last + 1, rules): + last += 1 + return last + + +def _joined_label(sheet: Sheet, header_row: int, last_row: int, col: int, rules: Rules) -> str: + parts = [cell_label(sheet, header_row, col)] + for row in range(header_row + 1, last_row + 1): + if is_numbering_row(sheet, row, rules): + continue + origin_row, _ = sheet.merged_origin(row, col) + if origin_row == header_row: # merged-продолжение той же ячейки + continue + parts.append(cell_label(sheet, row, col)) + return " ".join(part for part in parts if part) + + +def _period_columns( + sheet: Sheet, rules: Rules, header_row: int, header_last_row: int, code_col: int +) -> tuple[tuple[PeriodColumn, ...], tuple[str, ...]]: + periods: list[PeriodColumn] = [] + skipped: list[str] = [] + seen: set[int] = set() + for row in range(header_row, header_last_row + 1): + for col, _ in sheet.row_cells(row): + origin_row, origin_col = sheet.merged_origin(row, col) + if origin_col <= code_col or origin_col in seen or is_numbering_row(sheet, row, rules): + continue + seen.add(origin_col) + label = _joined_label(sheet, origin_row, header_last_row, origin_col, rules) + end = rules.statement_periods.period_end(label) + if end is None and not label: + continue + if end is None: + skipped.append(label) + else: + periods.append(PeriodColumn(origin_col, label, end)) + periods.sort(key=lambda column: column.col) + return tuple(periods), tuple(skipped) + + +def _value_columns( + sheet: Sheet, rules: Rules, code_col: int, first_row: int, last_row: int +) -> tuple[PeriodColumn, ...]: + """Продолжение без заголовка: колонки правее кодов, где в строках с кодами есть хотя бы одно число.""" + found: set[int] = set() + for row in range(first_row, last_row + 1): + if line_code(sheet.cell(row, code_col), rules) is None: + continue + found.update(col for col, cell in sheet.row_cells(row) if col > code_col and cell.kind == "number") + return tuple(PeriodColumn(col, "", None) for col in sorted(found)) + + +def _name_column( # noqa: PLR0913, PLR0917 - сигнатура соответствует вызовам + sheet: Sheet, rules: Rules, code_col: int, header_row: int | None, first_row: int, last_row: int +) -> int | None: + """Колонка левее кодов с наибольшим числом текстовых значений в строках с кодами; «Пояснения» (по подписи или по + коротким значениям) исключается; при равенстве — ближайшая к кодам.""" + texts: dict[int, int] = {} + long_texts: dict[int, int] = {} + for row in range(first_row, last_row + 1): + if line_code(sheet.cell(row, code_col), rules) is None: + continue + for col, cell in sheet.row_cells(row): + if col >= code_col or cell.kind != "text": + continue + texts[col] = texts.get(col, 0) + 1 + if len(cell.text) > _SHORT_TEXT: + long_texts[col] = long_texts.get(col, 0) + 1 + candidates = [ + col + for col in texts + if long_texts.get(col, 0) > 0 + and not ( + header_row is not None and rules.statement_headers.is_explanations_caption(sheet.text_at(header_row, col)) + ) + ] + if not candidates: + return None + return max(candidates, key=lambda col: (texts[col], col)) + + +def find_layout(sheet: Sheet, rules: Rules, first_row: int, last_row: int) -> TableLayout: + code_col = find_code_column(sheet, rules, first_row, last_row) + if code_col is None: + raise LayoutNotFoundError(NO_CODE_COLUMN) + first_code_row = _first_code_row(sheet, rules, code_col, first_row, last_row) + header_row = _find_header_row(sheet, rules, code_col, first_row, first_code_row) + if header_row is None: + periods = _value_columns(sheet, rules, code_col, first_row, last_row) + name_col = _name_column(sheet, rules, code_col, None, first_row, last_row) + return TableLayout(None, None, code_col, name_col, periods, (), first_row) + header_last_row = _header_last_row(sheet, rules, header_row, code_col, first_code_row) + periods, skipped = _period_columns(sheet, rules, header_row, header_last_row, code_col) + name_col = _name_column(sheet, rules, code_col, header_row, header_last_row + 1, last_row) + return TableLayout(header_row, header_last_row, code_col, name_col, periods, skipped, header_last_row + 1) diff --git a/api/tests/unit/parser/statements/__init__.py b/api/tests/unit/parser/statements/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/tests/unit/parser/statements/test_columns.py b/api/tests/unit/parser/statements/test_columns.py new file mode 100644 index 0000000..35cd418 --- /dev/null +++ b/api/tests/unit/parser/statements/test_columns.py @@ -0,0 +1,241 @@ +"""Заголовок таблицы формы, колонка кодов, наименований и периодов (design D-4; spec «Заголовок таблицы, колонка кодов +и колонки-периоды определяются по тексту»).""" + +import datetime +from decimal import Decimal + +import pytest +from gridbuilder import at, sheet_from_rows + +from src.parser.grid import Cell +from src.parser.rules import Rules, load_rules +from src.parser.statements.columns import ( + LayoutNotFoundError, + find_code_column, + find_layout, + line_code, +) + + +@pytest.fixture(scope="module") +def rules() -> Rules: + return load_rules() + + +# Test constants +CODE_COL_ONE_C = 4 +NAME_COL_ONE_C = 2 +CODE_COL_TDSHEET = 2 +NAME_COL_TDSHEET = 1 +HEADER_ROW_ONE_C = 4 +FIRST_CODE_ROW_ONE_C = 5 +PERIOD_COL_ONE_C_1 = 5 +PERIOD_COL_ONE_C_2 = 6 +HEADER_ROW_TDSHEET = 2 +HEADER_LAST_ROW_TDSHEET = 3 +FIRST_CODE_ROW_TDSHEET = 4 +HEADER_ROW_TWO_ROW = 1 +HEADER_LAST_ROW_TWO_ROW = 2 +FIRST_CODE_ROW_TWO_ROW = 3 +HEADER_ROW_NO_CAPTION = 1 +CODE_COL_NO_CAPTION = 2 +NAME_COL_NO_CAPTION = 1 +HEADER_ROW_CONTINUATION = None +FIRST_DATA_ROW_CONTINUATION = 1 +CODE_COL_CONTINUATION = 6 +PERIOD_COL_CONT_1 = 7 +PERIOD_COL_CONT_2 = 8 +PERIOD_COL_CONT_3 = 9 +HEADER_ROW_BARE_YEAR = 2 +PERIOD_COL_BARE_YEAR_1 = 15 +PERIOD_COL_BARE_YEAR_2 = 20 + + +def test_line_code_from_text_and_numeric_cells(rules: Rules) -> None: + assert line_code(Cell("1110", "text"), rules) == "1110" + assert line_code(Cell(Decimal("1110"), "number"), rules) == "1110" + assert line_code(Cell(Decimal("1110.0"), "number"), rules) == "1110" # Review Focus 1: xlrd даёт float + assert line_code(Cell("11101", "text"), rules) == "11101" + assert line_code(Cell("2110", "text"), rules) == "2110" + assert line_code(Cell("5200", "text"), rules) == "5200" + assert line_code(Cell("2023", "text"), rules) is None # год, не код + assert line_code(Cell("0710001", "text"), rules) is None + assert line_code(Cell("", "empty"), rules) is None + + +# Test data: ONE_C_BALANCE sheet +_HEADER_ROW_ONE_C = [ + "Пояснения", + "Наименование показателя", + None, + "Код", + "На 31 декабря 2024 г.", + "На 31 декабря 2023 г.", + "Изменение, %", +] + +ONE_C_BALANCE = [ + ["Бухгалтерский баланс"], + ["на 31 декабря 2024 г."], + [], + _HEADER_ROW_ONE_C, + [None, "АКТИВ"], + [None, "Нематериальные активы", None, "1110", 5971264, 27405634, 12], + [None, "Нематериальные активы в организации", None, "11101", 5852464, 27214614, 0], + [None, None, None, None, 0, 0, 0], + ["1", "Основные средства", None, "1150", 2039077663, 1809528568, 7], + [None, "Итого по разделу I", None, "1100", 2045048927, 1836934202, 7], +] + + +def test_code_column_by_share(rules: Rules) -> None: + sheet = sheet_from_rows(ONE_C_BALANCE) + + assert find_code_column(sheet, rules, 1, sheet.n_rows) == CODE_COL_ONE_C + + +def test_code_column_needs_three_codes_and_sixty_percent(rules: Rules) -> None: + sheet = sheet_from_rows([["x", "1110"], ["y", "abc"], ["z", "def"], ["w", "ghi"], ["v", "1150"]]) + + assert find_code_column(sheet, rules, 1, 5) is None + + +def test_layout_one_c(rules: Rules) -> None: + sheet = sheet_from_rows(ONE_C_BALANCE) + + layout = find_layout(sheet, rules, 1, sheet.n_rows) + + assert (layout.header_row, layout.header_last_row, layout.first_data_row) == ( + HEADER_ROW_ONE_C, + HEADER_ROW_ONE_C, + FIRST_CODE_ROW_ONE_C, + ) + assert layout.code_col == CODE_COL_ONE_C and layout.name_col == NAME_COL_ONE_C + expected_periods = [ + (PERIOD_COL_ONE_C_1, "На 31 декабря 2024 г.", datetime.date(2024, 12, 31)), + (PERIOD_COL_ONE_C_2, "На 31 декабря 2023 г.", datetime.date(2023, 12, 31)), + ] + assert [(column.col, column.label, column.period_date) for column in layout.periods] == expected_periods + assert layout.skipped_labels == ("Изменение, %",) + + +def test_layout_tdsheet_with_numbering_row_and_line_breaks(rules: Rules) -> None: + sheet = sheet_from_rows( + [ + ["БУХГАЛТЕРСКИЙ БАЛАНС"], + ["Наименование показателя", "Коды", "За 9 месяцев\n2025 года", "На 31 декабря \n2024 года"], + ["1", "2", "3", "4"], + ["Нематериальные активы", "1110", 0, 0], + ["Основные средства", "1150", Decimal("898852.21795"), 966270], + ["ИТОГО по разделу I", "1100", Decimal("898852.21795"), 966270], + ] + ) + + layout = find_layout(sheet, rules, 1, sheet.n_rows) + + assert (layout.header_row, layout.header_last_row, layout.first_data_row) == ( + HEADER_ROW_TDSHEET, + HEADER_LAST_ROW_TDSHEET, + FIRST_CODE_ROW_TDSHEET, + ) + assert layout.name_col == NAME_COL_TDSHEET and layout.code_col == CODE_COL_TDSHEET + expected_labels = ["За 9 месяцев 2025 года", "На 31 декабря 2024 года"] + assert [column.label for column in layout.periods] == expected_labels + assert layout.periods[0].period_date == datetime.date(2025, 9, 30) + + +def test_layout_two_row_header(rules: Rules) -> None: + sheet = sheet_from_rows( + [ + ["Наименование показателя", "Код", "На 31 декабря", "На 31 декабря"], + [None, None, "2024 г.", "2023 г."], + ["Запасы", "1210", 10, 20], + ["Дебиторская задолженность", "1230", 30, 40], + ["Итого по разделу II", "1200", 40, 60], + ] + ) + + layout = find_layout(sheet, rules, 1, sheet.n_rows) + + assert (layout.header_row, layout.header_last_row, layout.first_data_row) == ( + HEADER_ROW_TWO_ROW, + HEADER_LAST_ROW_TWO_ROW, + FIRST_CODE_ROW_TWO_ROW, + ) + expected_dates = [datetime.date(2024, 12, 31), datetime.date(2023, 12, 31)] + assert [column.period_date for column in layout.periods] == expected_dates + + +def test_layout_without_code_caption(rules: Rules) -> None: + sheet = sheet_from_rows( + [ + ["Показатель", None, "На 31.12.2024", "На 31.12.2023"], + ["Основные средства", "1150", 100, 90], + ["Запасы", "1210", 50, 40], + ["БАЛАНС", "1600", 150, 130], + ] + ) + + layout = find_layout(sheet, rules, 1, sheet.n_rows) + + assert layout.header_row == HEADER_ROW_NO_CAPTION + assert layout.code_col == CODE_COL_NO_CAPTION + assert layout.name_col == NAME_COL_NO_CAPTION + expected_dates = [datetime.date(2024, 12, 31), datetime.date(2023, 12, 31)] + assert [column.period_date for column in layout.periods] == expected_dates + + +def test_layout_continuation_without_header(rules: Rules) -> None: + sheet = sheet_from_rows( + [ + [None, None, None, None, "в том числе:"], + [None, None, None, None, "Биологические активы", "11901", 0, 0, 0], + [None, None, None, None, "Итого по разделу I", "1100", 11853698344, 11570097340, 9391157985], + [None, None, None, "II. Оборотные активы"], + [None, None, None, "Запасы", None, "1210", 1001705104, 1426756617, 3393013092], + ] + ) + + layout = find_layout(sheet, rules, 1, sheet.n_rows) + + assert layout.header_row is HEADER_ROW_CONTINUATION + assert layout.first_data_row == FIRST_DATA_ROW_CONTINUATION + assert layout.code_col == CODE_COL_CONTINUATION + expected_cols = [ + (PERIOD_COL_CONT_1, "", None), + (PERIOD_COL_CONT_2, "", None), + (PERIOD_COL_CONT_3, "", None), + ] + assert [(column.col, column.label, column.period_date) for column in layout.periods] == expected_cols + + +def test_explanations_column_is_not_name(rules: Rules) -> None: + sheet = sheet_from_rows(ONE_C_BALANCE) + + assert find_layout(sheet, rules, 1, sheet.n_rows).name_col == NAME_COL_ONE_C + + +def test_no_code_column_raises(rules: Rules) -> None: + sheet = sheet_from_rows([["Показатель", "Сумма"], ["Основные средства", 100], ["Запасы", 50]]) + + with pytest.raises(LayoutNotFoundError, match="колонка кодов не найдена"): + find_layout(sheet, rules, 1, 3) + + +def test_bare_year_in_title_block_is_not_a_header(rules: Rules) -> None: + """У НПЗ над таблицей стоит «Отчетная дата | 30 | 09 | 2025»: строка с подписью «Код» побеждает.""" + sheet = sheet_from_rows( + [ + [*at({6: "Отчетная дата", 22: "30", 24: "09", 25: "2025"})], + [*at({4: "Наименование показателя", 12: "Код", 15: "На 30 сентября 2025 г.", 20: "На 31 декабря 2024 г."})], + [*at({4: "Гудвил", 12: "1105", 15: 0, 20: 0})], + [*at({4: "Нематериальные активы", 12: "1110", 15: 586260, 20: 905786})], + [*at({4: "Итого по разделу I", 12: "1100", 15: 586260, 20: 905786})], + ] + ) + + layout = find_layout(sheet, rules, 1, sheet.n_rows) + + period_cols = [PERIOD_COL_BARE_YEAR_1, PERIOD_COL_BARE_YEAR_2] + assert layout.header_row == HEADER_ROW_BARE_YEAR + assert [column.col for column in layout.periods] == period_cols