From 10b5bda550c11b43150340fd4f3e45ff6341f554 Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Tue, 6 Oct 2026 18:12:27 +0400 Subject: [PATCH] =?UTF-8?q?feat(statements):=20=D0=BA=D0=BE=D0=BD=D0=B2?= =?UTF-8?q?=D0=B5=D0=B9=D0=B5=D1=80=20=D1=84=D0=BE=D1=80=D0=BC=20=E2=80=94?= =?UTF-8?q?=20=D0=B1=D0=BB=D0=BE=D0=BA=D0=B8,=20=D1=81=D0=BA=D0=BB=D0=B5?= =?UTF-8?q?=D0=B9=D0=BA=D0=B0=20=D0=BF=D1=80=D0=BE=D0=B4=D0=BE=D0=BB=D0=B6?= =?UTF-8?q?=D0=B5=D0=BD=D0=B8=D0=B9,=20=D1=81=D1=82=D0=B0=D1=82=D1=83?= =?UTF-8?q?=D1=81=20=D0=B4=D0=BE=D0=BA=D1=83=D0=BC=D0=B5=D0=BD=D1=82=D0=B0?= =?UTF-8?q?;=20=D0=BF=D0=BE=D0=B4=D0=BA=D0=BB=D1=8E=D1=87=D0=B5=D0=BD?= =?UTF-8?q?=D0=B8=D0=B5=20=D0=B2=20get=5Fparser?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit StatementParser заменяет SkippingStatementParser; продолжение присоединяется к последнему открытому документу; duplicate_line_codes — уникальные коды; потерянный лист-продолжение (нет колонки кодов) даёт needs_check. Refs #14 Co-Authored-By: Claude Sonnet Claude-Session: cedd7915-2efe-41e5-988c-a9117033598b --- api/src/parser/__init__.py | 5 +- api/src/parser/book.py | 11 +- api/src/parser/statements/header.py | 4 +- api/src/parser/statements/merge.py | 44 +++ api/src/parser/statements/statement.py | 163 ++++++++++ api/tests/fixtures/test_rules_coverage.py | 5 +- api/tests/synthetic/test_synthetic_files.py | 30 +- .../unit/parser/statements/test_statement.py | 303 ++++++++++++++++++ api/tests/unit/parser/test_book_parser.py | 29 +- api/tests/unit/parser/test_read_limits.py | 5 +- 10 files changed, 555 insertions(+), 44 deletions(-) create mode 100644 api/src/parser/statements/merge.py create mode 100644 api/src/parser/statements/statement.py create mode 100644 api/tests/unit/parser/statements/test_statement.py diff --git a/api/src/parser/__init__.py b/api/src/parser/__init__.py index bae4153..cddbf2c 100644 --- a/api/src/parser/__init__.py +++ b/api/src/parser/__init__.py @@ -69,8 +69,9 @@ def get_parser(settings: Settings | None = None, *, rules_dir: Path = DEFAULT_RU и ключом, сервис не стартует, а не падает на первом файле. """ # `book.py` и `osv.py` импортируют `ParseOutcome` из этого пакета: импорт на уровне модуля был бы циклическим - from src.parser.book import BookParser, SkippingStatementParser # noqa: PLC0415 - цикл импорта book → src.parser + from src.parser.book import BookParser # noqa: PLC0415 - цикл импорта book → src.parser from src.parser.osv import OsvParser # noqa: PLC0415 - цикл импорта osv → src.parser + from src.parser.statements.statement import StatementParser # noqa: PLC0415 - цикл импорта fields = Settings.model_fields max_bytes = fields["PARSE_MAX_BYTES"].default if settings is None else settings.PARSE_MAX_BYTES @@ -79,7 +80,7 @@ def get_parser(settings: Settings | None = None, *, rules_dir: Path = DEFAULT_RU return BookParser( rules, osv=OsvParser(rules), - statements=SkippingStatementParser(), + statements=StatementParser(rules), max_bytes=max_bytes, timeout_seconds=timeout_seconds, ) diff --git a/api/src/parser/book.py b/api/src/parser/book.py index 4892964..0d1b47b 100644 --- a/api/src/parser/book.py +++ b/api/src/parser/book.py @@ -24,7 +24,7 @@ from src.parser.readers._common import one_line_reason from src.parser.readers._limits import ZIP_UNCOMPRESSED_LIMIT_FACTOR from src.parser.rules import Rules -__all__ = ["BookParser", "SkippingStatementParser", "StatementSheetParser"] +__all__ = ["BookParser", "StatementSheetParser"] _MS_IN_SECOND = 1000 STATEMENT_KINDS = frozenset({SheetKind.STATEMENT, SheetKind.STATEMENT_CONTINUATION}) @@ -36,15 +36,6 @@ class StatementSheetParser(Protocol): ) -> tuple[list[StatementDocument], list[str], bool]: ... -class SkippingStatementParser: - """Поведение Среза 2a: листы форм пропускаются с предупреждением. Заменяется `StatementParser` (задача 12).""" - - def parse_sheets( - self, sheets: Sequence[tuple[Sheet, SheetKind]], deadline: Deadline - ) -> tuple[list[StatementDocument], list[str], bool]: - return [], [sheet_line(sheet, "skipped: statement (slice 2b)") for sheet, _ in sheets], True - - class BookParser: def __init__( # noqa: PLR0913 - все зависимости оркестратора именованные self, diff --git a/api/src/parser/statements/header.py b/api/src/parser/statements/header.py index d7c7aef..70579cc 100644 --- a/api/src/parser/statements/header.py +++ b/api/src/parser/statements/header.py @@ -145,7 +145,7 @@ def parse_header( ), None, ) - return StatementHeader(form_name=marker), [] + return (StatementHeader(form_name=marker) if marker is not None else StatementHeader()), [] auditor_from = _auditor_from(sheet, rules, first_row, last_row) organization = _organization(sheet, rules, first_row, last_row, auditor_from) inn = _inn(sheet, rules, first_row, last_row, auditor_from) @@ -158,7 +158,7 @@ def parse_header( if units is None: warnings.append(NO_UNITS_WARNING) header = StatementHeader( - form_name=block.form_name, + **({} if block.form_name is None else {"form_name": block.form_name}), organization_name=organization, organization_name_confidence=_FULL_CONFIDENCE if organization else None, inn=inn, diff --git a/api/src/parser/statements/merge.py b/api/src/parser/statements/merge.py new file mode 100644 index 0000000..541548c --- /dev/null +++ b/api/src/parser/statements/merge.py @@ -0,0 +1,44 @@ +"""Черновик документа формы и присоединение листов-продолжений (design D-8).""" + +from dataclasses import dataclass, field + +from src.contract.result import StatementHeader, StatementLine +from src.parser.rules import FormType +from src.parser.statements.columns import PeriodColumn, TableLayout + +__all__ = ["Draft", "can_join", "join"] + + +@dataclass +class Draft: + kind: FormType | None # None — продолжение без ОКУД в маркере → OTHER + header: StatementHeader + periods: tuple[PeriodColumn, ...] + sheet_names: list[str] + lines: list[StatementLine] = field(default_factory=list) + duplicates: list[str] = field(default_factory=list) + has_code_column: bool = True + lonely_continuation: bool = False + fractional_values: int = 0 + numeric_values: int = 0 + warnings: list[str] = field(default_factory=list) + + @property + def codes(self) -> set[str]: + return {line.line_code for line in self.lines} + + +def can_join(draft: Draft, layout: TableLayout) -> bool: + """Продолжение присоединяется, если число колонок-сумм совпало с открытым документом.""" + return draft.has_code_column and len(layout.periods) == len(draft.periods) + + +def join(draft: Draft, sheet_name: str, lines: list[StatementLine], duplicates: list[str]) -> None: + """Строки продолжения в конец документа; коды, уже бывшие в документе, — в дубликаты.""" + for line in lines: + if line.line_code in draft.codes: + draft.duplicates.append(line.line_code) + continue + draft.lines.append(line) + draft.duplicates.extend(duplicates) + draft.sheet_names.append(sheet_name) diff --git a/api/src/parser/statements/statement.py b/api/src/parser/statements/statement.py new file mode 100644 index 0000000..7ea187a --- /dev/null +++ b/api/src/parser/statements/statement.py @@ -0,0 +1,163 @@ +"""Конвейер форм (design D-3…D-8): листы книги по порядку → блоки форм → шапка, таблица, строки → продолжения → +соотношения §7 → `statementDocument`. `ok` документа: жёсткие соотношения сошлись, дубликатов нет, колонка кодов +найдена, есть колонка-период, в шапке есть организация или ИНН, документ не одинокое продолжение. DISCLOSURE на +статус не влияет. +""" + +from collections.abc import Sequence + +from src.contract.result import StatementColumn, StatementDocument, StatementHeader +from src.parser.deadline import Deadline +from src.parser.detect import SheetKind +from src.parser.grid import Sheet +from src.parser.rules import Rules +from src.parser.statements.checks import run_statement_checks +from src.parser.statements.columns import NO_CODE_COLUMN, LayoutNotFoundError, TableLayout, find_layout +from src.parser.statements.detect import FormBlock, continuation_form, find_forms +from src.parser.statements.header import parse_header +from src.parser.statements.lines import collect_lines +from src.parser.statements.merge import Draft, can_join, join + +__all__ = ["CONTINUATION_WARNING", "LOCATOR_PREFIX", "LOCATOR_SEPARATOR", "StatementParser"] + +LOCATOR_PREFIX = "sheet:" +LOCATOR_SEPARATOR = "," +CONTINUATION_WARNING = "страница-продолжение без шапки: загрузите форму одним файлом" +SKIPPED_COLUMN_WARNING = "колонка без периода пропущена" + + +def _line(sheet: Sheet, text: str) -> str: + return f"sheet={sheet.name} {text}" + + +class StatementParser: + def __init__(self, rules: Rules) -> None: + self._rules = rules + + def parse_sheets( + self, sheets: Sequence[tuple[Sheet, SheetKind]], deadline: Deadline + ) -> tuple[list[StatementDocument], list[str], bool]: + drafts: list[Draft] = [] + warnings: list[str] = [] + dropped = False + for sheet, kind in sheets: + deadline.check(f"лист «{sheet.name}»: форма") + if kind is SheetKind.STATEMENT: + blocks, block_warnings = find_forms(sheet, self._rules) + warnings.extend(_line(sheet, text) for text in block_warnings) + for block in blocks: + drafts.append(self._block_draft(sheet, block, warnings)) + else: + dropped = not self._continuation(sheet, drafts, warnings) or dropped + documents = [self._document(draft) for draft in drafts] + for draft in drafts: + warnings.extend(draft.warnings) + ok = not dropped and all( + self._is_ok(draft, document) for draft, document in zip(drafts, documents, strict=True) + ) + return documents, warnings, ok + + # --- блоки --- + + def _block_draft(self, sheet: Sheet, block: FormBlock, warnings: list[str]) -> Draft: + if block.kind == "DISCLOSURE": + header, _ = parse_header(sheet, self._rules, block.first_row, block.last_row, block) + return Draft(kind=block.kind, header=header, periods=(), sheet_names=[sheet.name]) + try: + layout = find_layout(sheet, self._rules, block.first_row, block.last_row) + except LayoutNotFoundError: + header, header_warnings = parse_header(sheet, self._rules, block.first_row, block.last_row, block) + warnings.extend(_line(sheet, text) for text in [NO_CODE_COLUMN, *header_warnings]) + return Draft(kind=block.kind, header=header, periods=(), sheet_names=[sheet.name], has_code_column=False) + header_last = (layout.header_row or layout.first_data_row) - 1 + header, header_warnings = parse_header(sheet, self._rules, block.first_row, header_last, block) + draft = Draft(kind=block.kind, header=header, periods=layout.periods, sheet_names=[sheet.name]) + self._fill(sheet, draft, layout, layout.first_data_row, block.last_row, header_warnings, warnings) + return draft + + def _continuation(self, sheet: Sheet, drafts: list[Draft], warnings: list[str]) -> bool: + """`False` — лист потерян (нет колонки кодов): строки не попали ни в один документ, статус не `done`.""" + try: + layout = find_layout(sheet, self._rules, 1, sheet.n_rows) + except LayoutNotFoundError: + warnings.append(_line(sheet, NO_CODE_COLUMN)) + return False + open_draft = next((draft for draft in reversed(drafts) if draft.kind != "DISCLOSURE"), None) + if open_draft is not None and can_join(open_draft, layout): + result = collect_lines( + sheet, self._rules, layout, layout.first_data_row, sheet.n_rows, units=open_draft.header.units + ) + join(open_draft, sheet.name, result.lines, result.duplicates) + open_draft.fractional_values += result.fractional_values + open_draft.numeric_values += result.numeric_values + warnings.extend(_line(sheet, text) for text in result.warnings) + return True + kind, _ = continuation_form(sheet, self._rules) + header, _ = parse_header(sheet, self._rules, 1, (layout.header_row or layout.first_data_row) - 1, None) + draft = Draft( + kind=kind, header=header, periods=layout.periods, sheet_names=[sheet.name], lonely_continuation=True + ) + warnings.append(_line(sheet, CONTINUATION_WARNING)) + self._fill(sheet, draft, layout, layout.first_data_row, sheet.n_rows, [], warnings) + drafts.append(draft) + return True + + def _fill( # noqa: PLR0913, PLR0917 - все части документа передаются явно + self, + sheet: Sheet, + draft: Draft, + layout: TableLayout, + first_row: int, + last_row: int, + header_warnings: list[str], + warnings: list[str], + ) -> None: + result = collect_lines(sheet, self._rules, layout, first_row, last_row, units=draft.header.units) + draft.lines = result.lines + draft.duplicates = result.duplicates + draft.fractional_values, draft.numeric_values = result.fractional_values, result.numeric_values + texts = [*header_warnings, *result.warnings] + if layout.skipped_labels: + texts.append(f"{SKIPPED_COLUMN_WARNING}: {', '.join(layout.skipped_labels)}") + warnings.extend(_line(sheet, text) for text in texts) + + # --- документ --- + + def _document(self, draft: Draft) -> StatementDocument: + kind = draft.kind or "OTHER" + columns = [ + StatementColumn(column_index=index, label=column.label, period_date=column.period_date) + for index, column in enumerate(draft.periods) + ] + locator = LOCATOR_PREFIX + LOCATOR_SEPARATOR.join(draft.sheet_names) + if kind == "DISCLOSURE" or not draft.has_code_column: + return StatementDocument( + document_type=kind, + locator=locator, + header=draft.header, + columns=columns if draft.has_code_column else [], + lines=[], + ) + checks, check_warnings = run_statement_checks( + draft.lines, len(columns), kind if kind != "OTHER" else "BALANCE", self._rules + ) + checks = checks.model_copy(update={"duplicate_line_codes": list(dict.fromkeys(draft.duplicates))}) + draft.warnings.extend(f"sheet={draft.sheet_names[0]} {text}" for text in check_warnings) + return StatementDocument( + document_type=kind, locator=locator, header=draft.header, columns=columns, lines=draft.lines, checks=checks + ) + + @staticmethod + def _is_ok(draft: Draft, document: StatementDocument) -> bool: + if document.document_type == "DISCLOSURE": + return True + header: StatementHeader = document.header + return ( + draft.has_code_column + and not draft.lonely_continuation + and bool(document.columns) + and bool(header.organization_name or header.inn) + and document.checks is not None + and not document.checks.relations_failed + and not document.checks.duplicate_line_codes + ) diff --git a/api/tests/fixtures/test_rules_coverage.py b/api/tests/fixtures/test_rules_coverage.py index e7ea8bb..cc1c9d7 100644 --- a/api/tests/fixtures/test_rules_coverage.py +++ b/api/tests/fixtures/test_rules_coverage.py @@ -20,9 +20,10 @@ from fixtures_helpers import osv_dkb_files, osv_fixture_files, synthetic_files from rules_coverage import DICTIONARIES, MatchCounter, iter_patterns, uncovered from src.parser import get_parser -from src.parser.book import BookParser, SkippingStatementParser +from src.parser.book import BookParser from src.parser.osv import OsvParser from src.parser.rules import DEFAULT_RULES_DIR, Rules, load_rules +from src.parser.statements.statement import StatementParser pytestmark = pytest.mark.fixtures @@ -39,7 +40,7 @@ def coverage(fixtures_dir: Path) -> tuple[Rules, MatchCounter, int]: parser = BookParser( rules, osv=OsvParser(rules), - statements=SkippingStatementParser(), + statements=StatementParser(rules), max_bytes=base.max_bytes, timeout_seconds=base.timeout_seconds, ) diff --git a/api/tests/synthetic/test_synthetic_files.py b/api/tests/synthetic/test_synthetic_files.py index 3f0089a..d079ae1 100644 --- a/api/tests/synthetic/test_synthetic_files.py +++ b/api/tests/synthetic/test_synthetic_files.py @@ -95,10 +95,10 @@ EXPECTED: dict[str, Expect] = { # «12 месяцев» "s11_header_repeat.xlsx": Expect("outline", 2025, "Q4", True), # «январь - сентябрь», данные «НУ» - "s12_statement_and_osv.xlsx": Expect("outline", 2025, "Q3", True, data_type="NU"), + "s12_statement_and_osv.xlsx": Expect("outline", 2025, "Q3", True, status=ResultStatus.NEEDS_CHECK, data_type="NU"), "s13_no_outline_indent.xlsb": Expect("indent", 2025, "Q2", False), # три листа форм по одному признаку + лист ОСВ - "s14_statement_forms.xlsx": Expect("outline", 2025, "Q1", True), + "s14_statement_forms.xlsx": Expect("outline", 2025, "Q1", True, status=ResultStatus.NEEDS_CHECK), # названия групп сумм: «Входящее/Конечное», «Начальное/Исходящее сальдо» "s15_group_title_synonyms.xlsx": Expect("outline", 2025, "Q1", True, documents=2), # шапка без организации, период «2024»; лист 1 — вариант 1, лист 2 — вариант 2 @@ -134,9 +134,8 @@ def parse_named(fixtures_dir: Path, name: str) -> ParseOutcome: def documents(outcome: ParseOutcome) -> list[OsvDocument]: assert outcome.error is None, outcome.error - docs = list(outcome.documents) - assert all(isinstance(doc, OsvDocument) for doc in docs) - return cast(list[OsvDocument], docs) + """Документы ОСВ результата; документы форм (`StatementDocument`) разбираются отдельными сценариями.""" + return [doc for doc in outcome.documents if isinstance(doc, OsvDocument)] def one_document(outcome: ParseOutcome) -> OsvDocument: @@ -422,14 +421,18 @@ def test_s11_header_repeat_is_skipped(fixtures_dir: Path) -> None: assert outcome.status is ResultStatus.DONE -def test_s12_statement_sheet_is_skipped_with_warning(fixtures_dir: Path) -> None: - """«Лист ББ + лист ОСВ»: один документ, лист формы пропущен с предупреждением о подсрезе 2b.""" +def test_s12_statement_sheet_goes_to_statement_pipeline(fixtures_dir: Path) -> None: + """«Лист ББ + лист ОСВ»: документ ОСВ и документ ББ; у синтетического ББ нет шапки и колонки кодов → needs_check.""" outcome = parse_named(fixtures_dir, "s12_statement_and_osv.xlsx") (doc,) = documents(outcome) assert doc.locator == "sheet:ОСВ 60" - assert any("Бухгалтерский баланс" in w and "skipped: statement" in w for w in warnings_of(outcome)) - assert outcome.status is ResultStatus.DONE + assert [(d.document_type, d.locator) for d in outcome.documents] == [ + ("OSV", "sheet:ОСВ 60"), + ("BALANCE", "sheet:Бухгалтерский баланс"), + ] + assert not any("skipped: statement" in w for w in warnings_of(outcome)) + assert outcome.status is ResultStatus.NEEDS_CHECK def test_s13_xlsb_equals_s02_except_locator_and_warnings(fixtures_dir: Path) -> None: @@ -449,14 +452,15 @@ def test_s13_xlsb_equals_s02_except_locator_and_warnings(fixtures_dir: Path) -> def test_s14_each_statement_form_is_recognised_by_its_own_marker(fixtures_dir: Path) -> None: - """Три листа форм (код ОКУД 0710001, «Отчет о финансовых результатах», код 0710002) пропущены, ОСВ разобран.""" + """Три листа форм (код ОКУД 0710001, «Отчет о финансовых результатах», код 0710002) дают три документа форм.""" outcome = parse_named(fixtures_dir, "s14_statement_forms.xlsx") (doc,) = documents(outcome) assert doc.locator == "sheet:ОСВ 60" - skipped = [w for w in warnings_of(outcome) if "skipped: statement" in w] - assert [w.split(" skipped")[0] for w in skipped] == ["sheet=Форма 1", "sheet=ОФР", "sheet=Форма 2"] - assert outcome.status is ResultStatus.DONE + statements = [(d.document_type, d.locator) for d in outcome.documents if d.document_type != "OSV"] + assert statements == [("BALANCE", "sheet:Форма 1"), ("INCOME", "sheet:ОФР"), ("INCOME", "sheet:Форма 2")] + assert not any("skipped: statement" in w for w in warnings_of(outcome)) + assert outcome.status is ResultStatus.NEEDS_CHECK def test_s15_group_title_synonyms_find_all_six_sum_columns(fixtures_dir: Path) -> None: diff --git a/api/tests/unit/parser/statements/test_statement.py b/api/tests/unit/parser/statements/test_statement.py new file mode 100644 index 0000000..5f9bc97 --- /dev/null +++ b/api/tests/unit/parser/statements/test_statement.py @@ -0,0 +1,303 @@ +# API/tests/unit/parser/statements/test_statement.py +"""Конвейер форм: блоки, склейка продолжений, статус документа (design D-8; spec «Склейка листов-продолжений и статус +документа»). Книги собираются `gridbuilder`, вид листа задаётся явно, как это делает `BookParser`.""" + +from decimal import Decimal + +import pytest +from gridbuilder import at, sheet_from_rows + +from src.parser.deadline import Deadline +from src.parser.detect import SheetKind +from src.parser.rules import load_rules +from src.parser.statements.statement import CONTINUATION_WARNING, StatementParser + +TWO_COLUMNS = TWO_DOCUMENTS = 2 + + +@pytest.fixture(scope="module") +def parser() -> StatementParser: + return StatementParser(load_rules()) + + +def deadline() -> Deadline: + return Deadline(120) + + +BALANCE_PAGE_1 = [ + ["Бухгалтерский баланс"], + ["на 31 декабря 2024 г."], + ["Форма по ОКУД", None, "0710001"], + ["Организация", None, 'ООО "Пример"', "ИНН", "7700000000"], + ["Единица измерения:", None, "тыс. руб."], + ["Наименование показателя", "Код", "На 31 декабря 2024 г.", "На 31 декабря 2023 г."], + ["Основные средства", "1150", 100, 90], + ["Итого по разделу I", "1100", 100, 90], + ["Запасы", "1210", 50, 40], + ["Итого по разделу II", "1200", 50, 40], + ["БАЛАНС", "1600", 150, 130], +] +BALANCE_PAGE_2 = [ + [*at({6: "Форма 0710001 с.2"})], + ["Наименование показателя", "Код", "На 31 декабря 2024 г.", "На 31 декабря 2023 г."], + ["Уставный капитал", "1310", 150, 130], + ["Итого по разделу III", "1300", 150, 130], + ["БАЛАНС", "1700", 150, 130], +] +INCOME = [ + ["Отчет о финансовых результатах"], + ["за 2024 г."], + ["Организация", None, 'ООО "Пример"'], + ["Единица измерения:", None, "тыс. руб."], + ["Наименование показателя", "Код", "За 2024 г."], + ["Выручка", "2110", 1000], + ["Себестоимость продаж", "2120", -600], + ["Валовая прибыль (убыток)", "2100", 400], + ["Коммерческие расходы", "2210", -50], + ["Управленческие расходы", "2220", -50], + ["Прибыль (убыток) от продаж", "2200", 300], + ["Проценты к уплате", "2330", -100], + ["Прибыль (убыток) до налогообложения", "2300", 200], + ["Налог на прибыль", "2410", -40], + ["Чистая прибыль (убыток)", "2400", 160], +] +# Полный набор: 2200 = 2100 − 2210 − 2220 и 2300 = 2200 − 2330 сходятся через abs_terms (знаки 1С), 2400 — мягкое +NPZ_CONTINUATION = [ + ["в том числе:"], + ["Биологические активы", "11901", 0, 0], + ["Расходы будущих периодов", "11903", 0, 0], + ["Прочие внеоборотные активы", "1190", 0, 0], +] + + +def test_full_balance_over_two_sheets_is_done(parser: StatementParser) -> None: + sheets = [ + (sheet_from_rows(BALANCE_PAGE_1, "ББ1"), SheetKind.STATEMENT), + (sheet_from_rows(BALANCE_PAGE_2, "ББ2"), SheetKind.STATEMENT_CONTINUATION), + ] + + documents, warnings, ok = parser.parse_sheets(sheets, deadline()) + + assert ok and len(documents) == 1 + document = documents[0] + assert document.document_type == "BALANCE" and document.locator == "sheet:ББ1,ББ2" + assert [line.line_code for line in document.lines] == [ + "1150", + "1100", + "1210", + "1200", + "1600", + "1310", + "1300", + "1700", + ] + assert document.checks is not None and document.checks.relations_failed == [] + assert document.header.inn == "7700000000" and document.header.units == "тыс. руб." + assert len(document.columns) == TWO_COLUMNS and document.columns[0].label == "На 31 декабря 2024 г." + + +def test_page_one_alone_fails_total_relation(parser: StatementParser) -> None: + documents, warnings, ok = parser.parse_sheets([(sheet_from_rows(BALANCE_PAGE_1), SheetKind.STATEMENT)], deadline()) + + assert not ok + assert [item.relation for item in documents[0].checks.relations_failed] == ["1600 = 1700", "1600 = 1700"] + assert any("1600 = 1700" in warning for warning in warnings) + + +def test_lonely_continuation_is_needs_check(parser: StatementParser) -> None: + documents, warnings, ok = parser.parse_sheets( + [(sheet_from_rows(BALANCE_PAGE_2, "с.2"), SheetKind.STATEMENT_CONTINUATION)], deadline() + ) + + assert not ok and documents[0].document_type == "BALANCE" + assert documents[0].header.organization_name is None and documents[0].header.form_name == "Форма 0710001 с.2" + assert [line.line_code for line in documents[0].lines] == ["1310", "1300", "1700"] + assert f"sheet=с.2 {CONTINUATION_WARNING}" in warnings + + +def test_continuation_joins_last_open_document(parser: StatementParser) -> None: + """Review Focus 4: продолжение после ОФР относится к ОФР, а не к балансу.""" + income_tail = [ + ["Налог", "2450", 0], + ["Прочее", "2460", 0], + ["Совокупный финансовый результат периода", "2500", 160], + ] + sheets = [ + (sheet_from_rows(BALANCE_PAGE_1, "ББ"), SheetKind.STATEMENT), + (sheet_from_rows(INCOME, "ОФР"), SheetKind.STATEMENT), + (sheet_from_rows(income_tail, "ОФР_2"), SheetKind.STATEMENT_CONTINUATION), + ] + + documents, _, _ = parser.parse_sheets(sheets, deadline()) + + assert [document.document_type for document in documents] == ["BALANCE", "INCOME"] + assert documents[1].locator == "sheet:ОФР,ОФР_2" and [line.line_code for line in documents[1].lines][-2:] == [ + "2460", + "2500", + ] + + +def test_continuation_without_header_maps_value_columns_by_order(parser: StatementParser) -> None: + sheets = [ + (sheet_from_rows(BALANCE_PAGE_1, "ББ"), SheetKind.STATEMENT), + (sheet_from_rows(NPZ_CONTINUATION, "ББ_2"), SheetKind.STATEMENT_CONTINUATION), + ] + + documents, _, _ = parser.parse_sheets(sheets, deadline()) + + tail = documents[0].lines[-1] + assert tail.line_code == "1190" and [value.column_index for value in tail.values] == [0, 1] + assert [line.line_code for line in documents[0].lines][-3:] == ["11901", "11903", "1190"] + + +def test_continuation_with_other_column_count_opens_new_document(parser: StatementParser) -> None: + three_columns = [ + ["Форма 0710001 с.2"], + ["Наименование", "Код", "На 31.12.2024", "На 31.12.2023", "На 31.12.2022"], + ["Уставный капитал", "1310", 1, 1, 1], + ["Добавочный капитал", "1350", 1, 1, 1], + ["Итого по разделу III", "1300", 2, 2, 2], + ] + sheets = [ + (sheet_from_rows(BALANCE_PAGE_1, "ББ"), SheetKind.STATEMENT), + (sheet_from_rows(three_columns, "с.2"), SheetKind.STATEMENT_CONTINUATION), + ] + + documents, warnings, ok = parser.parse_sheets(sheets, deadline()) + + assert len(documents) == TWO_DOCUMENTS and not ok and any(CONTINUATION_WARNING in warning for warning in warnings) + + +def test_two_forms_on_one_sheet(parser: StatementParser) -> None: + sheet = sheet_from_rows([*BALANCE_PAGE_1, *BALANCE_PAGE_2[2:], [], *INCOME], "Отчетность") + + documents, _, ok = parser.parse_sheets([(sheet, SheetKind.STATEMENT)], deadline()) + + assert [document.document_type for document in documents] == ["BALANCE", "INCOME"] and ok + assert all(document.locator == "sheet:Отчетность" for document in documents) + assert "2110" not in [line.line_code for line in documents[0].lines] + + +def test_disclosure_only_book_is_done(parser: StatementParser) -> None: + """Review Focus 5.""" + sheet = sheet_from_rows( + [ + ["Пояснения к бухгалтерскому балансу и отчету о финансовых результатах"], + ["1. Нематериальные активы"], + ["текст"], + ], + "Пояснения", + ) + + documents, warnings, ok = parser.parse_sheets([(sheet, SheetKind.STATEMENT)], deadline()) + + assert ( + ok and documents[0].document_type == "DISCLOSURE" and documents[0].lines == [] and documents[0].checks is None + ) + + +def test_no_code_column_is_needs_check_with_warning(parser: StatementParser) -> None: + sheet = sheet_from_rows( + [ + ["Бухгалтерский баланс"], + ["Организация", "ООО Х"], + ["Показатель", "На 31.12.2024"], + ["Основные средства", 100], + ["Запасы", 50], + ["БАЛАНС", 150], + ], + "ББ", + ) + + documents, warnings, ok = parser.parse_sheets([(sheet, SheetKind.STATEMENT)], deadline()) + + assert not ok and documents[0].lines == [] and documents[0].columns == [] + assert "sheet=ББ колонка кодов не найдена" in warnings + + +def test_duplicate_code_is_needs_check(parser: StatementParser) -> None: + rows = [*BALANCE_PAGE_1, ["Запасы", "1210", 1, 1]] + + documents, _, ok = parser.parse_sheets([(sheet_from_rows(rows), SheetKind.STATEMENT)], deadline()) + + assert not ok and documents[0].checks.duplicate_line_codes == ["1210"] + + +def test_template_without_sums_is_needs_check(parser: StatementParser) -> None: + rows = [ + ["Бухгалтерский баланс"], + ["Наименование показателя", "Код", "На 31 декабря 2024 г."], + ["Основные средства", "1150"], + ["Запасы", "1210"], + ["БАЛАНС", "1600"], + ] + + documents, _, ok = parser.parse_sheets([(sheet_from_rows(rows), SheetKind.STATEMENT)], deadline()) + + assert ( + not ok + and documents[0].checks.relations_failed == [] + and all(value.amount is None for line in documents[0].lines for value in line.values) + ) + + +def test_duplicate_codes_are_unique_in_order_of_first_duplicate(parser: StatementParser) -> None: + """spec: `duplicate_line_codes` — уникальные коды; повтор на каждое вхождение не плодит записей (T11-ревью).""" + rows = [*BALANCE_PAGE_1, ["Запасы", "1210", 1, 1], ["Запасы", "1210", 2, 2], ["Основные средства", "1150", 1, 1]] + + documents, _, ok = parser.parse_sheets([(sheet_from_rows(rows), SheetKind.STATEMENT)], deadline()) + + assert not ok and documents[0].checks.duplicate_line_codes == ["1210", "1150"] + + +def test_code_repeated_on_continuation_is_duplicate_and_first_value_kept(parser: StatementParser) -> None: + first = (sheet_from_rows(BALANCE_PAGE_1, "ББ"), SheetKind.STATEMENT) + again = [ + ["Наименование показателя", "Код", "На 31 декабря 2024 г.", "На 31 декабря 2023 г."], + ["Запасы", "1210", 7, 7], + ["Итого", "1200", 50, 40], + ["Прочее", "1230", 0, 0], + ] + + documents, _, ok = parser.parse_sheets( + [first, (sheet_from_rows(again, "ББ3"), SheetKind.STATEMENT_CONTINUATION)], deadline() + ) + + assert not ok and documents[0].checks.duplicate_line_codes == ["1210", "1200"] + assert [line.values[0].amount for line in documents[0].lines if line.line_code == "1210"] == [Decimal(50)] + + +def test_headerless_lonely_continuation_has_empty_header(parser: StatementParser) -> None: + documents, _, ok = parser.parse_sheets( + [(sheet_from_rows(NPZ_CONTINUATION, "хвост"), SheetKind.STATEMENT_CONTINUATION)], deadline() + ) + + assert not ok and documents[0].document_type == "OTHER" and documents[0].header.form_name is None + + +def test_continuation_without_code_column_is_dropped_and_not_ok(parser: StatementParser) -> None: + """Потерянный лист нельзя оставлять статусом `done`.""" + lost = [["Форма 0710001 с.2"], ["Код", "На 31.12.2024"], ["1700", 1]] + sheets = [ + (sheet_from_rows(BALANCE_PAGE_1, "ББ"), SheetKind.STATEMENT), + (sheet_from_rows(BALANCE_PAGE_2, "ББ2"), SheetKind.STATEMENT_CONTINUATION), + (sheet_from_rows(lost, "с.3"), SheetKind.STATEMENT_CONTINUATION), + ] + + documents, warnings, ok = parser.parse_sheets(sheets, deadline()) + + assert ( + not ok + and len(documents) == 1 + and documents[0].checks.relations_failed == [] + and "sheet=с.3 колонка кодов не найдена" in warnings + ) + + +def test_block_with_okud_but_without_title_has_no_form_name(parser: StatementParser) -> None: + """Блок, открытый только кодом ОКУД: `form_name` не передаётся (в контракте нет null).""" + rows = [["Форма по ОКУД", None, "0710001"], *BALANCE_PAGE_1[3:]] + + documents, _, _ = parser.parse_sheets([(sheet_from_rows(rows), SheetKind.STATEMENT)], deadline()) + + assert documents[0].header.form_name is None and documents[0].header.okud == "0710001" diff --git a/api/tests/unit/parser/test_book_parser.py b/api/tests/unit/parser/test_book_parser.py index 07ae8aa..c15ccce 100644 --- a/api/tests/unit/parser/test_book_parser.py +++ b/api/tests/unit/parser/test_book_parser.py @@ -25,7 +25,7 @@ from result_contract import assert_valid_result from src.contract import ErrorCode, OsvDocument, ResultStatus, StatementDocument from src.contract.result import StatementHeader from src.parser import ParseOutcome, hierarchy -from src.parser.book import BookParser, SkippingStatementParser, StatementSheetParser +from src.parser.book import BookParser, StatementSheetParser from src.parser.classify import SUM_FIELDS from src.parser.deadline import Deadline from src.parser.detect import SheetKind @@ -34,6 +34,7 @@ from src.parser.grid import Sheet from src.parser.hierarchy import BodyRow from src.parser.osv import OsvParser from src.parser.rules import DEFAULT_RULES_DIR, HEADERS_FILE, Rules, load_rules +from src.parser.statements.statement import StatementParser OPENING = "Сальдо на начало периода" TURNOVER = "Обороты за период" @@ -138,7 +139,7 @@ def book( return BookParser( rules, osv=OsvParser(rules), - statements=statements or SkippingStatementParser(), + statements=statements or StatementParser(rules), max_bytes=max_bytes, timeout_seconds=timeout_seconds, clock=clock, @@ -374,12 +375,12 @@ def test_total_row_without_caption_is_the_last_row_of_sums(rules: Rules) -> None # --- Несколько листов --- -def test_statement_sheet_is_skipped_and_osv_parsed(rules: Rules) -> None: +def test_statement_sheet_goes_to_real_pipeline_next_to_osv(rules: Rules) -> None: outcome = book(rules).parse(xlsx({"Баланс": statement(), "ОСВ 60": mini_osv()}), "XLSX") - assert outcome.status is ResultStatus.DONE - assert only_document(outcome).locator == "sheet:ОСВ 60" - assert warnings_of(outcome) == ["sheet=ОСВ 60 hierarchy=outline", "sheet=Баланс skipped: statement (slice 2b)"] + assert [type(document).__name__ for document in outcome.documents] == ["OsvDocument", "StatementDocument"] + assert outcome.documents[0].locator == "sheet:ОСВ 60" and outcome.documents[1].locator == "sheet:Баланс" + assert "sheet=ОСВ 60 hierarchy=outline" in warnings_of(outcome) assert_valid_result(outcome) @@ -416,7 +417,9 @@ def test_one_spoiled_sheet_makes_whole_result_needs_check(rules: Rules) -> None: def test_book_without_osv_is_invalid_document(rules: Rules) -> None: """Spec «Нет ОСВ»: сообщение называет число просмотренных листов.""" - outcome = book(rules).parse(xlsx({"Баланс": statement(), "Данные": [["a", "b"], [1, 2]]}), "XLSX") + outcome = book(rules).parse( + xlsx({"Справка": [["Наименование", "Сумма"], ["Аренда", 100]], "Данные": [["a", "b"], [1, 2]]}), "XLSX" + ) assert outcome.status is ResultStatus.ERROR assert outcome.error is not None @@ -673,20 +676,20 @@ def test_statement_sheets_go_to_statement_parser_in_sheet_order(rules: Rules) -> assert "sheet=ББ statements-fake" in (outcome.metrics.warnings if outcome.metrics else []) -def test_skipping_parser_keeps_slice_2a_behaviour(rules: Rules) -> None: +def test_balance_sheet_gives_statement_document_without_stub(rules: Rules) -> None: outcome = book(rules).parse(xlsx({"ББ": BALANCE, "Лист_1": mini_osv()}), "XLSX") - assert len(outcome.documents) == 1 - assert "sheet=ББ skipped: statement (slice 2b)" in warnings_of(outcome) + assert [type(document).__name__ for document in outcome.documents] == ["OsvDocument", "StatementDocument"] + assert not any("skipped: statement" in warning for warning in warnings_of(outcome)) def test_statement_continuation_is_not_parsed_as_osv(rules: Rules) -> None: """R-5: лист-продолжение формы идёт в конвейер форм (здесь заглушка), а не в разбор ОСВ.""" outcome = book(rules).parse(xlsx({"ОСВ 60": mini_osv(), "с.2": CONTINUATION}), "XLSX") - assert outcome.status is ResultStatus.DONE - assert only_document(outcome).locator == "sheet:ОСВ 60" - assert warnings_of(outcome) == ["sheet=ОСВ 60 hierarchy=outline", "sheet=с.2 skipped: statement (slice 2b)"] + assert outcome.status is ResultStatus.NEEDS_CHECK # лист-продолжение потерян: в нём меньше трёх кодов + assert [document.locator for document in outcome.documents] == ["sheet:ОСВ 60"] + assert "sheet=с.2 колонка кодов не найдена" in warnings_of(outcome) def test_book_without_osv_and_forms_is_invalid_document(rules: Rules) -> None: diff --git a/api/tests/unit/parser/test_read_limits.py b/api/tests/unit/parser/test_read_limits.py index c26bc98..06e8019 100644 --- a/api/tests/unit/parser/test_read_limits.py +++ b/api/tests/unit/parser/test_read_limits.py @@ -14,7 +14,7 @@ import openpyxl import pytest from src.contract import ErrorCode, ResultStatus -from src.parser.book import BookParser, SkippingStatementParser +from src.parser.book import BookParser from src.parser.errors import UnreadableError, UnsupportedFormatError from src.parser.grid import MAX_INDEXED_AREA, MergedRange, Sheet from src.parser.osv import OsvParser @@ -28,6 +28,7 @@ from src.parser.readers._limits import ( merged_area_of_xml, ) from src.parser.rules import Rules, load_rules +from src.parser.statements.statement import StatementParser FAST_SECONDS = 1.0 SHEET_PART = "xl/worksheets/sheet1.xml" @@ -125,7 +126,7 @@ def test_huge_merged_range_is_rejected_before_openpyxl() -> None: def _book(rules: Rules, max_bytes: int) -> BookParser: return BookParser( - rules, osv=OsvParser(rules), statements=SkippingStatementParser(), max_bytes=max_bytes, timeout_seconds=120 + rules, osv=OsvParser(rules), statements=StatementParser(rules), max_bytes=max_bytes, timeout_seconds=120 )