feat(statements): конвейер форм — блоки, склейка продолжений, статус документа; подключение в get_parser
StatementParser заменяет SkippingStatementParser; продолжение присоединяется к последнему открытому документу; duplicate_line_codes — уникальные коды; потерянный лист-продолжение (нет колонки кодов) даёт needs_check. Refs #14 Co-Authored-By: Claude Sonnet <noreply@anthropic.com> Claude-Session: cedd7915-2efe-41e5-988c-a9117033598b
This commit is contained in:
Andrey Epifancev
parent
b9e6e319cf
commit
10b5bda550
@ -69,8 +69,9 @@ def get_parser(settings: Settings | None = None, *, rules_dir: Path = DEFAULT_RU
|
||||
и ключом, сервис не стартует, а не падает на первом файле.
|
||||
"""
|
||||
# `book.py` и `osv.py` импортируют `ParseOutcome` из этого пакета: импорт на уровне модуля был бы циклическим
|
||||
from src.parser.book import BookParser, SkippingStatementParser # noqa: PLC0415 - цикл импорта book → src.parser
|
||||
from src.parser.book import BookParser # noqa: PLC0415 - цикл импорта book → src.parser
|
||||
from src.parser.osv import OsvParser # noqa: PLC0415 - цикл импорта osv → src.parser
|
||||
from src.parser.statements.statement import StatementParser # noqa: PLC0415 - цикл импорта
|
||||
|
||||
fields = Settings.model_fields
|
||||
max_bytes = fields["PARSE_MAX_BYTES"].default if settings is None else settings.PARSE_MAX_BYTES
|
||||
@ -79,7 +80,7 @@ def get_parser(settings: Settings | None = None, *, rules_dir: Path = DEFAULT_RU
|
||||
return BookParser(
|
||||
rules,
|
||||
osv=OsvParser(rules),
|
||||
statements=SkippingStatementParser(),
|
||||
statements=StatementParser(rules),
|
||||
max_bytes=max_bytes,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
|
||||
@ -24,7 +24,7 @@ from src.parser.readers._common import one_line_reason
|
||||
from src.parser.readers._limits import ZIP_UNCOMPRESSED_LIMIT_FACTOR
|
||||
from src.parser.rules import Rules
|
||||
|
||||
__all__ = ["BookParser", "SkippingStatementParser", "StatementSheetParser"]
|
||||
__all__ = ["BookParser", "StatementSheetParser"]
|
||||
|
||||
_MS_IN_SECOND = 1000
|
||||
STATEMENT_KINDS = frozenset({SheetKind.STATEMENT, SheetKind.STATEMENT_CONTINUATION})
|
||||
@ -36,15 +36,6 @@ class StatementSheetParser(Protocol):
|
||||
) -> tuple[list[StatementDocument], list[str], bool]: ...
|
||||
|
||||
|
||||
class SkippingStatementParser:
|
||||
"""Поведение Среза 2a: листы форм пропускаются с предупреждением. Заменяется `StatementParser` (задача 12)."""
|
||||
|
||||
def parse_sheets(
|
||||
self, sheets: Sequence[tuple[Sheet, SheetKind]], deadline: Deadline
|
||||
) -> tuple[list[StatementDocument], list[str], bool]:
|
||||
return [], [sheet_line(sheet, "skipped: statement (slice 2b)") for sheet, _ in sheets], True
|
||||
|
||||
|
||||
class BookParser:
|
||||
def __init__( # noqa: PLR0913 - все зависимости оркестратора именованные
|
||||
self,
|
||||
|
||||
@ -145,7 +145,7 @@ def parse_header(
|
||||
),
|
||||
None,
|
||||
)
|
||||
return StatementHeader(form_name=marker), []
|
||||
return (StatementHeader(form_name=marker) if marker is not None else StatementHeader()), []
|
||||
auditor_from = _auditor_from(sheet, rules, first_row, last_row)
|
||||
organization = _organization(sheet, rules, first_row, last_row, auditor_from)
|
||||
inn = _inn(sheet, rules, first_row, last_row, auditor_from)
|
||||
@ -158,7 +158,7 @@ def parse_header(
|
||||
if units is None:
|
||||
warnings.append(NO_UNITS_WARNING)
|
||||
header = StatementHeader(
|
||||
form_name=block.form_name,
|
||||
**({} if block.form_name is None else {"form_name": block.form_name}),
|
||||
organization_name=organization,
|
||||
organization_name_confidence=_FULL_CONFIDENCE if organization else None,
|
||||
inn=inn,
|
||||
|
||||
44
api/src/parser/statements/merge.py
Normal file
44
api/src/parser/statements/merge.py
Normal file
@ -0,0 +1,44 @@
|
||||
"""Черновик документа формы и присоединение листов-продолжений (design D-8)."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from src.contract.result import StatementHeader, StatementLine
|
||||
from src.parser.rules import FormType
|
||||
from src.parser.statements.columns import PeriodColumn, TableLayout
|
||||
|
||||
__all__ = ["Draft", "can_join", "join"]
|
||||
|
||||
|
||||
@dataclass
|
||||
class Draft:
|
||||
kind: FormType | None # None — продолжение без ОКУД в маркере → OTHER
|
||||
header: StatementHeader
|
||||
periods: tuple[PeriodColumn, ...]
|
||||
sheet_names: list[str]
|
||||
lines: list[StatementLine] = field(default_factory=list)
|
||||
duplicates: list[str] = field(default_factory=list)
|
||||
has_code_column: bool = True
|
||||
lonely_continuation: bool = False
|
||||
fractional_values: int = 0
|
||||
numeric_values: int = 0
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def codes(self) -> set[str]:
|
||||
return {line.line_code for line in self.lines}
|
||||
|
||||
|
||||
def can_join(draft: Draft, layout: TableLayout) -> bool:
|
||||
"""Продолжение присоединяется, если число колонок-сумм совпало с открытым документом."""
|
||||
return draft.has_code_column and len(layout.periods) == len(draft.periods)
|
||||
|
||||
|
||||
def join(draft: Draft, sheet_name: str, lines: list[StatementLine], duplicates: list[str]) -> None:
|
||||
"""Строки продолжения в конец документа; коды, уже бывшие в документе, — в дубликаты."""
|
||||
for line in lines:
|
||||
if line.line_code in draft.codes:
|
||||
draft.duplicates.append(line.line_code)
|
||||
continue
|
||||
draft.lines.append(line)
|
||||
draft.duplicates.extend(duplicates)
|
||||
draft.sheet_names.append(sheet_name)
|
||||
163
api/src/parser/statements/statement.py
Normal file
163
api/src/parser/statements/statement.py
Normal file
@ -0,0 +1,163 @@
|
||||
"""Конвейер форм (design D-3…D-8): листы книги по порядку → блоки форм → шапка, таблица, строки → продолжения →
|
||||
соотношения §7 → `statementDocument`. `ok` документа: жёсткие соотношения сошлись, дубликатов нет, колонка кодов
|
||||
найдена, есть колонка-период, в шапке есть организация или ИНН, документ не одинокое продолжение. DISCLOSURE на
|
||||
статус не влияет.
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from src.contract.result import StatementColumn, StatementDocument, StatementHeader
|
||||
from src.parser.deadline import Deadline
|
||||
from src.parser.detect import SheetKind
|
||||
from src.parser.grid import Sheet
|
||||
from src.parser.rules import Rules
|
||||
from src.parser.statements.checks import run_statement_checks
|
||||
from src.parser.statements.columns import NO_CODE_COLUMN, LayoutNotFoundError, TableLayout, find_layout
|
||||
from src.parser.statements.detect import FormBlock, continuation_form, find_forms
|
||||
from src.parser.statements.header import parse_header
|
||||
from src.parser.statements.lines import collect_lines
|
||||
from src.parser.statements.merge import Draft, can_join, join
|
||||
|
||||
__all__ = ["CONTINUATION_WARNING", "LOCATOR_PREFIX", "LOCATOR_SEPARATOR", "StatementParser"]
|
||||
|
||||
LOCATOR_PREFIX = "sheet:"
|
||||
LOCATOR_SEPARATOR = ","
|
||||
CONTINUATION_WARNING = "страница-продолжение без шапки: загрузите форму одним файлом"
|
||||
SKIPPED_COLUMN_WARNING = "колонка без периода пропущена"
|
||||
|
||||
|
||||
def _line(sheet: Sheet, text: str) -> str:
|
||||
return f"sheet={sheet.name} {text}"
|
||||
|
||||
|
||||
class StatementParser:
|
||||
def __init__(self, rules: Rules) -> None:
|
||||
self._rules = rules
|
||||
|
||||
def parse_sheets(
|
||||
self, sheets: Sequence[tuple[Sheet, SheetKind]], deadline: Deadline
|
||||
) -> tuple[list[StatementDocument], list[str], bool]:
|
||||
drafts: list[Draft] = []
|
||||
warnings: list[str] = []
|
||||
dropped = False
|
||||
for sheet, kind in sheets:
|
||||
deadline.check(f"лист «{sheet.name}»: форма")
|
||||
if kind is SheetKind.STATEMENT:
|
||||
blocks, block_warnings = find_forms(sheet, self._rules)
|
||||
warnings.extend(_line(sheet, text) for text in block_warnings)
|
||||
for block in blocks:
|
||||
drafts.append(self._block_draft(sheet, block, warnings))
|
||||
else:
|
||||
dropped = not self._continuation(sheet, drafts, warnings) or dropped
|
||||
documents = [self._document(draft) for draft in drafts]
|
||||
for draft in drafts:
|
||||
warnings.extend(draft.warnings)
|
||||
ok = not dropped and all(
|
||||
self._is_ok(draft, document) for draft, document in zip(drafts, documents, strict=True)
|
||||
)
|
||||
return documents, warnings, ok
|
||||
|
||||
# --- блоки ---
|
||||
|
||||
def _block_draft(self, sheet: Sheet, block: FormBlock, warnings: list[str]) -> Draft:
|
||||
if block.kind == "DISCLOSURE":
|
||||
header, _ = parse_header(sheet, self._rules, block.first_row, block.last_row, block)
|
||||
return Draft(kind=block.kind, header=header, periods=(), sheet_names=[sheet.name])
|
||||
try:
|
||||
layout = find_layout(sheet, self._rules, block.first_row, block.last_row)
|
||||
except LayoutNotFoundError:
|
||||
header, header_warnings = parse_header(sheet, self._rules, block.first_row, block.last_row, block)
|
||||
warnings.extend(_line(sheet, text) for text in [NO_CODE_COLUMN, *header_warnings])
|
||||
return Draft(kind=block.kind, header=header, periods=(), sheet_names=[sheet.name], has_code_column=False)
|
||||
header_last = (layout.header_row or layout.first_data_row) - 1
|
||||
header, header_warnings = parse_header(sheet, self._rules, block.first_row, header_last, block)
|
||||
draft = Draft(kind=block.kind, header=header, periods=layout.periods, sheet_names=[sheet.name])
|
||||
self._fill(sheet, draft, layout, layout.first_data_row, block.last_row, header_warnings, warnings)
|
||||
return draft
|
||||
|
||||
def _continuation(self, sheet: Sheet, drafts: list[Draft], warnings: list[str]) -> bool:
|
||||
"""`False` — лист потерян (нет колонки кодов): строки не попали ни в один документ, статус не `done`."""
|
||||
try:
|
||||
layout = find_layout(sheet, self._rules, 1, sheet.n_rows)
|
||||
except LayoutNotFoundError:
|
||||
warnings.append(_line(sheet, NO_CODE_COLUMN))
|
||||
return False
|
||||
open_draft = next((draft for draft in reversed(drafts) if draft.kind != "DISCLOSURE"), None)
|
||||
if open_draft is not None and can_join(open_draft, layout):
|
||||
result = collect_lines(
|
||||
sheet, self._rules, layout, layout.first_data_row, sheet.n_rows, units=open_draft.header.units
|
||||
)
|
||||
join(open_draft, sheet.name, result.lines, result.duplicates)
|
||||
open_draft.fractional_values += result.fractional_values
|
||||
open_draft.numeric_values += result.numeric_values
|
||||
warnings.extend(_line(sheet, text) for text in result.warnings)
|
||||
return True
|
||||
kind, _ = continuation_form(sheet, self._rules)
|
||||
header, _ = parse_header(sheet, self._rules, 1, (layout.header_row or layout.first_data_row) - 1, None)
|
||||
draft = Draft(
|
||||
kind=kind, header=header, periods=layout.periods, sheet_names=[sheet.name], lonely_continuation=True
|
||||
)
|
||||
warnings.append(_line(sheet, CONTINUATION_WARNING))
|
||||
self._fill(sheet, draft, layout, layout.first_data_row, sheet.n_rows, [], warnings)
|
||||
drafts.append(draft)
|
||||
return True
|
||||
|
||||
def _fill( # noqa: PLR0913, PLR0917 - все части документа передаются явно
|
||||
self,
|
||||
sheet: Sheet,
|
||||
draft: Draft,
|
||||
layout: TableLayout,
|
||||
first_row: int,
|
||||
last_row: int,
|
||||
header_warnings: list[str],
|
||||
warnings: list[str],
|
||||
) -> None:
|
||||
result = collect_lines(sheet, self._rules, layout, first_row, last_row, units=draft.header.units)
|
||||
draft.lines = result.lines
|
||||
draft.duplicates = result.duplicates
|
||||
draft.fractional_values, draft.numeric_values = result.fractional_values, result.numeric_values
|
||||
texts = [*header_warnings, *result.warnings]
|
||||
if layout.skipped_labels:
|
||||
texts.append(f"{SKIPPED_COLUMN_WARNING}: {', '.join(layout.skipped_labels)}")
|
||||
warnings.extend(_line(sheet, text) for text in texts)
|
||||
|
||||
# --- документ ---
|
||||
|
||||
def _document(self, draft: Draft) -> StatementDocument:
|
||||
kind = draft.kind or "OTHER"
|
||||
columns = [
|
||||
StatementColumn(column_index=index, label=column.label, period_date=column.period_date)
|
||||
for index, column in enumerate(draft.periods)
|
||||
]
|
||||
locator = LOCATOR_PREFIX + LOCATOR_SEPARATOR.join(draft.sheet_names)
|
||||
if kind == "DISCLOSURE" or not draft.has_code_column:
|
||||
return StatementDocument(
|
||||
document_type=kind,
|
||||
locator=locator,
|
||||
header=draft.header,
|
||||
columns=columns if draft.has_code_column else [],
|
||||
lines=[],
|
||||
)
|
||||
checks, check_warnings = run_statement_checks(
|
||||
draft.lines, len(columns), kind if kind != "OTHER" else "BALANCE", self._rules
|
||||
)
|
||||
checks = checks.model_copy(update={"duplicate_line_codes": list(dict.fromkeys(draft.duplicates))})
|
||||
draft.warnings.extend(f"sheet={draft.sheet_names[0]} {text}" for text in check_warnings)
|
||||
return StatementDocument(
|
||||
document_type=kind, locator=locator, header=draft.header, columns=columns, lines=draft.lines, checks=checks
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_ok(draft: Draft, document: StatementDocument) -> bool:
|
||||
if document.document_type == "DISCLOSURE":
|
||||
return True
|
||||
header: StatementHeader = document.header
|
||||
return (
|
||||
draft.has_code_column
|
||||
and not draft.lonely_continuation
|
||||
and bool(document.columns)
|
||||
and bool(header.organization_name or header.inn)
|
||||
and document.checks is not None
|
||||
and not document.checks.relations_failed
|
||||
and not document.checks.duplicate_line_codes
|
||||
)
|
||||
5
api/tests/fixtures/test_rules_coverage.py
vendored
5
api/tests/fixtures/test_rules_coverage.py
vendored
@ -20,9 +20,10 @@ from fixtures_helpers import osv_dkb_files, osv_fixture_files, synthetic_files
|
||||
from rules_coverage import DICTIONARIES, MatchCounter, iter_patterns, uncovered
|
||||
|
||||
from src.parser import get_parser
|
||||
from src.parser.book import BookParser, SkippingStatementParser
|
||||
from src.parser.book import BookParser
|
||||
from src.parser.osv import OsvParser
|
||||
from src.parser.rules import DEFAULT_RULES_DIR, Rules, load_rules
|
||||
from src.parser.statements.statement import StatementParser
|
||||
|
||||
pytestmark = pytest.mark.fixtures
|
||||
|
||||
@ -39,7 +40,7 @@ def coverage(fixtures_dir: Path) -> tuple[Rules, MatchCounter, int]:
|
||||
parser = BookParser(
|
||||
rules,
|
||||
osv=OsvParser(rules),
|
||||
statements=SkippingStatementParser(),
|
||||
statements=StatementParser(rules),
|
||||
max_bytes=base.max_bytes,
|
||||
timeout_seconds=base.timeout_seconds,
|
||||
)
|
||||
|
||||
@ -95,10 +95,10 @@ EXPECTED: dict[str, Expect] = {
|
||||
# «12 месяцев»
|
||||
"s11_header_repeat.xlsx": Expect("outline", 2025, "Q4", True),
|
||||
# «январь - сентябрь», данные «НУ»
|
||||
"s12_statement_and_osv.xlsx": Expect("outline", 2025, "Q3", True, data_type="NU"),
|
||||
"s12_statement_and_osv.xlsx": Expect("outline", 2025, "Q3", True, status=ResultStatus.NEEDS_CHECK, data_type="NU"),
|
||||
"s13_no_outline_indent.xlsb": Expect("indent", 2025, "Q2", False),
|
||||
# три листа форм по одному признаку + лист ОСВ
|
||||
"s14_statement_forms.xlsx": Expect("outline", 2025, "Q1", True),
|
||||
"s14_statement_forms.xlsx": Expect("outline", 2025, "Q1", True, status=ResultStatus.NEEDS_CHECK),
|
||||
# названия групп сумм: «Входящее/Конечное», «Начальное/Исходящее сальдо»
|
||||
"s15_group_title_synonyms.xlsx": Expect("outline", 2025, "Q1", True, documents=2),
|
||||
# шапка без организации, период «2024»; лист 1 — вариант 1, лист 2 — вариант 2
|
||||
@ -134,9 +134,8 @@ def parse_named(fixtures_dir: Path, name: str) -> ParseOutcome:
|
||||
|
||||
def documents(outcome: ParseOutcome) -> list[OsvDocument]:
|
||||
assert outcome.error is None, outcome.error
|
||||
docs = list(outcome.documents)
|
||||
assert all(isinstance(doc, OsvDocument) for doc in docs)
|
||||
return cast(list[OsvDocument], docs)
|
||||
"""Документы ОСВ результата; документы форм (`StatementDocument`) разбираются отдельными сценариями."""
|
||||
return [doc for doc in outcome.documents if isinstance(doc, OsvDocument)]
|
||||
|
||||
|
||||
def one_document(outcome: ParseOutcome) -> OsvDocument:
|
||||
@ -422,14 +421,18 @@ def test_s11_header_repeat_is_skipped(fixtures_dir: Path) -> None:
|
||||
assert outcome.status is ResultStatus.DONE
|
||||
|
||||
|
||||
def test_s12_statement_sheet_is_skipped_with_warning(fixtures_dir: Path) -> None:
|
||||
"""«Лист ББ + лист ОСВ»: один документ, лист формы пропущен с предупреждением о подсрезе 2b."""
|
||||
def test_s12_statement_sheet_goes_to_statement_pipeline(fixtures_dir: Path) -> None:
|
||||
"""«Лист ББ + лист ОСВ»: документ ОСВ и документ ББ; у синтетического ББ нет шапки и колонки кодов → needs_check."""
|
||||
outcome = parse_named(fixtures_dir, "s12_statement_and_osv.xlsx")
|
||||
|
||||
(doc,) = documents(outcome)
|
||||
assert doc.locator == "sheet:ОСВ 60"
|
||||
assert any("Бухгалтерский баланс" in w and "skipped: statement" in w for w in warnings_of(outcome))
|
||||
assert outcome.status is ResultStatus.DONE
|
||||
assert [(d.document_type, d.locator) for d in outcome.documents] == [
|
||||
("OSV", "sheet:ОСВ 60"),
|
||||
("BALANCE", "sheet:Бухгалтерский баланс"),
|
||||
]
|
||||
assert not any("skipped: statement" in w for w in warnings_of(outcome))
|
||||
assert outcome.status is ResultStatus.NEEDS_CHECK
|
||||
|
||||
|
||||
def test_s13_xlsb_equals_s02_except_locator_and_warnings(fixtures_dir: Path) -> None:
|
||||
@ -449,14 +452,15 @@ def test_s13_xlsb_equals_s02_except_locator_and_warnings(fixtures_dir: Path) ->
|
||||
|
||||
|
||||
def test_s14_each_statement_form_is_recognised_by_its_own_marker(fixtures_dir: Path) -> None:
|
||||
"""Три листа форм (код ОКУД 0710001, «Отчет о финансовых результатах», код 0710002) пропущены, ОСВ разобран."""
|
||||
"""Три листа форм (код ОКУД 0710001, «Отчет о финансовых результатах», код 0710002) дают три документа форм."""
|
||||
outcome = parse_named(fixtures_dir, "s14_statement_forms.xlsx")
|
||||
|
||||
(doc,) = documents(outcome)
|
||||
assert doc.locator == "sheet:ОСВ 60"
|
||||
skipped = [w for w in warnings_of(outcome) if "skipped: statement" in w]
|
||||
assert [w.split(" skipped")[0] for w in skipped] == ["sheet=Форма 1", "sheet=ОФР", "sheet=Форма 2"]
|
||||
assert outcome.status is ResultStatus.DONE
|
||||
statements = [(d.document_type, d.locator) for d in outcome.documents if d.document_type != "OSV"]
|
||||
assert statements == [("BALANCE", "sheet:Форма 1"), ("INCOME", "sheet:ОФР"), ("INCOME", "sheet:Форма 2")]
|
||||
assert not any("skipped: statement" in w for w in warnings_of(outcome))
|
||||
assert outcome.status is ResultStatus.NEEDS_CHECK
|
||||
|
||||
|
||||
def test_s15_group_title_synonyms_find_all_six_sum_columns(fixtures_dir: Path) -> None:
|
||||
|
||||
303
api/tests/unit/parser/statements/test_statement.py
Normal file
303
api/tests/unit/parser/statements/test_statement.py
Normal file
@ -0,0 +1,303 @@
|
||||
# API/tests/unit/parser/statements/test_statement.py
|
||||
"""Конвейер форм: блоки, склейка продолжений, статус документа (design D-8; spec «Склейка листов-продолжений и статус
|
||||
документа»). Книги собираются `gridbuilder`, вид листа задаётся явно, как это делает `BookParser`."""
|
||||
|
||||
from decimal import Decimal
|
||||
|
||||
import pytest
|
||||
from gridbuilder import at, sheet_from_rows
|
||||
|
||||
from src.parser.deadline import Deadline
|
||||
from src.parser.detect import SheetKind
|
||||
from src.parser.rules import load_rules
|
||||
from src.parser.statements.statement import CONTINUATION_WARNING, StatementParser
|
||||
|
||||
TWO_COLUMNS = TWO_DOCUMENTS = 2
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def parser() -> StatementParser:
|
||||
return StatementParser(load_rules())
|
||||
|
||||
|
||||
def deadline() -> Deadline:
|
||||
return Deadline(120)
|
||||
|
||||
|
||||
BALANCE_PAGE_1 = [
|
||||
["Бухгалтерский баланс"],
|
||||
["на 31 декабря 2024 г."],
|
||||
["Форма по ОКУД", None, "0710001"],
|
||||
["Организация", None, 'ООО "Пример"', "ИНН", "7700000000"],
|
||||
["Единица измерения:", None, "тыс. руб."],
|
||||
["Наименование показателя", "Код", "На 31 декабря 2024 г.", "На 31 декабря 2023 г."],
|
||||
["Основные средства", "1150", 100, 90],
|
||||
["Итого по разделу I", "1100", 100, 90],
|
||||
["Запасы", "1210", 50, 40],
|
||||
["Итого по разделу II", "1200", 50, 40],
|
||||
["БАЛАНС", "1600", 150, 130],
|
||||
]
|
||||
BALANCE_PAGE_2 = [
|
||||
[*at({6: "Форма 0710001 с.2"})],
|
||||
["Наименование показателя", "Код", "На 31 декабря 2024 г.", "На 31 декабря 2023 г."],
|
||||
["Уставный капитал", "1310", 150, 130],
|
||||
["Итого по разделу III", "1300", 150, 130],
|
||||
["БАЛАНС", "1700", 150, 130],
|
||||
]
|
||||
INCOME = [
|
||||
["Отчет о финансовых результатах"],
|
||||
["за 2024 г."],
|
||||
["Организация", None, 'ООО "Пример"'],
|
||||
["Единица измерения:", None, "тыс. руб."],
|
||||
["Наименование показателя", "Код", "За 2024 г."],
|
||||
["Выручка", "2110", 1000],
|
||||
["Себестоимость продаж", "2120", -600],
|
||||
["Валовая прибыль (убыток)", "2100", 400],
|
||||
["Коммерческие расходы", "2210", -50],
|
||||
["Управленческие расходы", "2220", -50],
|
||||
["Прибыль (убыток) от продаж", "2200", 300],
|
||||
["Проценты к уплате", "2330", -100],
|
||||
["Прибыль (убыток) до налогообложения", "2300", 200],
|
||||
["Налог на прибыль", "2410", -40],
|
||||
["Чистая прибыль (убыток)", "2400", 160],
|
||||
]
|
||||
# Полный набор: 2200 = 2100 − 2210 − 2220 и 2300 = 2200 − 2330 сходятся через abs_terms (знаки 1С), 2400 — мягкое
|
||||
NPZ_CONTINUATION = [
|
||||
["в том числе:"],
|
||||
["Биологические активы", "11901", 0, 0],
|
||||
["Расходы будущих периодов", "11903", 0, 0],
|
||||
["Прочие внеоборотные активы", "1190", 0, 0],
|
||||
]
|
||||
|
||||
|
||||
def test_full_balance_over_two_sheets_is_done(parser: StatementParser) -> None:
|
||||
sheets = [
|
||||
(sheet_from_rows(BALANCE_PAGE_1, "ББ1"), SheetKind.STATEMENT),
|
||||
(sheet_from_rows(BALANCE_PAGE_2, "ББ2"), SheetKind.STATEMENT_CONTINUATION),
|
||||
]
|
||||
|
||||
documents, warnings, ok = parser.parse_sheets(sheets, deadline())
|
||||
|
||||
assert ok and len(documents) == 1
|
||||
document = documents[0]
|
||||
assert document.document_type == "BALANCE" and document.locator == "sheet:ББ1,ББ2"
|
||||
assert [line.line_code for line in document.lines] == [
|
||||
"1150",
|
||||
"1100",
|
||||
"1210",
|
||||
"1200",
|
||||
"1600",
|
||||
"1310",
|
||||
"1300",
|
||||
"1700",
|
||||
]
|
||||
assert document.checks is not None and document.checks.relations_failed == []
|
||||
assert document.header.inn == "7700000000" and document.header.units == "тыс. руб."
|
||||
assert len(document.columns) == TWO_COLUMNS and document.columns[0].label == "На 31 декабря 2024 г."
|
||||
|
||||
|
||||
def test_page_one_alone_fails_total_relation(parser: StatementParser) -> None:
|
||||
documents, warnings, ok = parser.parse_sheets([(sheet_from_rows(BALANCE_PAGE_1), SheetKind.STATEMENT)], deadline())
|
||||
|
||||
assert not ok
|
||||
assert [item.relation for item in documents[0].checks.relations_failed] == ["1600 = 1700", "1600 = 1700"]
|
||||
assert any("1600 = 1700" in warning for warning in warnings)
|
||||
|
||||
|
||||
def test_lonely_continuation_is_needs_check(parser: StatementParser) -> None:
|
||||
documents, warnings, ok = parser.parse_sheets(
|
||||
[(sheet_from_rows(BALANCE_PAGE_2, "с.2"), SheetKind.STATEMENT_CONTINUATION)], deadline()
|
||||
)
|
||||
|
||||
assert not ok and documents[0].document_type == "BALANCE"
|
||||
assert documents[0].header.organization_name is None and documents[0].header.form_name == "Форма 0710001 с.2"
|
||||
assert [line.line_code for line in documents[0].lines] == ["1310", "1300", "1700"]
|
||||
assert f"sheet=с.2 {CONTINUATION_WARNING}" in warnings
|
||||
|
||||
|
||||
def test_continuation_joins_last_open_document(parser: StatementParser) -> None:
|
||||
"""Review Focus 4: продолжение после ОФР относится к ОФР, а не к балансу."""
|
||||
income_tail = [
|
||||
["Налог", "2450", 0],
|
||||
["Прочее", "2460", 0],
|
||||
["Совокупный финансовый результат периода", "2500", 160],
|
||||
]
|
||||
sheets = [
|
||||
(sheet_from_rows(BALANCE_PAGE_1, "ББ"), SheetKind.STATEMENT),
|
||||
(sheet_from_rows(INCOME, "ОФР"), SheetKind.STATEMENT),
|
||||
(sheet_from_rows(income_tail, "ОФР_2"), SheetKind.STATEMENT_CONTINUATION),
|
||||
]
|
||||
|
||||
documents, _, _ = parser.parse_sheets(sheets, deadline())
|
||||
|
||||
assert [document.document_type for document in documents] == ["BALANCE", "INCOME"]
|
||||
assert documents[1].locator == "sheet:ОФР,ОФР_2" and [line.line_code for line in documents[1].lines][-2:] == [
|
||||
"2460",
|
||||
"2500",
|
||||
]
|
||||
|
||||
|
||||
def test_continuation_without_header_maps_value_columns_by_order(parser: StatementParser) -> None:
|
||||
sheets = [
|
||||
(sheet_from_rows(BALANCE_PAGE_1, "ББ"), SheetKind.STATEMENT),
|
||||
(sheet_from_rows(NPZ_CONTINUATION, "ББ_2"), SheetKind.STATEMENT_CONTINUATION),
|
||||
]
|
||||
|
||||
documents, _, _ = parser.parse_sheets(sheets, deadline())
|
||||
|
||||
tail = documents[0].lines[-1]
|
||||
assert tail.line_code == "1190" and [value.column_index for value in tail.values] == [0, 1]
|
||||
assert [line.line_code for line in documents[0].lines][-3:] == ["11901", "11903", "1190"]
|
||||
|
||||
|
||||
def test_continuation_with_other_column_count_opens_new_document(parser: StatementParser) -> None:
|
||||
three_columns = [
|
||||
["Форма 0710001 с.2"],
|
||||
["Наименование", "Код", "На 31.12.2024", "На 31.12.2023", "На 31.12.2022"],
|
||||
["Уставный капитал", "1310", 1, 1, 1],
|
||||
["Добавочный капитал", "1350", 1, 1, 1],
|
||||
["Итого по разделу III", "1300", 2, 2, 2],
|
||||
]
|
||||
sheets = [
|
||||
(sheet_from_rows(BALANCE_PAGE_1, "ББ"), SheetKind.STATEMENT),
|
||||
(sheet_from_rows(three_columns, "с.2"), SheetKind.STATEMENT_CONTINUATION),
|
||||
]
|
||||
|
||||
documents, warnings, ok = parser.parse_sheets(sheets, deadline())
|
||||
|
||||
assert len(documents) == TWO_DOCUMENTS and not ok and any(CONTINUATION_WARNING in warning for warning in warnings)
|
||||
|
||||
|
||||
def test_two_forms_on_one_sheet(parser: StatementParser) -> None:
|
||||
sheet = sheet_from_rows([*BALANCE_PAGE_1, *BALANCE_PAGE_2[2:], [], *INCOME], "Отчетность")
|
||||
|
||||
documents, _, ok = parser.parse_sheets([(sheet, SheetKind.STATEMENT)], deadline())
|
||||
|
||||
assert [document.document_type for document in documents] == ["BALANCE", "INCOME"] and ok
|
||||
assert all(document.locator == "sheet:Отчетность" for document in documents)
|
||||
assert "2110" not in [line.line_code for line in documents[0].lines]
|
||||
|
||||
|
||||
def test_disclosure_only_book_is_done(parser: StatementParser) -> None:
|
||||
"""Review Focus 5."""
|
||||
sheet = sheet_from_rows(
|
||||
[
|
||||
["Пояснения к бухгалтерскому балансу и отчету о финансовых результатах"],
|
||||
["1. Нематериальные активы"],
|
||||
["текст"],
|
||||
],
|
||||
"Пояснения",
|
||||
)
|
||||
|
||||
documents, warnings, ok = parser.parse_sheets([(sheet, SheetKind.STATEMENT)], deadline())
|
||||
|
||||
assert (
|
||||
ok and documents[0].document_type == "DISCLOSURE" and documents[0].lines == [] and documents[0].checks is None
|
||||
)
|
||||
|
||||
|
||||
def test_no_code_column_is_needs_check_with_warning(parser: StatementParser) -> None:
|
||||
sheet = sheet_from_rows(
|
||||
[
|
||||
["Бухгалтерский баланс"],
|
||||
["Организация", "ООО Х"],
|
||||
["Показатель", "На 31.12.2024"],
|
||||
["Основные средства", 100],
|
||||
["Запасы", 50],
|
||||
["БАЛАНС", 150],
|
||||
],
|
||||
"ББ",
|
||||
)
|
||||
|
||||
documents, warnings, ok = parser.parse_sheets([(sheet, SheetKind.STATEMENT)], deadline())
|
||||
|
||||
assert not ok and documents[0].lines == [] and documents[0].columns == []
|
||||
assert "sheet=ББ колонка кодов не найдена" in warnings
|
||||
|
||||
|
||||
def test_duplicate_code_is_needs_check(parser: StatementParser) -> None:
|
||||
rows = [*BALANCE_PAGE_1, ["Запасы", "1210", 1, 1]]
|
||||
|
||||
documents, _, ok = parser.parse_sheets([(sheet_from_rows(rows), SheetKind.STATEMENT)], deadline())
|
||||
|
||||
assert not ok and documents[0].checks.duplicate_line_codes == ["1210"]
|
||||
|
||||
|
||||
def test_template_without_sums_is_needs_check(parser: StatementParser) -> None:
|
||||
rows = [
|
||||
["Бухгалтерский баланс"],
|
||||
["Наименование показателя", "Код", "На 31 декабря 2024 г."],
|
||||
["Основные средства", "1150"],
|
||||
["Запасы", "1210"],
|
||||
["БАЛАНС", "1600"],
|
||||
]
|
||||
|
||||
documents, _, ok = parser.parse_sheets([(sheet_from_rows(rows), SheetKind.STATEMENT)], deadline())
|
||||
|
||||
assert (
|
||||
not ok
|
||||
and documents[0].checks.relations_failed == []
|
||||
and all(value.amount is None for line in documents[0].lines for value in line.values)
|
||||
)
|
||||
|
||||
|
||||
def test_duplicate_codes_are_unique_in_order_of_first_duplicate(parser: StatementParser) -> None:
|
||||
"""spec: `duplicate_line_codes` — уникальные коды; повтор на каждое вхождение не плодит записей (T11-ревью)."""
|
||||
rows = [*BALANCE_PAGE_1, ["Запасы", "1210", 1, 1], ["Запасы", "1210", 2, 2], ["Основные средства", "1150", 1, 1]]
|
||||
|
||||
documents, _, ok = parser.parse_sheets([(sheet_from_rows(rows), SheetKind.STATEMENT)], deadline())
|
||||
|
||||
assert not ok and documents[0].checks.duplicate_line_codes == ["1210", "1150"]
|
||||
|
||||
|
||||
def test_code_repeated_on_continuation_is_duplicate_and_first_value_kept(parser: StatementParser) -> None:
|
||||
first = (sheet_from_rows(BALANCE_PAGE_1, "ББ"), SheetKind.STATEMENT)
|
||||
again = [
|
||||
["Наименование показателя", "Код", "На 31 декабря 2024 г.", "На 31 декабря 2023 г."],
|
||||
["Запасы", "1210", 7, 7],
|
||||
["Итого", "1200", 50, 40],
|
||||
["Прочее", "1230", 0, 0],
|
||||
]
|
||||
|
||||
documents, _, ok = parser.parse_sheets(
|
||||
[first, (sheet_from_rows(again, "ББ3"), SheetKind.STATEMENT_CONTINUATION)], deadline()
|
||||
)
|
||||
|
||||
assert not ok and documents[0].checks.duplicate_line_codes == ["1210", "1200"]
|
||||
assert [line.values[0].amount for line in documents[0].lines if line.line_code == "1210"] == [Decimal(50)]
|
||||
|
||||
|
||||
def test_headerless_lonely_continuation_has_empty_header(parser: StatementParser) -> None:
|
||||
documents, _, ok = parser.parse_sheets(
|
||||
[(sheet_from_rows(NPZ_CONTINUATION, "хвост"), SheetKind.STATEMENT_CONTINUATION)], deadline()
|
||||
)
|
||||
|
||||
assert not ok and documents[0].document_type == "OTHER" and documents[0].header.form_name is None
|
||||
|
||||
|
||||
def test_continuation_without_code_column_is_dropped_and_not_ok(parser: StatementParser) -> None:
|
||||
"""Потерянный лист нельзя оставлять статусом `done`."""
|
||||
lost = [["Форма 0710001 с.2"], ["Код", "На 31.12.2024"], ["1700", 1]]
|
||||
sheets = [
|
||||
(sheet_from_rows(BALANCE_PAGE_1, "ББ"), SheetKind.STATEMENT),
|
||||
(sheet_from_rows(BALANCE_PAGE_2, "ББ2"), SheetKind.STATEMENT_CONTINUATION),
|
||||
(sheet_from_rows(lost, "с.3"), SheetKind.STATEMENT_CONTINUATION),
|
||||
]
|
||||
|
||||
documents, warnings, ok = parser.parse_sheets(sheets, deadline())
|
||||
|
||||
assert (
|
||||
not ok
|
||||
and len(documents) == 1
|
||||
and documents[0].checks.relations_failed == []
|
||||
and "sheet=с.3 колонка кодов не найдена" in warnings
|
||||
)
|
||||
|
||||
|
||||
def test_block_with_okud_but_without_title_has_no_form_name(parser: StatementParser) -> None:
|
||||
"""Блок, открытый только кодом ОКУД: `form_name` не передаётся (в контракте нет null)."""
|
||||
rows = [["Форма по ОКУД", None, "0710001"], *BALANCE_PAGE_1[3:]]
|
||||
|
||||
documents, _, _ = parser.parse_sheets([(sheet_from_rows(rows), SheetKind.STATEMENT)], deadline())
|
||||
|
||||
assert documents[0].header.form_name is None and documents[0].header.okud == "0710001"
|
||||
@ -25,7 +25,7 @@ from result_contract import assert_valid_result
|
||||
from src.contract import ErrorCode, OsvDocument, ResultStatus, StatementDocument
|
||||
from src.contract.result import StatementHeader
|
||||
from src.parser import ParseOutcome, hierarchy
|
||||
from src.parser.book import BookParser, SkippingStatementParser, StatementSheetParser
|
||||
from src.parser.book import BookParser, StatementSheetParser
|
||||
from src.parser.classify import SUM_FIELDS
|
||||
from src.parser.deadline import Deadline
|
||||
from src.parser.detect import SheetKind
|
||||
@ -34,6 +34,7 @@ from src.parser.grid import Sheet
|
||||
from src.parser.hierarchy import BodyRow
|
||||
from src.parser.osv import OsvParser
|
||||
from src.parser.rules import DEFAULT_RULES_DIR, HEADERS_FILE, Rules, load_rules
|
||||
from src.parser.statements.statement import StatementParser
|
||||
|
||||
OPENING = "Сальдо на начало периода"
|
||||
TURNOVER = "Обороты за период"
|
||||
@ -138,7 +139,7 @@ def book(
|
||||
return BookParser(
|
||||
rules,
|
||||
osv=OsvParser(rules),
|
||||
statements=statements or SkippingStatementParser(),
|
||||
statements=statements or StatementParser(rules),
|
||||
max_bytes=max_bytes,
|
||||
timeout_seconds=timeout_seconds,
|
||||
clock=clock,
|
||||
@ -374,12 +375,12 @@ def test_total_row_without_caption_is_the_last_row_of_sums(rules: Rules) -> None
|
||||
# --- Несколько листов ---
|
||||
|
||||
|
||||
def test_statement_sheet_is_skipped_and_osv_parsed(rules: Rules) -> None:
|
||||
def test_statement_sheet_goes_to_real_pipeline_next_to_osv(rules: Rules) -> None:
|
||||
outcome = book(rules).parse(xlsx({"Баланс": statement(), "ОСВ 60": mini_osv()}), "XLSX")
|
||||
|
||||
assert outcome.status is ResultStatus.DONE
|
||||
assert only_document(outcome).locator == "sheet:ОСВ 60"
|
||||
assert warnings_of(outcome) == ["sheet=ОСВ 60 hierarchy=outline", "sheet=Баланс skipped: statement (slice 2b)"]
|
||||
assert [type(document).__name__ for document in outcome.documents] == ["OsvDocument", "StatementDocument"]
|
||||
assert outcome.documents[0].locator == "sheet:ОСВ 60" and outcome.documents[1].locator == "sheet:Баланс"
|
||||
assert "sheet=ОСВ 60 hierarchy=outline" in warnings_of(outcome)
|
||||
assert_valid_result(outcome)
|
||||
|
||||
|
||||
@ -416,7 +417,9 @@ def test_one_spoiled_sheet_makes_whole_result_needs_check(rules: Rules) -> None:
|
||||
|
||||
def test_book_without_osv_is_invalid_document(rules: Rules) -> None:
|
||||
"""Spec «Нет ОСВ»: сообщение называет число просмотренных листов."""
|
||||
outcome = book(rules).parse(xlsx({"Баланс": statement(), "Данные": [["a", "b"], [1, 2]]}), "XLSX")
|
||||
outcome = book(rules).parse(
|
||||
xlsx({"Справка": [["Наименование", "Сумма"], ["Аренда", 100]], "Данные": [["a", "b"], [1, 2]]}), "XLSX"
|
||||
)
|
||||
|
||||
assert outcome.status is ResultStatus.ERROR
|
||||
assert outcome.error is not None
|
||||
@ -673,20 +676,20 @@ def test_statement_sheets_go_to_statement_parser_in_sheet_order(rules: Rules) ->
|
||||
assert "sheet=ББ statements-fake" in (outcome.metrics.warnings if outcome.metrics else [])
|
||||
|
||||
|
||||
def test_skipping_parser_keeps_slice_2a_behaviour(rules: Rules) -> None:
|
||||
def test_balance_sheet_gives_statement_document_without_stub(rules: Rules) -> None:
|
||||
outcome = book(rules).parse(xlsx({"ББ": BALANCE, "Лист_1": mini_osv()}), "XLSX")
|
||||
|
||||
assert len(outcome.documents) == 1
|
||||
assert "sheet=ББ skipped: statement (slice 2b)" in warnings_of(outcome)
|
||||
assert [type(document).__name__ for document in outcome.documents] == ["OsvDocument", "StatementDocument"]
|
||||
assert not any("skipped: statement" in warning for warning in warnings_of(outcome))
|
||||
|
||||
|
||||
def test_statement_continuation_is_not_parsed_as_osv(rules: Rules) -> None:
|
||||
"""R-5: лист-продолжение формы идёт в конвейер форм (здесь заглушка), а не в разбор ОСВ."""
|
||||
outcome = book(rules).parse(xlsx({"ОСВ 60": mini_osv(), "с.2": CONTINUATION}), "XLSX")
|
||||
|
||||
assert outcome.status is ResultStatus.DONE
|
||||
assert only_document(outcome).locator == "sheet:ОСВ 60"
|
||||
assert warnings_of(outcome) == ["sheet=ОСВ 60 hierarchy=outline", "sheet=с.2 skipped: statement (slice 2b)"]
|
||||
assert outcome.status is ResultStatus.NEEDS_CHECK # лист-продолжение потерян: в нём меньше трёх кодов
|
||||
assert [document.locator for document in outcome.documents] == ["sheet:ОСВ 60"]
|
||||
assert "sheet=с.2 колонка кодов не найдена" in warnings_of(outcome)
|
||||
|
||||
|
||||
def test_book_without_osv_and_forms_is_invalid_document(rules: Rules) -> None:
|
||||
|
||||
@ -14,7 +14,7 @@ import openpyxl
|
||||
import pytest
|
||||
|
||||
from src.contract import ErrorCode, ResultStatus
|
||||
from src.parser.book import BookParser, SkippingStatementParser
|
||||
from src.parser.book import BookParser
|
||||
from src.parser.errors import UnreadableError, UnsupportedFormatError
|
||||
from src.parser.grid import MAX_INDEXED_AREA, MergedRange, Sheet
|
||||
from src.parser.osv import OsvParser
|
||||
@ -28,6 +28,7 @@ from src.parser.readers._limits import (
|
||||
merged_area_of_xml,
|
||||
)
|
||||
from src.parser.rules import Rules, load_rules
|
||||
from src.parser.statements.statement import StatementParser
|
||||
|
||||
FAST_SECONDS = 1.0
|
||||
SHEET_PART = "xl/worksheets/sheet1.xml"
|
||||
@ -125,7 +126,7 @@ def test_huge_merged_range_is_rejected_before_openpyxl() -> None:
|
||||
|
||||
def _book(rules: Rules, max_bytes: int) -> BookParser:
|
||||
return BookParser(
|
||||
rules, osv=OsvParser(rules), statements=SkippingStatementParser(), max_bytes=max_bytes, timeout_seconds=120
|
||||
rules, osv=OsvParser(rules), statements=StatementParser(rules), max_bytes=max_bytes, timeout_seconds=120
|
||||
)
|
||||
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user