diff --git a/api/src/parser/book.py b/api/src/parser/book.py index 7e4ec52..fb81b33 100644 --- a/api/src/parser/book.py +++ b/api/src/parser/book.py @@ -19,6 +19,7 @@ from src.parser.detect import SheetKind, classify_sheet from src.parser.errors import InvalidDocumentError, ParserError, UnsupportedFormatError from src.parser.grid import Sheet from src.parser.osv import OsvParser, data_rows, sheet_line, unique +from src.parser.osv_warnings import render_legacy from src.parser.readers import read_book_detected from src.parser.readers._common import one_line_reason from src.parser.readers._limits import ZIP_UNCOMPRESSED_LIMIT_FACTOR @@ -118,7 +119,7 @@ class BookParser: warnings.append(sheet_line(sheet, "skipped: нет признаков ОСВ или формы")) continue try: - document, sheet_warnings, sheet_ok = self._osv.parse_sheet(sheet, deadline) + result = self._osv.parse_sheet(sheet, deadline) except ParserError: # T7.1: ответ с ошибкой называет листы форм, которые из-за неё остались неразобранными warnings.extend( @@ -126,9 +127,9 @@ class BookParser: for form_sheet, _ in statement_sheets ) raise - osv_documents.append(document) - warnings.extend(sheet_warnings) - ok = ok and sheet_ok + osv_documents.append(result.document) + warnings.extend(render_legacy(sheet.name, result.notes)) + ok = ok and result.ok statement_documents, statement_warnings, statements_ok = self._statements.parse_sheets( statement_sheets, deadline, formats_unreadable=read.detected == "XLSB" ) diff --git a/api/src/parser/checks.py b/api/src/parser/checks.py index 057e69c..92b53e5 100644 --- a/api/src/parser/checks.py +++ b/api/src/parser/checks.py @@ -30,6 +30,7 @@ from decimal import Decimal from src.contract.result import OsvChecks from src.parser.classify import SUM_FIELDS, ClassifiedRow +from src.parser.osv_warnings import OSV_BALANCE_IDENTITY, OSV_PARENT_SUM, OSV_TOTAL, OsvNote from src.parser.rules import OsvCheckRules __all__ = [ @@ -216,23 +217,23 @@ def _total_warning(rows: Sequence[ClassifiedRow]) -> str: return message -def run_checks(rows: Sequence[ClassifiedRow], exempt: OsvCheckRules | None = None) -> tuple[OsvChecks, list[str]]: - """Три проверки §1.4 и предупреждения с номерами строк (пустой список — всё сошлось). +def run_checks(rows: Sequence[ClassifiedRow], exempt: OsvCheckRules | None = None) -> tuple[OsvChecks, list[OsvNote]]: + """Три проверки §1.4 и заметки с кодами `OSV_*` и номерами строк (пустой список — всё сошлось). `exempt` — счета без сальдо по аналитике (`rules.checks`, OQ-132); без него проверяются все строки полностью. """ identity = balance_identity(rows, exempt) parents = parent_sums(rows, exempt) matches = total_matches_root(rows) - warnings: list[str] = [] + warnings: list[OsvNote] = [] if identity: - warnings.append( - f"§1.4: сальдо на начало + обороты ≠ сальдо на конец (±{TOLERANCE}): строки {_rows_text(identity)}" - ) + text = f"§1.4: сальдо на начало + обороты ≠ сальдо на конец (±{TOLERANCE}): строки {_rows_text(identity)}" + warnings.append(OsvNote(OSV_BALANCE_IDENTITY, text)) if parents: - warnings.append(f"§1.4: суммы родителя ≠ сумме детей (±{TOLERANCE}): строки {_rows_text(parents)}") + text = f"§1.4: суммы родителя ≠ сумме детей (±{TOLERANCE}): строки {_rows_text(parents)}" + warnings.append(OsvNote(OSV_PARENT_SUM, text)) if not matches: - warnings.append(_total_warning(rows)) + warnings.append(OsvNote(OSV_TOTAL, _total_warning(rows))) checks = OsvChecks( total_row_matches_root=matches, balance_identity_violations=identity, diff --git a/api/src/parser/osv.py b/api/src/parser/osv.py index 252e66f..01fed38 100644 --- a/api/src/parser/osv.py +++ b/api/src/parser/osv.py @@ -7,7 +7,7 @@ import datetime from collections.abc import Iterable, Sequence -from dataclasses import replace +from dataclasses import dataclass, replace from src.contract import OsvDocument from src.contract.result import OsvChecks, OsvGroup, OsvHeader, OsvRow @@ -20,9 +20,18 @@ from src.parser.grid import Sheet, column_letter from src.parser.header import ACCOUNT_NOT_FOUND_WARNING, DocHeader, build_group_levels, parse_doc_header from src.parser.hierarchy import resolve_levels from src.parser.inn_rows import InnGlue, glue_inn_rows +from src.parser.osv_warnings import ( + OSV_HEADER, + OSV_INN_ROWS, + OSV_NO_DATA_ROWS, + OSV_ROWS, + OsvNote, + hierarchy_note, + notes_of, +) from src.parser.rules import Rules -__all__ = ["OsvParser", "data_rows", "sheet_line", "unique"] +__all__ = ["OsvParser", "SheetResult", "data_rows", "sheet_line", "unique"] LOCATOR_PREFIX = "sheet:" INN_COLUMN_KEY = "inn" @@ -120,6 +129,14 @@ def data_rows(document: OsvDocument) -> int: return sum(1 for row in document.rows if row.group_type != "TOTAL") +def _unique_notes(notes: Iterable[OsvNote]) -> list[OsvNote]: + """Без повторов по тексту, в порядке первого появления (как `unique` для строк вывода 1.x).""" + seen: dict[str, OsvNote] = {} + for note in notes: + seen.setdefault(note.message, note) + return list(seen.values()) + + def _invalid_format(sheet: Sheet, step: str, exc: InvalidFormatError) -> InvalidFormatError: """Сообщение `INVALID_FORMAT` с листом и шагом; префикс «лист «…»:» детектора не дублируется.""" reason = exc.message.removeprefix(f"лист «{sheet.name}»: ") @@ -140,15 +157,24 @@ def _check_inn_owners(glue: InnGlue, classified: list[ClassifiedRow]) -> InnGlue return replace(glue, warnings=[*glue.warnings, *extra], ok=False) +@dataclass(frozen=True) +class SheetResult: + """Итог листа ОСВ: документ, заметки с кодами `OSV_*` (первая — иерархия) и признак «контрольные суммы сошлись».""" + + document: OsvDocument + notes: tuple[OsvNote, ...] + ok: bool + + class OsvParser: """Конвейер листа ОСВ по словарям `rules`; книгу разбирает `BookParser`.""" def __init__(self, rules: Rules) -> None: self._rules = rules - def parse_sheet(self, sheet: Sheet, deadline: Deadline) -> tuple[OsvDocument, list[str], bool]: - """Документ листа ОСВ, его предупреждения (первое — `sheet=<имя> hierarchy=<стратегия>`) и признак - «контрольные суммы сошлись»; нет заголовка/колонок/иерархии — `InvalidFormatError` с листом и шагом. + def parse_sheet(self, sheet: Sheet, deadline: Deadline) -> SheetResult: + """Документ листа ОСВ, его заметки (первая — иерархия) и признак «контрольные суммы сошлись»; + нет заголовка/колонок/иерархии — `InvalidFormatError` с листом и шагом. """ rules = self._rules step = "заголовок таблицы" @@ -191,16 +217,21 @@ class OsvParser: glue = _check_inn_owners(glue, classified) doc = _with_table_account(doc, classified) deadline.check(f"лист «{sheet.name}»: контрольные суммы") - checks, check_warnings = run_checks(classified, rules.checks) + checks, check_notes = run_checks(classified, rules.checks) document = self._document( sheet=sheet, header=header, doc=doc, schema=schema.groups, rows=classified, checks=checks ) - warnings = [sheet_line(sheet, f"hierarchy={levels.strategy}"), *doc.warnings, *body_warnings, *glue.warnings] + notes = [ + hierarchy_note(levels.strategy), + *notes_of(OSV_HEADER, doc.warnings), + *notes_of(OSV_ROWS, body_warnings), + *notes_of(OSV_INN_ROWS, glue.warnings), + ] if not any(item.group_type != "TOTAL" for item in classified): - warnings.append(NO_DATA_ROWS_WARNING) - warnings.extend(check_warnings) + notes.append(OsvNote(OSV_NO_DATA_ROWS, NO_DATA_ROWS_WARNING)) + notes.extend(check_notes) # Строка ИНН без пары, похожая на ИНН строка или расхождение с колонкой (#35) — needs_check - return document, unique(warnings), _is_ok(checks) and glue.ok + return SheetResult(document, tuple(_unique_notes(notes)), _is_ok(checks) and glue.ok) # --- внутреннее --- diff --git a/api/src/parser/osv_warnings.py b/api/src/parser/osv_warnings.py new file mode 100644 index 0000000..882e0c3 --- /dev/null +++ b/api/src/parser/osv_warnings.py @@ -0,0 +1,63 @@ +"""Предупреждения листа ОСВ с кодами `OSV_*` (P4-12): код — по источнику заметки, уровень info. + +Пока вывод 1.x прежний: `render_legacy` собирает из заметок те же строки `metrics.warnings`, что были до кодов +(текст меняет задача 7). Заметка читателя «xls без кодовой страницы» — не код ОСВ, а предупреждение файла +`XLS_NO_CODEPAGE` пакета (P4-13, R-2); здесь её нет. +""" + +from collections.abc import Iterable, Sequence +from dataclasses import dataclass + +__all__ = [ + "OSV_BALANCE_IDENTITY", + "OSV_HEADER", + "OSV_HIERARCHY_PREFIX", + "OSV_INN_ROWS", + "OSV_NO_DATA_ROWS", + "OSV_PARENT_SUM", + "OSV_ROWS", + "OSV_TOTAL", + "OSV_XLSB_NO_OUTLINE", + "OsvNote", + "hierarchy_note", + "legacy_line", + "notes_of", + "render_legacy", +] + +OSV_HIERARCHY_PREFIX = "OSV_HIERARCHY_" +OSV_HEADER = "OSV_HEADER" +OSV_ROWS = "OSV_ROWS" +OSV_INN_ROWS = "OSV_INN_ROWS" +OSV_NO_DATA_ROWS = "OSV_NO_DATA_ROWS" +OSV_BALANCE_IDENTITY = "OSV_BALANCE_IDENTITY" +OSV_PARENT_SUM = "OSV_PARENT_SUM" +OSV_TOTAL = "OSV_TOTAL" +OSV_XLSB_NO_OUTLINE = "OSV_XLSB_NO_OUTLINE" # заметка читателя xlsb, подключается в задаче 7 + + +@dataclass(frozen=True) +class OsvNote: + code: str + message: str + + +def hierarchy_note(strategy: str) -> OsvNote: + """Стратегия иерархии: `outline|indent|account_pattern|sums|trivial` → `OSV_HIERARCHY_BY_<СТРАТЕГИЯ>`.""" + return OsvNote(f"{OSV_HIERARCHY_PREFIX}BY_{strategy.upper()}", f"hierarchy={strategy}") + + +def notes_of(code: str, messages: Iterable[str]) -> list[OsvNote]: + return [OsvNote(code, message) for message in messages] + + +def legacy_line(sheet_name: str, note: OsvNote) -> str: + return f"sheet={sheet_name} {note.message}" + + +def render_legacy(sheet_name: str, notes: Sequence[OsvNote]) -> list[str]: + """Строки вывода 1.x: заметка иерархии с префиксом листа, остальные — как были; без повторов, порядок прежний.""" + lines = [ + legacy_line(sheet_name, note) if note.code.startswith(OSV_HIERARCHY_PREFIX) else note.message for note in notes + ] + return list(dict.fromkeys(lines)) diff --git a/api/tests/unit/parser/test_checks.py b/api/tests/unit/parser/test_checks.py index b088714..b0f335e 100644 --- a/api/tests/unit/parser/test_checks.py +++ b/api/tests/unit/parser/test_checks.py @@ -79,7 +79,8 @@ def test_tolerance_is_one_kopeck() -> None: def test_consistent_tree_has_no_violations() -> None: - checks, warnings = run_checks(tree()) + checks, notes = run_checks(tree()) + warnings = [note.message for note in notes] assert checks.total_row_matches_root is True assert checks.balance_identity_violations == [] @@ -250,7 +251,8 @@ def multi_root_tree(total_sums: Sums = SUB_SUM) -> list[ClassifiedRow]: def test_total_matches_sum_of_level_zero_rows_when_there_are_several_roots() -> None: rows = multi_root_tree() - checks, warnings = run_checks(rows) + checks, notes = run_checks(rows) + warnings = [note.message for note in notes] assert total_matches_root(rows) is True assert checks.total_row_matches_root is True @@ -288,7 +290,8 @@ def test_expanded_balance_total_is_not_matched_by_net_when_total_has_both_sides( def test_total_that_equals_neither_first_root_nor_sum_of_roots_mismatches() -> None: rows = multi_root_tree(("140", None, "40", "20", "161", None)) - checks, warnings = run_checks(rows) + checks, notes = run_checks(rows) + warnings = [note.message for note in notes] assert checks.total_row_matches_root is False assert any("Итого" in warning and "11" in warning and "сумме 2 строк уровня 0" in warning for warning in warnings) @@ -326,7 +329,8 @@ def test_missing_total_fails_root_check_with_warning() -> None: """R-20: без «Итого» сверку выполнить нельзя — `false` и предупреждение.""" rows = tree()[:-1] - checks, warnings = run_checks(rows) + checks, notes = run_checks(rows) + warnings = [note.message for note in notes] assert total_matches_root(rows) is False assert checks.total_row_matches_root is False @@ -337,7 +341,8 @@ def test_run_checks_warnings_name_row_numbers() -> None: rows = tree() rows[2] = row(9, ("40", None, "30", "20", "51", None), level=1, parent=7) - checks, warnings = run_checks(rows) + checks, notes = run_checks(rows) + warnings = [note.message for note in notes] assert checks.balance_identity_violations == [9] assert checks.parent_sum_violations == [7] @@ -350,7 +355,8 @@ def test_run_checks_reports_total_mismatch_with_row_numbers() -> None: rows = tree() rows[-1] = total(10, ("100", None, "50", "30", "121", None)) - checks, warnings = run_checks(rows) + checks, notes = run_checks(rows) + warnings = [note.message for note in notes] assert checks.total_row_matches_root is False assert any("10" in warning and "7" in warning and "Итого" in warning for warning in warnings), warnings @@ -361,7 +367,8 @@ def test_run_checks_warning_lists_are_capped() -> None: broken: Sums = ("1", None, None, None, "2", None) rows = [row(number, broken) for number in range(1, 101)] - checks, warnings = run_checks(rows) + checks, notes = run_checks(rows) + warnings = [note.message for note in notes] assert checks.balance_identity_violations == list(range(1, 101)) (identity,) = [warning for warning in warnings if "сальдо" in warning] @@ -431,7 +438,8 @@ def test_total_row_is_still_checked_on_such_accounts() -> None: rows[-1] = total(9, (None, None, "31", "30", None, None)) assert total_matches_root(rows) is False - checks, warnings = run_checks(rows, NO_SALDO) + checks, notes = run_checks(rows, NO_SALDO) + warnings = [note.message for note in notes] assert checks.total_row_matches_root is False assert any("Итого" in warning for warning in warnings) @@ -449,7 +457,8 @@ def test_exemption_is_matched_by_account_root_not_by_prefix_of_digits() -> None: def test_run_checks_without_violations_on_account_91_gives_clean_status() -> None: - checks, warnings = run_checks(tree_91(), NO_SALDO) + checks, notes = run_checks(tree_91(), NO_SALDO) + warnings = [note.message for note in notes] assert checks.balance_identity_violations == [] assert checks.parent_sum_violations == [] diff --git a/api/tests/unit/parser/test_inn_rows_osv.py b/api/tests/unit/parser/test_inn_rows_osv.py index 395ab28..3c44a80 100644 --- a/api/tests/unit/parser/test_inn_rows_osv.py +++ b/api/tests/unit/parser/test_inn_rows_osv.py @@ -39,7 +39,9 @@ def test_inn_glued_to_row_that_is_not_a_counterparty_needs_check(rules: Rules) - ] ) - document, warnings, ok = OsvParser(rules).parse_sheet(sheet, Deadline(TIMEOUT_SECONDS)) + result = OsvParser(rules).parse_sheet(sheet, Deadline(TIMEOUT_SECONDS)) + document, ok = result.document, result.ok + warnings = [note.message for note in result.notes] assert [(row.row_value, row.group_type) for row in document.rows][1:2] == [("ООО Альфа", "DEPARTMENT")] assert not ok diff --git a/api/tests/unit/parser/test_inn_subkonto.py b/api/tests/unit/parser/test_inn_subkonto.py index d529612..1c2ef8b 100644 --- a/api/tests/unit/parser/test_inn_subkonto.py +++ b/api/tests/unit/parser/test_inn_subkonto.py @@ -49,7 +49,8 @@ def r(level: int, text: object, sums: Sums) -> list[object]: def _parse(sheet: Sheet, rules: Rules) -> tuple[OsvDocument, list[str], bool]: - return OsvParser(rules).parse_sheet(sheet, Deadline(TIMEOUT_SECONDS)) + result = OsvParser(rules).parse_sheet(sheet, Deadline(TIMEOUT_SECONDS)) + return result.document, [note.message for note in result.notes], result.ok def _values(document: OsvDocument) -> list[str]: diff --git a/api/tests/unit/parser/test_osv_warnings.py b/api/tests/unit/parser/test_osv_warnings.py new file mode 100644 index 0000000..0fc8619 --- /dev/null +++ b/api/tests/unit/parser/test_osv_warnings.py @@ -0,0 +1,80 @@ +"""Предупреждения ОСВ с кодами `OSV_*` (P4-12): коды по источнику, вывод 1.x прежний.""" + +import pytest +from gridbuilder import L, M, sheet_from_rows +from test_checks import row, total, tree + +from src.parser.checks import run_checks +from src.parser.deadline import Deadline +from src.parser.grid import Sheet +from src.parser.osv import OsvParser +from src.parser.osv_warnings import OsvNote, hierarchy_note, legacy_line, render_legacy +from src.parser.rules import Rules, load_rules + +SUMS = (None, None, 100, 100, None, None) +BROKEN = (None, None, 100, 100, None, 5) + + +@pytest.fixture(scope="module") +def rules() -> Rules: + return load_rules() + + +def _sheet(*rows: list[object]) -> Sheet: + head: list[list[object]] = [ + ["ООО Зерновая"], + ["Оборотно-сальдовая ведомость по счету 60"], + ["Период: 9 месяцев 2025 г."], + ["Детализация по субсчетам, субконто: Контрагенты"], + ["Выводимые данные: сумма"], + [], + ["Субконто", M("Сальдо на начало периода", 2), M("Оборот за период", 2), M("Сальдо на конец периода", 2)], + [None, "Дебет", "Кредит", "Дебет", "Кредит", "Дебет", "Кредит"], + ] + return sheet_from_rows([*head, *rows]) + + +def test_check_notes_have_codes() -> None: + rows = tree() + rows[2] = row(9, ("40", None, "30", "20", "51", None), level=1, parent=7) + rows[-1] = total(10, ("100", None, "50", "30", "121", None)) + + _, notes = run_checks(rows) + + assert [note.code for note in notes] == ["OSV_BALANCE_IDENTITY", "OSV_PARENT_SUM", "OSV_TOTAL"] + + +def test_hierarchy_note_code_by_strategy() -> None: + assert hierarchy_note("sums") == OsvNote("OSV_HIERARCHY_BY_SUMS", "hierarchy=sums") + assert hierarchy_note("account_pattern").code == "OSV_HIERARCHY_BY_ACCOUNT_PATTERN" + + +def test_legacy_line_and_render() -> None: + note = hierarchy_note("outline") + assert legacy_line("Лист1", note) == "sheet=Лист1 hierarchy=outline" + other = OsvNote("OSV_ROWS", "текст") + assert render_legacy("Лист1", [note, other, other]) == ["sheet=Лист1 hierarchy=outline", "текст"] + + +def test_parse_sheet_notes_start_with_hierarchy(rules: Rules) -> None: + sheet = _sheet([L(0), "60.01", *BROKEN], [L(1), "ООО Альфа", *SUMS], [L(0), "Итого", *BROKEN]) + + result = OsvParser(rules).parse_sheet(sheet, Deadline(60)) + + assert result.notes[0].code.startswith("OSV_HIERARCHY_BY_") + assert all(note.code.startswith("OSV_") for note in result.notes) + assert not result.ok + + +def test_legacy_lines_are_unchanged(rules: Rules) -> None: + sheet = _sheet([L(0), "60.01", *BROKEN], [L(1), "ООО Альфа", *SUMS], [L(0), "Итого", *BROKEN]) + + result = OsvParser(rules).parse_sheet(sheet, Deadline(60)) + + lines = render_legacy(sheet.name, result.notes) + assert lines == [ + "sheet=Лист_1 hierarchy=outline", + "валюта в шапке не найдена", + "§1.4: сальдо на начало + обороты ≠ сальдо на конец (±0.01): строки 9, 11", + "§1.4: суммы родителя ≠ сумме детей (±0.01): строки 9", + ]