RSHB/arm-tabular-parser
9
0
Fork 0
Code Issues 9 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

refactor(osv): предупреждения листа ОСВ с кодами OSV_* (вывод 1.x прежний) (#54)

Browse Source 
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W
This commit is contained in:
Andrey Epifancev 2026-10-18 20:35:29 +04:00
parent ebc1f438da
commit 702ea1dff9
8 changed files with 221 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
api/src/parser/book.py
View File

@ -19,6 +19,7 @@ from src.parser.detect import SheetKind, classify_sheet
from src.parser.errors import InvalidDocumentError, ParserError, UnsupportedFormatError
from src.parser.grid import Sheet
from src.parser.osv import OsvParser, data_rows, sheet_line, unique
from src.parser.osv_warnings import render_legacy
from src.parser.readers import read_book_detected
from src.parser.readers._common import one_line_reason
from src.parser.readers._limits import ZIP_UNCOMPRESSED_LIMIT_FACTOR
@ -118,7 +119,7 @@ class BookParser:
warnings.append(sheet_line(sheet, "skipped: нет признаков ОСВ или формы"))
continue
try:
document, sheet_warnings, sheet_ok = self._osv.parse_sheet(sheet, deadline)
result = self._osv.parse_sheet(sheet, deadline)
except ParserError:
# T7.1: ответ с ошибкой называет листы форм, которые из-за неё остались неразобранными
warnings.extend(
@ -126,9 +127,9 @@ class BookParser:
for form_sheet, _ in statement_sheets
)
raise
osv_documents.append(document)
warnings.extend(sheet_warnings)
ok = ok and sheet_ok
osv_documents.append(result.document)
warnings.extend(render_legacy(sheet.name, result.notes))
ok = ok and result.ok
statement_documents, statement_warnings, statements_ok = self._statements.parse_sheets(
statement_sheets, deadline, formats_unreadable=read.detected == "XLSB"
)

17
api/src/parser/checks.py
View File

@ -30,6 +30,7 @@ from decimal import Decimal
from src.contract.result import OsvChecks
from src.parser.classify import SUM_FIELDS, ClassifiedRow
from src.parser.osv_warnings import OSV_BALANCE_IDENTITY, OSV_PARENT_SUM, OSV_TOTAL, OsvNote
from src.parser.rules import OsvCheckRules
__all__ = [
@ -216,23 +217,23 @@ def _total_warning(rows: Sequence[ClassifiedRow]) -> str:
return message
def run_checks(rows: Sequence[ClassifiedRow], exempt: OsvCheckRules | None = None) -> tuple[OsvChecks, list[str]]:
"""Три проверки §1.4 и предупреждения с номерами строк (пустой список — всё сошлось).
def run_checks(rows: Sequence[ClassifiedRow], exempt: OsvCheckRules | None = None) -> tuple[OsvChecks, list[OsvNote]]:
"""Три проверки §1.4 и заметки с кодами `OSV_*` и номерами строк (пустой список — всё сошлось).
`exempt` — счета без сальдо по аналитике (`rules.checks`, OQ-132); без него проверяются все строки полностью.
"""
identity = balance_identity(rows, exempt)
parents = parent_sums(rows, exempt)
matches = total_matches_root(rows)
warnings: list[str] = []
warnings: list[OsvNote] = []
if identity:
warnings.append(
f"§1.4: сальдо на начало + обороты ≠ сальдо на конец (±{TOLERANCE}): строки {_rows_text(identity)}"
)
text = f"§1.4: сальдо на начало + обороты ≠ сальдо на конец (±{TOLERANCE}): строки {_rows_text(identity)}"
warnings.append(OsvNote(OSV_BALANCE_IDENTITY, text))
if parents:
warnings.append(f"§1.4: суммы родителя ≠ сумме детей (±{TOLERANCE}): строки {_rows_text(parents)}")
text = f"§1.4: суммы родителя ≠ сумме детей (±{TOLERANCE}): строки {_rows_text(parents)}"
warnings.append(OsvNote(OSV_PARENT_SUM, text))
if not matches:
warnings.append(_total_warning(rows))
warnings.append(OsvNote(OSV_TOTAL, _total_warning(rows)))
checks = OsvChecks(
total_row_matches_root=matches,
balance_identity_violations=identity,

51
api/src/parser/osv.py
View File

@ -7,7 +7,7 @@
import datetime
from collections.abc import Iterable, Sequence
from dataclasses import replace
from dataclasses import dataclass, replace
from src.contract import OsvDocument
from src.contract.result import OsvChecks, OsvGroup, OsvHeader, OsvRow
@ -20,9 +20,18 @@ from src.parser.grid import Sheet, column_letter
from src.parser.header import ACCOUNT_NOT_FOUND_WARNING, DocHeader, build_group_levels, parse_doc_header
from src.parser.hierarchy import resolve_levels
from src.parser.inn_rows import InnGlue, glue_inn_rows
from src.parser.osv_warnings import (
OSV_HEADER,
OSV_INN_ROWS,
OSV_NO_DATA_ROWS,
OSV_ROWS,
OsvNote,
hierarchy_note,
notes_of,
)
from src.parser.rules import Rules
__all__ = ["OsvParser", "data_rows", "sheet_line", "unique"]
__all__ = ["OsvParser", "SheetResult", "data_rows", "sheet_line", "unique"]
LOCATOR_PREFIX = "sheet:"
INN_COLUMN_KEY = "inn"
@ -120,6 +129,14 @@ def data_rows(document: OsvDocument) -> int:
return sum(1 for row in document.rows if row.group_type != "TOTAL")
def _unique_notes(notes: Iterable[OsvNote]) -> list[OsvNote]:
"""Без повторов по тексту, в порядке первого появления (как `unique` для строк вывода 1.x)."""
seen: dict[str, OsvNote] = {}
for note in notes:
seen.setdefault(note.message, note)
return list(seen.values())
def _invalid_format(sheet: Sheet, step: str, exc: InvalidFormatError) -> InvalidFormatError:
"""Сообщение `INVALID_FORMAT` с листом и шагом; префикс «лист «…»:» детектора не дублируется."""
reason = exc.message.removeprefix(f"лист «{sheet.name}»: ")
@ -140,15 +157,24 @@ def _check_inn_owners(glue: InnGlue, classified: list[ClassifiedRow]) -> InnGlue
return replace(glue, warnings=[*glue.warnings, *extra], ok=False)
@dataclass(frozen=True)
class SheetResult:
"""Итог листа ОСВ: документ, заметки с кодами `OSV_*` (первая — иерархия) и признак «контрольные суммы сошлись»."""
document: OsvDocument
notes: tuple[OsvNote, ...]
ok: bool
class OsvParser:
"""Конвейер листа ОСВ по словарям `rules`; книгу разбирает `BookParser`."""
def __init__(self, rules: Rules) -> None:
self._rules = rules
def parse_sheet(self, sheet: Sheet, deadline: Deadline) -> tuple[OsvDocument, list[str], bool]:
"""Документ листа ОСВ, его предупреждения (первое — `sheet=<имя> hierarchy=<стратегия>`) и признак
«контрольные суммы сошлись»; нет заголовка/колонок/иерархии — `InvalidFormatError` с листом и шагом.
def parse_sheet(self, sheet: Sheet, deadline: Deadline) -> SheetResult:
"""Документ листа ОСВ, его заметки (первая — иерархия) и признак «контрольные суммы сошлись»;
нет заголовка/колонок/иерархии — `InvalidFormatError` с листом и шагом.
"""
rules = self._rules
step = "заголовок таблицы"
@ -191,16 +217,21 @@ class OsvParser:
glue = _check_inn_owners(glue, classified)
doc = _with_table_account(doc, classified)
deadline.check(f"лист «{sheet.name}»: контрольные суммы")
checks, check_warnings = run_checks(classified, rules.checks)
checks, check_notes = run_checks(classified, rules.checks)
document = self._document(
sheet=sheet, header=header, doc=doc, schema=schema.groups, rows=classified, checks=checks
)
warnings = [sheet_line(sheet, f"hierarchy={levels.strategy}"), *doc.warnings, *body_warnings, *glue.warnings]
notes = [
hierarchy_note(levels.strategy),
*notes_of(OSV_HEADER, doc.warnings),
*notes_of(OSV_ROWS, body_warnings),
*notes_of(OSV_INN_ROWS, glue.warnings),
]
if not any(item.group_type != "TOTAL" for item in classified):
warnings.append(NO_DATA_ROWS_WARNING)
warnings.extend(check_warnings)
notes.append(OsvNote(OSV_NO_DATA_ROWS, NO_DATA_ROWS_WARNING))
notes.extend(check_notes)
# Строка ИНН без пары, похожая на ИНН строка или расхождение с колонкой (#35) — needs_check
return document, unique(warnings), _is_ok(checks) and glue.ok
return SheetResult(document, tuple(_unique_notes(notes)), _is_ok(checks) and glue.ok)
# --- внутреннее ---

63
api/src/parser/osv_warnings.py Normal file
View File

@ -0,0 +1,63 @@
"""Предупреждения листа ОСВ с кодами `OSV_*` (P4-12): код — по источнику заметки, уровень info.
Пока вывод 1.x прежний: `render_legacy` собирает из заметок те же строки `metrics.warnings`, что были до кодов
(текст меняет задача 7). Заметка читателя «xls без кодовой страницы» — не код ОСВ, а предупреждение файла
`XLS_NO_CODEPAGE` пакета (P4-13, R-2); здесь её нет.
"""
from collections.abc import Iterable, Sequence
from dataclasses import dataclass
__all__ = [
"OSV_BALANCE_IDENTITY",
"OSV_HEADER",
"OSV_HIERARCHY_PREFIX",
"OSV_INN_ROWS",
"OSV_NO_DATA_ROWS",
"OSV_PARENT_SUM",
"OSV_ROWS",
"OSV_TOTAL",
"OSV_XLSB_NO_OUTLINE",
"OsvNote",
"hierarchy_note",
"legacy_line",
"notes_of",
"render_legacy",
]
OSV_HIERARCHY_PREFIX = "OSV_HIERARCHY_"
OSV_HEADER = "OSV_HEADER"
OSV_ROWS = "OSV_ROWS"
OSV_INN_ROWS = "OSV_INN_ROWS"
OSV_NO_DATA_ROWS = "OSV_NO_DATA_ROWS"
OSV_BALANCE_IDENTITY = "OSV_BALANCE_IDENTITY"
OSV_PARENT_SUM = "OSV_PARENT_SUM"
OSV_TOTAL = "OSV_TOTAL"
OSV_XLSB_NO_OUTLINE = "OSV_XLSB_NO_OUTLINE" # заметка читателя xlsb, подключается в задаче 7
@dataclass(frozen=True)
class OsvNote:
code: str
message: str
def hierarchy_note(strategy: str) -> OsvNote:
"""Стратегия иерархии: `outline|indent|account_pattern|sums|trivial` → `OSV_HIERARCHY_BY_<СТРАТЕГИЯ>`."""
return OsvNote(f"{OSV_HIERARCHY_PREFIX}BY_{strategy.upper()}", f"hierarchy={strategy}")
def notes_of(code: str, messages: Iterable[str]) -> list[OsvNote]:
return [OsvNote(code, message) for message in messages]
def legacy_line(sheet_name: str, note: OsvNote) -> str:
return f"sheet={sheet_name} {note.message}"
def render_legacy(sheet_name: str, notes: Sequence[OsvNote]) -> list[str]:
"""Строки вывода 1.x: заметка иерархии с префиксом листа, остальные — как были; без повторов, порядок прежний."""
lines = [
legacy_line(sheet_name, note) if note.code.startswith(OSV_HIERARCHY_PREFIX) else note.message for note in notes
]
return list(dict.fromkeys(lines))

27
api/tests/unit/parser/test_checks.py
View File

@ -79,7 +79,8 @@ def test_tolerance_is_one_kopeck() -> None:
def test_consistent_tree_has_no_violations() -> None:
checks, warnings = run_checks(tree())
checks, notes = run_checks(tree())
warnings = [note.message for note in notes]
assert checks.total_row_matches_root is True
assert checks.balance_identity_violations == []
@ -250,7 +251,8 @@ def multi_root_tree(total_sums: Sums = SUB_SUM) -> list[ClassifiedRow]:
def test_total_matches_sum_of_level_zero_rows_when_there_are_several_roots() -> None:
rows = multi_root_tree()
checks, warnings = run_checks(rows)
checks, notes = run_checks(rows)
warnings = [note.message for note in notes]
assert total_matches_root(rows) is True
assert checks.total_row_matches_root is True
@ -288,7 +290,8 @@ def test_expanded_balance_total_is_not_matched_by_net_when_total_has_both_sides(
def test_total_that_equals_neither_first_root_nor_sum_of_roots_mismatches() -> None:
rows = multi_root_tree(("140", None, "40", "20", "161", None))
checks, warnings = run_checks(rows)
checks, notes = run_checks(rows)
warnings = [note.message for note in notes]
assert checks.total_row_matches_root is False
assert any("Итого" in warning and "11" in warning and "сумме 2 строк уровня 0" in warning for warning in warnings)
@ -326,7 +329,8 @@ def test_missing_total_fails_root_check_with_warning() -> None:
"""R-20: без «Итого» сверку выполнить нельзя — `false` и предупреждение."""
rows = tree()[:-1]
checks, warnings = run_checks(rows)
checks, notes = run_checks(rows)
warnings = [note.message for note in notes]
assert total_matches_root(rows) is False
assert checks.total_row_matches_root is False
@ -337,7 +341,8 @@ def test_run_checks_warnings_name_row_numbers() -> None:
rows = tree()
rows[2] = row(9, ("40", None, "30", "20", "51", None), level=1, parent=7)
checks, warnings = run_checks(rows)
checks, notes = run_checks(rows)
warnings = [note.message for note in notes]
assert checks.balance_identity_violations == [9]
assert checks.parent_sum_violations == [7]
@ -350,7 +355,8 @@ def test_run_checks_reports_total_mismatch_with_row_numbers() -> None:
rows = tree()
rows[-1] = total(10, ("100", None, "50", "30", "121", None))
checks, warnings = run_checks(rows)
checks, notes = run_checks(rows)
warnings = [note.message for note in notes]
assert checks.total_row_matches_root is False
assert any("10" in warning and "7" in warning and "Итого" in warning for warning in warnings), warnings
@ -361,7 +367,8 @@ def test_run_checks_warning_lists_are_capped() -> None:
broken: Sums = ("1", None, None, None, "2", None)
rows = [row(number, broken) for number in range(1, 101)]
checks, warnings = run_checks(rows)
checks, notes = run_checks(rows)
warnings = [note.message for note in notes]
assert checks.balance_identity_violations == list(range(1, 101))
(identity,) = [warning for warning in warnings if "сальдо" in warning]
@ -431,7 +438,8 @@ def test_total_row_is_still_checked_on_such_accounts() -> None:
rows[-1] = total(9, (None, None, "31", "30", None, None))
assert total_matches_root(rows) is False
checks, warnings = run_checks(rows, NO_SALDO)
checks, notes = run_checks(rows, NO_SALDO)
warnings = [note.message for note in notes]
assert checks.total_row_matches_root is False
assert any("Итого" in warning for warning in warnings)
@ -449,7 +457,8 @@ def test_exemption_is_matched_by_account_root_not_by_prefix_of_digits() -> None:
def test_run_checks_without_violations_on_account_91_gives_clean_status() -> None:
checks, warnings = run_checks(tree_91(), NO_SALDO)
checks, notes = run_checks(tree_91(), NO_SALDO)
warnings = [note.message for note in notes]
assert checks.balance_identity_violations == []
assert checks.parent_sum_violations == []

4
api/tests/unit/parser/test_inn_rows_osv.py
View File

@ -39,7 +39,9 @@ def test_inn_glued_to_row_that_is_not_a_counterparty_needs_check(rules: Rules) -
]
)
document, warnings, ok = OsvParser(rules).parse_sheet(sheet, Deadline(TIMEOUT_SECONDS))
result = OsvParser(rules).parse_sheet(sheet, Deadline(TIMEOUT_SECONDS))
document, ok = result.document, result.ok
warnings = [note.message for note in result.notes]
assert [(row.row_value, row.group_type) for row in document.rows][1:2] == [("ООО Альфа", "DEPARTMENT")]
assert not ok

3
api/tests/unit/parser/test_inn_subkonto.py
View File

@ -49,7 +49,8 @@ def r(level: int, text: object, sums: Sums) -> list[object]:
def _parse(sheet: Sheet, rules: Rules) -> tuple[OsvDocument, list[str], bool]:
return OsvParser(rules).parse_sheet(sheet, Deadline(TIMEOUT_SECONDS))
result = OsvParser(rules).parse_sheet(sheet, Deadline(TIMEOUT_SECONDS))
return result.document, [note.message for note in result.notes], result.ok
def _values(document: OsvDocument) -> list[str]:

80
api/tests/unit/parser/test_osv_warnings.py Normal file
View File

@ -0,0 +1,80 @@
"""Предупреждения ОСВ с кодами `OSV_*` (P4-12): коды по источнику, вывод 1.x прежний."""
import pytest
from gridbuilder import L, M, sheet_from_rows
from test_checks import row, total, tree
from src.parser.checks import run_checks
from src.parser.deadline import Deadline
from src.parser.grid import Sheet
from src.parser.osv import OsvParser
from src.parser.osv_warnings import OsvNote, hierarchy_note, legacy_line, render_legacy
from src.parser.rules import Rules, load_rules
SUMS = (None, None, 100, 100, None, None)
BROKEN = (None, None, 100, 100, None, 5)
@pytest.fixture(scope="module")
def rules() -> Rules:
return load_rules()
def _sheet(*rows: list[object]) -> Sheet:
head: list[list[object]] = [
["ООО Зерновая"],
["Оборотно-сальдовая ведомость по счету 60"],
["Период: 9 месяцев 2025 г."],
["Детализация по субсчетам, субконто: Контрагенты"],
["Выводимые данные: сумма"],
[],
["Субконто", M("Сальдо на начало периода", 2), M("Оборот за период", 2), M("Сальдо на конец периода", 2)],
[None, "Дебет", "Кредит", "Дебет", "Кредит", "Дебет", "Кредит"],
]
return sheet_from_rows([*head, *rows])
def test_check_notes_have_codes() -> None:
rows = tree()
rows[2] = row(9, ("40", None, "30", "20", "51", None), level=1, parent=7)
rows[-1] = total(10, ("100", None, "50", "30", "121", None))
_, notes = run_checks(rows)
assert [note.code for note in notes] == ["OSV_BALANCE_IDENTITY", "OSV_PARENT_SUM", "OSV_TOTAL"]
def test_hierarchy_note_code_by_strategy() -> None:
assert hierarchy_note("sums") == OsvNote("OSV_HIERARCHY_BY_SUMS", "hierarchy=sums")
assert hierarchy_note("account_pattern").code == "OSV_HIERARCHY_BY_ACCOUNT_PATTERN"
def test_legacy_line_and_render() -> None:
note = hierarchy_note("outline")
assert legacy_line("Лист1", note) == "sheet=Лист1 hierarchy=outline"
other = OsvNote("OSV_ROWS", "текст")
assert render_legacy("Лист1", [note, other, other]) == ["sheet=Лист1 hierarchy=outline", "текст"]
def test_parse_sheet_notes_start_with_hierarchy(rules: Rules) -> None:
sheet = _sheet([L(0), "60.01", *BROKEN], [L(1), "ООО Альфа", *SUMS], [L(0), "Итого", *BROKEN])
result = OsvParser(rules).parse_sheet(sheet, Deadline(60))
assert result.notes[0].code.startswith("OSV_HIERARCHY_BY_")
assert all(note.code.startswith("OSV_") for note in result.notes)
assert not result.ok
def test_legacy_lines_are_unchanged(rules: Rules) -> None:
sheet = _sheet([L(0), "60.01", *BROKEN], [L(1), "ООО Альфа", *SUMS], [L(0), "Итого", *BROKEN])
result = OsvParser(rules).parse_sheet(sheet, Deadline(60))
lines = render_legacy(sheet.name, result.notes)
assert lines == [
"sheet=Лист_1 hierarchy=outline",
"валюта в шапке не найдена",
"§1.4: сальдо на начало + обороты ≠ сальдо на конец (±0.01): строки 9, 11",
"§1.4: суммы родителя ≠ сумме детей (±0.01): строки 9",
]