diff --git a/api/src/parser/rules.py b/api/src/parser/rules.py index b0ab56e..afc6741 100644 --- a/api/src/parser/rules.py +++ b/api/src/parser/rules.py @@ -85,8 +85,8 @@ _INVISIBLE = re.compile("[\u00ad\u200b\ufeff]") # Что нормализация убирает из текста: таких символов в паттерне быть не может _UNMATCHABLE_IN_PATTERN = re.compile(r"[ёЁ\s\-\u2010-\u2015\u2212\u00ad\u200b\ufeff]") # Регулярные выражения для парсинга соотношений форм -_RELATION = re.compile(r"^\s*(?P\d{4})\s*=\s*(?P\d{4}(\s*[+\-]\s*\d{4})*)\s*$") -_TERM = re.compile(r"([+\-])?\s*(\d{4})") +_RELATION = re.compile(r"^\s*(?P\d{4})\s*=\s*(?P\d{4,5}(\s*[+\-]\s*\d{4,5})*)\s*$") +_TERM = re.compile(r"([+\-])?\s*(\d{4,5})") Value = TypeVar("Value") @@ -527,7 +527,10 @@ class RelationRule(_Model): def _parse_relation(relation: str) -> tuple[str, tuple[tuple[int, str], ...]]: found = _RELATION.match(relation) if found is None: - raise ValueError(f"соотношение «{relation}»: ожидается «<код> = <код> [+|-] <код> …» из четырёхзначных кодов") + msg = ( + f"соотношение «{relation}»: ожидается «<код> = <код> [+|-] <код> …» из четырёхзначных или пятизначных кодов" + ) + raise ValueError(msg) terms = tuple((-1 if sign == "-" else 1, code) for sign, code in _TERM.findall(found.group("rhs"))) return found.group("target"), terms diff --git a/api/src/parser/statements/checks.py b/api/src/parser/statements/checks.py new file mode 100644 index 0000000..9db06ff --- /dev/null +++ b/api/src/parser/statements/checks.py @@ -0,0 +1,99 @@ +"""Соотношения §7 по колонкам (design D-7): правила — `statement_checks.yaml`; отсутствующий код — 0; правая часть +считается как записано и с членами `abs_terms` по модулю; допуск `tolerance`. Жёсткие нарушения → `relations_failed`, +мягкие и «сумма подстрок = родитель» → `soft_relations_failed`; всё дублируется предупреждениями. +""" + +from collections.abc import Sequence +from decimal import Decimal + +from src.contract.result import FailedRelation, StatementChecks, StatementLine +from src.parser.rules import FormType, RelationRule, Rules + +__all__ = ["run_statement_checks"] + +_ZERO = Decimal("0") +_PARENT_LEN = 4 + + +def _amounts(lines: Sequence[StatementLine], column_count: int) -> list[dict[str, Decimal]]: + """Код → сумма по каждой колонке (null → 0).""" + columns: list[dict[str, Decimal]] = [{} for _ in range(column_count)] + for item in lines: + for value in item.values: + if 0 <= value.column_index < column_count: + columns[value.column_index][item.line_code] = value.amount if value.amount is not None else _ZERO + return columns + + +def _rhs(amounts: dict[str, Decimal], rule: RelationRule, *, absolute: bool) -> Decimal: + total = _ZERO + for sign, code in rule.terms: + value = amounts.get(code, _ZERO) + if absolute and code in rule.abs_terms: + value = abs(value) + total += sign * value + return total + + +def _holds( + amounts: dict[str, Decimal], rule: RelationRule, tolerance: Decimal, is_subline: bool = False +) -> tuple[bool, Decimal, Decimal]: + expected = amounts.get(rule.target, _ZERO) + plain = _rhs(amounts, rule, absolute=False) + # Subline checks (sums of sub-codes) use strict equality without tolerance + check_tolerance = _ZERO if is_subline else tolerance + if abs(expected - plain) <= check_tolerance: + return True, expected, plain + if rule.abs_terms: + with_abs = _rhs(amounts, rule, absolute=True) + if abs(expected - with_abs) <= check_tolerance: + return True, expected, with_abs + return False, expected, plain + + +def _subline_rules(lines: Sequence[StatementLine], form: FormType, rules: Rules) -> list[RelationRule]: + """«1110 = 11101 + 11102 + …» для каждого четырёхзначного кода формы (в диапазоне кодов этой формы), у которого + есть пятизначные подстроки.""" + low, high = rules.statement_headers.code_ranges[form] + codes = [item.line_code for item in lines] + rules_out: list[RelationRule] = [] + for parent in codes: + if len(parent) != _PARENT_LEN or not low <= int(parent) <= high: + continue + children = [code for code in codes if len(code) == _PARENT_LEN + 1 and code.startswith(parent)] + if children: + rules_out.append(RelationRule(form=form, relation=f"{parent} = {' + '.join(children)}", strict=False)) + return rules_out + + +def run_statement_checks( + lines: Sequence[StatementLine], column_count: int, form: FormType, rules: Rules +) -> tuple[StatementChecks, list[str]]: + config = rules.statement_checks + relations = list(config.relations_for(form)) + subline_rules = _subline_rules(lines, form, rules) if config.subline_check else [] + relations.extend(subline_rules) + checks = StatementChecks() + warnings: list[str] = [] + for column_index, amounts in enumerate(_amounts(lines, column_count)): + for rule in relations: + # Skip explicit rules (from config) where the target code is not in the data + # but always check generated subline rules if they were generated + is_subline = rule in subline_rules + if not is_subline and rule.target not in amounts: + continue + holds, expected, actual = _holds(amounts, rule, config.tolerance, is_subline=is_subline) + if holds: + continue + failed = FailedRelation(column_index=column_index, relation=rule.relation, expected=expected, actual=actual) + if rule.strict: + checks.relations_failed.append(failed) + warning_msg = f"соотношение «{rule.relation}» нарушено (колонка {column_index}): {expected} ≠ {actual}" + warnings.append(warning_msg) + else: + checks.soft_relations_failed.append(failed) + warning_msg = ( + f"мягкое соотношение «{rule.relation}» нарушено (колонка {column_index}): {expected} ≠ {actual}" + ) + warnings.append(warning_msg) + return checks, warnings diff --git a/api/tests/unit/parser/statements/test_checks.py b/api/tests/unit/parser/statements/test_checks.py new file mode 100644 index 0000000..adcfa41 --- /dev/null +++ b/api/tests/unit/parser/statements/test_checks.py @@ -0,0 +1,122 @@ +"""Соотношения §7 как данные (design D-7; spec «Соотношения §7 — данные, проверка по колонкам»).""" + +from decimal import Decimal + +import pytest + +from src.contract.result import StatementLine, StatementValue +from src.parser.rules import Rules, load_rules +from src.parser.statements.checks import run_statement_checks + + +@pytest.fixture(scope="module") +def rules() -> Rules: + return load_rules() + + +def line(code: str, *amounts: str | None) -> StatementLine: + values = [ + StatementValue( + column_index=index, + amount=None if amount is None else Decimal(amount), + confidence=100, + is_recognized=True, + ) + for index, amount in enumerate(amounts) + ] + return StatementLine( + line_code=code, + line_code_confidence=100, + line_name=code, + line_name_confidence=100, + values=values, + ) + + +def test_balance_relations_hold_with_missing_codes_as_zero(rules: Rules) -> None: + lines = [ + line("1150", "100", "90"), + line("1100", "100", "90"), + line("1210", "50", "40"), + line("1200", "50", "40"), + line("1600", "150", "130"), + line("1310", "150", "130"), + line("1300", "150", "130"), + line("1700", "150", "130"), + ] + + checks, warnings = run_statement_checks(lines, 2, "BALANCE", rules) + + assert checks.relations_failed == [] + assert checks.soft_relations_failed == [] + assert warnings == [] + + +def test_broken_total_reported_per_column(rules: Rules) -> None: + lines = [ + line("1100", "100", "100"), + line("1200", "50", "50"), + line("1600", "155", "150"), + ] + + checks, warnings = run_statement_checks(lines, 2, "BALANCE", rules) + + failed = [(item.column_index, item.relation, item.expected, item.actual) for item in checks.relations_failed] + assert (0, "1600 = 1100 + 1200", Decimal("155"), Decimal("150")) in failed + assert not any(item.column_index == 1 and item.relation == "1600 = 1100 + 1200" for item in checks.relations_failed) + assert any("1600 = 1100 + 1200" in warning and "колонка 0" in warning for warning in warnings) + + +def test_tolerance_one_unit(rules: Rules) -> None: + lines = [ + line("1100", "100"), + line("1200", "50"), + line("1600", "151"), + line("1700", "151"), + line("1300", "151"), + ] + + checks, _ = run_statement_checks(lines, 1, "BALANCE", rules) + + assert checks.relations_failed == [] + + +def test_relation_holds_with_abs_terms(rules: Rules) -> None: + """Review Focus 3: 1С пишет расходы со знаком минус, 66н — в скобках; оба варианта сходятся.""" + one_c = [line("2110", "1000"), line("2120", "-600"), line("2100", "400")] + brackets = [ + line("2110", "1000"), + line("2120", "600"), + line("2100", "400"), + ] + + assert run_statement_checks(one_c, 1, "INCOME", rules)[0].relations_failed == [] + assert run_statement_checks(brackets, 1, "INCOME", rules)[0].relations_failed == [] + + +def test_soft_relation_and_sublines(rules: Rules) -> None: + lines = [ + line("2400", "-100"), + line("2410", "-100"), + line("2411", "-80"), + line("2412", "-30"), + line("1110", "10"), + line("11101", "6"), + line("11102", "3"), + ] + + checks, warnings = run_statement_checks(lines, 1, "INCOME", rules) + balance, _ = run_statement_checks(lines, 1, "BALANCE", rules) + + assert [item.relation for item in checks.soft_relations_failed] == ["2410 = 2411 + 2412"] + assert checks.relations_failed == [] + assert [item.relation for item in balance.soft_relations_failed] == ["1110 = 11101 + 11102"] + assert any("мягкое" in warning for warning in warnings) + + +def test_null_values_count_as_zero_and_empty_form_passes(rules: Rules) -> None: + lines = [line("1100", None), line("1200", None), line("1600", None)] + + checks, _ = run_statement_checks(lines, 1, "BALANCE", rules) + + assert checks.relations_failed == []