diff --git a/arm_statements/amounts.py b/arm_statements/amounts.py new file mode 100644 index 0000000..1c40eac --- /dev/null +++ b/arm_statements/amounts.py @@ -0,0 +1,196 @@ +"""Сумма и значение ячейки (шаг 4): `parse_amount` — единственное место, где разбор зависит от источника.""" + +import re +from dataclasses import dataclass +from datetime import date +from decimal import ROUND_HALF_UP, Decimal + +from arm_statements.codes import code_parts +from arm_statements.loader import Rules +from arm_statements.model import Cell, SignForm, Source, StatementValue +from arm_statements.text import replace_lookalikes + +MAX_INTEGER_DIGITS = 16 +_CENT = Decimal("0.01") +_DIGITS = frozenset("0123456789") +_CORE_START = _DIGITS | frozenset("(-−+") +_CORE_END = _DIGITS | frozenset(")}]") +_DASHES = frozenset("-–—−") +_MINUSES = frozenset("-−") +_BODY = re.compile(r"[\d\s,.]*") +_DIGIT_SPACE_RUN = re.compile(r"\d+(?:\s+\d+)+") +_SPACE_GROUPS = re.compile(r"\d{1,3}(?:\s\d{3})+") +_COMMA_THOUSANDS = re.compile(r"[1-9]\d{0,2}(?:,\d{3})+") +_DOT_THOUSANDS = re.compile(r"[1-9]\d*(?:\.\d{3})+") +_SEPARATORS = ",." + + +@dataclass(frozen=True, slots=True) +class AmountParse: + amount: Decimal | None + string_value: str | None + is_recognized: bool + sign_form: SignForm | None + + +@dataclass(frozen=True, slots=True) +class ValueBuild: + value: StatementValue + out_of_range: bool + + +def _quantize(amount: Decimal) -> Decimal: + return amount.quantize(_CENT, rounding=ROUND_HALF_UP) + + +def _too_long(amount: Decimal) -> bool: + return len(str(abs(int(amount)))) > MAX_INTEGER_DIGITS + + +def _is_blank(text: str) -> bool: + return all(char.isspace() or char in _DASHES for char in text) + + +def _strip_edge_bars(text: str) -> tuple[str, bool]: + """Крайние `|` (след границы таблицы) — до замены двойников; второй элемент — было ли удаление.""" + stripped = text.strip() + while stripped.startswith("|"): + stripped = stripped[1:].strip() + while stripped.endswith("|"): + stripped = stripped[:-1].strip() + return stripped, stripped != text.strip() + + +def _core_bounds(text: str) -> tuple[int, int] | None: + starts = [index for index, char in enumerate(text) if char in _CORE_START] + ends = [index for index, char in enumerate(text) if char in _CORE_END] + if not starts or not ends or starts[0] > ends[-1]: + return None + return starts[0], ends[-1] + + +def _guess(digits: str, text: str, sign_form: SignForm, *, negative: bool) -> AmountParse: + """«Лучшая догадка»: цифры подряд, не длиннее 16, иначе значение не число.""" + if not digits or len(digits) > MAX_INTEGER_DIGITS: + return AmountParse(None, text, False, sign_form) + amount = Decimal(digits) + return AmountParse(-amount if negative else amount, None, False, sign_form) + + +def _split_core(core: str) -> tuple[str, SignForm, bool, bool] | None: + """Ядро → (тело без знака и скобок, форма знака, отрицательное ли, исправлена ли скобка) или None.""" + fixed = False + sign_form = SignForm.UNSIGNED + negative = False + if core[0] == "(": + if len(core) < 2 or core[-1] not in ")}]": # noqa: PLR2004 — скобки в ядре минимум пара + return None + fixed = core[-1] != ")" + core = core[1:-1].strip() + sign_form, negative = SignForm.PARENTHESES, True + if core and core[0] in "-−+": + core = core[1:].strip() + elif core[0] in "-−+": + if core[0] in _MINUSES: + sign_form, negative = SignForm.MINUS, True + core = core[1:].strip() + if not core or not _BODY.fullmatch(core) or not core[0].isdigit(): + return None + return core, sign_form, negative, fixed + + +def _number(body: str, source: Source, text: str, sign_form: SignForm, *, negative: bool) -> AmountParse: + """Стадия 6: разделители. Возвращает догадку с `is_recognized = false` или точный разбор.""" + runs = _DIGIT_SPACE_RUN.findall(body) + spaces_ok = all(_SPACE_GROUPS.fullmatch(run) for run in runs) + compact = "".join(body.split()) + digits = compact.replace(",", "").replace(".", "") + if not spaces_ok: + return _guess(digits, text, sign_form, negative=negative) + if source is Source.OCR and _COMMA_THOUSANDS.fullmatch(compact): + compact = compact.replace(",", "") + elif source is Source.OCR and _DOT_THOUSANDS.fullmatch(compact): + compact = compact.replace(".", "") + separators = [char for char in compact if char in _SEPARATORS] + if not separators: + integer, fraction = compact, "" + elif len(separators) > 1: # смешанные или повторяющиеся разделители + return _guess(digits, text, sign_form, negative=negative) + else: + integer, fraction = compact.replace(",", ".").split(".") + if not integer: + return AmountParse(None, text, False, sign_form) + if source is Source.OCR and len(fraction) >= 3: # noqa: PLR2004 — три знака: не копейки + return AmountParse(None, text, False, sign_form) + if len(integer) > MAX_INTEGER_DIGITS: + return AmountParse(None, text, False, sign_form) + amount = _quantize(Decimal(f"{integer}.{fraction}" if fraction else integer)) + return AmountParse(-amount if negative else amount, None, True, sign_form) + + +def parse_amount(text: str | None, source: Source, rules: Rules) -> AmountParse: + """Стадии 1–6 шага «значение ячейки»; от источника зависят только десятичный разделитель и тысячи.""" + if text is None or _is_blank(text): + return AmountParse(None, None, True, None) + work, fixed = _strip_edge_bars(text) + replaced = replace_lookalikes(work, rules.lookalikes) + fixed = fixed or replaced != work + if not any(char in _DIGITS for char in replaced): # стадия 5 + return AmountParse(None, text, False, None) + bounds = _core_bounds(replaced) + if bounds is None: + return AmountParse(None, text, False, None) + start, end = bounds + core = replaced[start : end + 1] + trash = bool((replaced[:start] + replaced[end + 1 :]).strip()) + split = _split_core(core) + if split is None: + return AmountParse(None, text, False, None) + body, sign_form, negative, bracket_fixed = split + parsed = _number(body, source, text, sign_form, negative=negative) + clean = parsed.is_recognized and not (fixed or trash or bracket_fixed) + return AmountParse(parsed.amount, parsed.string_value, clean, parsed.sign_form) + + +def sign_form_of(text: str | None, rules: Rules) -> SignForm | None: + return parse_amount(text, Source.OCR, rules).sign_form + + +def _plain(cell: Cell, column_index: int, parsed: AmountParse, *, recognized: bool | None = None) -> StatementValue: + return StatementValue( + column_index=column_index, + amount=parsed.amount, + string_value=parsed.string_value, + text=cell.text, + confidence=cell.confidence, + is_recognized=parsed.is_recognized if recognized is None else recognized, + ) + + +def build_value( + cell: Cell | None, + column_index: int, + source: Source, + rules: Rules, + *, + split_part: bool = False, +) -> ValueBuild: + if cell is None: + return ValueBuild(StatementValue(column_index, None, None, None, 100.0, True), out_of_range=False) + text = cell.text + string_text = text if text.strip() else None + value = cell.value + if isinstance(value, Decimal): + amount = _quantize(value) + if _too_long(amount): + return ValueBuild(_plain(cell, column_index, AmountParse(None, text, False, None)), out_of_range=True) + return ValueBuild( + _plain(cell, column_index, AmountParse(amount, None, True, None), recognized=not split_part), + out_of_range=False, + ) + if isinstance(value, date | bool): + return ValueBuild(_plain(cell, column_index, AmountParse(None, string_text, False, None)), out_of_range=False) + if len(code_parts(text)) > 1 and not split_part: + return ValueBuild(_plain(cell, column_index, AmountParse(None, text, False, None)), out_of_range=False) + parsed = parse_amount(text, source, rules) + return ValueBuild(_plain(cell, column_index, parsed, recognized=False if split_part else None), out_of_range=False) diff --git a/arm_statements/tests/test_amounts.py b/arm_statements/tests/test_amounts.py new file mode 100644 index 0000000..8025cf5 --- /dev/null +++ b/arm_statements/tests/test_amounts.py @@ -0,0 +1,84 @@ +from datetime import date +from decimal import Decimal as D + +import pytest + +from arm_statements.amounts import build_value, parse_amount, sign_form_of +from arm_statements.loader import Rules +from arm_statements.model import Cell, SignForm, Source + +OCR, EXCEL = Source.OCR, Source.EXCEL + + +@pytest.mark.parametrize( + ("raw", "amount", "string_value", "recognized"), + [ + ("1 2О5", D("1205"), None, False), + ("42831 1", D("428311"), None, False), + ("(7744536}", D("-7744536"), None, False), + ("211(1 |", None, "211(1 |", False), + ("(Ю4)", None, "(Ю4)", False), + (": : 2856983 = ■", D("2856983"), None, False), + ("| : 122477", D("122477"), None, False), + ("|122477", D("122477"), None, False), + ("{38378)", None, "{38378)", False), + ("4,200", D("4200"), None, True), + ("1,234,567", D("1234567"), None, True), + ("837.994", D("837994"), None, True), + ("1 234.567", D("1234567"), None, True), + ("4,2", D("4.2"), None, True), + ("1 234,56", D("1234.56"), None, True), + ("1.234,56", D("123456"), None, False), + ("0,125", None, "0,125", False), + ("1 234 567", D("1234567"), None, True), + ("(1 234)", D("-1234"), None, True), + ("−500", D("-500"), None, True), + ("12:34", None, "12:34", False), + ("1 234 руб.", D("1234"), None, False), + ("—", None, None, True), + ("-", None, None, True), + ("", None, None, True), + ("I", None, "I", False), + ], +) +def test_parse_amount_ocr(rules: Rules, raw: str, amount: D | None, string_value: str | None, recognized: bool) -> None: + parsed = parse_amount(raw, OCR, rules) + assert (parsed.amount, parsed.string_value, parsed.is_recognized) == (amount, string_value, recognized) + + +def test_comma_depends_on_source(rules: Rules) -> None: + assert parse_amount("4,200", EXCEL, rules).amount == D("4.20") + assert parse_amount("1.500", EXCEL, rules).amount == D("1.5") + + +def test_sign_forms(rules: Rules) -> None: + assert sign_form_of("(7744536}", rules) is SignForm.PARENTHESES + assert sign_form_of("-1 234", rules) is SignForm.MINUS + assert sign_form_of("38378", rules) is SignForm.UNSIGNED + assert sign_form_of("{38378)", rules) is None + + +def test_build_value_excel_number_and_out_of_range(rules: Rules) -> None: + ok = build_value(Cell("106 311", D("106311.456"), 100.0, "Баланс!D27"), 3, EXCEL, rules) + assert (ok.value.amount, ok.value.is_recognized, ok.out_of_range) == (D("106311.46"), True, False) + big = build_value(Cell("1E+17", D("123456789012345678"), 100.0, "Баланс!D28"), 3, EXCEL, rules) + assert (big.value.amount, big.value.string_value, big.value.is_recognized, big.out_of_range) == ( + None, + "1E+17", + False, + True, + ) + + +def test_build_value_date_missing_multiline_split(rules: Rules) -> None: + as_date = build_value(Cell("30.09.2025", date(2025, 9, 30), 100.0, "Баланс!D3"), 3, EXCEL, rules).value + assert (as_date.amount, as_date.string_value, as_date.is_recognized) == (None, "30.09.2025", False) + missing = build_value(None, 4, OCR, rules).value + assert (missing.amount, missing.string_value, missing.text, missing.is_recognized) == (None, None, None, True) + assert missing.confidence == pytest.approx(100.0) + multi = build_value(Cell("917 775\n2 371 333", None, 100.0, "row:9"), 2, OCR, rules).value + assert (multi.amount, multi.string_value, multi.is_recognized) == (None, "917 775\n2 371 333", False) + part = build_value(Cell("917 775", None, 100.0, "row:9#1"), 2, OCR, rules, split_part=True).value + assert (part.amount, part.is_recognized) == (D("917775"), False) + dash = build_value(Cell("—", None, 100.0, "row:3"), 2, OCR, rules).value + assert (dash.amount, dash.string_value, dash.is_recognized, dash.text) == (None, None, True, "—")