feat: буквы-двойники, нормализация подписей, разбор кода строки и отрезки кодов

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W

arm_statements/codes.py

@@ -0,0 +1,118 @@
+"""Разбор кода строки и отрезки кодов форм."""
+
+import re
+from collections import Counter
+from collections.abc import Iterable, Mapping, Sequence
+from typing import NamedTuple
+
+from arm_statements.loader import CodeSection, Rules
+from arm_statements.model import DocumentType
+from arm_statements.text import replace_lookalikes
+
+_CODE = re.compile(r"\d{4,5}(?:\.0)?")
+_BAR = "|"
+
+
+class CodeParse(NamedTuple):
+    code: str | None
+    fixed: bool  # при разборе заменён двойник
+
+
+def code_parts(text: str | None) -> tuple[str, ...]:
+    """Непустые строки ячейки, обрезанные."""
+    if text is None:
+        return ()
+    return tuple(part.strip() for part in text.splitlines() if part.strip())
+
+
+def section_of(code: str, rules: Rules) -> CodeSection | None:
+    head = int(code[:4])
+    for item in rules.code_ranges.ranges:
+        if item.low <= head <= item.high:
+            return item.section
+    return None
+
+
+def _strip_edges(text: str) -> str:
+    start, end = 0, len(text)
+    while start < end and not text[start].isalnum():
+        start += 1
+    while end > start and not text[end - 1].isalnum():
+        end -= 1
+    return text[start:end]
+
+
+def _as_code(text: str, rules: Rules) -> str | None:
+    core = _strip_edges(text)
+    if _CODE.fullmatch(core) is None:
+        return None
+    code = core.split(".", 1)[0]
+    return code if section_of(code, rules) is not None else None
+
+
+def _parse_line(line: str, rules: Rules) -> CodeParse:
+    compact = "".join(line.split())
+    # сначала `|` как двойник единицы; если результата-кода нет — `|` как краевая пунктуация
+    for candidate in (compact, compact.replace(_BAR, "")):
+        replaced = replace_lookalikes(candidate, rules.lookalikes)
+        code = _as_code(replaced, rules)
+        if code is not None:
+            return CodeParse(code, replaced != candidate)
+    return CodeParse(None, False)
+
+
+def parse_code(text: str | None, rules: Rules) -> CodeParse:
+    """Однострочный текст → код; многострочный → `CodeParse(None, False)`."""
+    parts = code_parts(text)
+    if len(parts) != 1:
+        return CodeParse(None, False)
+    return _parse_line(parts[0], rules)
+
+
+def is_code_cell(text: str | None, rules: Rules) -> bool:
+    parts = code_parts(text)
+    return bool(parts) and all(_parse_line(part, rules).code is not None for part in parts)
+
+
+def majority_section(codes: Iterable[str], rules: Rules) -> CodeSection | None:
+    """Отрезок большинства; ничья — первый по порядку отрезков словаря."""
+    counts: Counter[CodeSection] = Counter()
+    for code in codes:
+        section = section_of(code, rules)
+        if section is not None:
+            counts[section] += 1
+    best: CodeSection | None = None
+    for item in rules.code_ranges.ranges:
+        if counts[item.section] and (best is None or counts[item.section] > counts[best]):
+            best = item.section
+    return best
+
+
+def document_type_of(section: CodeSection) -> DocumentType:
+    mapping: Mapping[CodeSection, DocumentType] = {
+        CodeSection.BALANCE: DocumentType.BALANCE,
+        CodeSection.INCOME: DocumentType.INCOME,
+    }
+    return mapping.get(section, DocumentType.DISCLOSURE)
+
+
+def sheet_joins(
+    form_type: DocumentType,
+    sheet_codes: Sequence[str],
+    sheet_amount_columns: int,
+    form_amount_columns: int,
+    rules: Rules,
+) -> bool:
+    """Лист без маркера и шапки присоединяется к форме: коды в её отрезке; при ≥3 кодах колонки сумм совпадают."""
+    if len(sheet_codes) < rules.code_ranges.min_codes_continuation:
+        return False
+    form_section = {DocumentType.BALANCE: CodeSection.BALANCE, DocumentType.INCOME: CodeSection.INCOME}.get(form_type)
+    for code in sheet_codes:
+        section = section_of(code, rules)
+        if section is None or document_type_of(section) is not form_type:
+            return False
+        if form_section is not None and section is not form_section:
+            return False
+    if len(sheet_codes) >= rules.code_ranges.min_codes:
+        return sheet_amount_columns == form_amount_columns
+    return True

arm_statements/tests/test_text_codes.py

@@ -0,0 +1,74 @@
+import pytest
+
+from arm_statements.codes import is_code_cell, majority_section, parse_code, sheet_joins
+from arm_statements.loader import CodeSection, Rules
+from arm_statements.model import DocumentType
+from arm_statements.text import fix_lookalikes, is_disguised_digit, normalize_caption, normalize_label
+
+
+@pytest.mark.parametrize(
+    ("raw", "code", "fixed"),
+    [
+        ("1220 ,", "1220", False),
+        ("| 140", "1140", True),
+        ("I1О0", "1100", True),
+        ("1110.0", "1110", False),
+        ("12301", "12301", False),
+        ("3450", "3450", False),
+        ("2024", None, False),
+        ("1050", None, False),
+        ("7450", None, False),
+        ("ИЧ,250", None, False),
+        ("изо :", None, False),
+        ("НИ", None, False),
+        ("1500\n1700", None, False),
+        ("2", None, False),
+    ],
+)
+def test_parse_code(rules: Rules, raw: str, code: str | None, fixed: bool) -> None:
+    assert parse_code(raw, rules) == (code, fixed)
+
+
+def test_multiline_code_cell(rules: Rules) -> None:
+    assert is_code_cell("1500\n1700", rules)
+    assert not is_code_cell("44 30\nI |", rules)
+
+
+def test_lookalikes_only_near_digit(rules: Rules) -> None:
+    assert fix_lookalikes("2ОО5", rules.lookalikes) == "2005"
+    assert fix_lookalikes("2|0|2|4|", rules.lookalikes).split() == ["2", "0", "2", "4"]
+    assert fix_lookalikes("Основные средства", rules.lookalikes) == "Основные средства"
+    assert fix_lookalikes("З", rules.lookalikes) == "З"
+    assert fix_lookalikes("З", rules.lookalikes, anywhere=True) == "3"
+
+
+def test_disguised_digit(rules: Rules) -> None:
+    assert is_disguised_digit("I", rules.lookalikes) and is_disguised_digit("| О", rules.lookalikes)
+    assert not is_disguised_digit("Косарев", rules.lookalikes)
+    assert not is_disguised_digit("(расшифровка подписи)", rules.lookalikes)
+
+
+def test_caption_normalization() -> None:
+    assert normalize_caption("Код  строки") == "код строки"
+    assert normalize_caption("Изменение, %") == "изменение %"
+    assert normalize_caption("Наименование\nпоказателя") == "наименование показателя"
+
+
+def test_label_normalization_keeps_punctuation(rules: Rules) -> None:
+    assert normalize_label("Местонахождение (адрес)") == "местонахождение (адрес)"
+    assert normalize_label("по ОКОПФ / ОКФС") == "по окопф / окфс"
+    assert normalize_label("Форма 0710001 с.2") == "форма 0710001 с.2"
+    assert normalize_label("Отчёт о финансовых\nрезультатах") == "отчет о финансовых результатах"
+    assert normalize_label("за­дол­жен—ность") == "задолжен ность"
+    assert normalize_label(None) == ""
+    for key, patterns in rules.headers.labels.items():
+        assert patterns, key
+    assert any(p.search(normalize_label("Местонахождение (адрес)")) for p in rules.headers.labels["address"])
+    assert any(p.search(normalize_label("по ОКОПФ / ОКФС")) for p in rules.headers.labels["okopf_okfs"])
+
+
+def test_majority_section_and_join(rules: Rules) -> None:
+    assert majority_section(["1110", "1150", "2110"], rules) is CodeSection.BALANCE
+    assert sheet_joins(DocumentType.INCOME, ["2400"], 1, 3, rules)
+    assert not sheet_joins(DocumentType.BALANCE, ["1110", "1150", "1100"], 4, 3, rules)
+    assert not sheet_joins(DocumentType.BALANCE, ["2110"], 3, 3, rules)

arm_statements/text.py

@@ -0,0 +1,60 @@
+"""Очистка OCR-текста: буквы-двойники, нормализация подписей и меток (§11.1 п. 1, 6)."""
+
+import re
+from collections.abc import Mapping
+
+# первая строка из цифр, пробелов, `|`, точек, запятых и двоеточий: `|` в ней — разделитель клеток формы
+_CELL_SEPARATOR_LINE = re.compile(r"[\s\d|•·.:,]*")
+_INVISIBLE = re.compile("[\u00ad\u200b\ufeff]")
+_SEPARATORS = re.compile(r"[\s\-\u2010-\u2015\u2212]+")
+_NON_CAPTION = re.compile(r"[^\w%]|_")
+
+
+def replace_lookalikes(text: str, lookalikes: Mapping[str, str], *, anywhere: bool = False) -> str:
+    """Двойники → цифры вплотную к цифре (несколько проходов); `anywhere` — без условия соседства."""
+    if not lookalikes:
+        return text
+    table = str.maketrans(dict(lookalikes))
+    if anywhere:
+        return text.translate(table)
+    chars = re.escape("".join(lookalikes))
+    near_digit = re.compile(rf"(?<=\d)[{chars}]|[{chars}](?=\d)")
+    previous = None
+    while previous != text:  # «2ОО5» требует двух проходов
+        previous = text
+        text = near_digit.sub(lambda match: match.group(0).translate(table), text)
+    return text
+
+
+def fix_lookalikes(text: str, lookalikes: Mapping[str, str], *, anywhere: bool = False) -> str:
+    """NBSP → пробел; `|` — разделитель клеток, если первая строка из цифр и пунктуации; двойники → цифры."""
+    text = text.replace("\xa0", " ")
+    if _CELL_SEPARATOR_LINE.fullmatch(text.split("\n", 1)[0]):
+        text = text.replace("|", " ")
+    return replace_lookalikes(text, lookalikes, anywhere=anywhere)
+
+
+def has_digit(text: str | None) -> bool:
+    return text is not None and any(char.isdigit() for char in text)
+
+
+def is_disguised_digit(text: str, lookalikes: Mapping[str, str]) -> bool:
+    """После замены двойников везде есть цифра и нет других букв («I», «| О» — да; «Косарев» — нет)."""
+    fixed = fix_lookalikes(text, lookalikes, anywhere=True)
+    return has_digit(fixed) and not any(char.isalpha() for char in fixed)
+
+
+def normalize_caption(text: str | None) -> str:
+    """Нижний регистр, `ё → е`, всё кроме букв, цифр и `%` → пробел, пробелы схлопнуты."""
+    if text is None:
+        return ""
+    lowered = text.lower().replace("ё", "е")
+    return " ".join(_NON_CAPTION.sub(" ", lowered).split())
+
+
+def normalize_label(text: str | None) -> str:
+    """Метки шапки: невидимые удалены, нижний регистр, `ё → е`, пробелы и дефисы/тире → пробел; пунктуация сохранена."""
+    if text is None:
+        return ""
+    lowered = _INVISIBLE.sub("", text).lower().replace("ё", "е")
+    return _SEPARATORS.sub(" ", lowered).strip()