diff --git a/arm_statements/codes.py b/arm_statements/codes.py new file mode 100644 index 0000000..fb8859c --- /dev/null +++ b/arm_statements/codes.py @@ -0,0 +1,118 @@ +"""Разбор кода строки и отрезки кодов форм.""" + +import re +from collections import Counter +from collections.abc import Iterable, Mapping, Sequence +from typing import NamedTuple + +from arm_statements.loader import CodeSection, Rules +from arm_statements.model import DocumentType +from arm_statements.text import replace_lookalikes + +_CODE = re.compile(r"\d{4,5}(?:\.0)?") +_BAR = "|" + + +class CodeParse(NamedTuple): + code: str | None + fixed: bool # при разборе заменён двойник + + +def code_parts(text: str | None) -> tuple[str, ...]: + """Непустые строки ячейки, обрезанные.""" + if text is None: + return () + return tuple(part.strip() for part in text.splitlines() if part.strip()) + + +def section_of(code: str, rules: Rules) -> CodeSection | None: + head = int(code[:4]) + for item in rules.code_ranges.ranges: + if item.low <= head <= item.high: + return item.section + return None + + +def _strip_edges(text: str) -> str: + start, end = 0, len(text) + while start < end and not text[start].isalnum(): + start += 1 + while end > start and not text[end - 1].isalnum(): + end -= 1 + return text[start:end] + + +def _as_code(text: str, rules: Rules) -> str | None: + core = _strip_edges(text) + if _CODE.fullmatch(core) is None: + return None + code = core.split(".", 1)[0] + return code if section_of(code, rules) is not None else None + + +def _parse_line(line: str, rules: Rules) -> CodeParse: + compact = "".join(line.split()) + # сначала `|` как двойник единицы; если результата-кода нет — `|` как краевая пунктуация + for candidate in (compact, compact.replace(_BAR, "")): + replaced = replace_lookalikes(candidate, rules.lookalikes) + code = _as_code(replaced, rules) + if code is not None: + return CodeParse(code, replaced != candidate) + return CodeParse(None, False) + + +def parse_code(text: str | None, rules: Rules) -> CodeParse: + """Однострочный текст → код; многострочный → `CodeParse(None, False)`.""" + parts = code_parts(text) + if len(parts) != 1: + return CodeParse(None, False) + return _parse_line(parts[0], rules) + + +def is_code_cell(text: str | None, rules: Rules) -> bool: + parts = code_parts(text) + return bool(parts) and all(_parse_line(part, rules).code is not None for part in parts) + + +def majority_section(codes: Iterable[str], rules: Rules) -> CodeSection | None: + """Отрезок большинства; ничья — первый по порядку отрезков словаря.""" + counts: Counter[CodeSection] = Counter() + for code in codes: + section = section_of(code, rules) + if section is not None: + counts[section] += 1 + best: CodeSection | None = None + for item in rules.code_ranges.ranges: + if counts[item.section] and (best is None or counts[item.section] > counts[best]): + best = item.section + return best + + +def document_type_of(section: CodeSection) -> DocumentType: + mapping: Mapping[CodeSection, DocumentType] = { + CodeSection.BALANCE: DocumentType.BALANCE, + CodeSection.INCOME: DocumentType.INCOME, + } + return mapping.get(section, DocumentType.DISCLOSURE) + + +def sheet_joins( + form_type: DocumentType, + sheet_codes: Sequence[str], + sheet_amount_columns: int, + form_amount_columns: int, + rules: Rules, +) -> bool: + """Лист без маркера и шапки присоединяется к форме: коды в её отрезке; при ≥3 кодах колонки сумм совпадают.""" + if len(sheet_codes) < rules.code_ranges.min_codes_continuation: + return False + form_section = {DocumentType.BALANCE: CodeSection.BALANCE, DocumentType.INCOME: CodeSection.INCOME}.get(form_type) + for code in sheet_codes: + section = section_of(code, rules) + if section is None or document_type_of(section) is not form_type: + return False + if form_section is not None and section is not form_section: + return False + if len(sheet_codes) >= rules.code_ranges.min_codes: + return sheet_amount_columns == form_amount_columns + return True diff --git a/arm_statements/tests/test_text_codes.py b/arm_statements/tests/test_text_codes.py new file mode 100644 index 0000000..17567c7 --- /dev/null +++ b/arm_statements/tests/test_text_codes.py @@ -0,0 +1,74 @@ +import pytest + +from arm_statements.codes import is_code_cell, majority_section, parse_code, sheet_joins +from arm_statements.loader import CodeSection, Rules +from arm_statements.model import DocumentType +from arm_statements.text import fix_lookalikes, is_disguised_digit, normalize_caption, normalize_label + + +@pytest.mark.parametrize( + ("raw", "code", "fixed"), + [ + ("1220 ,", "1220", False), + ("| 140", "1140", True), + ("I1О0", "1100", True), + ("1110.0", "1110", False), + ("12301", "12301", False), + ("3450", "3450", False), + ("2024", None, False), + ("1050", None, False), + ("7450", None, False), + ("ИЧ,250", None, False), + ("изо :", None, False), + ("НИ", None, False), + ("1500\n1700", None, False), + ("2", None, False), + ], +) +def test_parse_code(rules: Rules, raw: str, code: str | None, fixed: bool) -> None: + assert parse_code(raw, rules) == (code, fixed) + + +def test_multiline_code_cell(rules: Rules) -> None: + assert is_code_cell("1500\n1700", rules) + assert not is_code_cell("44 30\nI |", rules) + + +def test_lookalikes_only_near_digit(rules: Rules) -> None: + assert fix_lookalikes("2ОО5", rules.lookalikes) == "2005" + assert fix_lookalikes("2|0|2|4|", rules.lookalikes).split() == ["2", "0", "2", "4"] + assert fix_lookalikes("Основные средства", rules.lookalikes) == "Основные средства" + assert fix_lookalikes("З", rules.lookalikes) == "З" + assert fix_lookalikes("З", rules.lookalikes, anywhere=True) == "3" + + +def test_disguised_digit(rules: Rules) -> None: + assert is_disguised_digit("I", rules.lookalikes) and is_disguised_digit("| О", rules.lookalikes) + assert not is_disguised_digit("Косарев", rules.lookalikes) + assert not is_disguised_digit("(расшифровка подписи)", rules.lookalikes) + + +def test_caption_normalization() -> None: + assert normalize_caption("Код строки") == "код строки" + assert normalize_caption("Изменение, %") == "изменение %" + assert normalize_caption("Наименование\nпоказателя") == "наименование показателя" + + +def test_label_normalization_keeps_punctuation(rules: Rules) -> None: + assert normalize_label("Местонахождение (адрес)") == "местонахождение (адрес)" + assert normalize_label("по ОКОПФ / ОКФС") == "по окопф / окфс" + assert normalize_label("Форма 0710001 с.2") == "форма 0710001 с.2" + assert normalize_label("Отчёт о финансовых\nрезультатах") == "отчет о финансовых результатах" + assert normalize_label("за­дол­жен—ность") == "задолжен ность" + assert normalize_label(None) == "" + for key, patterns in rules.headers.labels.items(): + assert patterns, key + assert any(p.search(normalize_label("Местонахождение (адрес)")) for p in rules.headers.labels["address"]) + assert any(p.search(normalize_label("по ОКОПФ / ОКФС")) for p in rules.headers.labels["okopf_okfs"]) + + +def test_majority_section_and_join(rules: Rules) -> None: + assert majority_section(["1110", "1150", "2110"], rules) is CodeSection.BALANCE + assert sheet_joins(DocumentType.INCOME, ["2400"], 1, 3, rules) + assert not sheet_joins(DocumentType.BALANCE, ["1110", "1150", "1100"], 4, 3, rules) + assert not sheet_joins(DocumentType.BALANCE, ["2110"], 3, 3, rules) diff --git a/arm_statements/text.py b/arm_statements/text.py new file mode 100644 index 0000000..70ac37d --- /dev/null +++ b/arm_statements/text.py @@ -0,0 +1,60 @@ +"""Очистка OCR-текста: буквы-двойники, нормализация подписей и меток (§11.1 п. 1, 6).""" + +import re +from collections.abc import Mapping + +# первая строка из цифр, пробелов, `|`, точек, запятых и двоеточий: `|` в ней — разделитель клеток формы +_CELL_SEPARATOR_LINE = re.compile(r"[\s\d|•·.:,]*") +_INVISIBLE = re.compile("[\u00ad\u200b\ufeff]") +_SEPARATORS = re.compile(r"[\s\-\u2010-\u2015\u2212]+") +_NON_CAPTION = re.compile(r"[^\w%]|_") + + +def replace_lookalikes(text: str, lookalikes: Mapping[str, str], *, anywhere: bool = False) -> str: + """Двойники → цифры вплотную к цифре (несколько проходов); `anywhere` — без условия соседства.""" + if not lookalikes: + return text + table = str.maketrans(dict(lookalikes)) + if anywhere: + return text.translate(table) + chars = re.escape("".join(lookalikes)) + near_digit = re.compile(rf"(?<=\d)[{chars}]|[{chars}](?=\d)") + previous = None + while previous != text: # «2ОО5» требует двух проходов + previous = text + text = near_digit.sub(lambda match: match.group(0).translate(table), text) + return text + + +def fix_lookalikes(text: str, lookalikes: Mapping[str, str], *, anywhere: bool = False) -> str: + """NBSP → пробел; `|` — разделитель клеток, если первая строка из цифр и пунктуации; двойники → цифры.""" + text = text.replace("\xa0", " ") + if _CELL_SEPARATOR_LINE.fullmatch(text.split("\n", 1)[0]): + text = text.replace("|", " ") + return replace_lookalikes(text, lookalikes, anywhere=anywhere) + + +def has_digit(text: str | None) -> bool: + return text is not None and any(char.isdigit() for char in text) + + +def is_disguised_digit(text: str, lookalikes: Mapping[str, str]) -> bool: + """После замены двойников везде есть цифра и нет других букв («I», «| О» — да; «Косарев» — нет).""" + fixed = fix_lookalikes(text, lookalikes, anywhere=True) + return has_digit(fixed) and not any(char.isalpha() for char in fixed) + + +def normalize_caption(text: str | None) -> str: + """Нижний регистр, `ё → е`, всё кроме букв, цифр и `%` → пробел, пробелы схлопнуты.""" + if text is None: + return "" + lowered = text.lower().replace("ё", "е") + return " ".join(_NON_CAPTION.sub(" ", lowered).split()) + + +def normalize_label(text: str | None) -> str: + """Метки шапки: невидимые удалены, нижний регистр, `ё → е`, пробелы и дефисы/тире → пробел; пунктуация сохранена.""" + if text is None: + return "" + lowered = _INVISIBLE.sub("", text).lower().replace("ё", "е") + return _SEPARATORS.sub(" ", lowered).strip()