From f5cb60a1cb2997ef190867a3fc7b7919cc4e56e8 Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Sun, 18 Oct 2026 13:11:38 +0400 Subject: [PATCH 01/28] =?UTF-8?q?feat:=20=D0=BA=D0=B0=D1=80=D0=BA=D0=B0?= =?UTF-8?q?=D1=81=20=D0=BF=D0=B0=D0=BA=D0=B5=D1=82=D0=B0=20arm=5Fstatement?= =?UTF-8?q?s,=20make=20check,=20=D0=BC=D0=B0=D1=80=D0=BA=D0=B5=D1=80=20?= =?UTF-8?q?=D0=B2=D0=B5=D1=80=D1=81=D0=B8=D0=B8=20=D0=BA=D0=BE=D0=BF=D0=B8?= =?UTF-8?q?=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W --- .gitignore | 18 +++++++++++++++ Makefile | 23 +++++++++++++++++++ README.md | 10 ++++++++ arm_statements/__init__.py | 5 ++++ arm_statements/tests/__init__.py | 0 arm_statements/tests/conftest.py | 1 + arm_statements/tests/test_version.py | 21 +++++++++++++++++ arm_statements/version.py | 14 ++++++++++++ pyproject.toml | 34 ++++++++++++++++++++++++++++ 9 files changed, 126 insertions(+) create mode 100644 Makefile create mode 100644 arm_statements/__init__.py create mode 100644 arm_statements/tests/__init__.py create mode 100644 arm_statements/tests/conftest.py create mode 100644 arm_statements/tests/test_version.py create mode 100644 arm_statements/version.py create mode 100644 pyproject.toml diff --git a/.gitignore b/.gitignore index 4b7e182..bff27ec 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,21 @@ .superpowers/ .bounded .DS_Store + +# Python +__pycache__/ +*.py[cod] +*.egg-info/ +build/ +dist/ +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.coverage +htmlcov/ +.venv/ +.env + +# Проект +arm_statements/VERSION +.codegraph/ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..6f3bc05 --- /dev/null +++ b/Makefile @@ -0,0 +1,23 @@ +BIN := $(CURDIR)/.venv/bin + +.PHONY: help venv lint test check golden-update + +help: ## Список целей + @grep -E '^[a-z-]+:.*##' $(MAKEFILE_LIST) | sed 's/:.*##/ —/' + +venv: ## Создать .venv и поставить зависимости (с dev) + uv venv --python 3.11 --allow-existing .venv + uv pip install --python .venv/bin/python -r pyproject.toml --extra dev + +lint: ## ruff check, ruff format --check, mypy + $(BIN)/ruff check . + $(BIN)/ruff format --check . + $(BIN)/mypy arm_statements + +test: ## pytest + $(BIN)/pytest + +check: lint test ## lint + test + +golden-update: ## Обновить эталоны golden-теста (задача 20) + GOLDEN_UPDATE=1 $(BIN)/pytest arm_statements/tests/test_golden.py diff --git a/README.md b/README.md index 208fa1d..b5b0bbe 100644 --- a/README.md +++ b/README.md @@ -3,3 +3,13 @@ Общий код нормализации форм ББ/ОФР (пакет `arm_statements`) для tabular-parser, scan-gateway и arm-core. Раскладывается копией в `api/arm_statements/` хостов скриптом `sync-statements` из arm-ka-docs. Спека: arm-ka-docs, `openspec/changes/unified-statement-normalization/`. + +Копия раскладывается `make -C docs sync-statements`; прямые правки копии в хостах запрещены. + +## Как проверить + +```bash +make venv && make check +``` + +Раздел «Публичный API» дописывается в задаче 20. diff --git a/arm_statements/__init__.py b/arm_statements/__init__.py new file mode 100644 index 0000000..6339e26 --- /dev/null +++ b/arm_statements/__init__.py @@ -0,0 +1,5 @@ +"""Общий код нормализации форм ББ/ОФР.""" + +from arm_statements.version import statements_version + +__all__ = ["statements_version"] diff --git a/arm_statements/tests/__init__.py b/arm_statements/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/arm_statements/tests/conftest.py b/arm_statements/tests/conftest.py new file mode 100644 index 0000000..c3a717e --- /dev/null +++ b/arm_statements/tests/conftest.py @@ -0,0 +1 @@ +"""Общие фикстуры тестов пакета (фикстура `rules` появится в задаче 3).""" diff --git a/arm_statements/tests/test_version.py b/arm_statements/tests/test_version.py new file mode 100644 index 0000000..be00685 --- /dev/null +++ b/arm_statements/tests/test_version.py @@ -0,0 +1,21 @@ +from pathlib import Path + +from arm_statements.version import statements_version + +FULL_HASH = "abc1234def5678abc1234def5678abc1234def56" + + +def test_version_absent_in_source() -> None: + assert statements_version() is None + + +def test_version_read_from_file(tmp_path: Path) -> None: + marker = tmp_path / "VERSION" + marker.write_text(f"{FULL_HASH}\n", encoding="utf-8") + assert statements_version(marker) == FULL_HASH + + +def test_version_empty_file_is_none(tmp_path: Path) -> None: + marker = tmp_path / "VERSION" + marker.write_text(" \n", encoding="utf-8") + assert statements_version(marker) is None diff --git a/arm_statements/version.py b/arm_statements/version.py new file mode 100644 index 0000000..d9e2ad5 --- /dev/null +++ b/arm_statements/version.py @@ -0,0 +1,14 @@ +"""Маркер версии копии пакета: файл VERSION пишет только sync-statements в хосте.""" + +from pathlib import Path + +_DEFAULT_MARKER = Path(__file__).parent / "VERSION" + + +def statements_version(path: Path | None = None) -> str | None: + """Вернуть версию копии (хеш коммита) или None, если файла нет или он пуст.""" + marker = _DEFAULT_MARKER if path is None else path + if not marker.is_file(): + return None + text = marker.read_text(encoding="utf-8").strip() + return text or None diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..119e89a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,34 @@ +[project] +name = "arm-statements" +version = "0" +requires-python = ">=3.11,<3.12" +dependencies = ["PyYAML==6.0.3"] + +[project.optional-dependencies] +dev = [ + "ruff==0.16.10", + "mypy==2.4.0", + "pytest==9.1.1", + "types-PyYAML==6.0.12.20260906", +] + +[tool.ruff] +line-length = 120 +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "F", "I", "UP", "B", "PL"] + +[tool.ruff.lint.isort] +known-first-party = ["arm_statements"] + +[tool.mypy] +python_version = "3.11" +strict = true +mypy_path = "." +explicit_package_bases = true + +[tool.pytest.ini_options] +testpaths = ["arm_statements/tests"] +pythonpath = ["."] +addopts = "--tb=short --strict-markers" -- 2.47.2 From 03a7ae1a596deb95f202d6fcdddac10a64406288 Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Sun, 18 Oct 2026 13:13:31 +0400 Subject: [PATCH 02/28] =?UTF-8?q?feat:=20=D1=82=D0=B8=D0=BF=D1=8B=20=D1=88?= =?UTF-8?q?=D0=B0=D0=B3=D0=B0=201=20=D0=B8=20=D0=B2=D1=8B=D1=85=D0=BE?= =?UTF-8?q?=D0=B4=D0=B0=20=D1=88=D0=B0=D0=B3=D0=BE=D0=B2=202=E2=80=938,=20?= =?UTF-8?q?=D1=81=D0=B5=D1=80=D0=B8=D0=B0=D0=BB=D0=B8=D0=B7=D0=B0=D1=86?= =?UTF-8?q?=D0=B8=D1=8F=20=D0=BA=D0=B0=D0=BD=D0=B4=D0=B8=D0=B4=D0=B0=D1=82?= =?UTF-8?q?=D0=B0=20=D0=B2=20json?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W --- arm_statements/model.py | 246 +++++++++++++++++++++++++++++ arm_statements/serialize.py | 108 +++++++++++++ arm_statements/tests/builders.py | 144 +++++++++++++++++ arm_statements/tests/test_model.py | 37 +++++ 4 files changed, 535 insertions(+) create mode 100644 arm_statements/model.py create mode 100644 arm_statements/serialize.py create mode 100644 arm_statements/tests/builders.py create mode 100644 arm_statements/tests/test_model.py diff --git a/arm_statements/model.py b/arm_statements/model.py new file mode 100644 index 0000000..8b6e451 --- /dev/null +++ b/arm_statements/model.py @@ -0,0 +1,246 @@ +"""Типы входа (кандидат формы) и выхода (нормализованный документ) пакета.""" + +from dataclasses import dataclass +from datetime import date +from decimal import Decimal +from enum import StrEnum + + +class Source(StrEnum): + OCR = "ocr" + EXCEL = "excel" + + +class DocumentType(StrEnum): + BALANCE = "BALANCE" + INCOME = "INCOME" + DISCLOSURE = "DISCLOSURE" + + +class FormVariant(StrEnum): + FULL = "full" + SIMPLIFIED = "simplified" + + +class Edition(StrEnum): + RAS_66N = "66n" + FSBU4 = "fsbu4" + + +class PeriodKind(StrEnum): + ABSOLUTE = "absolute" + CURRENT = "relative:current" + PREV_YEAR_END = "relative:prev_year_end" + PREV_PREV_YEAR_END = "relative:prev_prev_year_end" + PREV_PERIOD = "relative:prev_period" + NONE = "none" + + +class LineStatus(StrEnum): + OK = "ok" + UNREADABLE = "unreadable" + MISSING = "missing" + DETAIL = "detail" + + +class Severity(StrEnum): + ERROR = "error" + INFO = "info" + + +class DocStatus(StrEnum): + DONE = "done" + NEEDS_CHECK = "needs_check" + + +class EnvelopeStatus(StrEnum): + DONE = "done" + NEEDS_CHECK = "needs_check" + ERROR = "error" + + +class SignForm(StrEnum): + PARENTHESES = "parentheses" + MINUS = "minus" + UNSIGNED = "unsigned" + + +class RowRole(StrEnum): + TITLE = "title" + HEADER = "header" + NUMBERING = "numbering" + DATA = "data" + + +CellValue = Decimal | date | bool | None + + +@dataclass(frozen=True, slots=True) +class Cell: + text: str + value: CellValue + confidence: float + ref: str + + +@dataclass(frozen=True, slots=True) +class Evidence: + okud: Cell | None = None + title: Cell | None = None + + +@dataclass(frozen=True, slots=True) +class HeaderCells: + organization_name: Cell | None = None + inn: Cell | None = None + report_date: Cell | None = None + units: Cell | None = None + okud: Cell | None = None + okpo: Cell | None = None + okopf: Cell | None = None + okfs: Cell | None = None + okei: Cell | None = None + address: Cell | None = None + + +HEADER_FIELDS: tuple[str, ...] = ( + "organization_name", + "inn", + "report_date", + "units", + "okud", + "okpo", + "okopf", + "okfs", + "okei", + "address", +) + + +@dataclass(frozen=True, slots=True) +class StatementWarning: + code: str + severity: Severity + message: str + line_refs: tuple[str, ...] = () + column_index: int | None = None + field: str | None = None + locator: str | None = None + + +@dataclass(frozen=True, slots=True) +class FormCandidate: + source: Source + locator: str + type_hint: DocumentType | None + evidence: Evidence + header: HeaderCells + rows: tuple[tuple[Cell | None, ...], ...] + segment_breaks: tuple[int, ...] = () + warnings: tuple[StatementWarning, ...] = () + + +@dataclass(frozen=True, slots=True) +class HeaderField: + text: str | None + confidence: float | None + value: str | None + + +@dataclass(frozen=True, slots=True) +class DateField: + text: str | None + confidence: float | None + value: date | None + + +@dataclass(frozen=True, slots=True) +class StatementHeader: + organization_name: HeaderField + inn: HeaderField + report_date: DateField + units: HeaderField + okud: HeaderField + okpo: HeaderField + okopf: HeaderField + okfs: HeaderField + okei: HeaderField + address: HeaderField + form_name: str | None = None + needs_review_fields: tuple[str, ...] = () + + +@dataclass(frozen=True, slots=True) +class StatementColumn: + column_index: int + label: str + label_confidence: float | None + period_date: date | None + period_kind: PeriodKind + + +@dataclass(frozen=True, slots=True) +class StatementValue: + column_index: int + amount: Decimal | None + string_value: str | None + text: str | None + confidence: float + is_recognized: bool + needs_review: bool = False + + +@dataclass(frozen=True, slots=True) +class StatementLine: + row_index: int + ref: str + line_status: LineStatus + line_code: str | None + line_code_text: str | None + line_code_confidence: float | None + line_name: str | None + line_name_confidence: float | None + parent_ref: str | None + needs_review: bool + values: tuple[StatementValue, ...] + + +@dataclass(frozen=True, slots=True) +class FailedRelation: + column_index: int | None + column_key: str + relation: str + codes: tuple[str, ...] + expected: Decimal | None + actual: Decimal | None + + +@dataclass(frozen=True, slots=True) +class StatementChecks: + relations_failed: tuple[FailedRelation, ...] + soft_relations_failed: tuple[FailedRelation, ...] + + +@dataclass(frozen=True, slots=True) +class StatementDocument: + document_type: DocumentType + form_variant: FormVariant | None + locator: str + header: StatementHeader + columns: tuple[StatementColumn, ...] + lines: tuple[StatementLine, ...] + checks: StatementChecks | None + status: DocStatus + warnings: tuple[StatementWarning, ...] + + +@dataclass(frozen=True, slots=True) +class CandidateResult: + document: StatementDocument | None + file_warnings: tuple[StatementWarning, ...] + + +@dataclass(frozen=True, slots=True) +class NormalizationResult: + documents: tuple[StatementDocument, ...] + file_warnings: tuple[StatementWarning, ...] diff --git a/arm_statements/serialize.py b/arm_statements/serialize.py new file mode 100644 index 0000000..03ebf8a --- /dev/null +++ b/arm_statements/serialize.py @@ -0,0 +1,108 @@ +"""Сериализация в json-совместимые значения и обратно (для golden-тестов и обмена).""" + +from collections.abc import Mapping +from dataclasses import fields, is_dataclass +from datetime import date +from decimal import Decimal +from enum import Enum +from typing import Any + +from arm_statements.model import ( + Cell, + CellValue, + DocumentType, + Evidence, + FormCandidate, + HeaderCells, + Severity, + Source, + StatementWarning, +) + + +def _value_to_jsonable(value: CellValue) -> Any: + """Тег типа значения ячейки; bool проверяется раньше прочего (не подкласс Decimal/date, но явно).""" + if value is None: + return None + if isinstance(value, bool): + return {"bool": value} + if isinstance(value, Decimal): + return {"decimal": str(value)} + return {"date": value.isoformat()} + + +def _scalar_to_jsonable(obj: object) -> Any: + if isinstance(obj, Enum): + return obj.value + if isinstance(obj, Decimal): + return str(obj) + if isinstance(obj, date): + return obj.isoformat() + return obj + + +def to_jsonable(obj: object) -> Any: + """Dataclass пакета → json-совместимое значение.""" + if isinstance(obj, Cell): + return { + "text": obj.text, + "value": _value_to_jsonable(obj.value), + "confidence": obj.confidence, + "ref": obj.ref, + } + if is_dataclass(obj) and not isinstance(obj, type): + return {f.name: to_jsonable(getattr(obj, f.name)) for f in fields(obj)} + if isinstance(obj, tuple | list): + return [to_jsonable(item) for item in obj] + return _scalar_to_jsonable(obj) + + +def _value_from_dict(raw: Mapping[str, Any] | None) -> CellValue: + if raw is None: + return None + if "decimal" in raw: + return Decimal(raw["decimal"]) + if "date" in raw: + return date.fromisoformat(raw["date"]) + if "bool" in raw: + return bool(raw["bool"]) + raise ValueError(f"неизвестный тег значения ячейки: {sorted(raw)}") + + +def _cell_from_dict(raw: Mapping[str, Any] | None) -> Cell | None: + if raw is None: + return None + return Cell( + text=raw["text"], + value=_value_from_dict(raw["value"]), + confidence=float(raw["confidence"]), + ref=raw["ref"], + ) + + +def _warning_from_dict(raw: Mapping[str, Any]) -> StatementWarning: + return StatementWarning( + code=raw["code"], + severity=Severity(raw["severity"]), + message=raw["message"], + line_refs=tuple(raw.get("line_refs", ())), + column_index=raw.get("column_index"), + field=raw.get("field"), + locator=raw.get("locator"), + ) + + +def candidate_from_dict(data: Mapping[str, Any]) -> FormCandidate: + """Обратная к `to_jsonable` для кандидата.""" + type_hint = data["type_hint"] + evidence = data["evidence"] + return FormCandidate( + source=Source(data["source"]), + locator=data["locator"], + type_hint=None if type_hint is None else DocumentType(type_hint), + evidence=Evidence(okud=_cell_from_dict(evidence["okud"]), title=_cell_from_dict(evidence["title"])), + header=HeaderCells(**{name: _cell_from_dict(raw) for name, raw in data["header"].items()}), + rows=tuple(tuple(_cell_from_dict(raw) for raw in row) for row in data["rows"]), + segment_breaks=tuple(data.get("segment_breaks", ())), + warnings=tuple(_warning_from_dict(raw) for raw in data.get("warnings", ())), + ) diff --git a/arm_statements/tests/builders.py b/arm_statements/tests/builders.py new file mode 100644 index 0000000..476b2f5 --- /dev/null +++ b/arm_statements/tests/builders.py @@ -0,0 +1,144 @@ +"""Сборщики тестовых кандидатов (только для тестов пакета).""" + +from collections.abc import Mapping, Sequence +from decimal import Decimal + +from arm_statements.model import ( + HEADER_FIELDS, + Cell, + DocumentType, + Evidence, + FormCandidate, + HeaderCells, + Source, + StatementWarning, +) + +OcrInput = str | tuple[str, float] | None +ExcelInput = str | Decimal | None +PlaceInput = str | Decimal | tuple[str, float] + +_FULL_CONFIDENCE = 100.0 +_LATIN_LETTERS = 26 + + +def _text_conf(item: str | tuple[str, float]) -> tuple[str, float]: + if isinstance(item, tuple): + return item + return item, _FULL_CONFIDENCE + + +def hdr(**fields: str | tuple[str, float]) -> HeaderCells: + """Шапка из текстов; ref — `hdr:<поле>`.""" + unknown = set(fields) - set(HEADER_FIELDS) + if unknown: + raise TypeError(f"неизвестные поля шапки: {sorted(unknown)}") + cells: dict[str, Cell] = {} + for name, item in fields.items(): + text, conf = _text_conf(item) + cells[name] = Cell(text, None, conf, f"hdr:{name}") + return HeaderCells(**cells) + + +def _evidence(okud: str | None, title: str | None) -> Evidence: + return Evidence( + okud=None if okud is None else Cell(okud, None, _FULL_CONFIDENCE, "evidence:okud"), + title=None if title is None else Cell(title, None, _FULL_CONFIDENCE, "evidence:title"), + ) + + +def ocr( # noqa: PLR0913 - сигнатура задана планом (Task 2) + rows: Sequence[Sequence[OcrInput]], + *, + type_hint: DocumentType | None = None, + header: HeaderCells | None = None, + title: str | None = None, + okud: str | None = None, + locator: str = "file:t.json;table:0", +) -> FormCandidate: + """Кандидат OCR: строка r → ячейки с ref `row:`.""" + grid: list[tuple[Cell | None, ...]] = [] + for r, row in enumerate(rows): + cells: list[Cell | None] = [] + for item in row: + if item is None: + cells.append(None) + else: + text, conf = _text_conf(item) + cells.append(Cell(text, None, conf, f"row:{r}")) + grid.append(tuple(cells)) + return FormCandidate( + source=Source.OCR, + locator=locator, + type_hint=type_hint, + evidence=_evidence(okud, title), + header=header if header is not None else HeaderCells(), + rows=tuple(grid), + ) + + +def _column_letters(index: int) -> str: + """0 → A, 25 → Z, 26 → AA.""" + letters = "" + n = index + 1 + while n > 0: + n, rem = divmod(n - 1, _LATIN_LETTERS) + letters = chr(ord("A") + rem) + letters + return letters + + +def _sheet_prefix(name: str) -> str: + """Имя листа в ссылке по правилу Excel: кавычки, если не только буквы/цифры/_ или начинается с цифры.""" + plain = all(ch.isalpha() or ch.isdigit() or ch == "_" for ch in name) and not name[:1].isdigit() + if plain: + return name + return "'" + name.replace("'", "''") + "'" + + +def _thousands(value: Decimal) -> str: + return f"{value:,}".replace(",", " ") + + +def excel( # noqa: PLR0913 - сигнатура задана планом (Task 2) + sheets: Sequence[tuple[str, Sequence[Sequence[ExcelInput]]]], + *, + type_hint: DocumentType | None = None, + header: HeaderCells | None = None, + title: str | None = None, + okud: str | None = None, + warnings: Sequence[StatementWarning] = (), +) -> FormCandidate: + """Кандидат Excel: листы подряд в одной сетке, ref `<лист>!`.""" + grid: list[tuple[Cell | None, ...]] = [] + breaks: list[int] = [] + for sheet_no, (name, sheet_rows) in enumerate(sheets): + if sheet_no > 0: + breaks.append(len(grid)) + prefix = _sheet_prefix(name) + for r, row in enumerate(sheet_rows): + cells: list[Cell | None] = [] + for c, item in enumerate(row): + ref = f"{prefix}!{_column_letters(c)}{r + 1}" + if item is None: + cells.append(None) + elif isinstance(item, Decimal): + cells.append(Cell(_thousands(item), item, _FULL_CONFIDENCE, ref)) + else: + cells.append(Cell(item, None, _FULL_CONFIDENCE, ref)) + grid.append(tuple(cells)) + return FormCandidate( + source=Source.EXCEL, + locator="sheet:" + ",".join(name for name, _ in sheets), + type_hint=type_hint, + evidence=_evidence(okud, title), + header=header if header is not None else HeaderCells(), + rows=tuple(grid), + segment_breaks=tuple(breaks), + warnings=tuple(warnings), + ) + + +def place(cells: Mapping[int, PlaceInput], width: int | None = None) -> list[PlaceInput | None]: + """Строка с ячейками в заданных столбцах, остальные None.""" + size = width if width is not None else (max(cells) + 1 if cells else 0) + return [cells.get(i) for i in range(size)] diff --git a/arm_statements/tests/test_model.py b/arm_statements/tests/test_model.py new file mode 100644 index 0000000..945eeb1 --- /dev/null +++ b/arm_statements/tests/test_model.py @@ -0,0 +1,37 @@ +import json +from datetime import date +from decimal import Decimal as D + +from arm_statements.model import Cell, DocumentType, Evidence, FormCandidate, HeaderCells, Source +from arm_statements.serialize import candidate_from_dict, to_jsonable +from arm_statements.tests.builders import excel, ocr + + +def test_candidate_roundtrip_through_json() -> None: + cand = FormCandidate( + source=Source.EXCEL, + locator="sheet:Баланс", + type_hint=DocumentType.BALANCE, + evidence=Evidence(okud=Cell("0710001", None, 100.0, "Баланс!F3"), title=None), + header=HeaderCells(report_date=Cell("30.09.2025", date(2025, 9, 30), 100.0, "Баланс!C5")), + rows=((Cell("106 311", D("106311"), 100.0, "Баланс!D27"), None, Cell("да", True, 100.0, "Баланс!F27")),), + ) + data = json.loads(json.dumps(to_jsonable(cand), ensure_ascii=False)) + assert data["rows"][0][0]["value"] == {"decimal": "106311"} + assert data["header"]["report_date"]["value"] == {"date": "2025-09-30"} + assert candidate_from_dict(data) == cand + + +def test_excel_builder_refs_follow_excel_quoting() -> None: + cand = excel([("Баланс", [["", "", "", "1110"]]), ("Бухгалтерский баланс. Лист 1", [["", "", "1150"]])]) + assert cand.rows[0][3] is not None and cand.rows[0][3].ref == "Баланс!D1" + assert cand.rows[1][2] is not None and cand.rows[1][2].ref == "'Бухгалтерский баланс. Лист 1'!C1" + assert cand.segment_breaks == (1,) + assert cand.locator == "sheet:Баланс,Бухгалтерский баланс. Лист 1" + + +def test_ocr_builder_cells() -> None: + cand = ocr([["Выручка", ("[190 :", 25.0), None]]) + cell = cand.rows[0][1] + assert cell == Cell("[190 :", None, 25.0, "row:0") + assert cand.rows[0][2] is None and cand.source is Source.OCR -- 2.47.2 From 25b8bc4abfba399a6ac45033e9cf64e597dc1673 Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Sun, 18 Oct 2026 13:17:18 +0400 Subject: [PATCH 03/28] =?UTF-8?q?feat:=20=D1=81=D0=BB=D0=BE=D0=B2=D0=B0?= =?UTF-8?q?=D1=80=D0=B8=20=D0=BD=D0=BE=D1=80=D0=BC=D0=B0=D0=BB=D0=B8=D0=B7?= =?UTF-8?q?=D0=B0=D1=86=D0=B8=D0=B8=20(=C2=A77,=20=D0=BF=D0=B5=D1=80=D0=B8?= =?UTF-8?q?=D0=BE=D0=B4=D1=8B,=20=D0=B7=D0=B0=D0=B3=D0=BE=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=D0=BA=D0=B8,=20=D0=B5=D0=B4=D0=B8=D0=BD=D0=B8=D1=86?= =?UTF-8?q?=D1=8B,=20=D0=BA=D0=BE=D0=B4=D1=8B,=20=D0=BF=D1=80=D0=B5=D0=B4?= =?UTF-8?q?=D1=83=D0=BF=D1=80=D0=B5=D0=B6=D0=B4=D0=B5=D0=BD=D0=B8=D1=8F)?= =?UTF-8?q?=20=D0=B8=20=D0=B7=D0=B0=D0=B3=D1=80=D1=83=D0=B7=D1=87=D0=B8?= =?UTF-8?q?=D0=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W --- arm_statements/issues.py | 72 +++ arm_statements/loader.py | 509 ++++++++++++++++++ arm_statements/rules/ocr_lookalikes.yaml | 16 + arm_statements/rules/statement_checks.yaml | 29 + .../rules/statement_code_orders.yaml | 194 +++++++ .../rules/statement_code_ranges.yaml | 13 + arm_statements/rules/statement_headers.yaml | 78 +++ arm_statements/rules/statement_periods.yaml | 44 ++ arm_statements/rules/statement_units.yaml | 16 + arm_statements/rules/statement_warnings.yaml | 36 ++ arm_statements/tests/conftest.py | 11 +- arm_statements/tests/test_loader.py | 141 +++++ 12 files changed, 1158 insertions(+), 1 deletion(-) create mode 100644 arm_statements/issues.py create mode 100644 arm_statements/loader.py create mode 100644 arm_statements/rules/ocr_lookalikes.yaml create mode 100644 arm_statements/rules/statement_checks.yaml create mode 100644 arm_statements/rules/statement_code_orders.yaml create mode 100644 arm_statements/rules/statement_code_ranges.yaml create mode 100644 arm_statements/rules/statement_headers.yaml create mode 100644 arm_statements/rules/statement_periods.yaml create mode 100644 arm_statements/rules/statement_units.yaml create mode 100644 arm_statements/rules/statement_warnings.yaml create mode 100644 arm_statements/tests/test_loader.py diff --git a/arm_statements/issues.py b/arm_statements/issues.py new file mode 100644 index 0000000..4f54512 --- /dev/null +++ b/arm_statements/issues.py @@ -0,0 +1,72 @@ +"""Предупреждения нормализации: перечень кодов и фабрика с уровнем из словаря.""" + +from collections.abc import Sequence +from enum import StrEnum +from typing import TYPE_CHECKING + +from arm_statements.model import StatementWarning + +if TYPE_CHECKING: + from arm_statements.loader import Rules + + +class WarningCode(StrEnum): + """Коды предупреждений: по члену на код `rules/statement_warnings.yaml` (имя = значение).""" + + JSON_FAILED = "JSON_FAILED" + TABLE_PARSE_ERROR = "TABLE_PARSE_ERROR" + TABLE_LOST = "TABLE_LOST" + TABLE_SKIPPED = "TABLE_SKIPPED" + CODE_COLUMN_NOT_FOUND = "CODE_COLUMN_NOT_FOUND" + SEGMENT_WITHOUT_CODES = "SEGMENT_WITHOUT_CODES" + SHEET_NOT_JOINED = "SHEET_NOT_JOINED" + CONTINUATION_TYPE_CONFLICT = "CONTINUATION_TYPE_CONFLICT" + SEGMENT_COLUMNS_MISMATCH = "SEGMENT_COLUMNS_MISMATCH" + COLUMN_WITHOUT_DATE = "COLUMN_WITHOUT_DATE" + COLUMN_DATE_UNCERTAIN = "COLUMN_DATE_UNCERTAIN" + HEADER_ROW_NOT_FOUND = "HEADER_ROW_NOT_FOUND" + TYPE_CONFLICT = "TYPE_CONFLICT" + NO_ORGANIZATION_NO_INN = "NO_ORGANIZATION_NO_INN" + PERIOD_UNKNOWN = "PERIOD_UNKNOWN" + REPORT_DATE_UNCERTAIN = "REPORT_DATE_UNCERTAIN" + XLSB_SCALE_SUSPECT = "XLSB_SCALE_SUSPECT" + INN_NOT_FOUND = "INN_NOT_FOUND" + INN_UNCERTAIN = "INN_UNCERTAIN" + ORGANIZATION_NOT_FOUND = "ORGANIZATION_NOT_FOUND" + ORGANIZATION_UNCERTAIN = "ORGANIZATION_UNCERTAIN" + REPORT_DATE_UNPARSED = "REPORT_DATE_UNPARSED" + UNITS_NOT_FOUND = "UNITS_NOT_FOUND" + FILE_TYPE_MISMATCH = "FILE_TYPE_MISMATCH" + XLS_NO_CODEPAGE = "XLS_NO_CODEPAGE" + COLUMN_SKIPPED = "COLUMN_SKIPPED" + FRACTIONAL_IN_THOUSANDS = "FRACTIONAL_IN_THOUSANDS" + XLSB_NO_FORMATS = "XLSB_NO_FORMATS" + NUMBER_FORMAT_UNPARSED = "NUMBER_FORMAT_UNPARSED" + VALUE_OUT_OF_RANGE = "VALUE_OUT_OF_RANGE" + SHEET_OTHER = "SHEET_OTHER" + ROW_SPLIT = "ROW_SPLIT" + + +def make_warning( # noqa: PLR0913 -- сигнатура задана публичным API пакета + code: str, + message: str, + rules: "Rules", + *, + line_refs: Sequence[str] = (), + column_index: int | None = None, + field: str | None = None, + locator: str | None = None, +) -> StatementWarning: + """Предупреждение с уровнем из словаря; неизвестный код — `KeyError` (выдуманный код не создаётся).""" + rule = rules.warnings.get(code) + if rule is None: + raise KeyError(f"код предупреждения {code} не задан в statement_warnings.yaml") + return StatementWarning( + code=str(code), + severity=rule.severity, + message=message, + line_refs=tuple(line_refs), + column_index=column_index, + field=field, + locator=locator, + ) diff --git a/arm_statements/loader.py b/arm_statements/loader.py new file mode 100644 index 0000000..7cfbe34 --- /dev/null +++ b/arm_statements/loader.py @@ -0,0 +1,509 @@ +"""Загрузчик словарей `rules/*.yaml` в типизированный `Rules` с проверкой при загрузке.""" + +import re +from collections.abc import Mapping +from dataclasses import dataclass +from decimal import Decimal +from enum import StrEnum +from functools import cache +from pathlib import Path +from typing import Literal, TypeVar + +import yaml + +from arm_statements.issues import WarningCode +from arm_statements.model import DocumentType, Edition, FormVariant, PeriodKind, Severity + +LABEL_KEYS: tuple[str, ...] = ( + "organization", + "inn", + "okpo", + "okopf", + "okfs", + "okopf_okfs", + "okei", + "units", + "address", + "date", +) + +_CHECKS = "statement_checks.yaml" +_PERIODS = "statement_periods.yaml" +_HEADERS = "statement_headers.yaml" +_UNITS = "statement_units.yaml" +_RANGES = "statement_code_ranges.yaml" +_ORDERS = "statement_code_orders.yaml" +_WARNINGS = "statement_warnings.yaml" +_LOOKALIKES = "ocr_lookalikes.yaml" + +_RELATION = re.compile(r"^(\d{4}) = (\d{4})((?: [+-] \d{4})*)$") +_RELATION_TERM = re.compile(r"([+-]) (\d{4})") +_PERIOD_GROUPS = frozenset({"d", "m", "mw", "y", "q", "n"}) +_ABSOLUTE_KINDS = frozenset({"range_end", "day_month_year", "quarter", "half", "months", "month_range", "year"}) +_MONTH_MAX = 12 +_PERCENT_MAX = 100 + +_E = TypeVar("_E", bound=StrEnum) + + +class RulesError(ValueError): + """Словарь не прошёл проверку (в сообщении — файл и ключ).""" + + +class CodeSection(StrEnum): + BALANCE = "BALANCE" + INCOME = "INCOME" + CAPITAL = "CAPITAL" + CASH_FLOW = "CASH_FLOW" + NOTES = "NOTES" + TARGET_USE = "TARGET_USE" + + +@dataclass(frozen=True, slots=True) +class CodeRange: + section: CodeSection + low: int + high: int + + +@dataclass(frozen=True, slots=True) +class CodeRanges: + ranges: tuple[CodeRange, ...] + min_share: Decimal + min_codes: int + min_codes_continuation: int + + +@dataclass(frozen=True, slots=True) +class RelationRule: + form: DocumentType + variant: FormVariant + relation: str + target: str + terms: tuple[tuple[int, str], ...] + strict: bool + abs_terms: frozenset[str] + + +@dataclass(frozen=True, slots=True) +class CheckRules: + tolerance: Decimal + rounding_tolerance_percent: Decimal + subline_check: bool + relations: tuple[RelationRule, ...] + + def relations_for(self, form: DocumentType, variant: FormVariant) -> tuple[RelationRule, ...]: + return tuple(r for r in self.relations if r.form == form and r.variant == variant) + + def abs_codes(self, form: DocumentType, variant: FormVariant) -> frozenset[str]: + found: set[str] = set() + for rule in self.relations_for(form, variant): + found |= rule.abs_terms + return frozenset(found) + + +@dataclass(frozen=True, slots=True) +class PeriodPattern: + kind: str + pattern: re.Pattern[str] + + +@dataclass(frozen=True, slots=True) +class PeriodRules: + relative: tuple[tuple[PeriodKind, re.Pattern[str]], ...] + absolute: tuple[PeriodPattern, ...] + months: tuple[tuple[str, int], ...] + min_year: int + max_year: int + max_text_length: int + + +@dataclass(frozen=True, slots=True) +class HeaderRules: + form_titles: Mapping[DocumentType, tuple[re.Pattern[str], ...]] + okud: Mapping[DocumentType, tuple[re.Pattern[str], ...]] + okud_code: re.Pattern[str] + continuation: tuple[re.Pattern[str], ...] + code_caption: re.Pattern[str] + name_caption: re.Pattern[str] + explanations_caption: re.Pattern[str] + column_exclusions: tuple[re.Pattern[str], ...] + labels: Mapping[str, tuple[re.Pattern[str], ...]] + auditor: tuple[re.Pattern[str], ...] + legal_form: tuple[re.Pattern[str], ...] + + +@dataclass(frozen=True, slots=True) +class UnitRules: + spellings: tuple[tuple[re.Pattern[str], str], ...] + okei: Mapping[str, str] + thousands: str + fractional_share_percent: int + + +@dataclass(frozen=True, slots=True) +class CodeOrders: + editions: tuple[Edition, ...] + sequences: Mapping[tuple[Edition, DocumentType, FormVariant], tuple[str, ...]] + + def sequence(self, edition: Edition, form: DocumentType, variant: FormVariant) -> tuple[str, ...]: + return self.sequences.get((edition, form, variant), ()) + + +@dataclass(frozen=True, slots=True) +class WarningRule: + code: str + severity: Severity + scope: Literal["document", "file"] + + +@dataclass(frozen=True, slots=True) +class Rules: + checks: CheckRules + periods: PeriodRules + headers: HeaderRules + units: UnitRules + code_ranges: CodeRanges + code_orders: CodeOrders + lookalikes: Mapping[str, str] + warnings: Mapping[str, WarningRule] + + +# --- проверяющие помощники: Any из YAML дальше них не уходит --- + + +def _fail(file: str, key: str, message: str) -> RulesError: + return RulesError(f"{file}: {key}: {message}") + + +def _read(directory: Path, name: str) -> dict[str, object]: + path = directory / name + try: + data = yaml.safe_load(path.read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError) as exc: + raise _fail(name, "<файл>", f"не удалось прочитать: {exc}") from exc + if not isinstance(data, dict): + raise _fail(name, "<корень>", "ожидается словарь") + return {str(k): v for k, v in data.items()} + + +def _map(file: str, key: str, node: object) -> dict[str, object]: + if not isinstance(node, dict): + raise _fail(file, key, "ожидается словарь") + return {str(k): v for k, v in node.items()} + + +def _seq(file: str, key: str, node: object) -> list[object]: + if not isinstance(node, list): + raise _fail(file, key, "ожидается список") + return list(node) + + +def _get(file: str, data: Mapping[str, object], key: str) -> object: + if key not in data: + raise _fail(file, key, "ключ отсутствует") + return data[key] + + +def _str(file: str, key: str, node: object) -> str: + if not isinstance(node, str): + raise _fail(file, key, "ожидается строка") + return node + + +def _int(file: str, key: str, node: object) -> int: + if isinstance(node, bool) or not isinstance(node, int): + raise _fail(file, key, "ожидается целое число") + return node + + +def _bool(file: str, key: str, node: object) -> bool: + if not isinstance(node, bool): + raise _fail(file, key, "ожидается true/false") + return node + + +def _number(file: str, key: str, node: object) -> Decimal: + if isinstance(node, bool) or not isinstance(node, int | float): + raise _fail(file, key, "ожидается число") + return Decimal(str(node)) + + +def _rx(file: str, key: str, node: object) -> re.Pattern[str]: + text = _str(file, key, node) + try: + return re.compile(text, re.IGNORECASE) + except re.error as exc: + raise _fail(file, key, f"регулярка не компилируется: {exc}") from exc + + +def _rx_list(file: str, key: str, node: object) -> tuple[re.Pattern[str], ...]: + return tuple(_rx(file, f"{key}[{i}]", item) for i, item in enumerate(_seq(file, key, node))) + + +def _enum(file: str, key: str, enum_type: type[_E], node: object) -> _E: + try: + return enum_type(_str(file, key, node)) + except ValueError as exc: + raise _fail(file, key, f"недопустимое значение {node!r}") from exc + + +def _doc_type_patterns(file: str, key: str, node: object) -> dict[DocumentType, tuple[re.Pattern[str], ...]]: + return { + _enum(file, f"{key}.{name}", DocumentType, name): _rx_list(file, f"{key}.{name}", value) + for name, value in _map(file, key, node).items() + } + + +# --- разделы --- + + +def _parse_relation(file: str, key: str, text: str) -> tuple[str, tuple[tuple[int, str], ...]]: + match = _RELATION.match(text) + if match is None: + raise _fail(file, key, f"соотношение {text!r} не вида «<код> = <код> [+|-] <код> …»") + terms = [(1, match.group(2))] + terms += [(1 if sign == "+" else -1, code) for sign, code in _RELATION_TERM.findall(match.group(3))] + return match.group(1), tuple(terms) + + +def _load_checks(directory: Path) -> CheckRules: + file = _CHECKS + data = _read(directory, file) + relations: list[RelationRule] = [] + for i, item in enumerate(_seq(file, "relations", _get(file, data, "relations"))): + key = f"relations[{i}]" + entry = _map(file, key, item) + form = _enum(file, f"{key}.form", DocumentType, _get(file, entry, "form")) + variant = _enum(file, f"{key}.variant", FormVariant, _get(file, entry, "variant")) + text = _str(file, f"{key}.relation", _get(file, entry, "relation")) + target, terms = _parse_relation(file, f"{key}.relation", text) + raw_abs = _seq(file, f"{key}.abs_terms", entry.get("abs_terms", [])) + abs_terms = frozenset(_str(file, f"{key}.abs_terms", code) for code in raw_abs) + if not abs_terms <= {code for _, code in terms}: + raise _fail(file, f"{key}.abs_terms", "коды должны быть членами правой части соотношения") + relations.append( + RelationRule( + form=form, + variant=variant, + relation=text, + target=target, + terms=terms, + strict=_bool(file, f"{key}.strict", _get(file, entry, "strict")), + abs_terms=abs_terms, + ) + ) + return CheckRules( + tolerance=_number(file, "tolerance", _get(file, data, "tolerance")), + rounding_tolerance_percent=_number( + file, "rounding_tolerance_percent", _get(file, data, "rounding_tolerance_percent") + ), + subline_check=_bool(file, "subline_check", _get(file, data, "subline_check")), + relations=tuple(relations), + ) + + +def _load_periods(directory: Path) -> PeriodRules: + file = _PERIODS + data = _read(directory, file) + relative: list[tuple[PeriodKind, re.Pattern[str]]] = [] + for i, item in enumerate(_seq(file, "relative", _get(file, data, "relative"))): + key = f"relative[{i}]" + entry = _map(file, key, item) + kind = _enum(file, f"{key}.kind", PeriodKind, _get(file, entry, "kind")) + relative.append((kind, _rx(file, f"{key}.pattern", _get(file, entry, "pattern")))) + absolute: list[PeriodPattern] = [] + for i, item in enumerate(_seq(file, "absolute", _get(file, data, "absolute"))): + key = f"absolute[{i}]" + entry = _map(file, key, item) + abs_kind = _str(file, f"{key}.kind", _get(file, entry, "kind")) + if abs_kind not in _ABSOLUTE_KINDS: + raise _fail(file, f"{key}.kind", f"недопустимый вид {abs_kind!r}") + pattern = _rx(file, f"{key}.pattern", _get(file, entry, "pattern")) + if not set(pattern.groupindex) <= _PERIOD_GROUPS: + raise _fail(file, f"{key}.pattern", f"допустимые имена групп: {sorted(_PERIOD_GROUPS)}") + absolute.append(PeriodPattern(kind, pattern)) + months: list[tuple[str, int]] = [] + for stem, number in _map(file, "months", _get(file, data, "months")).items(): + month = _int(file, f"months.{stem}", number) + if not 1 <= month <= _MONTH_MAX: + raise _fail(file, f"months.{stem}", "номер месяца вне 1..12") + months.append((stem, month)) + years = _map(file, "years", _get(file, data, "years")) + min_year = _int(file, "years.min", _get(file, years, "min")) + max_year = _int(file, "years.max", _get(file, years, "max")) + if min_year > max_year: + raise _fail(file, "years", "min больше max") + return PeriodRules( + relative=tuple(relative), + absolute=tuple(absolute), + months=tuple(months), + min_year=min_year, + max_year=max_year, + max_text_length=_int(file, "max_text_length", _get(file, data, "max_text_length")), + ) + + +def _load_headers(directory: Path) -> HeaderRules: + file = _HEADERS + data = _read(directory, file) + captions = _map(file, "captions", _get(file, data, "captions")) + labels_raw = _map(file, "labels", _get(file, data, "labels")) + if tuple(labels_raw) != LABEL_KEYS: + raise _fail(file, "labels", f"ключи должны быть ровно {LABEL_KEYS}, получены {tuple(labels_raw)}") + return HeaderRules( + form_titles=_doc_type_patterns(file, "form_titles", _get(file, data, "form_titles")), + okud=_doc_type_patterns(file, "okud", _get(file, data, "okud")), + okud_code=_rx(file, "okud_code", _get(file, data, "okud_code")), + continuation=_rx_list(file, "continuation", _get(file, data, "continuation")), + code_caption=_rx(file, "captions.code", _get(file, captions, "code")), + name_caption=_rx(file, "captions.name", _get(file, captions, "name")), + explanations_caption=_rx(file, "captions.explanations", _get(file, captions, "explanations")), + column_exclusions=_rx_list(file, "captions.exclusions", _get(file, captions, "exclusions")), + labels={name: _rx_list(file, f"labels.{name}", value) for name, value in labels_raw.items()}, + auditor=_rx_list(file, "auditor", _get(file, data, "auditor")), + legal_form=_rx_list(file, "legal_form", _get(file, data, "legal_form")), + ) + + +def _load_units(directory: Path) -> UnitRules: + file = _UNITS + data = _read(directory, file) + spellings: list[tuple[re.Pattern[str], str]] = [] + canonical_all: set[str] = set() + for i, item in enumerate(_seq(file, "units", _get(file, data, "units"))): + key = f"units[{i}]" + entry = _map(file, key, item) + canonical = _str(file, f"{key}.canonical", _get(file, entry, "canonical")) + canonical_all.add(canonical) + spellings.extend((rx, canonical) for rx in _rx_list(file, f"{key}.patterns", _get(file, entry, "patterns"))) + okei = { + code: _str(file, f"okei.{code}", value) for code, value in _map(file, "okei", _get(file, data, "okei")).items() + } + for code, canonical in okei.items(): + if canonical not in canonical_all: + raise _fail(file, f"okei.{code}", f"единица {canonical!r} не среди canonical") + thousands = _str(file, "thousands", _get(file, data, "thousands")) + if thousands not in canonical_all: + raise _fail(file, "thousands", f"единица {thousands!r} не среди canonical") + percent = _int(file, "fractional_share_percent", _get(file, data, "fractional_share_percent")) + if not 1 <= percent <= _PERCENT_MAX: + raise _fail(file, "fractional_share_percent", "ожидается 1..100") + return UnitRules(tuple(spellings), okei, thousands, percent) + + +def _load_ranges(directory: Path) -> CodeRanges: + file = _RANGES + data = _read(directory, file) + ranges: list[CodeRange] = [] + for name, bounds in _map(file, "sections", _get(file, data, "sections")).items(): + section = _enum(file, f"sections.{name}", CodeSection, name) + pair = _seq(file, f"sections.{name}", bounds) + if len(pair) != len(("low", "high")): + raise _fail(file, f"sections.{name}", "ожидается [low, high]") + low, high = (_int(file, f"sections.{name}", v) for v in pair) + if low > high: + raise _fail(file, f"sections.{name}", "low больше high") + ranges.append(CodeRange(section, low, high)) + min_codes = _int(file, "min_codes", _get(file, data, "min_codes")) + continuation = _int(file, "min_codes_continuation", _get(file, data, "min_codes_continuation")) + if not 1 <= continuation <= min_codes: + raise _fail(file, "min_codes_continuation", "должно быть 1 <= min_codes_continuation <= min_codes") + return CodeRanges( + ranges=tuple(ranges), + min_share=_number(file, "min_share", _get(file, data, "min_share")), + min_codes=min_codes, + min_codes_continuation=continuation, + ) + + +def _load_orders(directory: Path) -> CodeOrders: + file = _ORDERS + data = _read(directory, file) + if _get(file, data, "version") != 1: + raise _fail(file, "version", "поддерживается только версия 1") + repeats = _map(file, "repeats", _get(file, data, "repeats")) + editions_raw = _map(file, "editions", _get(file, data, "editions")) + editions: list[Edition] = [] + sequences: dict[tuple[Edition, DocumentType, FormVariant], tuple[str, ...]] = {} + for ed_name, forms_node in editions_raw.items(): + edition = _enum(file, f"editions.{ed_name}", Edition, ed_name) + editions.append(edition) + forms = _map(file, f"editions.{ed_name}", forms_node) + edition_repeats = _map(file, f"repeats.{ed_name}", repeats.get(ed_name, {})) + for form_name, variants_node in forms.items(): + form = _enum(file, f"editions.{ed_name}.{form_name}", DocumentType, form_name) + repeat_codes = [ + _str(file, f"repeats.{ed_name}.{form_name}", c) + for c in _seq(file, f"repeats.{ed_name}.{form_name}", edition_repeats.get(form_name, [])) + ] + variants = _map(file, f"editions.{ed_name}.{form_name}", variants_node) + for var_name, codes_node in variants.items(): + key = f"editions.{ed_name}.{form_name}.{var_name}" + variant = _enum(file, key, FormVariant, var_name) + codes = tuple(_str(file, key, c) for c in _seq(file, key, codes_node)) + for code in codes: + if not re.fullmatch(r"\d{4}", code): + raise _fail(file, key, f"код {code!r} не из четырёх цифр") + _check_repeat_counts(file, key, variant, codes, repeat_codes) + sequences[(edition, form, variant)] = codes + return CodeOrders(tuple(editions), sequences) + + +def _check_repeat_counts(file: str, key: str, variant: FormVariant, codes: tuple[str, ...], repeats: list[str]) -> None: + expected = {code: 1 + repeats.count(code) for code in set(codes) | set(repeats)} + for code, limit in sorted(expected.items()): + actual = codes.count(code) + if variant is FormVariant.FULL and actual != limit: + raise _fail(file, key, f"код {code} встречается {actual} раз, ожидается {limit} (1 + repeats)") + if variant is FormVariant.SIMPLIFIED and actual > limit: + raise _fail(file, key, f"код {code} встречается {actual} раз, допустимо не более {limit}") + + +def _load_warnings(directory: Path) -> dict[str, WarningRule]: + file = _WARNINGS + data = _read(directory, file) + warnings: dict[str, WarningRule] = {} + for code, node in _map(file, "warnings", _get(file, data, "warnings")).items(): + entry = _map(file, f"warnings.{code}", node) + severity = _enum(file, f"warnings.{code}.severity", Severity, _get(file, entry, "severity")) + scope = _str(file, f"warnings.{code}.scope", _get(file, entry, "scope")) + if scope == "document": + warnings[code] = WarningRule(code, severity, "document") + elif scope == "file": + warnings[code] = WarningRule(code, severity, "file") + else: + raise _fail(file, f"warnings.{code}.scope", f"недопустимая область {scope!r}") + if set(warnings) != {c.value for c in WarningCode}: + diff = sorted(set(warnings) ^ {c.value for c in WarningCode}) + raise _fail(file, "warnings", f"коды словаря и WarningCode расходятся: {diff}") + return warnings + + +def _load_lookalikes(directory: Path) -> dict[str, str]: + file = _LOOKALIKES + data = _read(directory, file) + return { + char: _str(file, f"lookalikes.{char}", value) + for char, value in _map(file, "lookalikes", _get(file, data, "lookalikes")).items() + } + + +def load_rules(directory: Path | None = None) -> Rules: + """Прочитать и проверить словари каталога (по умолчанию — `rules/` пакета).""" + base = directory if directory is not None else Path(__file__).parent / "rules" + return Rules( + checks=_load_checks(base), + periods=_load_periods(base), + headers=_load_headers(base), + units=_load_units(base), + code_ranges=_load_ranges(base), + code_orders=_load_orders(base), + lookalikes=_load_lookalikes(base), + warnings=_load_warnings(base), + ) + + +@cache +def default_rules() -> Rules: + return load_rules() diff --git a/arm_statements/rules/ocr_lookalikes.yaml b/arm_statements/rules/ocr_lookalikes.yaml new file mode 100644 index 0000000..486a8a9 --- /dev/null +++ b/arm_statements/rules/ocr_lookalikes.yaml @@ -0,0 +1,16 @@ +# Буквы и знаки, которые OCR ставит вместо цифр (§11.1 п. 1). Замена — только вплотную к цифре, в несколько проходов. +lookalikes: + "О": "0" + "о": "0" + "O": "0" + "o": "0" + "З": "3" + "з": "3" + "б": "6" + "S": "5" + "s": "5" + "I": "1" + "l": "1" + "|": "1" + "і": "1" + "Ч": "4" diff --git a/arm_statements/rules/statement_checks.yaml b/arm_statements/rules/statement_checks.yaml new file mode 100644 index 0000000..8466325 --- /dev/null +++ b/arm_statements/rules/statement_checks.yaml @@ -0,0 +1,29 @@ +# Соотношения §7 (05-domain-rules.md), ББ и ОФР. Данные, не код: новое соотношение — строка здесь. +# variant: full — полный набор; simplified — упрощённый набор (спека «Шаг 7», §7 «Упрощённый набор»). +# relation: «<код> = <код> [+|-] <код> …», только четырёхзначные коды; strict: нарушение переводит документ в needs_check +# (false — только мягкое предупреждение, ASSUMPTION(OQ-12)); abs_terms: коды, которые 1С выгружает со знаком минус, а 66н — +# в скобках: правая часть считается и как записано, и с этими членами по модулю. +tolerance: 1 # допуск в единицах формы (тыс. руб. округляются построчно) +# OQ-138: единицы шапки верны, расхождение до 0,001 % итоговой строки — округление (мягкое нарушение, статус не портит). +rounding_tolerance_percent: 0.001 +subline_check: true # «сумма пятизначных подстрок = родитель» — мягкая проверка, генерируется по коду +relations: + - {form: BALANCE, variant: full, relation: "1100 = 1105 + 1110 + 1120 + 1130 + 1140 + 1150 + 1160 + 1170 + 1180 + 1190", strict: true} + - {form: BALANCE, variant: full, relation: "1200 = 1210 + 1215 + 1220 + 1230 + 1240 + 1250 + 1260", strict: true} + - {form: BALANCE, variant: full, relation: "1300 = 1310 + 1320 + 1340 + 1350 + 1360 + 1370", strict: true} + - {form: BALANCE, variant: full, relation: "1400 = 1410 + 1420 + 1430 + 1450", strict: true} + - {form: BALANCE, variant: full, relation: "1500 = 1510 + 1520 + 1530 + 1540 + 1550", strict: true} + - {form: BALANCE, variant: full, relation: "1600 = 1100 + 1200", strict: true} + - {form: BALANCE, variant: full, relation: "1700 = 1300 + 1400 + 1500", strict: true} + - {form: BALANCE, variant: full, relation: "1600 = 1700", strict: true} + - {form: INCOME, variant: full, relation: "2100 = 2110 - 2120", strict: true, abs_terms: ["2120"]} + - {form: INCOME, variant: full, relation: "2200 = 2100 - 2210 - 2220", strict: true, abs_terms: ["2210", "2220"]} + - {form: INCOME, variant: full, relation: "2300 = 2200 + 2310 + 2320 - 2330 + 2340 - 2350", strict: true, abs_terms: ["2330", "2350"]} + - {form: INCOME, variant: full, relation: "2400 = 2300 - 2410 + 2420", strict: false, abs_terms: ["2410"]} + - {form: INCOME, variant: full, relation: "2400 = 2300 + 2410 + 2460", strict: false} + - {form: INCOME, variant: full, relation: "2410 = 2411 + 2412", strict: false} + - {form: INCOME, variant: full, relation: "2500 = 2400 + 2510 + 2520 + 2530", strict: false} + - {form: BALANCE, variant: simplified, relation: "1600 = 1150 + 1170 + 1210 + 1230 + 1250", strict: true} + - {form: BALANCE, variant: simplified, relation: "1700 = 1300 + 1350 + 1360 + 1410 + 1450 + 1510 + 1520 + 1550", strict: true} + - {form: BALANCE, variant: simplified, relation: "1600 = 1700", strict: true} + - {form: INCOME, variant: simplified, relation: "2400 = 2110 - 2120 - 2330 + 2340 - 2350 - 2410", strict: false, abs_terms: ["2120", "2330", "2350", "2410"]} diff --git a/arm_statements/rules/statement_code_orders.yaml b/arm_statements/rules/statement_code_orders.yaml new file mode 100644 index 0000000..de9bce1 --- /dev/null +++ b/arm_statements/rules/statement_code_orders.yaml @@ -0,0 +1,194 @@ +# Упорядоченные перечни кодов строк форм ББ/ОФР по редакциям и вариантам — данные для структурного подтверждения +# неуверенного кода (05-domain-rules.md §11.9 «Код строки»; change unified-statement-normalization, K-5, Р-20). +# +# Что это: порядок кодов бланка (итоги после слагаемых: 1110 … 1190, 1100), без наименований (Р-3: «код → наименование» +# знает только ядро, arm-core/api/rules/statement_lines.yaml). Только четырёхзначные коды: пятизначная подстрока +# подтверждается через родителя (§11.9). Перечень — объединение кодов шаблонов банка (словарь имён ядра, +# fixtures/statements-xlsx/templates/) и кодов печатных форм 1С и сканов, найденных в fixtures (источник — комментарий +# у кода). Код без образца в fixtures сюда не добавляется: кандидаты — OQ-168. +# +# Где живёт: этот файл — источник; в пакете arm_statements лежит его байтовая копия (rules/statement_code_orders.yaml), +# идентичность проверяет sync-statements --check (как копии схем контракта в сервисах). Ядро берёт порядок и состав +# редакций из копии пакета; sync-statements --check сверяет, что все коды словаря имён ядра есть в перечне своей +# редакции. Правка перечня — PR сюда, затем в arm-statements, затем sync-statements в потребителях («сначала ядро»). +# +# Как читается (§11.9): четырёхзначные коды строк ok документа — последовательность; для каждой редакции берётся +# наибольшая подпоследовательность, согласованная с порядком перечня варианта формы документа (form_variant); выбирается +# редакция с более длинной (при равенстве — по эффективной дате: fsbu4 после 2024-12-31). Повтор кода допустим только +# там, где он есть в перечне (repeats): тогда структура подтверждает обе копии; повтор вне перечня с неуверенной копией — +# needs_review у неуверенной копии (Р-16, Р-20). +# +# Проверка формы файла — scripts/validate_contracts.py (make validate-contracts). +version: 1 + +# Законные повторы кода в одном перечне (Р-20). Ключ — редакция и форма; значение — коды, которые встречаются в +# перечне больше одного раза: в варианте full код стоит ровно 1 + (число его вхождений в этот список) раз, в simplified — +# не чаще (сейчас повторов в simplified нет; validate_contracts.py проверяет оба правила). +repeats: + 66n: + # раздел III для некоммерческих организаций («Целевое финансирование») печатается после раздела III коммерческих + # (fixtures/ocr/aep_3.json, 31.12.2024: 1310 «Паевой фонд», 1320 «Целевой капитал», 1350 «Целевые средства», + # 1360 «Фонд недвижимого и особо ценного движимого имущества», 1370 «Резервный и иные целевые фонды») + BALANCE: ["1310", "1320", "1350", "1360", "1370", "1300"] + INCOME: [] + fsbu4: + # печатная форма 1С по ФСБУ 4/2023 (fixtures/ocr/aep_6.json, «БО 1 кв 2025.json», «Бухгалтерская отчетность + # 1 кв.25.json»): 1220 «НДС, начисленный по отгрузке», 1520 «Обязательство по аренде» (AURORA-1431) + BALANCE: ["1220", "1520"] + # 2410 «Налог при упрощенной системе налогообложения» после 2412 (fixtures/ocr/aep_6.json, OQ-152) + INCOME: ["2410"] + +editions: + 66n: + BALANCE: + full: + - "1110" + - "1120" + - "1130" + - "1140" + - "1150" + - "1160" + - "1170" + - "1180" + - "1190" + - "1100" + - "1210" + - "1220" + - "1230" + - "1231" # код клиента: unagrande-group/2024-FY/ББ. Лист 1 .xls + - "1240" + - "1250" + - "1260" + - "1200" + - "1600" + - "1310" + - "1320" + - "1340" + - "1350" + - "1360" + - "1370" + - "1300" + - "1310" # раздел III НКО (aep_3), см. repeats + - "1320" + - "1350" + - "1360" + - "1370" + - "1300" + - "1410" + - "1420" + - "1430" + - "1450" + - "1400" + - "1510" + - "1520" + - "1530" + - "1540" + - "1550" + - "1500" + - "1700" + # упрощённая форма, приказ 66н прил. 5 — по бланку, образца в fixtures нет (OQ-166) + simplified: ["1150", "1170", "1210", "1250", "1230", "1600", "1300", "1350", "1360", "1410", "1450", "1510", + "1520", "1550", "1700"] + INCOME: + full: + - "2110" + - "2120" + - "2100" + - "2210" + - "2220" + - "2200" + - "2310" + - "2320" + - "2330" + - "2340" + - "2350" + - "2300" + - "2410" + - "2411" + - "2412" + - "2460" + - "2400" + - "2510" + - "2520" + - "2530" # печатная форма 1С: unagrande-co/2024-FY/ОФР. Лист 2 .xls, mezheninovskaya/2025-Q3 + - "2500" + - "2900" + - "2910" + - "2930" # печатная форма 1С: fixtures/ocr/aep_3.json (после 2900) + # упрощённая форма, приказ 66н прил. 5 — по бланку, образца нет (OQ-166) + simplified: ["2110", "2120", "2330", "2340", "2350", "2410", "2400"] + fsbu4: + BALANCE: + full: + - "1105" # печатная форма 1С: npz/2025-Q3, fixtures/ocr/aep_6.json, «БО 1 кв 2025.json» + - "1110" + - "1130" # печатная форма 1С: npz/2025-Q3, «БО 1 кв 2025.json», «Бухгалтерская отчетность 1 кв.25.json» + - "1140" # то же и aep_6.json + - "1150" + - "1160" + - "1170" + - "1180" + - "1190" + - "1100" + - "1210" + - "1215" + - "1220" + - "1220" # «НДС, начисленный по отгрузке», см. repeats + - "1230" + - "1240" + - "1250" + - "1260" + - "1200" + - "1600" + - "1310" + - "1320" + - "1340" + - "1350" + - "1360" + - "1370" + - "1300" + - "1410" + - "1420" + - "1430" + - "1450" + - "1400" + - "1510" + - "1520" + - "1520" # «Обязательство по аренде», см. repeats + - "1530" # печатная форма 1С: npz/2025-Q3, aep_6.json, оба json 1 кв. 2025 + - "1540" + - "1550" + - "1500" + - "1700" + # упрощённая форма по ФСБУ 4/2023 — состав принят как у 66н до образца (OQ-166, OQ-168) + simplified: ["1150", "1170", "1210", "1250", "1230", "1600", "1300", "1350", "1360", "1410", "1450", "1510", + "1520", "1550", "1700"] + INCOME: + full: + - "2110" + - "2120" + - "2100" + - "2210" + - "2220" + - "2200" + - "2310" + - "2320" + - "2330" + - "2340" + - "2350" + - "2300" + - "2410" + - "2411" + - "2412" + - "2410" # «Налог при упрощенной системе налогообложения», см. repeats + - "2420" + - "2460" # печатная форма 1С: npz/2025-Q3, aep_6.json, оба json 1 кв. 2025 + - "2400" + - "2510" + - "2520" + - "2530" + - "2500" + - "2900" + - "2910" + # упрощённая форма по ФСБУ 4/2023 — как у 66н до образца (OQ-166, OQ-168) + simplified: ["2110", "2120", "2330", "2340", "2350", "2410", "2400"] diff --git a/arm_statements/rules/statement_code_ranges.yaml b/arm_statements/rules/statement_code_ranges.yaml new file mode 100644 index 0000000..bf994db --- /dev/null +++ b/arm_statements/rules/statement_code_ranges.yaml @@ -0,0 +1,13 @@ +# Коды строк форм: диапазоны четырёхзначных кодов (пятизначные — по первым четырём цифрам), доля кодов в столбце, +# минимум кодов, чтобы столбец считался столбцом кодов. +sections: + BALANCE: [1100, 1799] + INCOME: [2100, 2999] + CAPITAL: [3100, 3699] + CASH_FLOW: [4100, 4499] + NOTES: [5000, 5999] + TARGET_USE: [6100, 6499] +min_share: 0.6 +min_codes: 3 +# слабое продолжение (отрезок формы без шапки, лист-продолжение без маркера): хватает одного кода +min_codes_continuation: 1 diff --git a/arm_statements/rules/statement_headers.yaml b/arm_statements/rules/statement_headers.yaml new file mode 100644 index 0000000..2bc897b --- /dev/null +++ b/arm_statements/rules/statement_headers.yaml @@ -0,0 +1,78 @@ +# Словарь форм ББ/ОФР (05-domain-rules.md §12): заголовки форм и ОКУД, маркер продолжения, подписи колонок, метки шапки. +# Шаблоны ищутся (re.search) без учёта регистра. Текст перед поиском: normalize_label (пунктуация сохраняется) для +# form_titles, okud, okud_code, continuation, labels, auditor, legal_form; normalize_caption (пунктуация -> пробел) +# для captions. Новый синоним = строка в YAML. + +# Заголовок формы -> тип документа. Титул «Бухгалтерская отчетность» (НПЗ) типом не является и здесь не перечислен. +form_titles: + BALANCE: + - 'бухгалтерский\s+баланс' + INCOME: + - 'отчет\s+о\s+финансовых\s+результатах' + - 'отчет\s+о\s+прибылях\s+и\s+убытках' + DISCLOSURE: + - 'пояснени\w*\s+к\s+бухгалтерскому\s+балансу' + - '^пояснени' + - '^расшифровк' + - 'отчет\s+об\s+изменениях\s+капитала' + - 'отчет\s+о\s+движении\s+денежных\s+средств' + - 'отчет\s+о\s+целевом\s+использовании' + +# Код ОКУД -> тип (ячейка «Форма по ОКУД 0710001» или сам код). Прочие формы 0710xxx — расшифровки +okud: + BALANCE: + - '\b0710001\b' + INCOME: + - '\b0710002\b' + DISCLOSURE: + - '\b07100(0(3|4|5|6|7|8|9)|(1|2|3|4|5|6|7|8|9)\d)\b' + +# Сам код ОКУД формы внутри ячейки («Форма по ОКУД 0710001» -> «0710001») +okud_code: '0710\d{3}' + +# Маркер страницы-продолжения в 1С-выгрузке: «Форма 0710001 с.2» +continuation: + - '\bформа\s+0710\d{3}\s+с\.?\s*\d+' + +# Подписи колонок заголовка таблицы (ячейка целиком; исключения — подстроки) +captions: + code: '^код(ы|\s+строки|\s+показателя)?$' + name: '^наименование' + explanations: '^пояснени' + exclusions: ['изменени', 'отклонени', '%', 'темп', 'доля', 'примечани'] + +# Метки шапки документа (значение — правее в той же строке либо в следующей строке) +labels: + organization: + - '^организация\b' + inn: + - '^инн\b' + - 'идентификационный\s+номер\s+налогоплательщика' + okpo: + - 'по\s+окпо' + okopf: + - 'по\s+окопф\b(?!\s*/)' + okfs: + - 'по\s+окфс' + okopf_okfs: + - 'по\s+окопф\s*/\s*окфс' + okei: + - 'по\s+океи' + units: + - 'единица\s+измерения' + address: + - 'местонахождение\s*\(адрес\)' + - 'адрес\s+в\s+пределах\s+места\s+нахождения' + - '^адрес\b' + date: + - '^(отчетная\s+)?дата\b' + +# Строки блока аудитора: ИНН ниже этой метки — ИНН аудитора, не организации +auditor: + - 'аудитор' + +# Признак, что строка содержит название организации (для имени над меткой «Организация», НПЗ) +legal_form: + - '\b(ооо|ао|пао|зао|оао|нао|ип|гуп|муп|фгуп)\b' + - '"' + - '«' diff --git a/arm_statements/rules/statement_periods.yaml b/arm_statements/rules/statement_periods.yaml new file mode 100644 index 0000000..4adfad2 --- /dev/null +++ b/arm_statements/rules/statement_periods.yaml @@ -0,0 +1,44 @@ +# Периоды форм ББ/ОФР: подпись колонки или подзаголовок формы -> дата (конец периода). §11.3, §12 п. 4. +# Регулярки ищутся без учёта регистра. Относительные подписи применяются к подписи после normalize_caption. +max_text_length: 200 # длиннее — не дата (защита от backtracking) +years: {min: 1990, max: 2100} # одинокий год вне диапазона датой не считается + +# Основа названия месяца -> номер (сопоставляется по началу слова) +months: + январ: 1 + феврал: 2 + март: 3 + апрел: 4 + ма: 5 + июн: 6 + июл: 7 + август: 8 + сентябр: 9 + октябр: 10 + ноябр: 11 + декабр: 12 + +# Относительные подписи колонок (по порядку проверки) +relative: + - {kind: "relative:prev_prev_year_end", pattern: 'предшествующ\w*\s+предыдущ'} + - {kind: "relative:prev_year_end", pattern: 'декабр\w*\s+предыдущ'} + - {kind: "relative:prev_period", pattern: 'аналогичн\w*\s+период\w*\s+предыдущ'} + - {kind: "relative:current", pattern: 'отч[её]тн\w*\s+(дат|период)'} + +# Абсолютные формы по приоритету (§11.3 п. 3). Именованные группы: d (день), m (месяц числом), mw (месяц словом), +# y (год, 2 или 4 цифры), q (номер квартала/полугодия словом, римской или арабской цифрой), n (число месяцев). +absolute: + - kind: range_end # «с 01.01.2025 по 30.09.2025»: конец диапазона + pattern: 'по\s*(?P\d{1,2})[.\-/ ](?P\d{1,2})[.\-/ ](?P\d{4}|\d{2})' + - kind: day_month_year # «31 декабря 2025 г.», «на 30 сентября 2025 года» + pattern: '(?P\d{1,2})\s+(?P[а-яё]{3,8})\s+(?P\d{4}|\d{2})\s*(?:г\.?|r\.?|год\w*)?' + - kind: quarter # «I квартал 2026», «1-й кв. 2026», «первый квартал» + pattern: '(?PI{1,3}V?|IV|\d|перв\w*|втор\w*|трет\w*|четв\w*)\s*-?\s*(?:й|ый|ого|го)?\s*кв\w*\.?\s*(?P\d{4}|\d{2})\s*(?:г\.?|r\.?|год\w*)?' + - kind: half # «1 полугодие 2026», «первое полугодие», «1 п/г 2026» + pattern: '(?PI{1,3}V?|IV|\d|перв\w*|втор\w*|трет\w*|четв\w*)\s*-?\s*(?:е|ое|го)?\s*(?:полугоди\w*|п/г)\.?\s*(?P\d{4}|\d{2})\s*(?:г\.?|r\.?|год\w*)?' + - kind: months # «9 месяцев 2025», «9 мес. 2025» + pattern: '(?P\d{1,2})\s*мес\w*\.?\s*(?P\d{4}|\d{2})\s*(?:г\.?|r\.?|год\w*)?' + - kind: month_range # «январь - сентябрь 2025»: последний месяц + pattern: '[а-яё]{3,8}\s*[-–—]\s*(?P[а-яё]{3,8})\s*(?P\d{4}|\d{2})\s*(?:г\.?|r\.?|год\w*)?' + - kind: year # «за 2025 г.», «2025 год», «2025г» + pattern: '(?:за\s*)?(?:отч[её]тный\s*)?(?P\d{4})\s*(?:г\b|г\.|r\b|r\.|год)' diff --git a/arm_statements/rules/statement_units.yaml b/arm_statements/rules/statement_units.yaml new file mode 100644 index 0000000..dd2c217 --- /dev/null +++ b/arm_statements/rules/statement_units.yaml @@ -0,0 +1,16 @@ +# Единицы измерения форм ББ/ОФР. Подпись проверяется по порядку: первая совпавшая запись определяет единицу. +units: + - canonical: "тыс. руб." + patterns: ['тыс\w*\.?\s*руб', 'тысяч\w*\s+рубл'] + - canonical: "млн руб." + patterns: ['млн\w*\.?\s*руб', 'миллион\w*\s+рубл'] + - canonical: "руб." + patterns: ['руб'] +# Код ОКЕИ -> каноническая единица +okei: + "384": "тыс. руб." + "383": "руб." + "385": "млн руб." +thousands: "тыс. руб." # единица, для которой проверяется доля дробных значений +# Порог для предупреждения о дробных значениях при единицах в тыс. руб. (процент; 1..100) +fractional_share_percent: 10 diff --git a/arm_statements/rules/statement_warnings.yaml b/arm_statements/rules/statement_warnings.yaml new file mode 100644 index 0000000..d81a271 --- /dev/null +++ b/arm_statements/rules/statement_warnings.yaml @@ -0,0 +1,36 @@ +# Предупреждения нормализации: код -> уровень и область (спека «Шаг 8 — предупреждения»). +# scope: document — предупреждение документа; file — предупреждение файла (metrics.warnings). +# Множество кодов равно перечислению WarningCode (arm_statements/issues.py); код добавляется в обоих местах. +warnings: + JSON_FAILED: {severity: error, scope: file} + TABLE_PARSE_ERROR: {severity: error, scope: file} + TABLE_LOST: {severity: error, scope: file} + TABLE_SKIPPED: {severity: info, scope: file} + CODE_COLUMN_NOT_FOUND: {severity: error, scope: document} + SEGMENT_WITHOUT_CODES: {severity: error, scope: document} + SHEET_NOT_JOINED: {severity: error, scope: document} + CONTINUATION_TYPE_CONFLICT: {severity: error, scope: document} + SEGMENT_COLUMNS_MISMATCH: {severity: error, scope: document} + COLUMN_WITHOUT_DATE: {severity: error, scope: document} + COLUMN_DATE_UNCERTAIN: {severity: error, scope: document} + HEADER_ROW_NOT_FOUND: {severity: error, scope: document} + TYPE_CONFLICT: {severity: error, scope: document} + NO_ORGANIZATION_NO_INN: {severity: error, scope: document} + PERIOD_UNKNOWN: {severity: error, scope: document} + REPORT_DATE_UNCERTAIN: {severity: error, scope: document} + XLSB_SCALE_SUSPECT: {severity: error, scope: document} + INN_NOT_FOUND: {severity: info, scope: document} + INN_UNCERTAIN: {severity: info, scope: document} + ORGANIZATION_NOT_FOUND: {severity: info, scope: document} + ORGANIZATION_UNCERTAIN: {severity: info, scope: document} + REPORT_DATE_UNPARSED: {severity: info, scope: document} + UNITS_NOT_FOUND: {severity: info, scope: document} + FILE_TYPE_MISMATCH: {severity: info, scope: file} + XLS_NO_CODEPAGE: {severity: info, scope: file} + COLUMN_SKIPPED: {severity: info, scope: document} + FRACTIONAL_IN_THOUSANDS: {severity: info, scope: document} + XLSB_NO_FORMATS: {severity: info, scope: document} + NUMBER_FORMAT_UNPARSED: {severity: info, scope: document} + VALUE_OUT_OF_RANGE: {severity: info, scope: document} + SHEET_OTHER: {severity: info, scope: file} + ROW_SPLIT: {severity: info, scope: document} diff --git a/arm_statements/tests/conftest.py b/arm_statements/tests/conftest.py index c3a717e..6e1508e 100644 --- a/arm_statements/tests/conftest.py +++ b/arm_statements/tests/conftest.py @@ -1 +1,10 @@ -"""Общие фикстуры тестов пакета (фикстура `rules` появится в задаче 3).""" +"""Общие фикстуры тестов пакета.""" + +import pytest + +from arm_statements.loader import Rules, default_rules + + +@pytest.fixture +def rules() -> Rules: + return default_rules() diff --git a/arm_statements/tests/test_loader.py b/arm_statements/tests/test_loader.py new file mode 100644 index 0000000..84a9e5e --- /dev/null +++ b/arm_statements/tests/test_loader.py @@ -0,0 +1,141 @@ +import re +from decimal import Decimal +from pathlib import Path + +import pytest +import yaml + +from arm_statements.issues import WarningCode, make_warning +from arm_statements.loader import LABEL_KEYS, Rules, RulesError, load_rules +from arm_statements.model import DocumentType, Edition, FormVariant, Severity + +RULES_DIR = Path(__file__).parent.parent / "rules" + +SPEC_WARNINGS = { + "JSON_FAILED": ("error", "file"), + "TABLE_PARSE_ERROR": ("error", "file"), + "TABLE_LOST": ("error", "file"), + "TABLE_SKIPPED": ("info", "file"), + "CODE_COLUMN_NOT_FOUND": ("error", "document"), + "SEGMENT_WITHOUT_CODES": ("error", "document"), + "SHEET_NOT_JOINED": ("error", "document"), + "CONTINUATION_TYPE_CONFLICT": ("error", "document"), + "SEGMENT_COLUMNS_MISMATCH": ("error", "document"), + "COLUMN_WITHOUT_DATE": ("error", "document"), + "COLUMN_DATE_UNCERTAIN": ("error", "document"), + "HEADER_ROW_NOT_FOUND": ("error", "document"), + "TYPE_CONFLICT": ("error", "document"), + "NO_ORGANIZATION_NO_INN": ("error", "document"), + "PERIOD_UNKNOWN": ("error", "document"), + "REPORT_DATE_UNCERTAIN": ("error", "document"), + "XLSB_SCALE_SUSPECT": ("error", "document"), + "INN_NOT_FOUND": ("info", "document"), + "INN_UNCERTAIN": ("info", "document"), + "ORGANIZATION_NOT_FOUND": ("info", "document"), + "ORGANIZATION_UNCERTAIN": ("info", "document"), + "REPORT_DATE_UNPARSED": ("info", "document"), + "UNITS_NOT_FOUND": ("info", "document"), + "FILE_TYPE_MISMATCH": ("info", "file"), + "COLUMN_SKIPPED": ("info", "document"), + "FRACTIONAL_IN_THOUSANDS": ("info", "document"), + "XLSB_NO_FORMATS": ("info", "document"), + "NUMBER_FORMAT_UNPARSED": ("info", "document"), + "VALUE_OUT_OF_RANGE": ("info", "document"), + "SHEET_OTHER": ("info", "file"), + "ROW_SPLIT": ("info", "document"), + "XLS_NO_CODEPAGE": ("info", "file"), +} + + +def test_warning_table_matches_spec(rules: Rules) -> None: + assert {c: (w.severity.value, w.scope) for c, w in rules.warnings.items()} == SPEC_WARNINGS + + +def test_simplified_income_relation_is_soft(rules: Rules) -> None: + (rule,) = rules.checks.relations_for(DocumentType.INCOME, FormVariant.SIMPLIFIED) + assert rule.target == "2400" and not rule.strict + assert rule.abs_terms == frozenset({"2120", "2330", "2350", "2410"}) + assert rules.checks.abs_codes(DocumentType.INCOME, FormVariant.FULL) == frozenset( + {"2120", "2210", "2220", "2330", "2350", "2410"} + ) + + +def test_code_orders_with_repeats(rules: Rules) -> None: + fsbu = rules.code_orders.sequence(Edition.FSBU4, DocumentType.BALANCE, FormVariant.FULL) + assert fsbu.count("1220") == len(("1220", "1220")) and "1120" not in fsbu + ras = rules.code_orders.sequence(Edition.RAS_66N, DocumentType.BALANCE, FormVariant.FULL) + assert "1120" in ras and ras.index("1110") < ras.index("1100") + assert rules.code_orders.editions == (Edition.RAS_66N, Edition.FSBU4) + assert rules.code_orders.sequence(Edition.FSBU4, DocumentType.DISCLOSURE, FormVariant.FULL) == () + + +def test_public_rule_types(rules: Rules) -> None: + assert isinstance(rules.checks.tolerance, Decimal) and rules.checks.tolerance == Decimal("1") + assert all(isinstance(e, Edition) for e in rules.code_orders.editions) + fsbu = rules.code_orders.sequence(Edition.FSBU4, DocumentType.INCOME, FormVariant.SIMPLIFIED) + assert isinstance(fsbu, tuple) and all(isinstance(c, str) for c in fsbu) + assert (rules.code_ranges.min_codes_continuation, rules.code_ranges.min_codes) == (1, 3) + assert tuple(rules.headers.labels) == LABEL_KEYS + + +def test_warning_code_enum_matches_dictionary(rules: Rules) -> None: + assert {c.value for c in WarningCode} == set(rules.warnings) == set(SPEC_WARNINGS) + assert WarningCode.XLS_NO_CODEPAGE == "XLS_NO_CODEPAGE" + + +def test_no_code_to_name_mapping_anywhere() -> None: + code = re.compile(r"^\d{4,5}$") + + def walk(node: object) -> None: + if isinstance(node, dict): + for key, value in node.items(): + assert not (code.match(str(key)) and isinstance(value, str) and re.search(r"[А-Яа-яЁё]", value)) + walk(value) + elif isinstance(node, list): + for item in node: + walk(item) + + for path in RULES_DIR.glob("*.yaml"): + walk(yaml.safe_load(path.read_text(encoding="utf-8"))) + + +def test_broken_repeat_count_rejected(tmp_path: Path) -> None: + for path in RULES_DIR.glob("*.yaml"): + (tmp_path / path.name).write_bytes(path.read_bytes()) + orders = tmp_path / "statement_code_orders.yaml" + orders.write_text( + orders.read_text(encoding="utf-8").replace('- "1520" # «Обязательство', '# "1520"'), encoding="utf-8" + ) + with pytest.raises(RulesError, match="statement_code_orders"): + load_rules(tmp_path) + + +def test_unknown_severity_rejected(tmp_path: Path) -> None: + for path in RULES_DIR.glob("*.yaml"): + (tmp_path / path.name).write_bytes(path.read_bytes()) + target = tmp_path / "statement_warnings.yaml" + target.write_text( + target.read_text(encoding="utf-8").replace("severity: info", "severity: warn", 1), encoding="utf-8" + ) + with pytest.raises(RulesError, match="statement_warnings"): + load_rules(tmp_path) + + +def test_make_warning_takes_severity_from_dictionary(rules: Rules) -> None: + warning = make_warning(WarningCode.COLUMN_SKIPPED, "колонка «Изменение, %» пропущена", rules, column_index=5) + assert (warning.code, warning.severity, warning.column_index) == ("COLUMN_SKIPPED", Severity.INFO, 5) + assert make_warning("JSON_FAILED", "x", rules).severity is Severity.ERROR + + +def test_make_warning_unknown_code_raises(rules: Rules) -> None: + with pytest.raises(KeyError, match="NOT_A_CODE"): + make_warning("NOT_A_CODE", "x", rules) + + +def test_label_keys_are_checked(tmp_path: Path) -> None: + for path in RULES_DIR.glob("*.yaml"): + (tmp_path / path.name).write_bytes(path.read_bytes()) + target = tmp_path / "statement_headers.yaml" + target.write_text(target.read_text(encoding="utf-8").replace(" okpo:", " okpo_x:", 1), encoding="utf-8") + with pytest.raises(RulesError, match="statement_headers"): + load_rules(tmp_path) -- 2.47.2 From 33ec43d78d1c7122f5c06767feb0c6715d2cd050 Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Sun, 18 Oct 2026 13:18:49 +0400 Subject: [PATCH 04/28] =?UTF-8?q?fix:=20=D0=B2=D0=B8=D0=B4=20=D0=B0=D0=B1?= =?UTF-8?q?=D1=81=D0=BE=D0=BB=D1=8E=D1=82=D0=BD=D0=BE=D0=B3=D0=BE=20=D0=BF?= =?UTF-8?q?=D0=B5=D1=80=D0=B8=D0=BE=D0=B4=D0=B0=20=D0=B1=D0=B5=D1=80=D1=91?= =?UTF-8?q?=D1=82=D1=81=D1=8F=20=D0=B8=D0=B7=20=D1=81=D0=BB=D0=BE=D0=B2?= =?UTF-8?q?=D0=B0=D1=80=D1=8F;=20=D0=BF=D1=80=D0=BE=D0=B2=D0=B5=D1=80?= =?UTF-8?q?=D0=BA=D0=B0=20=D0=BA=D0=BB=D1=8E=D1=87=D0=B5=D0=B9=20repeats?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W --- arm_statements/loader.py | 9 ++++++++- arm_statements/tests/test_loader.py | 26 ++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/arm_statements/loader.py b/arm_statements/loader.py index 7cfbe34..2ffc8f2 100644 --- a/arm_statements/loader.py +++ b/arm_statements/loader.py @@ -322,7 +322,7 @@ def _load_periods(directory: Path) -> PeriodRules: pattern = _rx(file, f"{key}.pattern", _get(file, entry, "pattern")) if not set(pattern.groupindex) <= _PERIOD_GROUPS: raise _fail(file, f"{key}.pattern", f"допустимые имена групп: {sorted(_PERIOD_GROUPS)}") - absolute.append(PeriodPattern(kind, pattern)) + absolute.append(PeriodPattern(abs_kind, pattern)) months: list[tuple[str, int]] = [] for stem, number in _map(file, "months", _get(file, data, "months")).items(): month = _int(file, f"months.{stem}", number) @@ -424,6 +424,13 @@ def _load_orders(directory: Path) -> CodeOrders: raise _fail(file, "version", "поддерживается только версия 1") repeats = _map(file, "repeats", _get(file, data, "repeats")) editions_raw = _map(file, "editions", _get(file, data, "editions")) + for ed_name, forms_node in repeats.items(): + if ed_name not in editions_raw: + raise _fail(file, f"repeats.{ed_name}", "редакции нет в editions") + known_forms = _map(file, f"editions.{ed_name}", editions_raw[ed_name]) + for form_name in _map(file, f"repeats.{ed_name}", forms_node): + if form_name not in known_forms: + raise _fail(file, f"repeats.{ed_name}.{form_name}", "формы нет в editions этой редакции") editions: list[Edition] = [] sequences: dict[tuple[Edition, DocumentType, FormVariant], tuple[str, ...]] = {} for ed_name, forms_node in editions_raw.items(): diff --git a/arm_statements/tests/test_loader.py b/arm_statements/tests/test_loader.py index 84a9e5e..f5df35c 100644 --- a/arm_statements/tests/test_loader.py +++ b/arm_statements/tests/test_loader.py @@ -139,3 +139,29 @@ def test_label_keys_are_checked(tmp_path: Path) -> None: target.write_text(target.read_text(encoding="utf-8").replace(" okpo:", " okpo_x:", 1), encoding="utf-8") with pytest.raises(RulesError, match="statement_headers"): load_rules(tmp_path) + + +def test_absolute_period_kinds_come_from_dictionary(rules: Rules) -> None: + expected = [ + "range_end", + "day_month_year", + "quarter", + "half", + "months", + "month_range", + "year", + ] + data = yaml.safe_load((RULES_DIR / "statement_periods.yaml").read_text(encoding="utf-8")) + assert [p["kind"] for p in data["absolute"]] == expected + assert [p.kind for p in rules.periods.absolute] == expected + + +def test_repeats_for_unknown_form_rejected(tmp_path: Path) -> None: + for path in RULES_DIR.glob("*.yaml"): + (tmp_path / path.name).write_bytes(path.read_bytes()) + orders = tmp_path / "statement_code_orders.yaml" + orders.write_text( + orders.read_text(encoding="utf-8").replace(" INCOME: []", " DISCLOSURE: []", 1), encoding="utf-8" + ) + with pytest.raises(RulesError, match="repeats"): + load_rules(tmp_path) -- 2.47.2 From aed37cc189e9db62d0b5898d4147351ef7b1bd6b Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Sun, 18 Oct 2026 13:20:30 +0400 Subject: [PATCH 05/28] =?UTF-8?q?feat:=20=D0=B1=D1=83=D0=BA=D0=B2=D1=8B-?= =?UTF-8?q?=D0=B4=D0=B2=D0=BE=D0=B9=D0=BD=D0=B8=D0=BA=D0=B8,=20=D0=BD?= =?UTF-8?q?=D0=BE=D1=80=D0=BC=D0=B0=D0=BB=D0=B8=D0=B7=D0=B0=D1=86=D0=B8?= =?UTF-8?q?=D1=8F=20=D0=BF=D0=BE=D0=B4=D0=BF=D0=B8=D1=81=D0=B5=D0=B9,=20?= =?UTF-8?q?=D1=80=D0=B0=D0=B7=D0=B1=D0=BE=D1=80=20=D0=BA=D0=BE=D0=B4=D0=B0?= =?UTF-8?q?=20=D1=81=D1=82=D1=80=D0=BE=D0=BA=D0=B8=20=D0=B8=20=D0=BE=D1=82?= =?UTF-8?q?=D1=80=D0=B5=D0=B7=D0=BA=D0=B8=20=D0=BA=D0=BE=D0=B4=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W --- arm_statements/codes.py | 118 ++++++++++++++++++++++++ arm_statements/tests/test_text_codes.py | 74 +++++++++++++++ arm_statements/text.py | 60 ++++++++++++ 3 files changed, 252 insertions(+) create mode 100644 arm_statements/codes.py create mode 100644 arm_statements/tests/test_text_codes.py create mode 100644 arm_statements/text.py diff --git a/arm_statements/codes.py b/arm_statements/codes.py new file mode 100644 index 0000000..fb8859c --- /dev/null +++ b/arm_statements/codes.py @@ -0,0 +1,118 @@ +"""Разбор кода строки и отрезки кодов форм.""" + +import re +from collections import Counter +from collections.abc import Iterable, Mapping, Sequence +from typing import NamedTuple + +from arm_statements.loader import CodeSection, Rules +from arm_statements.model import DocumentType +from arm_statements.text import replace_lookalikes + +_CODE = re.compile(r"\d{4,5}(?:\.0)?") +_BAR = "|" + + +class CodeParse(NamedTuple): + code: str | None + fixed: bool # при разборе заменён двойник + + +def code_parts(text: str | None) -> tuple[str, ...]: + """Непустые строки ячейки, обрезанные.""" + if text is None: + return () + return tuple(part.strip() for part in text.splitlines() if part.strip()) + + +def section_of(code: str, rules: Rules) -> CodeSection | None: + head = int(code[:4]) + for item in rules.code_ranges.ranges: + if item.low <= head <= item.high: + return item.section + return None + + +def _strip_edges(text: str) -> str: + start, end = 0, len(text) + while start < end and not text[start].isalnum(): + start += 1 + while end > start and not text[end - 1].isalnum(): + end -= 1 + return text[start:end] + + +def _as_code(text: str, rules: Rules) -> str | None: + core = _strip_edges(text) + if _CODE.fullmatch(core) is None: + return None + code = core.split(".", 1)[0] + return code if section_of(code, rules) is not None else None + + +def _parse_line(line: str, rules: Rules) -> CodeParse: + compact = "".join(line.split()) + # сначала `|` как двойник единицы; если результата-кода нет — `|` как краевая пунктуация + for candidate in (compact, compact.replace(_BAR, "")): + replaced = replace_lookalikes(candidate, rules.lookalikes) + code = _as_code(replaced, rules) + if code is not None: + return CodeParse(code, replaced != candidate) + return CodeParse(None, False) + + +def parse_code(text: str | None, rules: Rules) -> CodeParse: + """Однострочный текст → код; многострочный → `CodeParse(None, False)`.""" + parts = code_parts(text) + if len(parts) != 1: + return CodeParse(None, False) + return _parse_line(parts[0], rules) + + +def is_code_cell(text: str | None, rules: Rules) -> bool: + parts = code_parts(text) + return bool(parts) and all(_parse_line(part, rules).code is not None for part in parts) + + +def majority_section(codes: Iterable[str], rules: Rules) -> CodeSection | None: + """Отрезок большинства; ничья — первый по порядку отрезков словаря.""" + counts: Counter[CodeSection] = Counter() + for code in codes: + section = section_of(code, rules) + if section is not None: + counts[section] += 1 + best: CodeSection | None = None + for item in rules.code_ranges.ranges: + if counts[item.section] and (best is None or counts[item.section] > counts[best]): + best = item.section + return best + + +def document_type_of(section: CodeSection) -> DocumentType: + mapping: Mapping[CodeSection, DocumentType] = { + CodeSection.BALANCE: DocumentType.BALANCE, + CodeSection.INCOME: DocumentType.INCOME, + } + return mapping.get(section, DocumentType.DISCLOSURE) + + +def sheet_joins( + form_type: DocumentType, + sheet_codes: Sequence[str], + sheet_amount_columns: int, + form_amount_columns: int, + rules: Rules, +) -> bool: + """Лист без маркера и шапки присоединяется к форме: коды в её отрезке; при ≥3 кодах колонки сумм совпадают.""" + if len(sheet_codes) < rules.code_ranges.min_codes_continuation: + return False + form_section = {DocumentType.BALANCE: CodeSection.BALANCE, DocumentType.INCOME: CodeSection.INCOME}.get(form_type) + for code in sheet_codes: + section = section_of(code, rules) + if section is None or document_type_of(section) is not form_type: + return False + if form_section is not None and section is not form_section: + return False + if len(sheet_codes) >= rules.code_ranges.min_codes: + return sheet_amount_columns == form_amount_columns + return True diff --git a/arm_statements/tests/test_text_codes.py b/arm_statements/tests/test_text_codes.py new file mode 100644 index 0000000..17567c7 --- /dev/null +++ b/arm_statements/tests/test_text_codes.py @@ -0,0 +1,74 @@ +import pytest + +from arm_statements.codes import is_code_cell, majority_section, parse_code, sheet_joins +from arm_statements.loader import CodeSection, Rules +from arm_statements.model import DocumentType +from arm_statements.text import fix_lookalikes, is_disguised_digit, normalize_caption, normalize_label + + +@pytest.mark.parametrize( + ("raw", "code", "fixed"), + [ + ("1220 ,", "1220", False), + ("| 140", "1140", True), + ("I1О0", "1100", True), + ("1110.0", "1110", False), + ("12301", "12301", False), + ("3450", "3450", False), + ("2024", None, False), + ("1050", None, False), + ("7450", None, False), + ("ИЧ,250", None, False), + ("изо :", None, False), + ("НИ", None, False), + ("1500\n1700", None, False), + ("2", None, False), + ], +) +def test_parse_code(rules: Rules, raw: str, code: str | None, fixed: bool) -> None: + assert parse_code(raw, rules) == (code, fixed) + + +def test_multiline_code_cell(rules: Rules) -> None: + assert is_code_cell("1500\n1700", rules) + assert not is_code_cell("44 30\nI |", rules) + + +def test_lookalikes_only_near_digit(rules: Rules) -> None: + assert fix_lookalikes("2ОО5", rules.lookalikes) == "2005" + assert fix_lookalikes("2|0|2|4|", rules.lookalikes).split() == ["2", "0", "2", "4"] + assert fix_lookalikes("Основные средства", rules.lookalikes) == "Основные средства" + assert fix_lookalikes("З", rules.lookalikes) == "З" + assert fix_lookalikes("З", rules.lookalikes, anywhere=True) == "3" + + +def test_disguised_digit(rules: Rules) -> None: + assert is_disguised_digit("I", rules.lookalikes) and is_disguised_digit("| О", rules.lookalikes) + assert not is_disguised_digit("Косарев", rules.lookalikes) + assert not is_disguised_digit("(расшифровка подписи)", rules.lookalikes) + + +def test_caption_normalization() -> None: + assert normalize_caption("Код строки") == "код строки" + assert normalize_caption("Изменение, %") == "изменение %" + assert normalize_caption("Наименование\nпоказателя") == "наименование показателя" + + +def test_label_normalization_keeps_punctuation(rules: Rules) -> None: + assert normalize_label("Местонахождение (адрес)") == "местонахождение (адрес)" + assert normalize_label("по ОКОПФ / ОКФС") == "по окопф / окфс" + assert normalize_label("Форма 0710001 с.2") == "форма 0710001 с.2" + assert normalize_label("Отчёт о финансовых\nрезультатах") == "отчет о финансовых результатах" + assert normalize_label("за­дол­жен—ность") == "задолжен ность" + assert normalize_label(None) == "" + for key, patterns in rules.headers.labels.items(): + assert patterns, key + assert any(p.search(normalize_label("Местонахождение (адрес)")) for p in rules.headers.labels["address"]) + assert any(p.search(normalize_label("по ОКОПФ / ОКФС")) for p in rules.headers.labels["okopf_okfs"]) + + +def test_majority_section_and_join(rules: Rules) -> None: + assert majority_section(["1110", "1150", "2110"], rules) is CodeSection.BALANCE + assert sheet_joins(DocumentType.INCOME, ["2400"], 1, 3, rules) + assert not sheet_joins(DocumentType.BALANCE, ["1110", "1150", "1100"], 4, 3, rules) + assert not sheet_joins(DocumentType.BALANCE, ["2110"], 3, 3, rules) diff --git a/arm_statements/text.py b/arm_statements/text.py new file mode 100644 index 0000000..70ac37d --- /dev/null +++ b/arm_statements/text.py @@ -0,0 +1,60 @@ +"""Очистка OCR-текста: буквы-двойники, нормализация подписей и меток (§11.1 п. 1, 6).""" + +import re +from collections.abc import Mapping + +# первая строка из цифр, пробелов, `|`, точек, запятых и двоеточий: `|` в ней — разделитель клеток формы +_CELL_SEPARATOR_LINE = re.compile(r"[\s\d|•·.:,]*") +_INVISIBLE = re.compile("[\u00ad\u200b\ufeff]") +_SEPARATORS = re.compile(r"[\s\-\u2010-\u2015\u2212]+") +_NON_CAPTION = re.compile(r"[^\w%]|_") + + +def replace_lookalikes(text: str, lookalikes: Mapping[str, str], *, anywhere: bool = False) -> str: + """Двойники → цифры вплотную к цифре (несколько проходов); `anywhere` — без условия соседства.""" + if not lookalikes: + return text + table = str.maketrans(dict(lookalikes)) + if anywhere: + return text.translate(table) + chars = re.escape("".join(lookalikes)) + near_digit = re.compile(rf"(?<=\d)[{chars}]|[{chars}](?=\d)") + previous = None + while previous != text: # «2ОО5» требует двух проходов + previous = text + text = near_digit.sub(lambda match: match.group(0).translate(table), text) + return text + + +def fix_lookalikes(text: str, lookalikes: Mapping[str, str], *, anywhere: bool = False) -> str: + """NBSP → пробел; `|` — разделитель клеток, если первая строка из цифр и пунктуации; двойники → цифры.""" + text = text.replace("\xa0", " ") + if _CELL_SEPARATOR_LINE.fullmatch(text.split("\n", 1)[0]): + text = text.replace("|", " ") + return replace_lookalikes(text, lookalikes, anywhere=anywhere) + + +def has_digit(text: str | None) -> bool: + return text is not None and any(char.isdigit() for char in text) + + +def is_disguised_digit(text: str, lookalikes: Mapping[str, str]) -> bool: + """После замены двойников везде есть цифра и нет других букв («I», «| О» — да; «Косарев» — нет).""" + fixed = fix_lookalikes(text, lookalikes, anywhere=True) + return has_digit(fixed) and not any(char.isalpha() for char in fixed) + + +def normalize_caption(text: str | None) -> str: + """Нижний регистр, `ё → е`, всё кроме букв, цифр и `%` → пробел, пробелы схлопнуты.""" + if text is None: + return "" + lowered = text.lower().replace("ё", "е") + return " ".join(_NON_CAPTION.sub(" ", lowered).split()) + + +def normalize_label(text: str | None) -> str: + """Метки шапки: невидимые удалены, нижний регистр, `ё → е`, пробелы и дефисы/тире → пробел; пунктуация сохранена.""" + if text is None: + return "" + lowered = _INVISIBLE.sub("", text).lower().replace("ё", "е") + return _SEPARATORS.sub(" ", lowered).strip() -- 2.47.2 From 19f70fea65e2fa5b2d327cc2ea81cc7700d2c607 Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Sun, 18 Oct 2026 13:23:13 +0400 Subject: [PATCH 06/28] =?UTF-8?q?feat:=20=D1=81=D0=BB=D0=BE=D0=B2=D0=B0?= =?UTF-8?q?=D1=80=D1=8C=20=D0=BF=D0=B5=D1=80=D0=B8=D0=BE=D0=B4=D0=BE=D0=B2?= =?UTF-8?q?=20=E2=80=94=20=D0=B4=D0=B0=D1=82=D1=8B,=20=D0=BE=D1=82=D0=BD?= =?UTF-8?q?=D0=BE=D1=81=D0=B8=D1=82=D0=B5=D0=BB=D1=8C=D0=BD=D1=8B=D0=B5=20?= =?UTF-8?q?=D0=BF=D0=BE=D0=B4=D0=BF=D0=B8=D1=81=D0=B8=20=D0=B1=D0=BB=D0=B0?= =?UTF-8?q?=D0=BD=D0=BA=D0=B0=2066=D0=BD,=20=D0=B8=D1=81=D0=BA=D0=BB=D1=8E?= =?UTF-8?q?=D1=87=D0=B5=D0=BD=D0=B8=D1=8F=20=D0=BA=D0=BE=D0=BB=D0=BE=D0=BD?= =?UTF-8?q?=D0=BE=D0=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W --- arm_statements/dates.py | 276 +++++++++++++++++++++++++++++ arm_statements/tests/test_dates.py | 94 ++++++++++ 2 files changed, 370 insertions(+) create mode 100644 arm_statements/dates.py create mode 100644 arm_statements/tests/test_dates.py diff --git a/arm_statements/dates.py b/arm_statements/dates.py new file mode 100644 index 0000000..b76a216 --- /dev/null +++ b/arm_statements/dates.py @@ -0,0 +1,276 @@ +"""Даты и подписи колонок: разбор текстовых и числовых форм, относительные подписи бланка 66н (§11.3, §12 п. 4). + +Вход `parse_date` — текст после `fix_lookalikes`. Регулярки и месяцы — данные (`rules/statement_periods.yaml`), +здесь только обработчики на каждый вид абсолютной формы.""" + +import calendar +import re +from collections.abc import Callable +from dataclasses import dataclass +from datetime import date + +from arm_statements.loader import PeriodRules, Rules +from arm_statements.model import Cell, PeriodKind +from arm_statements.text import fix_lookalikes, normalize_caption + +__all__ = ["PeriodLabel", "expected_dates", "parse_date", "parse_period_label", "resolve_relative"] + +_BARE_YEAR = re.compile(r"\d{4}") +_DIGITS = re.compile(r"\d+") +_ROMAN = {"I": 1, "II": 2, "III": 3, "IV": 4, "1V": 4, "11": 2, "111": 3} +_ORDINAL = {"перв": 1, "втор": 2, "трет": 3, "четв": 4} +_QUARTER_MONTHS = 3 +_HALF_MONTHS = 6 +_DECEMBER = 12 +_SHORT_YEAR_LIMIT = 100 # меньше — двузначный год → 20YY +_CENTURY = 2000 +_DIGITS_FULL = 8 +_DIGITS_YEAR = 4 +_DIGITS_SHORT = 6 +_PARTS = 3 +_SHORT_PART = 2 + + +@dataclass(frozen=True, slots=True) +class PeriodLabel: + kind: PeriodKind + date: date | None + excluded: bool + bare_year: bool + + +def _year_full(year: str) -> int: + value = int(year) + return value if value >= _SHORT_YEAR_LIMIT else _CENTURY + value + + +def _month_num(word: str, rules: PeriodRules) -> int | None: + lowered = word.lower() + for stem, number in rules.months: + if lowered.startswith(stem): + return number + return None + + +def _month_end(year: int, month: int) -> date | None: + if not 1 <= month <= _DECEMBER: + return None + try: + return date(year, month, calendar.monthrange(year, month)[1]) + except ValueError: + return None + + +def _period_end(year: str, month: int | None) -> date | None: + return _month_end(_year_full(year), month) if month else None + + +def _safe(year: int, month: int | None, day: int) -> date | None: + try: + return date(year, month, day) if month else None + except ValueError: + return None + + +def _ordinal(word: str) -> int | None: + """Номер квартала/полугодия из «1», «I», «IV», «первый», «1-й».""" + token = word.strip().upper().rstrip("-Й") + if token in _ROMAN: + return _ROMAN[token] + if token.isdigit(): + return int(token) + lowered = token.lower() + for stem, number in _ORDINAL.items(): + if lowered.startswith(stem): + return number + return None + + +def _group(match: re.Match[str], name: str) -> str | None: + return match.group(name) if name in match.re.groupindex else None + + +def _handle_range_end(match: re.Match[str], rules: PeriodRules) -> date | None: + return _handle_day_month_year(match, rules) + + +def _handle_day_month_year(match: re.Match[str], rules: PeriodRules) -> date | None: + year, day = _group(match, "y"), _group(match, "d") + if year is None or day is None: + return None + month_word = _group(match, "mw") + month = _month_num(month_word, rules) if month_word is not None else int(_group(match, "m") or 0) + return _safe(_year_full(year), month, int(day)) + + +def _handle_quarter(match: re.Match[str], rules: PeriodRules) -> date | None: + del rules + year, number = _group(match, "y"), _ordinal(_group(match, "q") or "") + return _period_end(year, (number or 0) * _QUARTER_MONTHS) if year else None + + +def _handle_half(match: re.Match[str], rules: PeriodRules) -> date | None: + del rules + year, number = _group(match, "y"), _ordinal(_group(match, "q") or "") + return _period_end(year, (number or 0) * _HALF_MONTHS) if year else None + + +def _handle_months(match: re.Match[str], rules: PeriodRules) -> date | None: + del rules + year, count = _group(match, "y"), _group(match, "n") + return _period_end(year, int(count)) if year and count else None + + +def _handle_month_range(match: re.Match[str], rules: PeriodRules) -> date | None: + year, word = _group(match, "y"), _group(match, "mw") + return _period_end(year, _month_num(word, rules)) if year and word else None + + +def _handle_year(match: re.Match[str], rules: PeriodRules) -> date | None: + year = _group(match, "y") + return _period_end(year, _DECEMBER) if year and _year_ok(int(year), rules) else None + + +_HANDLERS: dict[str, Callable[[re.Match[str], PeriodRules], date | None]] = { + "range_end": _handle_range_end, + "day_month_year": _handle_day_month_year, + "quarter": _handle_quarter, + "half": _handle_half, + "months": _handle_months, + "month_range": _handle_month_range, + "year": _handle_year, +} + + +def _year_ok(year: int, rules: PeriodRules) -> bool: + return rules.min_year <= year <= rules.max_year + + +def _text_date(text: str, rules: PeriodRules) -> date | None: + for item in rules.absolute: + match = item.pattern.search(text) + if match: + found = _HANDLERS[item.kind](match, rules) + if found: + return found + return None + + +def _from_parts(first: str, second: str, third: str) -> date | None: + """ДД ММ ГГ[ГГ] и ГГ[ГГ] ММ ДД (год двух- или четырёхзначный только в позиции года).""" + for day, month, year in ((first, second, third), (third, second, first)): + if len(year) in (_SHORT_PART, _DIGITS_YEAR) and len(day) <= _SHORT_PART and len(month) <= _SHORT_PART: + try: + return date(_year_full(year), int(month), int(day)) + except ValueError: + continue + return None + + +def _from_digits(digits: str) -> date | None: + """8 слитных цифр: ДДММГГГГ, ГГГГММДД, ГГГГДДММ — первый валидный вариант.""" + return ( + _from_parts(digits[:2], digits[2:4], digits[4:]) + or _from_parts(digits[6:], digits[4:6], digits[:4]) + or _from_parts(digits[4:6], digits[6:], digits[:4]) + ) + + +def _from_digits_run(digits: str, groups: list[str], rules: PeriodRules) -> date | None: + """Склеенные цифры «2 0 2 4», «30 09 2 0 2 5»: решает их число.""" + if len(digits) == _DIGITS_FULL: + return _from_digits(digits) + if len(digits) == _DIGITS_YEAR: + return date(int(digits), 12, 31) if _year_ok(int(digits), rules) else None + if len(digits) == _DIGITS_SHORT: + return _from_parts(digits[:2], digits[2:4], digits[4:]) + if len(groups) > _PARTS: # «30.09.2025 0:00:00» + return _from_parts(*groups[:_PARTS]) + return None + + +def _numeric_date(text: str, rules: PeriodRules) -> date | None: + groups = _DIGITS.findall(text) + if not groups: + return None + if len(groups) == _PARTS: + return _from_parts(*groups) + return _from_digits_run("".join(groups), groups, rules) + + +def parse_date(text: str | None, rules: Rules) -> date | None: + """Дата из текста после очистки или None. Длиннее `max_text_length` не разбирается (проверка до регулярок).""" + if text is None: + return None + stripped = text.strip() + if not stripped or len(stripped) > rules.periods.max_text_length: + return None + periods = rules.periods + return ( + _text_date(stripped, periods) + or _numeric_date(stripped.split("\n", 1)[0], periods) + or _numeric_date(stripped.replace("\n", " "), periods) + ) + + +def _is_excluded(caption: str, rules: Rules) -> bool: + return any(rx.search(caption) for rx in rules.headers.column_exclusions) + + +def _classify(text: str, rules: Rules) -> PeriodLabel: + caption = normalize_caption(text) + bare_year = _BARE_YEAR.fullmatch(text.strip()) is not None + if _is_excluded(caption, rules): + return PeriodLabel(PeriodKind.NONE, None, True, False) + for kind, pattern in rules.periods.relative: + if pattern.search(caption): + return PeriodLabel(kind, None, False, bare_year) + found = parse_date(text, rules) + return PeriodLabel(PeriodKind.ABSOLUTE if found else PeriodKind.NONE, found, False, bare_year) + + +def parse_period_label(cell: Cell | None, rules: Rules) -> PeriodLabel: + """Подпись колонки → вид периода и дата. Дата-значение ячейки Excel — абсолютная.""" + if cell is not None and isinstance(cell.value, date): + return PeriodLabel(PeriodKind.ABSOLUTE, cell.value, False, False) + raw = cell.text.strip() if cell is not None else "" + if not raw: + return PeriodLabel(PeriodKind.NONE, None, False, False) + # длинный текст не чистим от двойников (backtracking); parse_date всё равно вернёт None + text = raw if len(raw) > rules.periods.max_text_length else fix_lookalikes(raw, rules.lookalikes) + return _classify(text, rules) + + +def _shift_years(day: date, years: int) -> date: + try: + return day.replace(year=day.year - years) + except ValueError: # 29.02 → 28.02 + return day.replace(year=day.year - years, day=28) + + +def resolve_relative(kind: PeriodKind, effective: date | None) -> date | None: + """Дата относительной подписи; для ABSOLUTE и NONE — None (дата из подписи или от аналитика, P1-24).""" + if effective is None: + return None + if kind is PeriodKind.CURRENT: + return effective + if kind is PeriodKind.PREV_YEAR_END: + return date(effective.year - 1, 12, 31) + if kind is PeriodKind.PREV_PREV_YEAR_END: + return date(effective.year - 2, 12, 31) + if kind is PeriodKind.PREV_PERIOD: + return _shift_years(effective, 1) + return None + + +def expected_dates(effective: date) -> frozenset[date]: + """Даты колонок формы при отчётной дате: она сама, она − 1 и − 2 года, 31.12 года − 1 и года − 2.""" + return frozenset( + { + effective, + _shift_years(effective, 1), + _shift_years(effective, 2), + date(effective.year - 1, 12, 31), + date(effective.year - 2, 12, 31), + } + ) diff --git a/arm_statements/tests/test_dates.py b/arm_statements/tests/test_dates.py new file mode 100644 index 0000000..887de26 --- /dev/null +++ b/arm_statements/tests/test_dates.py @@ -0,0 +1,94 @@ +from datetime import date + +import pytest + +from arm_statements.dates import expected_dates, parse_date, parse_period_label, resolve_relative +from arm_statements.loader import Rules +from arm_statements.model import Cell, PeriodKind +from arm_statements.text import fix_lookalikes + +# Обязательные строки scan-gateway test_dates.py::test_parses_real_forms (спека «Шаг 2 — подпись и дата колонки») +REAL_FORMS = [ + ("на 30 сентября 2025 г.", date(2025, 9, 30)), + ("На 30 июня\n2025 г", date(2025, 6, 30)), + ("31 декабря 2024 года", date(2024, 12, 31)), + ("30 сентября 2025 r.", date(2025, 9, 30)), + ("с 01.01.2025 по 30.09.2025", date(2025, 9, 30)), + ("I квартал 2026", date(2026, 3, 31)), + ("1-й кв. 2026", date(2026, 3, 31)), + ("первый квартал 2025 г.", date(2025, 3, 31)), + ("1 полугодие 2026", date(2026, 6, 30)), + ("1 п/г 2026", date(2026, 6, 30)), + ("9 месяцев 2025", date(2025, 9, 30)), + ("9 мес. 2025", date(2025, 9, 30)), + ("За январь - июнь\n2025 г.", date(2025, 6, 30)), + ("январь – сентябрь 2025", date(2025, 9, 30)), + ("за 2025 г.", date(2025, 12, 31)), + ("2024 год", date(2024, 12, 31)), + ("30.09.2025", date(2025, 9, 30)), + ("2025 03 31", date(2025, 3, 31)), + ("30/09/25", date(2025, 9, 30)), + ("2 0 2 4", date(2024, 12, 31)), + ("30 09 2 0 2 5", date(2025, 9, 30)), + ("30092025", date(2025, 9, 30)), + ("20250930", date(2025, 9, 30)), + ("30 09 25", date(2025, 9, 30)), + ("30.09.2025 0:00:00", date(2025, 9, 30)), + ("2|0|2|4|", date(2024, 12, 31)), + ("3l.12.2О24", date(2024, 12, 31)), +] + + +@pytest.mark.parametrize(("raw", "expected"), REAL_FORMS) +def test_parses_real_forms(rules: Rules, raw: str, expected: date) -> None: + assert parse_date(fix_lookalikes(raw, rules.lookalikes), rules) == expected + label = parse_period_label(Cell(raw, None, 100.0, "row:0"), rules) + assert (label.kind, label.date) == (PeriodKind.ABSOLUTE, expected) + + +@pytest.mark.parametrize("raw", ["Общество с ограниченной ответственностью", "", "+", "тыс. руб.", "1999999999999"]) +def test_not_a_date(rules: Rules, raw: str) -> None: + assert parse_date(raw, rules) is None + + +def test_long_text_not_parsed(rules: Rules) -> None: + assert parse_date("на 30 сентября 2025 г." + "x" * 200, rules) is None + + +@pytest.mark.parametrize( + ("raw", "kind"), + [ + ("На отчетную дату", PeriodKind.CURRENT), + ("За отчётный период", PeriodKind.CURRENT), + ("На 31 декабря предыдущего года", PeriodKind.PREV_YEAR_END), + ("На 31 декабря года, предшествующего предыдущему", PeriodKind.PREV_PREV_YEAR_END), + ("За аналогичный период предыдущего года", PeriodKind.PREV_PERIOD), + ], +) +def test_relative_labels(rules: Rules, raw: str, kind: PeriodKind) -> None: + assert parse_period_label(Cell(raw, None, 100.0, "row:0"), rules).kind is kind + + +def test_exclusion_and_excel_date_value(rules: Rules) -> None: + assert parse_period_label(Cell("Изменение, %", None, 100.0, "A!F1"), rules).excluded + label = parse_period_label(Cell("30.09.2025", date(2025, 9, 30), 100.0, "A!D1"), rules) + assert (label.kind, label.date) == (PeriodKind.ABSOLUTE, date(2025, 9, 30)) + assert parse_period_label(Cell("2024", None, 100.0, "A!E1"), rules).bare_year + + +def test_empty_label(rules: Rules) -> None: + assert parse_period_label(None, rules).kind is PeriodKind.NONE + label = parse_period_label(Cell(" ", None, 100.0, "A!G1"), rules) + assert (label.kind, label.date, label.excluded, label.bare_year) == (PeriodKind.NONE, None, False, False) + + +def test_resolve_relative_and_expected() -> None: + eff = date(2025, 9, 30) + assert resolve_relative(PeriodKind.PREV_YEAR_END, eff) == date(2024, 12, 31) + assert resolve_relative(PeriodKind.PREV_PREV_YEAR_END, eff) == date(2023, 12, 31) + assert resolve_relative(PeriodKind.PREV_PERIOD, date(2024, 2, 29)) == date(2023, 2, 28) + assert resolve_relative(PeriodKind.CURRENT, None) is None + assert resolve_relative(PeriodKind.ABSOLUTE, eff) is None and resolve_relative(PeriodKind.NONE, eff) is None + assert expected_dates(eff) == frozenset( + {eff, date(2024, 9, 30), date(2023, 9, 30), date(2024, 12, 31), date(2023, 12, 31)} + ) -- 2.47.2 From 7b14276b1aa578c5d40d0179dc60eab70a0515ff Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Sun, 18 Oct 2026 13:26:07 +0400 Subject: [PATCH 07/28] =?UTF-8?q?feat:=20parse=5Famount=20=D0=BF=D0=BE=20?= =?UTF-8?q?=D1=81=D1=82=D0=B0=D0=B4=D0=B8=D1=8F=D0=BC=20(=D1=8F=D0=B4?= =?UTF-8?q?=D1=80=D0=BE=20=D1=87=D0=B8=D1=81=D0=BB=D0=B0,=20=D0=BA=D1=80?= =?UTF-8?q?=D0=B0=D0=B5=D0=B2=D0=BE=D0=B9=20=D0=BC=D1=83=D1=81=D0=BE=D1=80?= =?UTF-8?q?,=20=D1=81=D0=BA=D0=BE=D0=B1=D0=BA=D0=B8,=20=D1=80=D0=B0=D0=B7?= =?UTF-8?q?=D0=B4=D0=B5=D0=BB=D0=B8=D1=82=D0=B5=D0=BB=D0=B8)=20=D0=B8=20?= =?UTF-8?q?=D0=B7=D0=BD=D0=B0=D1=87=D0=B5=D0=BD=D0=B8=D0=B5=20=D1=8F=D1=87?= =?UTF-8?q?=D0=B5=D0=B9=D0=BA=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W --- arm_statements/amounts.py | 196 +++++++++++++++++++++++++++ arm_statements/tests/test_amounts.py | 84 ++++++++++++ 2 files changed, 280 insertions(+) create mode 100644 arm_statements/amounts.py create mode 100644 arm_statements/tests/test_amounts.py diff --git a/arm_statements/amounts.py b/arm_statements/amounts.py new file mode 100644 index 0000000..1c40eac --- /dev/null +++ b/arm_statements/amounts.py @@ -0,0 +1,196 @@ +"""Сумма и значение ячейки (шаг 4): `parse_amount` — единственное место, где разбор зависит от источника.""" + +import re +from dataclasses import dataclass +from datetime import date +from decimal import ROUND_HALF_UP, Decimal + +from arm_statements.codes import code_parts +from arm_statements.loader import Rules +from arm_statements.model import Cell, SignForm, Source, StatementValue +from arm_statements.text import replace_lookalikes + +MAX_INTEGER_DIGITS = 16 +_CENT = Decimal("0.01") +_DIGITS = frozenset("0123456789") +_CORE_START = _DIGITS | frozenset("(-−+") +_CORE_END = _DIGITS | frozenset(")}]") +_DASHES = frozenset("-–—−") +_MINUSES = frozenset("-−") +_BODY = re.compile(r"[\d\s,.]*") +_DIGIT_SPACE_RUN = re.compile(r"\d+(?:\s+\d+)+") +_SPACE_GROUPS = re.compile(r"\d{1,3}(?:\s\d{3})+") +_COMMA_THOUSANDS = re.compile(r"[1-9]\d{0,2}(?:,\d{3})+") +_DOT_THOUSANDS = re.compile(r"[1-9]\d*(?:\.\d{3})+") +_SEPARATORS = ",." + + +@dataclass(frozen=True, slots=True) +class AmountParse: + amount: Decimal | None + string_value: str | None + is_recognized: bool + sign_form: SignForm | None + + +@dataclass(frozen=True, slots=True) +class ValueBuild: + value: StatementValue + out_of_range: bool + + +def _quantize(amount: Decimal) -> Decimal: + return amount.quantize(_CENT, rounding=ROUND_HALF_UP) + + +def _too_long(amount: Decimal) -> bool: + return len(str(abs(int(amount)))) > MAX_INTEGER_DIGITS + + +def _is_blank(text: str) -> bool: + return all(char.isspace() or char in _DASHES for char in text) + + +def _strip_edge_bars(text: str) -> tuple[str, bool]: + """Крайние `|` (след границы таблицы) — до замены двойников; второй элемент — было ли удаление.""" + stripped = text.strip() + while stripped.startswith("|"): + stripped = stripped[1:].strip() + while stripped.endswith("|"): + stripped = stripped[:-1].strip() + return stripped, stripped != text.strip() + + +def _core_bounds(text: str) -> tuple[int, int] | None: + starts = [index for index, char in enumerate(text) if char in _CORE_START] + ends = [index for index, char in enumerate(text) if char in _CORE_END] + if not starts or not ends or starts[0] > ends[-1]: + return None + return starts[0], ends[-1] + + +def _guess(digits: str, text: str, sign_form: SignForm, *, negative: bool) -> AmountParse: + """«Лучшая догадка»: цифры подряд, не длиннее 16, иначе значение не число.""" + if not digits or len(digits) > MAX_INTEGER_DIGITS: + return AmountParse(None, text, False, sign_form) + amount = Decimal(digits) + return AmountParse(-amount if negative else amount, None, False, sign_form) + + +def _split_core(core: str) -> tuple[str, SignForm, bool, bool] | None: + """Ядро → (тело без знака и скобок, форма знака, отрицательное ли, исправлена ли скобка) или None.""" + fixed = False + sign_form = SignForm.UNSIGNED + negative = False + if core[0] == "(": + if len(core) < 2 or core[-1] not in ")}]": # noqa: PLR2004 — скобки в ядре минимум пара + return None + fixed = core[-1] != ")" + core = core[1:-1].strip() + sign_form, negative = SignForm.PARENTHESES, True + if core and core[0] in "-−+": + core = core[1:].strip() + elif core[0] in "-−+": + if core[0] in _MINUSES: + sign_form, negative = SignForm.MINUS, True + core = core[1:].strip() + if not core or not _BODY.fullmatch(core) or not core[0].isdigit(): + return None + return core, sign_form, negative, fixed + + +def _number(body: str, source: Source, text: str, sign_form: SignForm, *, negative: bool) -> AmountParse: + """Стадия 6: разделители. Возвращает догадку с `is_recognized = false` или точный разбор.""" + runs = _DIGIT_SPACE_RUN.findall(body) + spaces_ok = all(_SPACE_GROUPS.fullmatch(run) for run in runs) + compact = "".join(body.split()) + digits = compact.replace(",", "").replace(".", "") + if not spaces_ok: + return _guess(digits, text, sign_form, negative=negative) + if source is Source.OCR and _COMMA_THOUSANDS.fullmatch(compact): + compact = compact.replace(",", "") + elif source is Source.OCR and _DOT_THOUSANDS.fullmatch(compact): + compact = compact.replace(".", "") + separators = [char for char in compact if char in _SEPARATORS] + if not separators: + integer, fraction = compact, "" + elif len(separators) > 1: # смешанные или повторяющиеся разделители + return _guess(digits, text, sign_form, negative=negative) + else: + integer, fraction = compact.replace(",", ".").split(".") + if not integer: + return AmountParse(None, text, False, sign_form) + if source is Source.OCR and len(fraction) >= 3: # noqa: PLR2004 — три знака: не копейки + return AmountParse(None, text, False, sign_form) + if len(integer) > MAX_INTEGER_DIGITS: + return AmountParse(None, text, False, sign_form) + amount = _quantize(Decimal(f"{integer}.{fraction}" if fraction else integer)) + return AmountParse(-amount if negative else amount, None, True, sign_form) + + +def parse_amount(text: str | None, source: Source, rules: Rules) -> AmountParse: + """Стадии 1–6 шага «значение ячейки»; от источника зависят только десятичный разделитель и тысячи.""" + if text is None or _is_blank(text): + return AmountParse(None, None, True, None) + work, fixed = _strip_edge_bars(text) + replaced = replace_lookalikes(work, rules.lookalikes) + fixed = fixed or replaced != work + if not any(char in _DIGITS for char in replaced): # стадия 5 + return AmountParse(None, text, False, None) + bounds = _core_bounds(replaced) + if bounds is None: + return AmountParse(None, text, False, None) + start, end = bounds + core = replaced[start : end + 1] + trash = bool((replaced[:start] + replaced[end + 1 :]).strip()) + split = _split_core(core) + if split is None: + return AmountParse(None, text, False, None) + body, sign_form, negative, bracket_fixed = split + parsed = _number(body, source, text, sign_form, negative=negative) + clean = parsed.is_recognized and not (fixed or trash or bracket_fixed) + return AmountParse(parsed.amount, parsed.string_value, clean, parsed.sign_form) + + +def sign_form_of(text: str | None, rules: Rules) -> SignForm | None: + return parse_amount(text, Source.OCR, rules).sign_form + + +def _plain(cell: Cell, column_index: int, parsed: AmountParse, *, recognized: bool | None = None) -> StatementValue: + return StatementValue( + column_index=column_index, + amount=parsed.amount, + string_value=parsed.string_value, + text=cell.text, + confidence=cell.confidence, + is_recognized=parsed.is_recognized if recognized is None else recognized, + ) + + +def build_value( + cell: Cell | None, + column_index: int, + source: Source, + rules: Rules, + *, + split_part: bool = False, +) -> ValueBuild: + if cell is None: + return ValueBuild(StatementValue(column_index, None, None, None, 100.0, True), out_of_range=False) + text = cell.text + string_text = text if text.strip() else None + value = cell.value + if isinstance(value, Decimal): + amount = _quantize(value) + if _too_long(amount): + return ValueBuild(_plain(cell, column_index, AmountParse(None, text, False, None)), out_of_range=True) + return ValueBuild( + _plain(cell, column_index, AmountParse(amount, None, True, None), recognized=not split_part), + out_of_range=False, + ) + if isinstance(value, date | bool): + return ValueBuild(_plain(cell, column_index, AmountParse(None, string_text, False, None)), out_of_range=False) + if len(code_parts(text)) > 1 and not split_part: + return ValueBuild(_plain(cell, column_index, AmountParse(None, text, False, None)), out_of_range=False) + parsed = parse_amount(text, source, rules) + return ValueBuild(_plain(cell, column_index, parsed, recognized=False if split_part else None), out_of_range=False) diff --git a/arm_statements/tests/test_amounts.py b/arm_statements/tests/test_amounts.py new file mode 100644 index 0000000..8025cf5 --- /dev/null +++ b/arm_statements/tests/test_amounts.py @@ -0,0 +1,84 @@ +from datetime import date +from decimal import Decimal as D + +import pytest + +from arm_statements.amounts import build_value, parse_amount, sign_form_of +from arm_statements.loader import Rules +from arm_statements.model import Cell, SignForm, Source + +OCR, EXCEL = Source.OCR, Source.EXCEL + + +@pytest.mark.parametrize( + ("raw", "amount", "string_value", "recognized"), + [ + ("1 2О5", D("1205"), None, False), + ("42831 1", D("428311"), None, False), + ("(7744536}", D("-7744536"), None, False), + ("211(1 |", None, "211(1 |", False), + ("(Ю4)", None, "(Ю4)", False), + (": : 2856983 = ■", D("2856983"), None, False), + ("| : 122477", D("122477"), None, False), + ("|122477", D("122477"), None, False), + ("{38378)", None, "{38378)", False), + ("4,200", D("4200"), None, True), + ("1,234,567", D("1234567"), None, True), + ("837.994", D("837994"), None, True), + ("1 234.567", D("1234567"), None, True), + ("4,2", D("4.2"), None, True), + ("1 234,56", D("1234.56"), None, True), + ("1.234,56", D("123456"), None, False), + ("0,125", None, "0,125", False), + ("1 234 567", D("1234567"), None, True), + ("(1 234)", D("-1234"), None, True), + ("−500", D("-500"), None, True), + ("12:34", None, "12:34", False), + ("1 234 руб.", D("1234"), None, False), + ("—", None, None, True), + ("-", None, None, True), + ("", None, None, True), + ("I", None, "I", False), + ], +) +def test_parse_amount_ocr(rules: Rules, raw: str, amount: D | None, string_value: str | None, recognized: bool) -> None: + parsed = parse_amount(raw, OCR, rules) + assert (parsed.amount, parsed.string_value, parsed.is_recognized) == (amount, string_value, recognized) + + +def test_comma_depends_on_source(rules: Rules) -> None: + assert parse_amount("4,200", EXCEL, rules).amount == D("4.20") + assert parse_amount("1.500", EXCEL, rules).amount == D("1.5") + + +def test_sign_forms(rules: Rules) -> None: + assert sign_form_of("(7744536}", rules) is SignForm.PARENTHESES + assert sign_form_of("-1 234", rules) is SignForm.MINUS + assert sign_form_of("38378", rules) is SignForm.UNSIGNED + assert sign_form_of("{38378)", rules) is None + + +def test_build_value_excel_number_and_out_of_range(rules: Rules) -> None: + ok = build_value(Cell("106 311", D("106311.456"), 100.0, "Баланс!D27"), 3, EXCEL, rules) + assert (ok.value.amount, ok.value.is_recognized, ok.out_of_range) == (D("106311.46"), True, False) + big = build_value(Cell("1E+17", D("123456789012345678"), 100.0, "Баланс!D28"), 3, EXCEL, rules) + assert (big.value.amount, big.value.string_value, big.value.is_recognized, big.out_of_range) == ( + None, + "1E+17", + False, + True, + ) + + +def test_build_value_date_missing_multiline_split(rules: Rules) -> None: + as_date = build_value(Cell("30.09.2025", date(2025, 9, 30), 100.0, "Баланс!D3"), 3, EXCEL, rules).value + assert (as_date.amount, as_date.string_value, as_date.is_recognized) == (None, "30.09.2025", False) + missing = build_value(None, 4, OCR, rules).value + assert (missing.amount, missing.string_value, missing.text, missing.is_recognized) == (None, None, None, True) + assert missing.confidence == pytest.approx(100.0) + multi = build_value(Cell("917 775\n2 371 333", None, 100.0, "row:9"), 2, OCR, rules).value + assert (multi.amount, multi.string_value, multi.is_recognized) == (None, "917 775\n2 371 333", False) + part = build_value(Cell("917 775", None, 100.0, "row:9#1"), 2, OCR, rules, split_part=True).value + assert (part.amount, part.is_recognized) == (D("917775"), False) + dash = build_value(Cell("—", None, 100.0, "row:3"), 2, OCR, rules).value + assert (dash.amount, dash.string_value, dash.is_recognized, dash.text) == (None, None, True, "—") -- 2.47.2 From 5f259be9ace0d2503f8215b662c7a59ebad5d3bb Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Sun, 18 Oct 2026 13:28:21 +0400 Subject: [PATCH 08/28] =?UTF-8?q?fix:=20out-of-range=20=D0=B4=D0=BB=D1=8F?= =?UTF-8?q?=20=D0=BB=D1=8E=D0=B1=D1=8B=D1=85=20Decimal,=20=D0=B7=D0=BD?= =?UTF-8?q?=D0=B0=D0=BA=20=D0=B2=20=D1=81=D0=BA=D0=BE=D0=B1=D0=BA=D0=B0?= =?UTF-8?q?=D1=85=20=D0=BD=D0=B5=D0=B4=D0=BE=D0=BF=D1=83=D1=81=D1=82=D0=B8?= =?UTF-8?q?=D0=BC,=20=D1=82=D0=B5=D1=81=D1=82=D1=8B=20=D0=BE=D0=BA=D1=80?= =?UTF-8?q?=D1=83=D0=B3=D0=BB=D0=B5=D0=BD=D0=B8=D1=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W --- arm_statements/amounts.py | 12 ++++------ arm_statements/tests/test_amounts.py | 35 ++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/arm_statements/amounts.py b/arm_statements/amounts.py index 1c40eac..208eab6 100644 --- a/arm_statements/amounts.py +++ b/arm_statements/amounts.py @@ -88,8 +88,6 @@ def _split_core(core: str) -> tuple[str, SignForm, bool, bool] | None: fixed = core[-1] != ")" core = core[1:-1].strip() sign_form, negative = SignForm.PARENTHESES, True - if core and core[0] in "-−+": - core = core[1:].strip() elif core[0] in "-−+": if core[0] in _MINUSES: sign_form, negative = SignForm.MINUS, True @@ -118,8 +116,6 @@ def _number(body: str, source: Source, text: str, sign_form: SignForm, *, negati return _guess(digits, text, sign_form, negative=negative) else: integer, fraction = compact.replace(",", ".").split(".") - if not integer: - return AmountParse(None, text, False, sign_form) if source is Source.OCR and len(fraction) >= 3: # noqa: PLR2004 — три знака: не копейки return AmountParse(None, text, False, sign_form) if len(integer) > MAX_INTEGER_DIGITS: @@ -181,9 +177,11 @@ def build_value( string_text = text if text.strip() else None value = cell.value if isinstance(value, Decimal): - amount = _quantize(value) - if _too_long(amount): - return ValueBuild(_plain(cell, column_index, AmountParse(None, text, False, None)), out_of_range=True) + # длину целой части проверяем до квантования: для 1E+30 и нефинитных значений quantize падает + amount = _quantize(value) if value.is_finite() and value.adjusted() < MAX_INTEGER_DIGITS else None + if amount is None or _too_long(amount): + overflow = AmountParse(None, string_text, False, None) + return ValueBuild(_plain(cell, column_index, overflow), out_of_range=True) return ValueBuild( _plain(cell, column_index, AmountParse(amount, None, True, None), recognized=not split_part), out_of_range=False, diff --git a/arm_statements/tests/test_amounts.py b/arm_statements/tests/test_amounts.py index 8025cf5..859880f 100644 --- a/arm_statements/tests/test_amounts.py +++ b/arm_statements/tests/test_amounts.py @@ -82,3 +82,38 @@ def test_build_value_date_missing_multiline_split(rules: Rules) -> None: assert (part.amount, part.is_recognized) == (D("917775"), False) dash = build_value(Cell("—", None, 100.0, "row:3"), 2, OCR, rules).value assert (dash.amount, dash.string_value, dash.is_recognized, dash.text) == (None, None, True, "—") + + +@pytest.mark.parametrize("raw", ["1E+30", "Infinity", "NaN", "-Infinity", "10000000000000000"]) +def test_build_value_decimal_out_of_range(rules: Rules, raw: str) -> None: + built = build_value(Cell(raw, D(raw), 100.0, "Баланс!D1"), 3, EXCEL, rules) + assert (built.value.amount, built.value.string_value, built.value.is_recognized, built.out_of_range) == ( + None, + raw, + False, + True, + ) + + +def test_build_value_decimal_empty_text_and_boundary(rules: Rules) -> None: + empty = build_value(Cell("", D("1E+30"), 100.0, "Баланс!D1"), 3, EXCEL, rules) + assert (empty.value.string_value, empty.out_of_range) == (None, True) + edge = build_value(Cell("x", D("9999999999999999"), 100.0, "Баланс!D1"), 3, EXCEL, rules) + assert (edge.value.amount, edge.out_of_range) == (D("9999999999999999"), False) + carry = build_value(Cell("x", D("9999999999999999.999"), 100.0, "Баланс!D1"), 3, EXCEL, rules) + assert (carry.value.amount, carry.out_of_range) == (None, True) + + +def test_build_value_rounds_half_up(rules: Rules) -> None: + assert build_value(Cell("0,125", D("0.125"), 100.0, "Баланс!D1"), 3, EXCEL, rules).value.amount == D("0.13") + assert build_value(Cell("-0,125", D("-0.125"), 100.0, "Баланс!D1"), 3, EXCEL, rules).value.amount == D("-0.13") + + +def test_sign_inside_parentheses_is_invalid(rules: Rules) -> None: + parsed = parse_amount("(-5)", OCR, rules) + assert (parsed.amount, parsed.string_value, parsed.is_recognized, parsed.sign_form) == (None, "(-5)", False, None) + + +def test_leading_comma_is_edge_trash(rules: Rules) -> None: + parsed = parse_amount(",5", OCR, rules) + assert (parsed.amount, parsed.is_recognized) == (D("5"), False) -- 2.47.2 From f73f3863784206cc59286f456d4e021a91a1c9e9 Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Sun, 18 Oct 2026 13:30:00 +0400 Subject: [PATCH 09/28] =?UTF-8?q?feat:=20=D1=80=D0=B0=D0=B7=D0=B1=D0=BE?= =?UTF-8?q?=D1=80=20=D1=88=D0=B0=D0=BF=D0=BA=D0=B8=20=D1=84=D0=BE=D1=80?= =?UTF-8?q?=D0=BC=D1=8B=20(=D0=98=D0=9D=D0=9D=20=D0=B1=D0=B5=D0=B7=20?= =?UTF-8?q?=D0=B4=D0=B2=D0=BE=D0=B9=D0=BD=D0=B8=D0=BA=D0=BE=D0=B2,=20?= =?UTF-8?q?=D0=B5=D0=B4=D0=B8=D0=BD=D0=B8=D1=86=D1=8B=20=D0=BF=D0=BE=20?= =?UTF-8?q?=D0=9E=D0=9A=D0=95=D0=98)=20=D0=B8=20=D1=82=D0=B8=D0=BF-=D0=BF?= =?UTF-8?q?=D0=BE=D0=B4=D1=81=D0=BA=D0=B0=D0=B7=D0=BA=D0=B0=20=D0=BF=D0=BE?= =?UTF-8?q?=20=D0=9E=D0=9A=D0=A3=D0=94=20=D0=B8=20=D0=B7=D0=B0=D0=B3=D0=BE?= =?UTF-8?q?=D0=BB=D0=BE=D0=B2=D0=BA=D1=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W --- arm_statements/header.py | 125 ++++++++++++++++++++++++++++ arm_statements/tests/test_header.py | 56 +++++++++++++ 2 files changed, 181 insertions(+) create mode 100644 arm_statements/header.py create mode 100644 arm_statements/tests/test_header.py diff --git a/arm_statements/header.py b/arm_statements/header.py new file mode 100644 index 0000000..a27c8e9 --- /dev/null +++ b/arm_statements/header.py @@ -0,0 +1,125 @@ +"""Шапка кандидата формы и тип-подсказка по ОКУД и заголовку (§12).""" + +import re +from datetime import date + +from arm_statements.dates import parse_date +from arm_statements.loader import Rules +from arm_statements.model import Cell, DateField, DocumentType, HeaderCells, HeaderField, StatementHeader +from arm_statements.text import fix_lookalikes, normalize_caption, normalize_label + +_INN_LENGTHS = (10, 12) +_NOT_DIGIT = re.compile(r"\D") + + +def _field(cell: Cell | None) -> HeaderField: + if cell is None: + return HeaderField(None, None, None) + return HeaderField(cell.text, cell.confidence, None) + + +def _stripped(cell: Cell | None) -> HeaderField: + if cell is None: + return HeaderField(None, None, None) + return HeaderField(cell.text, cell.confidence, cell.text.strip() or None) + + +def _digits(cell: Cell | None) -> HeaderField: + """Только цифры текста (пусто — None); двойники не заменяются.""" + if cell is None: + return HeaderField(None, None, None) + return HeaderField(cell.text, cell.confidence, _NOT_DIGIT.sub("", cell.text) or None) + + +def _inn(cell: Cell | None) -> HeaderField: + base = _digits(cell) + if base.value is not None and len(base.value) not in _INN_LENGTHS: + return HeaderField(base.text, base.confidence, None) + return base + + +def _report_date(cell: Cell | None, rules: Rules) -> DateField: + if cell is None: + return DateField(None, None, None) + value: date | None + if isinstance(cell.value, date): + value = cell.value + else: + value = parse_date(fix_lookalikes(cell.text, rules.lookalikes), rules) + return DateField(cell.text, cell.confidence, value) + + +def _okud(cell: Cell | None, rules: Rules) -> HeaderField: + if cell is None: + return HeaderField(None, None, None) + match = rules.headers.okud_code.search(normalize_label(cell.text)) + return HeaderField(cell.text, cell.confidence, match.group(0) if match else None) + + +def _units(cell: Cell | None, okei: HeaderField, rules: Rules) -> HeaderField: + base = _field(cell) + if cell is not None: + caption = normalize_caption(cell.text) + for pattern, canonical in rules.units.spellings: + if pattern.search(caption): + return HeaderField(base.text, base.confidence, canonical) + if okei.value is not None: + return HeaderField(base.text, base.confidence, rules.units.okei.get(okei.value)) + return base + + +def parse_header(cells: HeaderCells, rules: Rules) -> StatementHeader: + """Шапка кандидата: текст и уверенность — из ячейки, значение — по правилам поля.""" + okei = _digits(cells.okei) + return StatementHeader( + organization_name=_stripped(cells.organization_name), + inn=_inn(cells.inn), + report_date=_report_date(cells.report_date, rules), + units=_units(cells.units, okei, rules), + okud=_okud(cells.okud, rules), + okpo=_digits(cells.okpo), + okopf=_digits(cells.okopf), + okfs=_digits(cells.okfs), + okei=okei, + address=_stripped(cells.address), + needs_review_fields=(), + ) + + +def _type_by_okud(text: str | None, rules: Rules) -> DocumentType | None: + if not text: + return None + label = normalize_label(text) + for doc_type, patterns in rules.headers.okud.items(): + if any(pattern.search(label) for pattern in patterns): + return doc_type + return None + + +def type_hint(okud_text: str | None, title_text: str | None, rules: Rules) -> DocumentType | None: + """Тип по ОКУД, иначе по заголовку формы; заголовки колонок не участвуют.""" + by_okud = _type_by_okud(okud_text, rules) + if by_okud is not None: + return by_okud + if not title_text: + return None + label = normalize_label(title_text) + for doc_type, patterns in rules.headers.form_titles.items(): + if any(pattern.search(label) for pattern in patterns): + return doc_type + return None + + +def is_continuation(text: str | None, rules: Rules) -> bool: + """Маркер страницы-продолжения «Форма 0710001 с.2» (ОКУД из словаря не обязателен).""" + if not text: + return False + label = normalize_label(text) + return any(pattern.search(label) for pattern in rules.headers.continuation) + + +def continuation_type(text: str | None, rules: Rules) -> DocumentType | None: + """Тип по ОКУД из маркера продолжения; не маркер или неизвестный ОКУД — None.""" + if not is_continuation(text, rules): + return None + return _type_by_okud(text, rules) diff --git a/arm_statements/tests/test_header.py b/arm_statements/tests/test_header.py new file mode 100644 index 0000000..071216e --- /dev/null +++ b/arm_statements/tests/test_header.py @@ -0,0 +1,56 @@ +from datetime import date + +from arm_statements.header import continuation_type, is_continuation, parse_header, type_hint +from arm_statements.loader import Rules +from arm_statements.model import DocumentType, HeaderCells +from arm_statements.tests.builders import hdr + + +def test_inn_digits_only_without_lookalikes(rules: Rules) -> None: + head = parse_header(hdr(inn="|7700000000|"), rules) + assert head.inn.value == "7700000000" + dirty = parse_header(hdr(inn=("3|1 21 2|", 80.0)), rules) + assert dirty.inn.value is None + assert (dirty.inn.text, dirty.inn.confidence) == ("3|1 21 2|", 80.0) + + +def test_units_by_text_then_okei(rules: Rules) -> None: + assert parse_header(hdr(units="в тыс. рублей"), rules).units.value == "тыс. руб." + by_okei = parse_header(hdr(okei="384"), rules) + assert (by_okei.units.value, by_okei.okei.value, by_okei.units.text) == ("тыс. руб.", "384", None) + assert parse_header(hdr(units="руб."), rules).units.value == "руб." + + +def test_report_date_and_codes(rules: Rules) -> None: + head = parse_header(hdr(report_date="30.09.2О25", okud="Форма по ОКУД 0710001", okopf="12267"), rules) + assert head.report_date.value == date(2025, 9, 30) + assert (head.okud.value, head.okopf.value) == ("0710001", "12267") + + +def test_type_hint_okud_wins(rules: Rules) -> None: + assert type_hint("0710002", "Бухгалтерский баланс", rules) is DocumentType.INCOME + assert type_hint("Форма по ОКУД 0710001", None, rules) is DocumentType.BALANCE + assert type_hint("0710005", None, rules) is DocumentType.DISCLOSURE + assert type_hint(None, "Пояснения к бухгалтерскому балансу", rules) is DocumentType.DISCLOSURE + assert type_hint(None, "Отчет о прибылях и убытках", rules) is DocumentType.INCOME + assert type_hint(None, "Бухгалтерская отчетность", rules) is None + + +def test_continuation_marker(rules: Rules) -> None: + assert continuation_type("Форма 0710002 с.2", rules) is DocumentType.INCOME + assert continuation_type("Бухгалтерский баланс", rules) is None + assert is_continuation("Форма 0710000 с. 3", rules) and continuation_type("Форма 0710000 с. 3", rules) is None + assert not is_continuation("Бухгалтерский баланс", rules) and not is_continuation(None, rules) + + +def test_type_hint_uses_label_normalization(rules: Rules) -> None: + assert type_hint(None, "ОТЧЁТ О ФИНАНСОВЫХ\nРЕЗУЛЬТАТАХ", rules) is DocumentType.INCOME + + +def test_absent_cells_and_empty_values(rules: Rules) -> None: + head = parse_header(HeaderCells(), rules) + assert (head.inn.text, head.inn.confidence, head.inn.value) == (None, None, None) + assert head.report_date.value is None and head.needs_review_fields == () + blank = parse_header(hdr(okpo=" ", organization_name=" ООО «Пример» "), rules) + assert blank.okpo.value is None + assert blank.organization_name.value == "ООО «Пример»" -- 2.47.2 From f4533f45e1dfa7971d67009422fd5b90c7e7ad68 Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Sun, 18 Oct 2026 13:33:47 +0400 Subject: [PATCH 10/28] =?UTF-8?q?feat:=20=D1=88=D0=B0=D0=B3=202=20?= =?UTF-8?q?=E2=80=94=20=D1=81=D1=82=D0=BE=D0=BB=D0=B1=D0=B5=D1=86=20=D0=BA?= =?UTF-8?q?=D0=BE=D0=B4=D0=BE=D0=B2=20=D1=81=20=D0=B7=D0=B0=D1=89=D0=B8?= =?UTF-8?q?=D1=82=D0=BE=D0=B9=20=D0=BE=D1=82=20=D0=B1=D0=BB=D0=BE=D0=BA?= =?UTF-8?q?=D0=B0=20=C2=AB=D0=9A=D0=BE=D0=B4=D1=8B=C2=BB,=20=D0=BD=D1=83?= =?UTF-8?q?=D0=BC=D0=B5=D1=80=D0=B0=D1=86=D0=B8=D1=8F,=20=D1=81=D1=82?= =?UTF-8?q?=D1=80=D0=BE=D0=BA=D0=B0=20=D0=B7=D0=B0=D0=B3=D0=BE=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=D0=BA=D0=B0,=20=D1=80=D0=BE=D0=BB=D0=B8=20=D1=81=D1=82?= =?UTF-8?q?=D1=80=D0=BE=D0=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W --- arm_statements/layout.py | 306 ++++++++++++++++++++++++++++ arm_statements/tests/builders.py | 13 +- arm_statements/tests/test_layout.py | 139 +++++++++++++ 3 files changed, 457 insertions(+), 1 deletion(-) create mode 100644 arm_statements/layout.py create mode 100644 arm_statements/tests/test_layout.py diff --git a/arm_statements/layout.py b/arm_statements/layout.py new file mode 100644 index 0000000..f79798f --- /dev/null +++ b/arm_statements/layout.py @@ -0,0 +1,306 @@ +"""Шаг 2: столбец кодов, нумерация, строка заголовка, роли строк, столбец наименования.""" + +import re +from collections import Counter +from collections.abc import Sequence +from decimal import Decimal + +from arm_statements.codes import code_parts, is_code_cell, parse_code, section_of +from arm_statements.dates import parse_period_label +from arm_statements.loader import CodeSection, Rules +from arm_statements.model import Cell, DocumentType, PeriodKind, RowRole +from arm_statements.text import fix_lookalikes, normalize_caption + +Grid = Sequence[Sequence[Cell | None]] + +_MIN_NUMBERING_CELLS = 3 +_SHORT_NOTE = 3 # «Пояснения» без подписи: значения не длиннее 3 символов +_SKIP_STEP = 2 # скачок через одну пропущенную цифру нумерации +_NUMBER = re.compile(r"[0-9]+") +_FULL_CONFIDENCE = 100.0 + + +def _text(cell: Cell | None) -> str: + return "" if cell is None else cell.text.strip() + + +def cell_at(rows: Grid, row: int, col: int) -> Cell | None: + """Ячейка сетки; вне строки или сетки — `None`.""" + if row < 0 or row >= len(rows) or col < 0: + return None + line = rows[row] + return line[col] if col < len(line) else None + + +def _width(rows: Grid, first: int, last: int) -> int: + return max((len(rows[r]) for r in range(max(first, 0), min(last, len(rows) - 1) + 1)), default=0) + + +def _relaxed_count(rows: Grid, span: range, col: int, section: CodeSection, rules: Rules) -> int: + count = 0 + for row in span: + text = _text(cell_at(rows, row, col)) + if not text or not is_code_cell(text, rules): + continue + codes = [parse_code(part, rules).code for part in code_parts(text)] + if all(code is not None and section_of(code, rules) is section for code in codes): + count += 1 + return count + + +def _share_ok(codes: int, nonempty: int, min_share: Decimal) -> bool: + return nonempty > 0 and Decimal(codes) >= min_share * Decimal(nonempty) + + +def _tally(rows: Grid, start: int, last: int, col: int, rules: Rules) -> tuple[int, int]: + """(кодов, непустых ячеек) столбца в строках `start..last`.""" + codes = nonempty = 0 + for row in range(start, last + 1): + text = _text(cell_at(rows, row, col)) + if text: + nonempty += 1 + if is_code_cell(text, rules): + codes += 1 + return codes, nonempty + + +def _best(candidates: dict[int, int]) -> int | None: + """Столбец с наибольшим числом кодов; ничья — левый.""" + best: int | None = None + for col in sorted(candidates): + if best is None or candidates[col] > candidates[best]: + best = col + return best + + +def _caption_candidates(rows: Grid, first: int, last: int, width: int, rules: Rules) -> dict[int, int]: + """Правило (а): столбец с подписью кода; под подписью — ячейки ниже неё до `last`.""" + ranges = rules.code_ranges + found: dict[int, int] = {} + for col in range(width): + first_code = next( + (r for r in range(first, last + 1) if is_code_cell(_text(cell_at(rows, r, col)), rules)), last + 1 + ) + captions = [ + r + for r in range(first, min(first_code, last + 1)) + if rules.headers.code_caption.search(normalize_caption(_text(cell_at(rows, r, col)))) + ] + if not captions: + continue + codes, nonempty = _tally(rows, captions[-1] + 1, last, col, rules) + if codes > 0 and (codes >= ranges.min_codes or _share_ok(codes, nonempty, ranges.min_share)): + found[col] = codes + return found + + +def find_code_column( + rows: Grid, first: int, last: int, rules: Rules, *, relaxed_section: CodeSection | None = None +) -> int | None: + """Столбец кодов по правилу (а)/(б) шага 2; `relaxed_section` — ослабленный поиск продолжения.""" + width = _width(rows, first, last) + if relaxed_section is not None: + counts = { + col: _relaxed_count(rows, range(first, last + 1), col, relaxed_section, rules) for col in range(width) + } + counts = {col: n for col, n in counts.items() if n >= rules.code_ranges.min_codes_continuation} + return _best(counts) + by_caption = _best(_caption_candidates(rows, first, last, width, rules)) + if by_caption is not None: + return by_caption + ranges = rules.code_ranges + scored: dict[int, int] = {} + for col in range(width): + codes, nonempty = _tally(rows, first, last, col, rules) + if codes >= ranges.min_codes and _share_ok(codes, nonempty, ranges.min_share): + scored[col] = codes + return _best(scored) + + +def code_column(rows: Grid, rules: Rules, *, relaxed: DocumentType | None = None) -> int | None: + """Столбец кодов всей сетки — то же правило, что у шага 2 (для адаптеров).""" + section: CodeSection | None = None + if relaxed is DocumentType.DISCLOSURE: + raise ValueError("relaxed=DISCLOSURE не поддерживается: у пояснений несколько отрезков кодов") + if relaxed is DocumentType.BALANCE: + section = CodeSection.BALANCE + elif relaxed is DocumentType.INCOME: + section = CodeSection.INCOME + if not rows: + return None + return find_code_column(rows, 0, len(rows) - 1, rules, relaxed_section=section) + + +def first_code_row(rows: Grid, first: int, last: int, code_col: int, rules: Rules) -> int | None: + for row in range(first, last + 1): + if is_code_cell(_text(cell_at(rows, row, code_col)), rules): + return row + return None + + +def is_numbering_row(cells: Sequence[Cell | None], rules: Rules) -> bool: + """≥ 3 непустых ячеек-целых, слева направо +1, начало 1 или 2, не больше одного пропуска.""" + numbers: list[int] = [] + for cell in cells: + text = _text(cell) + if not text: + continue + fixed = fix_lookalikes(text, rules.lookalikes, anywhere=True).strip() + if _NUMBER.fullmatch(fixed) is None: + return False + numbers.append(int(fixed)) + if len(numbers) < _MIN_NUMBERING_CELLS or numbers[0] not in (1, 2): + return False + skips = 0 + for prev, current in zip(numbers, numbers[1:], strict=False): + step = current - prev + if step == 1: + continue + if step == _SKIP_STEP and skips == 0: + skips += 1 + continue + return False + return True + + +def _is_period(text: str, ref: str, rules: Rules) -> bool: + cell = Cell(text, None, _FULL_CONFIDENCE, ref) + return parse_period_label(cell, rules).kind is not PeriodKind.NONE + + +def _has_header_periods(rows: Grid, row: int, code_col: int, rules: Rules) -> bool: + """Две и более подписи-периода правее кода, либо одна — не голый год.""" + labels = [ + parse_period_label(cell, rules) + for col, cell in enumerate(rows[row]) + if col > code_col and cell is not None and cell.text.strip() + ] + periods = [label for label in labels if label.kind is not PeriodKind.NONE] + strong = [label for label in periods if not label.bare_year] + return bool(strong) or len(periods) >= 2 # noqa: PLR2004 - две подписи подряд — заголовок, одна «2025» — титул + + +def _glued_with_upper(rows: Grid, upper: int, lower: int, code_col: int, rules: Rules) -> bool: + """P1-9: подпись верхней строки отдельно — не период, а «верхняя + нижняя» — период.""" + for col, cell in enumerate(rows[upper]): + if col <= code_col: + continue + top = _text(cell) + bottom = _text(cell_at(rows, lower, col)) + if not top or not bottom: + continue + if not _is_period(top, "", rules) and _is_period(f"{top} {bottom}", "", rules): + return True + return False + + +def _upper_candidate(rows: Grid, row: int, first: int, rules: Rules) -> int | None: + above = row - 1 + while above >= first and is_numbering_row(rows[above], rules): + above -= 1 + return above if above >= first else None + + +def find_header_rows(rows: Grid, first: int, first_code: int, code_col: int, rules: Rules) -> tuple[int, ...]: + """Строка заголовка (одна или две, склеенные); не найдена — `()`.""" + candidates = [r for r in range(first, first_code) if not is_numbering_row(rows[r], rules)] + captioned = [ + r for r in candidates if rules.headers.code_caption.search(normalize_caption(_text(cell_at(rows, r, code_col)))) + ] + header: int | None = captioned[-1] if captioned else None + if header is None: + header = next((r for r in reversed(candidates) if _has_header_periods(rows, r, code_col, rules)), None) + if header is None: + return () + upper = _upper_candidate(rows, header, first, rules) + if upper is not None and _glued_with_upper(rows, upper, header, code_col, rules): + return (upper, header) + return (header,) + + +def segment_roles(rows: Grid, first: int, last: int, header_rows: tuple[int, ...], rules: Rules) -> list[RowRole]: + """Роли строк `first..last`: выше заголовка — TITLE (только при найденном заголовке).""" + roles: list[RowRole] = [] + top = min(header_rows) if header_rows else None + for row in range(first, last + 1): + if row in header_rows: + roles.append(RowRole.HEADER) + elif top is not None and row < top: + roles.append(RowRole.TITLE) + elif is_numbering_row(rows[row], rules): + roles.append(RowRole.NUMBERING) + else: + roles.append(RowRole.DATA) + return roles + + +def _caption_column( + rows: Grid, header_rows: tuple[int, ...], code_col: int, pattern: re.Pattern[str], *, left_of_code: bool +) -> int | None: + for row in header_rows: + for col, cell in enumerate(rows[row]): + if col == code_col or (left_of_code and col > code_col): + continue + if pattern.search(normalize_caption(_text(cell))): + return col + return None + + +def _code_rows(rows: Grid, span: range, skip: tuple[int, ...], code_col: int, rules: Rules) -> list[int]: + return [r for r in span if r not in skip and is_code_cell(_text(cell_at(rows, r, code_col)), rules)] + + +def _short_columns(rows: Grid, data: Sequence[int], code_col: int, skip: int | None) -> list[int]: + result: list[int] = [] + for col in range(code_col): + if col == skip: + continue + values = [_text(cell_at(rows, r, col)) for r in data] + filled = [v for v in values if v] + if filled and all(len(v) <= _SHORT_NOTE for v in filled): + result.append(col) + return result + + +def _text_column(rows: Grid, data: Sequence[int], code_col: int, skip: set[int]) -> int | None: + counts: Counter[int] = Counter() + for col in range(code_col): + if col in skip: + continue + counts[col] = sum(1 for r in data if any(ch.isalpha() for ch in _text(cell_at(rows, r, col)))) + best: int | None = None + for col in sorted(counts): # ничья — ближайший к коду + if counts[col] > 0 and (best is None or counts[col] >= counts[best]): + best = col + return best + + +def find_name_column( # noqa: PLR0913, PLR0917 - сигнатура задана планом (Task 8) + rows: Grid, header_rows: tuple[int, ...], first: int, last: int, code_col: int, rules: Rules +) -> tuple[int | None, int | None]: + """(столбец наименования, столбец «Пояснения»).""" + name_col = _caption_column(rows, header_rows, code_col, rules.headers.name_caption, left_of_code=False) + notes_col = _caption_column(rows, header_rows, code_col, rules.headers.explanations_caption, left_of_code=True) + data = _code_rows(rows, range(first, last + 1), header_rows, code_col, rules) + if name_col is None: + skip = {notes_col} if notes_col is not None else set(_short_columns(rows, data, code_col, None)) + name_col = _text_column(rows, data, code_col, skip) + if notes_col is None: + short = _short_columns(rows, data, code_col, name_col) + notes_col = short[-1] if short else None + return name_col, notes_col + + +def name_cell( + cells: Sequence[Cell | None], name_col: int | None, code_col: int, explanations_col: int | None +) -> Cell | None: + """Ячейка наименования; пустая — ближайшая левее кода ячейка с буквами, кроме «Пояснений».""" + if name_col is not None and name_col < len(cells): + cell = cells[name_col] + if cell is not None and cell.text.strip(): + return cell + for col in range(min(code_col, len(cells)) - 1, -1, -1): + cell = cells[col] + if col != explanations_col and cell is not None and any(ch.isalpha() for ch in cell.text): + return cell + return None diff --git a/arm_statements/tests/builders.py b/arm_statements/tests/builders.py index 476b2f5..7b54b00 100644 --- a/arm_statements/tests/builders.py +++ b/arm_statements/tests/builders.py @@ -2,6 +2,7 @@ from collections.abc import Mapping, Sequence from decimal import Decimal +from typing import overload from arm_statements.model import ( HEADER_FIELDS, @@ -138,7 +139,17 @@ def excel( # noqa: PLR0913 - сигнатура задана планом (Task ) -def place(cells: Mapping[int, PlaceInput], width: int | None = None) -> list[PlaceInput | None]: +@overload +def place(cells: Mapping[int, str | Decimal], width: int | None = None) -> list[str | Decimal | None]: ... + + +@overload +def place(cells: Mapping[int, PlaceInput], width: int | None = None) -> list[PlaceInput | None]: ... + + +def place( + cells: Mapping[int, PlaceInput] | Mapping[int, str | Decimal], width: int | None = None +) -> list[PlaceInput | None] | list[str | Decimal | None]: """Строка с ячейками в заданных столбцах, остальные None.""" size = width if width is not None else (max(cells) + 1 if cells else 0) return [cells.get(i) for i in range(size)] diff --git a/arm_statements/tests/test_layout.py b/arm_statements/tests/test_layout.py new file mode 100644 index 0000000..74c15f3 --- /dev/null +++ b/arm_statements/tests/test_layout.py @@ -0,0 +1,139 @@ +from decimal import Decimal as D + +import pytest + +from arm_statements.layout import ( + code_column, + find_code_column, + find_header_rows, + find_name_column, + first_code_row, + is_numbering_row, + name_cell, + segment_roles, +) +from arm_statements.loader import CodeSection, Rules +from arm_statements.model import Cell, DocumentType, RowRole +from arm_statements.tests.builders import excel, ocr, place + +BALANCE_ROWS = [ + ["Наименование показателя", "Код", "На 30.09.2025", "На 31.12.2024"], + ["1", "2", "3", "4"], + ["Нематериальные активы", "1110", "10", "9"], + ["Основные средства", "1150", "20", "19"], + ["Итого по разделу I", "1100", "30", "28"], +] + + +def test_code_column_by_caption(rules: Rules) -> None: + cand = ocr(BALANCE_ROWS) + assert (find_code_column(cand.rows, 0, 4, rules), first_code_row(cand.rows, 0, 4, 1, rules)) == (1, 2) + + +def test_title_block_kody_is_not_code_column(rules: Rules) -> None: + title = [ + place({5: v}) for v in ["Коды", "0710001", "30 09 2025", "12345678", "7700000000", "12267", "16", "384", "", ""] + ] + table = [ + place({0: n, 2: c, 3: D(a)}) + for n, c, a in [ + ("Запасы", "1210", "5"), + ("НДС", "1220", "1"), + ("Дебиторы", "1230", "4"), + ("Итого", "1200", "10"), + ] + ] + cand = excel([("Баланс", [*title, *table])]) + assert [find_code_column(cand.rows, 0, len(cand.rows) - 1, rules)] == [2] + + +def test_two_line_document(rules: Rules) -> None: + cand = ocr([["Наименование", "Код", "2025"], ["Выручка", "2110", "5"], ["Прибыль", "2400", "1"]]) + assert find_code_column(cand.rows, 0, 2, rules) == 1 + + +def test_relaxed_search_finds_single_code(rules: Rules) -> None: + cand = ocr([["Чистая прибыль", "", "2400", "7", "6"]]) + assert find_code_column(cand.rows, 0, 0, rules) is None + assert [find_code_column(cand.rows, 0, 0, rules, relaxed_section=CodeSection.INCOME)] == [2] + + +def test_public_code_column(rules: Rules) -> None: + assert code_column(ocr(BALANCE_ROWS).rows, rules) == 1 + single = ocr([["Чистая прибыль", "", "2400", "7", "6"]]).rows + assert (code_column(single, rules), code_column(single, rules, relaxed=DocumentType.INCOME)) == (None, 2) + assert code_column(single, rules, relaxed=DocumentType.BALANCE) is None and code_column((), rules) is None + with pytest.raises(ValueError, match="DISCLOSURE"): + code_column(single, rules, relaxed=DocumentType.DISCLOSURE) + + +def test_numbering_rows(rules: Rules) -> None: + def row(*texts: str) -> tuple[Cell | None, ...]: + return ocr([list(texts)]).rows[0] + + assert is_numbering_row(row("1", "2", "3", "4", "5"), rules) + assert is_numbering_row(row("1", "2", "З", "4"), rules) + assert is_numbering_row(row("2", "3", "4", "5"), rules) + assert is_numbering_row(row("1", "2", "4", "5"), rules) + assert not is_numbering_row(row("1", "3", "5"), rules) + assert not is_numbering_row(row("1", "2"), rules) + + +def test_header_row_roles_and_title(rules: Rules) -> None: + rows = [["Бухгалтерский баланс", "", "", ""], *BALANCE_ROWS] + cand = ocr(rows) + header = find_header_rows(cand.rows, 0, 3, 1, rules) + assert header == (1,) + roles = segment_roles(cand.rows, 0, 5, header, rules) + assert roles == [RowRole.TITLE, RowRole.HEADER, RowRole.NUMBERING, RowRole.DATA, RowRole.DATA, RowRole.DATA] + assert RowRole.TITLE not in segment_roles(cand.rows, 3, 5, (), rules) + + +def test_two_row_header_is_glued(rules: Rules) -> None: + cand = ocr( + [ + ["", "", "На 30 сентября", "На 31 декабря"], + ["Наименование", "Код", "2025 г.", "2024 г."], + ["Запасы", "1210", "5", "4"], + ] + ) + assert find_header_rows(cand.rows, 0, 2, 1, rules) == (0, 1) + single = ocr( + [ + ["Раздел", "", "", ""], + ["Наименование", "Код", "На 30.09.2025", "На 31.12.2024"], + ["Запасы", "1210", "5", "4"], + ] + ) + assert find_header_rows(single.rows, 0, 2, 1, rules) == (1,) + + +def test_name_column_without_caption(rules: Rules) -> None: + cand = ocr( + [ + ["5.1", "Запасы", "1210", "5"], + ["5.2", "Дебиторская задолженность", "1230", "4"], + ["", "Итого", "1200", "9"], + ] + ) + assert find_name_column(cand.rows, (), 0, 2, 2, rules) == (1, 0) + + +def test_header_without_code_caption_skips_numbering(rules: Rules) -> None: + cand = ocr( + [ + ["Титул 2025", "", "", ""], + ["Показатель", "", "На 30.09.2025", "На 31.12.2024"], + ["1", "2", "3", "4"], + ["Запасы", "1210", "5", "4"], + ] + ) + assert find_header_rows(cand.rows, 0, 3, 1, rules) == (1,) + assert find_header_rows(cand.rows, 3, 3, 1, rules) == () + + +def test_name_cell_falls_back_to_left_text(rules: Rules) -> None: + row = ocr([["5.1", "Запасы", "", "1210"]]).rows[0] + cell = name_cell(row, 2, 3, 0) + assert cell is not None and cell.text == "Запасы" + assert name_cell(ocr([["5.1", "", "", "1210"]]).rows[0], 2, 3, 0) is None -- 2.47.2 From 995a778fa4b351fd0f064a8b16922dccebe5bd92 Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Sun, 18 Oct 2026 13:37:12 +0400 Subject: [PATCH 11/28] =?UTF-8?q?feat:=20=D1=88=D0=B0=D0=B3=202=20?= =?UTF-8?q?=E2=80=94=20=D0=BA=D0=BE=D0=BB=D0=BE=D0=BD=D0=BA=D0=B8=20=D0=BF?= =?UTF-8?q?=D0=B5=D1=80=D0=B8=D0=BE=D0=B4=D0=BE=D0=B2,=20=D0=BE=D1=82?= =?UTF-8?q?=D0=BD=D0=BE=D1=81=D0=B8=D1=82=D0=B5=D0=BB=D1=8C=D0=BD=D1=8B?= =?UTF-8?q?=D0=B5=20=D0=B4=D0=B0=D1=82=D1=8B=20=D0=B1=D0=BB=D0=B0=D0=BD?= =?UTF-8?q?=D0=BA=D0=B0=2066=D0=BD,=20=D0=BD=D0=B5=D1=83=D0=B2=D0=B5=D1=80?= =?UTF-8?q?=D0=B5=D0=BD=D0=BD=D1=8B=D0=B5=20=D0=B8=20=D0=BC=D1=83=D1=81?= =?UTF-8?q?=D0=BE=D1=80=D0=BD=D1=8B=D0=B5=20=D0=BF=D0=BE=D0=B4=D0=BF=D0=B8?= =?UTF-8?q?=D1=81=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W --- arm_statements/columns.py | 162 +++++++++++++++++++++++++++ arm_statements/tests/test_columns.py | 146 ++++++++++++++++++++++++ 2 files changed, 308 insertions(+) create mode 100644 arm_statements/columns.py create mode 100644 arm_statements/tests/test_columns.py diff --git a/arm_statements/columns.py b/arm_statements/columns.py new file mode 100644 index 0000000..75ae3ff --- /dev/null +++ b/arm_statements/columns.py @@ -0,0 +1,162 @@ +"""Шаг 2: колонки значений, подписи и даты колонок.""" + +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import date +from decimal import Decimal + +from arm_statements.amounts import parse_amount +from arm_statements.codes import is_code_cell +from arm_statements.dates import expected_dates, parse_period_label, resolve_relative +from arm_statements.issues import WarningCode, make_warning +from arm_statements.layout import Grid, cell_at +from arm_statements.loader import Rules +from arm_statements.model import Cell, PeriodKind, Source, StatementColumn, StatementHeader, StatementWarning + +_FULL_CONFIDENCE = 100.0 + + +@dataclass(frozen=True, slots=True) +class ColumnDraft: + """Колонка сегмента до сопоставления; `date` — только у абсолютной подписи.""" + + grid_col: int + label: str + label_confidence: float | None + kind: PeriodKind + date: date | None + + +@dataclass(frozen=True, slots=True) +class DocumentDates: + effective: date | None + confident_effective: date | None + latest_absolute: date | None + confident_latest_absolute: date | None + + +def _text(cell: Cell | None) -> str: + return "" if cell is None else cell.text.strip() + + +def _code_rows(rows: Grid, data_rows: Sequence[int], code_col: int, rules: Rules) -> list[int]: + return [r for r in data_rows if is_code_cell(_text(cell_at(rows, r, code_col)), rules)] + + +def _is_number(cell: Cell | None, source: Source, rules: Rules) -> bool: + if cell is None: + return False + if isinstance(cell.value, Decimal): + return True + return parse_amount(cell.text, source, rules).amount is not None + + +def has_numbers( # noqa: PLR0913, PLR0917 - сигнатура задана планом (Task 9) + rows: Grid, data_rows: Sequence[int], col: int, code_col: int, source: Source, rules: Rules +) -> bool: + """В строках с кодом есть ячейка столбца с числом (значение-число или разобранная сумма).""" + return any(_is_number(cell_at(rows, r, col), source, rules) for r in _code_rows(rows, data_rows, code_col, rules)) + + +def _width(rows: Grid, row_indexes: Sequence[int]) -> int: + return max((len(rows[r]) for r in row_indexes if 0 <= r < len(rows)), default=0) + + +def amount_columns(rows: Grid, code_col: int, rules: Rules, *, source: Source = Source.EXCEL) -> tuple[int, ...]: + """Столбцы правее кодов, где в строках с кодом есть числа, по возрастанию.""" + every = range(len(rows)) + return tuple( + col + for col in range(code_col + 1, _width(rows, every)) + if has_numbers(rows, every, col, code_col, source, rules) + ) + + +def _label(rows: Grid, header_rows: tuple[int, ...], col: int) -> Cell | None: + """Подпись колонки: строки заголовка через пробел, уверенность — минимум; одна ячейка берётся как есть.""" + cells = [c for r in header_rows if (c := cell_at(rows, r, col)) is not None and c.text.strip()] + if not cells: + return None + if len(cells) == 1: + return cells[0] + text = " ".join(c.text.strip() for c in cells) + return Cell(text, None, min(c.confidence for c in cells), cells[0].ref) + + +def segment_columns( # noqa: PLR0913, PLR0917 - сигнатура задана планом (Task 9) + rows: Grid, + header_rows: tuple[int, ...], + data_rows: Sequence[int], + code_col: int, + source: Source, + rules: Rules, +) -> tuple[list[ColumnDraft], list[StatementWarning]]: + """Колонки значений сегмента и предупреждения о пропущенных/недатированных столбцах.""" + drafts: list[ColumnDraft] = [] + warnings: list[StatementWarning] = [] + for col in range(code_col + 1, _width(rows, [*header_rows, *data_rows])): + numbers = has_numbers(rows, data_rows, col, code_col, source, rules) + if not header_rows: + if numbers: + drafts.append(ColumnDraft(col, "", None, PeriodKind.NONE, None)) + continue + cell = _label(rows, header_rows, col) + parsed = parse_period_label(cell, rules) + if cell is not None and parsed.excluded: + message = f"столбец {col} («{cell.text}») пропущен: не период" + warnings.append(make_warning(WarningCode.COLUMN_SKIPPED, message, rules, column_index=col)) + elif cell is not None and parsed.kind is not PeriodKind.NONE: + drafts.append(ColumnDraft(col, cell.text, cell.confidence, parsed.kind, parsed.date)) + elif numbers: + label, conf = (cell.text, cell.confidence) if cell is not None else ("", None) + message = f"у столбца {col} с числами нет разобранной подписи периода" + warnings.append(make_warning(WarningCode.COLUMN_WITHOUT_DATE, message, rules, column_index=col)) + drafts.append(ColumnDraft(col, label, conf, PeriodKind.NONE, None)) + return drafts, warnings + + +def _latest(drafts: Sequence[ColumnDraft], *, confident: bool) -> date | None: + days = [ + d.date + for d in drafts + if d.kind is PeriodKind.ABSOLUTE + and d.date is not None + and (not confident or d.label_confidence == _FULL_CONFIDENCE) + ] + return max(days, default=None) + + +def document_dates(drafts: Sequence[ColumnDraft], header: StatementHeader) -> DocumentDates: + """Эффективная дата и её «уверенный» двойник: неуверенное само себя не подтверждает.""" + latest = _latest(drafts, confident=False) + confident_latest = _latest(drafts, confident=True) + report = header.report_date + sure_report = report.value if report.confidence == _FULL_CONFIDENCE else None + return DocumentDates( + effective=report.value or latest, + confident_effective=sure_report or confident_latest, + latest_absolute=latest, + confident_latest_absolute=confident_latest, + ) + + +def finalize_columns( + drafts: Sequence[ColumnDraft], dates: DocumentDates, rules: Rules +) -> tuple[tuple[StatementColumn, ...], list[StatementWarning]]: + """Колонки документа: даты относительных подписей и проверка неуверенных абсолютных.""" + expected = None if dates.confident_effective is None else expected_dates(dates.confident_effective) + columns: list[StatementColumn] = [] + warnings: list[StatementWarning] = [] + for draft in drafts: + if draft.kind is PeriodKind.ABSOLUTE: + period = draft.date + unsure = draft.label_confidence is not None and draft.label_confidence < _FULL_CONFIDENCE + if unsure and (expected is None or period not in expected): + message = f"дата подписи столбца {draft.grid_col} («{draft.label}») не подтверждена" + warnings.append( + make_warning(WarningCode.COLUMN_DATE_UNCERTAIN, message, rules, column_index=draft.grid_col) + ) + else: + period = resolve_relative(draft.kind, dates.effective) + columns.append(StatementColumn(draft.grid_col, draft.label, draft.label_confidence, period, draft.kind)) + return tuple(columns), warnings diff --git a/arm_statements/tests/test_columns.py b/arm_statements/tests/test_columns.py new file mode 100644 index 0000000..2a12edc --- /dev/null +++ b/arm_statements/tests/test_columns.py @@ -0,0 +1,146 @@ +from collections.abc import Sequence +from datetime import date +from decimal import Decimal as D +from typing import Any + +from arm_statements.columns import amount_columns, document_dates, finalize_columns, segment_columns +from arm_statements.header import parse_header +from arm_statements.loader import Rules +from arm_statements.model import HeaderCells, PeriodKind, Severity, Source, StatementColumn, StatementWarning +from arm_statements.tests.builders import excel, hdr, ocr + +OCR, EXCEL = Source.OCR, Source.EXCEL + + +def run( # noqa: PLR0913, PLR0917 - вспомогательная функция тестов, сигнатура задана планом (Task 9) + rows: Sequence[Sequence[Any]], + source: Source, + rules: Rules, + header: HeaderCells | None = None, + header_rows: tuple[int, ...] = (0,), + data_rows: tuple[int, ...] = (1, 2), +) -> tuple[tuple[StatementColumn, ...], list[StatementWarning]]: + cand = ocr(rows) if source is OCR else excel([("Лист1", rows)]) + drafts, warnings = segment_columns(cand.rows, header_rows, data_rows, 1 if source is OCR else 2, source, rules) + head = parse_header(header or hdr(), rules) + columns, more = finalize_columns(drafts, document_dates(drafts, head), rules) + return columns, [*warnings, *more] + + +def test_blank_66n_relative_labels(rules: Rules) -> None: + rows: list[list[Any]] = [ + ["Наименование", "Код", "На отчетную дату", "На 31 декабря предыдущего года", + "На 31 декабря года, предшествующего предыдущему"], + ["Запасы", "1210", "5", "4", "3"], + ["Итого", "1200", "5", "4", "3"], + ] # fmt: skip + columns, warnings = run(rows, OCR, rules, header=hdr(report_date="30.09.2025")) + assert [c.period_date for c in columns] == [date(2025, 9, 30), date(2024, 12, 31), date(2023, 12, 31)] + assert [c.period_kind for c in columns] == [ + PeriodKind.CURRENT, + PeriodKind.PREV_YEAR_END, + PeriodKind.PREV_PREV_YEAR_END, + ] + assert warnings == [] + + +def test_excel_skipped_change_column(rules: Rules) -> None: + rows: list[list[Any]] = [ + ["Наименование", "", "Код", "На 30.09.2025", "На 31.12.2024", "Изменение, %"], + ["Запасы", "", "1210", D("5"), D("4"), D("25")], + ["Итого", "", "1200", D("5"), D("4"), D("25")], + ] + columns, warnings = run(rows, EXCEL, rules) + assert [c.column_index for c in columns] == [3, 4] + assert [(w.code, w.severity, w.column_index) for w in warnings] == [("COLUMN_SKIPPED", Severity.INFO, 5)] + assert {c.label_confidence for c in columns} == {100.0} + + +def test_junk_label_over_numbers_is_kept(rules: Rules) -> None: + rows: list[list[Any]] = [ + ["Наименование", "Код", "На 30.09.2025", ("| | г", 17.0)], + ["Запасы", "1210", "5", "4"], + ["Итого", "1200", "5", "4"], + ] + columns, warnings = run(rows, OCR, rules, header=hdr(report_date="30.09.2025")) + junk = columns[1] + assert (junk.label, junk.period_date, junk.period_kind) == ("| | г", None, PeriodKind.NONE) + assert [(w.code, w.severity, w.column_index) for w in warnings] == [("COLUMN_WITHOUT_DATE", Severity.ERROR, 3)] + + +def test_empty_label_over_numbers_is_kept(rules: Rules) -> None: + rows: list[list[Any]] = [ + ["Наименование", "Код", "На 30.09.2025", ""], + ["Запасы", "1210", "5", "4"], + ["Итого", "1200", "5", "4"], + ] + columns, warnings = run(rows, OCR, rules) + assert [(c.column_index, c.label, c.label_confidence, c.period_kind) for c in columns] == [ + (2, "На 30.09.2025", 100.0, PeriodKind.ABSOLUTE), + (3, "", None, PeriodKind.NONE), + ] + assert [(w.code, w.column_index) for w in warnings] == [("COLUMN_WITHOUT_DATE", 3)] + + +def test_two_row_header_is_joined_with_min_confidence(rules: Rules) -> None: + rows: list[list[Any]] = [ + ["", "", ("На 31 декабря", 90.0)], + ["Наименование", "Код", ("2024 г.", 80.0)], + ["Запасы", "1210", "5"], + ] + cand = ocr(rows) + drafts, _ = segment_columns(cand.rows, (0, 1), (2,), 1, OCR, rules) + assert [(d.grid_col, d.label, d.label_confidence, d.date) for d in drafts] == [ + (2, "На 31 декабря 2024 г.", 80.0, date(2024, 12, 31)) + ] + + +def test_sparse_ocr_columns(rules: Rules) -> None: + rows: list[list[Any]] = [ + ["Наименование", "Код", "На 30.09.2025", None, "На 31.12.2024", "На 31.12.2023"], + ["Запасы", "1210", "5", None, "4", "3"], + ["Итого", "1200", "5", None, "4", "3"], + ] + columns, _ = run(rows, OCR, rules) + assert [c.column_index for c in columns] == [2, 4, 5] + + +def test_uncertain_label_does_not_confirm_itself(rules: Rules) -> None: + rows: list[list[Any]] = [ + ["Наименование", "Код", ("На 30.09.2О25", 60.0), ""], + ["Запасы", "1210", "5", "4"], + ["Итого", "1200", "5", "4"], + ] + cand = ocr(rows) + drafts, _ = segment_columns(cand.rows, (0,), (1, 2), 1, OCR, rules) + dates = document_dates(drafts, parse_header(hdr(), rules)) + assert (dates.effective, dates.confident_effective) == (date(2025, 9, 30), None) + _, warnings = finalize_columns(drafts, dates, rules) + assert [(w.code, w.column_index) for w in warnings] == [("COLUMN_DATE_UNCERTAIN", 2)] + + +def test_uncertain_label_confirmed_by_report_date(rules: Rules) -> None: + rows: list[list[Any]] = [ + ["Наименование", "Код", "На 30.09.2025", ("На 31.12.2024", 60.0), ("На 31.03.2025", 60.0)], + ["Запасы", "1210", "5", "4", "3"], + ["Итого", "1200", "5", "4", "3"], + ] + _, warnings = run(rows, OCR, rules, header=hdr(report_date="30.09.2025")) + assert [(w.code, w.column_index) for w in warnings] == [("COLUMN_DATE_UNCERTAIN", 4)] + + +def test_segment_without_header(rules: Rules) -> None: + cand = ocr([["Запасы", "1210", "5", "", "4"], ["Итого", "1200", "5", "", "4"]]) + drafts, warnings = segment_columns(cand.rows, (), (0, 1), 1, OCR, rules) + assert [(d.grid_col, d.label, d.kind) for d in drafts] == [(2, "", PeriodKind.NONE), (4, "", PeriodKind.NONE)] + assert warnings == [] + + +def test_public_amount_columns(rules: Rules) -> None: + rows: list[list[Any]] = [ + ["Наименование", "", "Код", "На 30.09.2025", "На 31.12.2024", "Изменение"], + ["Запасы", "", "1210", D("5"), D("4"), "рост"], + ["Итого", "", "1200", D("5"), "", "рост"], + ] + assert amount_columns(excel([("Лист1", rows)]).rows, 2, rules) == (3, 4) + assert amount_columns(ocr([["Запасы", "1210", "5", "", "4"]]).rows, 1, rules, source=OCR) == (2, 4) -- 2.47.2 From ed85759b9bc060e7207870a3d7a857fb6c85d188 Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Sun, 18 Oct 2026 13:41:12 +0400 Subject: [PATCH 12/28] =?UTF-8?q?feat:=20=D1=88=D0=B0=D0=B3=202=20?= =?UTF-8?q?=E2=80=94=20=D1=81=D0=B5=D0=B3=D0=BC=D0=B5=D0=BD=D1=82=D1=8B,?= =?UTF-8?q?=20=D0=BF=D0=BE=D0=B2=D1=82=D0=BE=D1=80=D0=BD=D0=B0=D1=8F=20?= =?UTF-8?q?=D1=88=D0=B0=D0=BF=D0=BA=D0=B0,=20=D0=BB=D0=B8=D1=81=D1=82?= =?UTF-8?q?=D1=8B-=D0=BF=D1=80=D0=BE=D0=B4=D0=BE=D0=BB=D0=B6=D0=B5=D0=BD?= =?UTF-8?q?=D0=B8=D1=8F=20=D0=B8=20=D1=81=D0=BE=D0=BF=D0=BE=D1=81=D1=82?= =?UTF-8?q?=D0=B0=D0=B2=D0=BB=D0=B5=D0=BD=D0=B8=D0=B5=20=D0=BA=D0=BE=D0=BB?= =?UTF-8?q?=D0=BE=D0=BD=D0=BE=D0=BA=20=D1=81=20=D0=BF=D0=B5=D1=80=D0=B2?= =?UTF-8?q?=D1=8B=D0=BC=20=D1=81=D0=B5=D0=B3=D0=BC=D0=B5=D0=BD=D1=82=D0=BE?= =?UTF-8?q?=D0=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W --- arm_statements/segments.py | 252 ++++++++++++++++++++++++++ arm_statements/tests/test_segments.py | 115 ++++++++++++ 2 files changed, 367 insertions(+) create mode 100644 arm_statements/segments.py create mode 100644 arm_statements/tests/test_segments.py diff --git a/arm_statements/segments.py b/arm_statements/segments.py new file mode 100644 index 0000000..9cbedec --- /dev/null +++ b/arm_statements/segments.py @@ -0,0 +1,252 @@ +"""Шаг 2: сегменты сетки, повторные шапки, границы листов и сопоставление колонок с первым сегментом.""" + +from collections.abc import Sequence +from dataclasses import dataclass +from typing import NamedTuple + +from arm_statements.codes import code_parts, is_code_cell, majority_section, parse_code +from arm_statements.columns import ColumnDraft, DocumentDates, document_dates, finalize_columns, segment_columns +from arm_statements.issues import WarningCode, make_warning +from arm_statements.layout import ( + Grid, + cell_at, + find_code_column, + find_header_rows, + find_name_column, + first_code_row, + is_numbering_row, + segment_roles, +) +from arm_statements.loader import CodeSection, Rules +from arm_statements.model import ( + FormCandidate, + RowRole, + StatementColumn, + StatementHeader, + StatementWarning, +) +from arm_statements.text import normalize_caption + + +@dataclass(frozen=True, slots=True) +class SegmentLayout: + """Раскладка сегмента; `value_cols` — (столбец сетки сегмента, `column_index` колонки документа).""" + + first_row: int + last_row: int + header_rows: tuple[int, ...] + code_col: int | None + name_col: int | None + explanations_col: int | None + value_cols: tuple[tuple[int, int], ...] + extra_cols: tuple[int, ...] + + +@dataclass(frozen=True, slots=True) +class Layout: + """Раскладка кандидата; `roles` — по всем строкам сетки.""" + + code_found: bool + column_section: CodeSection | None + columns: tuple[StatementColumn, ...] + segments: tuple[SegmentLayout, ...] + roles: tuple[RowRole, ...] + dates: DocumentDates + warnings: tuple[StatementWarning, ...] + + +@dataclass(frozen=True, slots=True) +class _Analysed: + """Сегмент до сопоставления колонок.""" + + first: int + last: int + header_rows: tuple[int, ...] + code_col: int | None + name_col: int | None + explanations_col: int | None + roles: tuple[RowRole, ...] + drafts: tuple[ColumnDraft, ...] + draft_warnings: tuple[StatementWarning, ...] + has_content: bool + + +class _Bounds(NamedTuple): + last: int + reference: frozenset[str] + + +def _text(rows: Grid, row: int, col: int) -> str: + cell = cell_at(rows, row, col) + return "" if cell is None else cell.text.strip() + + +def _chunks(rows: Grid, breaks: Sequence[int]) -> list[tuple[int, int]]: + """Куски сетки по границам листов; границы вне сетки и повторы отбрасываются.""" + cuts = sorted({b for b in breaks if 0 < b < len(rows)}) + starts = [0, *cuts] + ends = [*(c - 1 for c in cuts), len(rows) - 1] + return list(zip(starts, ends, strict=True)) + + +def _caption_cells(rows: Grid, header_rows: Sequence[int]) -> frozenset[str]: + cells = (normalize_caption(_text(rows, r, c)) for r in header_rows for c in range(len(rows[r]))) + return frozenset(c for c in cells if c) + + +def _is_repeat_header(rows: Grid, row: int, code_col: int | None, reference: frozenset[str], rules: Rules) -> bool: + """Строка-шапка: ячейка подписи кода или не менее двух ячеек, равных ячейкам шапки первого сегмента.""" + if is_numbering_row(rows[row], rules): + return False + if code_col is not None and is_code_cell(_text(rows, row, code_col), rules): + return False + normalized = [normalize_caption(c.text) for c in rows[row] if c is not None and c.text.strip()] + if any(rules.headers.code_caption.search(text) for text in normalized): + return True + return sum(1 for text in normalized if text and text in reference) >= _MIN_SAME_CELLS + + +_MIN_SAME_CELLS = 2 + + +def _segment_bounds(rows: Grid, start: int, chunk_last: int, reference: frozenset[str], rules: Rules) -> _Bounds: + """Последняя строка сегмента: до следующей шапки, стоящей после первой строки с кодом.""" + code_col = find_code_column(rows, start, chunk_last, rules) + first_code = None if code_col is None else first_code_row(rows, start, chunk_last, code_col, rules) + if code_col is None or first_code is None: + return _Bounds(chunk_last, reference) + ref = reference or _caption_cells(rows, find_header_rows(rows, start, first_code, code_col, rules)) + for row in range(first_code + 1, chunk_last + 1): + if _is_repeat_header(rows, row, code_col, ref, rules): + return _Bounds(row - 1, ref) + return _Bounds(chunk_last, ref) + + +def _segment_codes(rows: Grid, first: int, last: int, code_col: int, rules: Rules) -> list[str]: + codes: list[str] = [] + for row in range(first, last + 1): + text = _text(rows, row, code_col) + if is_code_cell(text, rules): + codes.extend(c for part in code_parts(text) if (c := parse_code(part, rules).code) is not None) + return codes + + +def _analyse( # noqa: PLR0913, PLR0917 - внутренняя сборка сегмента + candidate: FormCandidate, first: int, last: int, section: CodeSection | None, rules: Rules +) -> _Analysed: + rows = candidate.rows + code_col = find_code_column(rows, first, last, rules) + if code_col is None and section is not None: + code_col = find_code_column(rows, first, last, rules, relaxed_section=section) + has_content = any(c is not None and c.text.strip() for r in range(first, last + 1) for c in rows[r]) + first_code = None if code_col is None else first_code_row(rows, first, last, code_col, rules) + if code_col is None or first_code is None: + return _Analysed(first, last, (), None, None, None, (RowRole.DATA,) * (last - first + 1), (), (), has_content) + header_rows = find_header_rows(rows, first, first_code, code_col, rules) + roles = tuple(segment_roles(rows, first, last, header_rows, rules)) + data_rows = [r for r, role in zip(range(first, last + 1), roles, strict=True) if role is RowRole.DATA] + name_col, notes_col = find_name_column(rows, header_rows, first, last, code_col, rules) + drafts, warnings = segment_columns(rows, header_rows, data_rows, code_col, candidate.source, rules) + return _Analysed( + first, last, header_rows, code_col, name_col, notes_col, roles, tuple(drafts), tuple(warnings), has_content + ) + + +def _split_and_analyse(candidate: FormCandidate, rules: Rules) -> tuple[list[_Analysed], CodeSection | None]: + rows = candidate.rows + result: list[_Analysed] = [] + reference: frozenset[str] = frozenset() + section: CodeSection | None = None + for chunk_first, chunk_last in _chunks(rows, candidate.segment_breaks): + start = chunk_first + while start <= chunk_last: + bounds = _segment_bounds(rows, start, chunk_last, reference, rules) + reference = reference or bounds.reference + seg = _analyse(candidate, start, bounds.last, section, rules) + if section is None and seg.code_col is not None: + section = majority_section(_segment_codes(rows, seg.first, seg.last, seg.code_col, rules), rules) + result.append(seg) + start = bounds.last + 1 + return result, section + + +def _map_by_date( + drafts: Sequence[ColumnDraft], columns: Sequence[StatementColumn] +) -> tuple[tuple[int, int], ...] | None: + """Сопоставление по `period_date`: у каждой колонки дата есть и совпадает ровно с одной колонкой документа.""" + if not drafts or any(d.date is None for d in drafts): + return None + pairs: list[tuple[int, int]] = [] + for draft in drafts: + found = [c.column_index for c in columns if c.period_date == draft.date] + if len(found) != 1: + return None + pairs.append((draft.grid_col, found[0])) + if len({target for _, target in pairs}) != len(pairs): + return None + return tuple(pairs) + + +def _map_segment( + seg: _Analysed, columns: tuple[StatementColumn, ...], rules: Rules +) -> tuple[SegmentLayout, StatementWarning | None]: + """Колонки сегмента → колонки документа; по порядку и разное число — `SEGMENT_COLUMNS_MISMATCH`.""" + drafts = seg.drafts + mapped = _map_by_date(drafts, columns) if seg.header_rows else None + warning: StatementWarning | None = None + extra: tuple[int, ...] = () + if mapped is None: + size = min(len(drafts), len(columns)) + mapped = tuple((d.grid_col, c.column_index) for d, c in zip(drafts[:size], columns[:size], strict=False)) + extra = tuple(d.grid_col for d in drafts[size:]) + if len(drafts) != len(columns): + tail = f"; лишние столбцы сегмента: {', '.join(str(c) for c in extra)}" if extra else "" + message = ( + f"в сегменте со строки {seg.first} колонок значений {len(drafts)}, " + f"в первом сегменте {len(columns)}{tail}" + ) + warning = make_warning(WarningCode.SEGMENT_COLUMNS_MISMATCH, message, rules, locator=None) + layout = SegmentLayout( + seg.first, seg.last, seg.header_rows, seg.code_col, seg.name_col, seg.explanations_col, mapped, extra + ) + return layout, warning + + +def find_layout(candidate: FormCandidate, header: StatementHeader, rules: Rules) -> Layout: + """Раскладка кандидата: сегменты, колонки документа, роли всех строк и предупреждения шага 2.""" + segments, section = _split_and_analyse(candidate, rules) + coded = [s for s in segments if s.code_col is not None] + defining = next((s for s in coded if s.header_rows), coded[0] if coded else None) + warnings: list[StatementWarning] = [] + columns: tuple[StatementColumn, ...] = () + dates = document_dates([], header) + if defining is not None: + dates = document_dates(defining.drafts, header) + columns, finalize_warnings = finalize_columns(defining.drafts, dates, rules) + warnings.extend(defining.draft_warnings) + warnings.extend(finalize_warnings) + if not any(s.header_rows for s in coded): + warnings.append( + make_warning(WarningCode.HEADER_ROW_NOT_FOUND, "строка заголовка таблицы не найдена", rules) + ) + laid: list[SegmentLayout] = [] + for seg in segments: + if seg.code_col is None: + laid.append(SegmentLayout(seg.first, seg.last, (), None, None, None, (), ())) + if coded and seg.has_content: + message = f"в сегменте со строки {seg.first} не найден столбец кодов" + warnings.append(make_warning(WarningCode.SEGMENT_WITHOUT_CODES, message, rules)) + elif seg is defining: + own = tuple((c.column_index, c.column_index) for c in columns) + laid.append( + SegmentLayout( + seg.first, seg.last, seg.header_rows, seg.code_col, seg.name_col, seg.explanations_col, own, () + ) + ) + else: + segment_layout, mismatch = _map_segment(seg, columns, rules) + laid.append(segment_layout) + if mismatch is not None: + warnings.append(mismatch) + roles = tuple(role for seg in segments for role in seg.roles) + return Layout(bool(coded), section, columns, tuple(laid), roles, dates, tuple(warnings)) diff --git a/arm_statements/tests/test_segments.py b/arm_statements/tests/test_segments.py new file mode 100644 index 0000000..fe86101 --- /dev/null +++ b/arm_statements/tests/test_segments.py @@ -0,0 +1,115 @@ +from decimal import Decimal as D + +from arm_statements.header import parse_header +from arm_statements.loader import Rules +from arm_statements.model import RowRole +from arm_statements.segments import find_layout +from arm_statements.tests.builders import excel, hdr, ocr, place + +DATES3 = ["На 30 сентября 2025 г.", "На 31 декабря 2024 г.", "На 31 декабря 2023 г."] + + +def test_npz_sheet_with_own_header_maps_by_date(rules: Rules) -> None: + first = [ + place({1: "Наименование показателя", 12: "Код", 16: DATES3[0], 21: DATES3[1], 24: DATES3[2]}), + place({1: "Нематериальные активы", 12: "1110", 16: D("10"), 21: D("9"), 24: D("8")}), + place({1: "Итого по разделу I", 12: "1100", 16: D("10"), 21: D("9"), 24: D("8")}), + ] + sheet4 = [ + place({0: "Наименование показателя", 6: "Код", 8: DATES3[0], 9: DATES3[1], 10: DATES3[2]}), + place({0: "Заемные средства", 6: "1510", 8: D("3"), 9: D("2"), 10: D("1")}), + ] + cand = excel([("Бухгалтерский баланс", first), ("Бухгалтерский баланс. Лист 4", sheet4)]) + layout = find_layout(cand, parse_header(hdr(report_date="30.09.2025"), rules), rules) + assert [c.column_index for c in layout.columns] == [16, 21, 24] + assert layout.segments[1].value_cols == ((8, 16), (9, 21), (10, 24)) + assert not [w for w in layout.warnings if w.code.startswith("SEGMENT_")] + + +def test_repeat_header_mid_table(rules: Rules) -> None: + rows = [ + ["Наименование показателя", "Код", "На 30.09.2025", "На 31.12.2024"], + ["Запасы", "1210", "5", "4"], + ["Наименование показателя", "Код", "На 30.09.2025", "На 31.12.2024"], + ["Итого", "1200", "5", "4"], + ] + layout = find_layout(ocr(rows), parse_header(hdr(), rules), rules) + assert [s.first_row for s in layout.segments] == [0, 2] + assert layout.roles[2] is RowRole.HEADER + + +def test_repeat_header_without_dates_maps_by_order(rules: Rules) -> None: + rows = [ + ["Наименование", "Код", "На 30.09.2025", "На 31.12.2024"], + ["Запасы", "1210", "5", "4"], + ["Наименование", "Код", "отчетн", "прошл"], + ["Итого", "1200", "5", "4"], + ] + layout = find_layout(ocr(rows), parse_header(hdr(), rules), rules) + assert layout.segments[1].value_cols == ((2, 2), (3, 3)) + assert layout.warnings == () + + +def test_wider_segment_is_mismatch(rules: Rules) -> None: + rows = [ + ["Наименование", "Код", "На 30.09.2025", "На 31.12.2024", "На 31.12.2023"], + ["Запасы", "1210", "5", "4", "3"], + ["Наименование", "Код", "графа 3", "графа 4", "графа 5", "графа 6"], + ["Итого", "1200", "5", "4", "3", "2"], + ] + layout = find_layout(ocr(rows), parse_header(hdr(), rules), rules) + (warning,) = [w for w in layout.warnings if w.code == "SEGMENT_COLUMNS_MISMATCH"] + assert warning.severity.value == "error" and "5" in warning.message + assert layout.segments[1].value_cols == ((2, 2), (3, 3), (4, 4)) and layout.segments[1].extra_cols == (5,) + + +def test_second_sheet_single_code_in_other_column(rules: Rules) -> None: + first = [ + place({0: "Наименование", 3: "Код", 4: "За 9 месяцев 2025 г.", 5: "За 9 месяцев 2024 г."}), + place({0: "Выручка", 3: "2110", 4: D("100"), 5: D("90")}), + place({0: "Себестоимость", 3: "2120", 4: D("-60"), 5: D("-50")}), + place({0: "Валовая прибыль", 3: "2100", 4: D("40"), 5: D("40")}), + ] + second = [place({0: "Чистая прибыль", 1: "2400", 2: D("7"), 3: D("6")})] + cand = excel([("ОФР", first), ("ОФР_2", second)]) + layout = find_layout(cand, parse_header(hdr(), rules), rules) + assert layout.segments[1].code_col == 1 + assert layout.segments[1].value_cols == ((2, 4), (3, 5)) + assert not [w for w in layout.warnings if w.code.startswith(("SEGMENT_", "HEADER_"))] + + +def test_header_not_found_anywhere(rules: Rules) -> None: + rows = [["Запасы", "1210", "5", "4"], ["Дебиторы", "1230", "3", "2"], ["Итого", "1200", "8", "6"]] + layout = find_layout(ocr(rows), parse_header(hdr(), rules), rules) + assert [w.code for w in layout.warnings] == ["HEADER_ROW_NOT_FOUND"] + assert [(c.column_index, c.label) for c in layout.columns] == [(2, ""), (3, "")] + + +def test_segment_without_codes(rules: Rules) -> None: + first = [["Наименование", "Код", "На 30.09.2025"], ["Запасы", "1210", "5"], ["Итого", "1200", "5"]] + tail = [["Примечание к форме", "", "7"]] + layout = find_layout(excel([("Баланс", first), ("Хвост", tail)]), parse_header(hdr(), rules), rules) + assert [w.code for w in layout.warnings] == ["SEGMENT_WITHOUT_CODES"] + assert layout.roles[3] is RowRole.DATA and layout.code_found + + +def test_first_segment_without_header_columns_from_first_headed(rules: Rules) -> None: + """P1-18: колонки документа задаёт первый сегмент с заголовком, у первого значения — по порядку.""" + rows = [ + ["Запасы", "1210", "5", "4"], + ["Дебиторы", "1230", "3", "2"], + ["Итого", "1200", "8", "6"], + ["Наименование", "Код", "На 30.09.2025", "На 31.12.2024"], + ["Прочее", "1250", "1", "1"], + ] + layout = find_layout(ocr(rows), parse_header(hdr(), rules), rules) + assert [s.first_row for s in layout.segments] == [0, 3] + assert [c.column_index for c in layout.columns] == [2, 3] + assert layout.segments[0].value_cols == ((2, 2), (3, 3)) + assert layout.warnings == () + + +def test_no_codes_anywhere_has_no_segment_warnings(rules: Rules) -> None: + layout = find_layout(ocr([["Просто", "текст"], ["ещё", "строка"]]), parse_header(hdr(), rules), rules) + assert not layout.code_found and layout.columns == () and layout.warnings == () + assert layout.roles == (RowRole.DATA, RowRole.DATA) -- 2.47.2 From 91e96795d5814a9ba359357f664d7a3d50327fea Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Sun, 18 Oct 2026 13:44:47 +0400 Subject: [PATCH 13/28] =?UTF-8?q?feat:=20=D1=88=D0=B0=D0=B3=203=20?= =?UTF-8?q?=E2=80=94=20=D1=81=D1=82=D1=80=D0=BE=D0=BA=D0=B8=20=D0=B4=D0=B0?= =?UTF-8?q?=D0=BD=D0=BD=D1=8B=D1=85=20=D1=81=D0=BE=20=D1=81=D1=82=D0=B0?= =?UTF-8?q?=D1=82=D1=83=D1=81=D0=B0=D0=BC=D0=B8=20ok/unreadable/detail/mis?= =?UTF-8?q?sing,=20ref,=20=D1=80=D0=B0=D0=B7=D0=BB=D0=B5=D0=BF=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D0=B5=20=D1=81=D0=BA=D0=BB=D0=B5=D0=B5=D0=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W --- arm_statements/rows.py | 253 ++++++++++++++++++++++++++++++ arm_statements/tests/test_rows.py | 115 ++++++++++++++ 2 files changed, 368 insertions(+) create mode 100644 arm_statements/rows.py create mode 100644 arm_statements/tests/test_rows.py diff --git a/arm_statements/rows.py b/arm_statements/rows.py new file mode 100644 index 0000000..cf05c63 --- /dev/null +++ b/arm_statements/rows.py @@ -0,0 +1,253 @@ +"""Шаг 3: строки данных, их статусы, `parent_ref` и разлепление склеенных строк.""" + +from collections.abc import Sequence +from dataclasses import dataclass + +from arm_statements.amounts import parse_amount +from arm_statements.codes import code_parts, parse_code, section_of +from arm_statements.issues import WarningCode, make_warning +from arm_statements.layout import Grid, cell_at, name_cell +from arm_statements.loader import Rules +from arm_statements.model import ( + Cell, + FormCandidate, + LineStatus, + RowRole, + Source, + StatementWarning, +) +from arm_statements.segments import Layout, SegmentLayout +from arm_statements.text import has_digit, is_disguised_digit + +_FULL_CONFIDENCE = 100.0 +_MIN_SPLIT_PARTS = 2 + +CellsByColumn = tuple[tuple[int, Cell | None], ...] + + +@dataclass(frozen=True, slots=True) +class RawLine: + """Строка данных до разбора значений: ячейки и статус; `cells` — по всем колонкам документа по порядку.""" + + row_index: int + ref: str + grid_row: int + code_cell: Cell | None + code: str | None + code_fixed: bool + line_status: LineStatus + name: Cell | None + parent_ref: str | None + cells: CellsByColumn + split: bool + + +@dataclass(slots=True) +class _State: + """Сквозное состояние документа: счётчик строк и ближайшая строка `ok`.""" + + next_index: int = 0 + last_ok_ref: str | None = None + + +def _blank(cell: Cell | None) -> bool: + return cell is None or not cell.text.strip() + + +def _is_content(cell: Cell | None, source: Source, rules: Rules) -> bool: + """Содержимое ячейки значения: сумма, неуверенный непустой текст или «цифра под видом буквы».""" + if cell is None: + return False + if cell.value is not None: + return True + text = cell.text + if not text.strip(): + return False + if parse_amount(text, source, rules).amount is not None: + return True + return cell.confidence < _FULL_CONFIDENCE or is_disguised_digit(text, rules.lookalikes) + + +def _first_ref(row: Sequence[Cell | None], code_cell: Cell | None) -> str: + if code_cell is not None: + return code_cell.ref + for cell in row: + if not _blank(cell): + return cell.ref if cell is not None else "" + return next((c.ref for c in row if c is not None), "") + + +def _grid_cols(seg: SegmentLayout, layout: Layout) -> dict[int, int]: + """`column_index` колонки документа → столбец сетки сегмента.""" + by_column = {column_index: grid_col for grid_col, column_index in seg.value_cols} + return {c.column_index: by_column[c.column_index] for c in layout.columns if c.column_index in by_column} + + +def _segment_cells(rows: Grid, row: int, mapping: dict[int, int], layout: Layout) -> CellsByColumn: + return tuple( + (c.column_index, cell_at(rows, row, mapping[c.column_index]) if c.column_index in mapping else None) + for c in layout.columns + ) + + +def _plain_name(cells: Sequence[Cell | None]) -> tuple[Cell | None, int]: + """Сегмент без столбца кодов: наименование — левая ячейка с буквами; второй элемент — её столбец.""" + for col, cell in enumerate(cells): + if cell is not None and any(ch.isalpha() for ch in cell.text): + return cell, col + return None, -1 + + +def _has_value_content(cells: CellsByColumn, source: Source, rules: Rules) -> bool: + return any(_is_content(cell, source, rules) for _, cell in cells) + + +def _part(cell: Cell, k: int, text: str) -> Cell: + return Cell(text, None, cell.confidence, f"{cell.ref}#{k}") + + +def _split_parts(code_cell: Cell | None, cells: CellsByColumn, layout: Layout, rules: Rules) -> tuple[str, ...] | None: + """Части кода склеенной строки или `None`, если разлеплять нельзя.""" + if code_cell is None or layout.column_section is None: + return None + parts = code_parts(code_cell.text) + count = len(parts) + if count < _MIN_SPLIT_PARTS: + return None + if any(not _blank(cell) and len(code_parts(cell.text if cell else None)) != count for _, cell in cells): + return None + in_range = ( + (code := parse_code(part, rules).code) is not None and section_of(code, rules) is layout.column_section + for part in parts + ) + return parts if any(in_range) else None + + +def _status(code_cell: Cell | None, code: str | None, *, has_content: bool, state: _State) -> LineStatus | None: + """Статус строки без разлепления; `None` — строка данных без содержимого.""" + if code_cell is not None and has_digit(code_cell.text): + return LineStatus.OK if code is not None else LineStatus.UNREADABLE + if not has_content: + return None + return LineStatus.DETAIL if state.last_ok_ref is not None else LineStatus.MISSING + + +def _emit( # noqa: PLR0913 - сборка одной строки + state: _State, + *, + ref: str, + grid_row: int, + code_cell: Cell | None, + code: str | None, + fixed: bool, + status: LineStatus, + name: Cell | None, + cells: CellsByColumn, + split: bool, +) -> RawLine: + parent = state.last_ok_ref if status is LineStatus.DETAIL else None + line = RawLine(state.next_index, ref, grid_row, code_cell, code, fixed, status, name, parent, cells, split) + state.next_index += 1 + if status is LineStatus.OK: + state.last_ok_ref = ref + return line + + +def _split_row( # noqa: PLR0913, PLR0917 - разлепление одной строки сетки + state: _State, + grid_row: int, + base_ref: str, + parts: tuple[str, ...], + code_cell: Cell, + name: Cell | None, + cells: CellsByColumn, + rules: Rules, +) -> list[RawLine]: + count = len(parts) + name_parts = code_parts(name.text) if name is not None else () + lines: list[RawLine] = [] + for k, part in enumerate(parts, start=1): + parsed = parse_code(part, rules) + derived_cells: CellsByColumn = tuple( + (index, cell if _blank(cell) or cell is None else _part(cell, k, code_parts(cell.text)[k - 1])) + for index, cell in cells + ) + derived_name = name + if name is not None and len(name_parts) == count: + derived_name = _part(name, k, name_parts[k - 1]) + lines.append( + _emit( + state, + ref=f"{base_ref}#{k}", + grid_row=grid_row, + code_cell=_part(code_cell, k, part), + code=parsed.code, + fixed=parsed.fixed, + status=LineStatus.OK if parsed.code is not None else LineStatus.UNREADABLE, + name=derived_name, + cells=derived_cells, + split=True, + ) + ) + return lines + + +def _row_lines( # noqa: PLR0913, PLR0917 - разбор одной строки сетки + candidate: FormCandidate, layout: Layout, seg: SegmentLayout, grid_row: int, state: _State, rules: Rules +) -> list[RawLine]: + rows = candidate.rows + row = rows[grid_row] + cells = _segment_cells(rows, grid_row, _grid_cols(seg, layout), layout) + code_cell = cell_at(rows, grid_row, seg.code_col) if seg.code_col is not None else None + if seg.code_col is not None: + name = name_cell(row, seg.name_col, seg.code_col, seg.explanations_col) + content_cells = cells + else: + name, name_col = _plain_name(row) + content_cells = ( + cells if seg.value_cols else tuple((col, cell) for col, cell in enumerate(row) if col > name_col) + ) + base_ref = _first_ref(row, code_cell) + parts = _split_parts(code_cell, cells, layout, rules) + if parts is not None and code_cell is not None: + return _split_row(state, grid_row, base_ref, parts, code_cell, name, cells, rules) + parsed = parse_code(code_cell.text if code_cell is not None else None, rules) + has_content = _has_value_content(content_cells, candidate.source, rules) + status = _status(code_cell, parsed.code, has_content=has_content, state=state) + if status is None: + return [] + ref = base_ref or _first_ref(row, None) + return [ + _emit( + state, + ref=ref, + grid_row=grid_row, + code_cell=code_cell, + code=parsed.code, + fixed=parsed.fixed, + status=status, + name=name, + cells=cells, + split=False, + ) + ] + + +def collect_raw_lines( + candidate: FormCandidate, layout: Layout, rules: Rules +) -> tuple[list[RawLine], list[StatementWarning]]: + """Строки данных по сегментам и предупреждения `ROW_SPLIT` (`info`) на каждую разлепленную строку сетки.""" + state = _State() + lines: list[RawLine] = [] + warnings: list[StatementWarning] = [] + for seg in layout.segments: + for grid_row in range(seg.first_row, seg.last_row + 1): + if grid_row >= len(candidate.rows) or layout.roles[grid_row] is not RowRole.DATA: + continue + produced = _row_lines(candidate, layout, seg, grid_row, state, rules) + lines.extend(produced) + if produced and produced[0].split: + message = f"склеенная строка сетки {grid_row} разделена на {len(produced)} строк" + refs = tuple(line.ref for line in produced) + warnings.append(make_warning(WarningCode.ROW_SPLIT, message, rules, line_refs=refs)) + return lines, warnings diff --git a/arm_statements/tests/test_rows.py b/arm_statements/tests/test_rows.py new file mode 100644 index 0000000..2c33bff --- /dev/null +++ b/arm_statements/tests/test_rows.py @@ -0,0 +1,115 @@ +from typing import Any + +from arm_statements.header import parse_header +from arm_statements.loader import Rules +from arm_statements.model import LineStatus, StatementWarning +from arm_statements.rows import RawLine, collect_raw_lines +from arm_statements.segments import find_layout +from arm_statements.tests.builders import excel, hdr, ocr + +HEAD = ["Наименование показателя", "Код", "На 30.09.2025", "На 31.12.2024"] +OK, UNREADABLE, DETAIL, MISSING = LineStatus.OK, LineStatus.UNREADABLE, LineStatus.DETAIL, LineStatus.MISSING + + +def lines_of(rows: Any, rules: Rules, source: str = "ocr") -> tuple[list[RawLine], list[StatementWarning]]: + cand = ocr(rows) if source == "ocr" else excel([("Баланс", rows)]) + layout = find_layout(cand, parse_header(hdr(), rules), rules) + return collect_raw_lines(cand, layout, rules) + + +def test_junk_code_lines_survive(rules: Rules) -> None: + lines, _ = lines_of( + [ + HEAD, + ["НМА", "1110", "5", "4"], + ["Запасы", "1150", "5", "4"], + ["Мусор", "ИЧ,250", "7", "6"], + ["в том числе: прочее", "", "1", ""], + ["", "44 30\nI |", "", ""], + ["Итого", "1100", "10", "8"], + ], + rules, + ) + assert [(ln.line_status, ln.code) for ln in lines] == [ + (OK, "1110"), + (OK, "1150"), + (UNREADABLE, None), + (DETAIL, None), + (UNREADABLE, None), + (OK, "1100"), + ] + assert lines[3].parent_ref == lines[1].ref == "row:2" + assert [ln.row_index for ln in lines] == [0, 1, 2, 3, 4, 5] + + +def test_lookalike_only_code_cells_are_empty(rules: Rules) -> None: + lines, _ = lines_of( + [ + HEAD, + ["НМА", "1110", "5", "4"], + ["Запасы", "1150", "5", "4"], + ["", "изо :", ("3", 0.0), ""], + ["", "НИ", "I", ""], + ["", "| 140", "2", "1"], + ["Итого", "1100", "12", "9"], + ], + rules, + ) + assert [(ln.line_status, ln.code, ln.code_fixed) for ln in lines] == [ + (OK, "1110", False), + (OK, "1150", False), + (DETAIL, None, False), + (DETAIL, None, False), + (OK, "1140", True), + (OK, "1100", False), + ] + assert [ln.parent_ref for ln in lines[2:4]] == ["row:2", "row:2"] + assert len({ln.ref for ln in lines}) == len(lines) + + +def test_missing_before_first_code_and_name_only_row(rules: Rules) -> None: + lines, _ = lines_of( + [HEAD, ["Выручка", "", "100", "90"], ["Раздел I. Активы", "", "", ""], ["Запасы", "1150", "5", "4"]], rules + ) + assert [ln.line_status for ln in lines] == [MISSING, OK] + + +def test_duplicate_code_both_ok(rules: Rules) -> None: + lines, _ = lines_of([HEAD, ["Налог", "2410", "100", "1"], ["Налог УСН", "2410", "649", "2"]], rules) + assert [(ln.line_status, ln.code) for ln in lines] == [(OK, "2410"), (OK, "2410")] + + +def test_split_glued_rows(rules: Rules) -> None: + lines, warnings = lines_of( + [HEAD, ["Запасы", "1150", "5", "4"], ["Займы\nКредиторы", "1500\n1700", "917 775\n2 371 333", "1\n2"]], rules + ) + split = lines[1:] + assert [(ln.ref, ln.code, ln.split) for ln in split] == [("row:2#1", "1500", True), ("row:2#2", "1700", True)] + assert [ln.cells[0][1].text for ln in split if ln.cells[0][1] is not None] == ["917 775", "2 371 333"] + assert [ln.name.text for ln in split if ln.name is not None] == ["Займы", "Кредиторы"] + assert [(w.code, w.line_refs) for w in warnings] == [("ROW_SPLIT", ("row:2#1", "row:2#2"))] + + +def test_not_split_when_no_part_in_column_range(rules: Rules) -> None: + lines, warnings = lines_of( + [ + HEAD, + ["НМА", "1110", "5", "4"], + ["Запасы", "1150", "5", "4"], + ["", "44 30\nI |", "1\n2", ""], + ["Итого", "1100", "10", "8"], + ], + rules, + ) + assert [ln.line_status for ln in lines] == [OK, OK, UNREADABLE, OK] and warnings == [] + + +def test_excel_signature_text_is_not_content(rules: Rules) -> None: + rows = [ + ["Наименование", "", "Код", "На 30.09.2025"], + ["Запасы", "", "1150", "5"], + ["Руководитель", "", "", "Косарев"], + ["", "", "", "(расшифровка подписи)"], + ] + lines, _ = lines_of(rows, rules, source="excel") + assert [ln.ref for ln in lines] == ["Баланс!C2"] -- 2.47.2 From 0c4dc6d2217bc97f1de001abb4dc0de231c2210e Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Sun, 18 Oct 2026 13:47:37 +0400 Subject: [PATCH 14/28] =?UTF-8?q?feat:=20=D1=88=D0=B0=D0=B3=D0=B8=204?= =?UTF-8?q?=E2=80=935=20=E2=80=94=20=D1=81=D1=82=D1=80=D0=BE=D0=BA=D0=B8?= =?UTF-8?q?=20=D1=84=D0=BE=D1=80=D0=BC=D1=8B=20=D1=81=20=D1=81=D1=8B=D1=80?= =?UTF-8?q?=D1=8B=D0=BC=20=D1=82=D0=B5=D0=BA=D1=81=D1=82=D0=BE=D0=BC=20?= =?UTF-8?q?=D0=B8=20=D1=83=D0=B2=D0=B5=D1=80=D0=B5=D0=BD=D0=BD=D0=BE=D1=81?= =?UTF-8?q?=D1=82=D1=8C=D1=8E,=20=D0=BF=D1=80=D0=B5=D0=B4=D1=83=D0=BF?= =?UTF-8?q?=D1=80=D0=B5=D0=B6=D0=B4=D0=B5=D0=BD=D0=B8=D1=8F=20=D0=B5=D0=B4?= =?UTF-8?q?=D0=B8=D0=BD=D0=B8=D1=86=20=D0=B8=20=D0=BC=D0=B0=D1=81=D1=88?= =?UTF-8?q?=D1=82=D0=B0=D0=B1=D0=B0=20xlsb?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W --- arm_statements/lines.py | 88 ++++++++++++++++++++++++++++++ arm_statements/tests/test_lines.py | 63 +++++++++++++++++++++ 2 files changed, 151 insertions(+) create mode 100644 arm_statements/lines.py create mode 100644 arm_statements/tests/test_lines.py diff --git a/arm_statements/lines.py b/arm_statements/lines.py new file mode 100644 index 0000000..0702806 --- /dev/null +++ b/arm_statements/lines.py @@ -0,0 +1,88 @@ +"""Шаги 4–5: значения ячеек в строки формы и предупреждения единиц измерения.""" + +from collections.abc import Sequence +from decimal import Decimal + +from arm_statements.amounts import build_value +from arm_statements.issues import WarningCode, make_warning +from arm_statements.loader import Rules +from arm_statements.model import ( + LineStatus, + Source, + StatementHeader, + StatementLine, + StatementValue, + StatementWarning, +) +from arm_statements.rows import RawLine + +CODE_CONFIDENCE_CAP = 99.0 +_PERCENT = 100 + + +def _code_confidence(raw: RawLine) -> float | None: + if raw.code_cell is None: + return None + confidence = raw.code_cell.confidence + return min(confidence, CODE_CONFIDENCE_CAP) if raw.code_fixed else confidence + + +def build_lines( + raw: Sequence[RawLine], source: Source, rules: Rules +) -> tuple[list[StatementLine], list[StatementWarning]]: + """Строки формы со значениями; `VALUE_OUT_OF_RANGE` (`info`) на каждое значение вне диапазона.""" + lines: list[StatementLine] = [] + warnings: list[StatementWarning] = [] + for item in raw: + values: list[StatementValue] = [] + for column_index, cell in item.cells: + built = build_value(cell, column_index, source, rules, split_part=item.split) + values.append(built.value) + if built.out_of_range and cell is not None: + message = f"значение ячейки {cell.ref} вне диапазона numeric(18,2)" + warnings.append( + make_warning( + WarningCode.VALUE_OUT_OF_RANGE, message, rules, line_refs=(cell.ref,), column_index=column_index + ) + ) + lines.append( + StatementLine( + row_index=item.row_index, + ref=item.ref, + line_status=item.line_status, + line_code=item.code if item.line_status is LineStatus.OK else None, + line_code_text=item.code_cell.text if item.code_cell is not None else None, + line_code_confidence=_code_confidence(item), + line_name=item.name.text if item.name is not None else None, + line_name_confidence=item.name.confidence if item.name is not None else None, + parent_ref=item.parent_ref, + needs_review=False, + values=tuple(values), + ) + ) + return lines, warnings + + +def _has_fraction(amount: Decimal) -> bool: + return amount != amount.to_integral_value() + + +def units_warnings( + lines: Sequence[StatementLine], + header: StatementHeader, + adapter_warnings: Sequence[StatementWarning], + rules: Rules, +) -> list[StatementWarning]: + """Единицы «тыс. руб.» и доля дробных не ниже порога: `FRACTIONAL_IN_THOUSANDS` / `XLSB_SCALE_SUSPECT`.""" + if header.units.value != rules.units.thousands: + return [] + amounts = [v.amount for line in lines for v in line.values if v.amount is not None] + if not amounts: + return [] + fractional = sum(1 for amount in amounts if _has_fraction(amount)) + if fractional * _PERCENT < rules.units.fractional_share_percent * len(amounts): + return [] + no_formats = any(w.code == WarningCode.XLSB_NO_FORMATS for w in adapter_warnings) + code = WarningCode.XLSB_SCALE_SUSPECT if no_formats else WarningCode.FRACTIONAL_IN_THOUSANDS + message = f"единицы «{rules.units.thousands}», но {fractional} из {len(amounts)} значений с дробной частью" + return [make_warning(code, message, rules)] diff --git a/arm_statements/tests/test_lines.py b/arm_statements/tests/test_lines.py new file mode 100644 index 0000000..1c868e8 --- /dev/null +++ b/arm_statements/tests/test_lines.py @@ -0,0 +1,63 @@ +from decimal import Decimal as D + +from arm_statements.header import parse_header +from arm_statements.issues import make_warning +from arm_statements.lines import build_lines, units_warnings +from arm_statements.loader import Rules +from arm_statements.model import FormCandidate, Severity, StatementLine, StatementWarning +from arm_statements.rows import collect_raw_lines +from arm_statements.segments import find_layout +from arm_statements.tests.builders import excel, hdr, ocr + +HEAD = ["Наименование показателя", "Код", "На 30.09.2025"] +CODE_CONFIDENCE = 99.0 +NAME_CONFIDENCE = 71.0 + + +def statement_lines(cand: FormCandidate, rules: Rules) -> tuple[list[StatementLine], list[StatementWarning]]: + layout = find_layout(cand, parse_header(hdr(), rules), rules) + raw, _ = collect_raw_lines(cand, layout, rules) + return build_lines(raw, cand.source, rules) + + +def test_ocr_name_and_code_confidence(rules: Rules) -> None: + cand = ocr([HEAD, [("Дебиторская задолжен-ность", 71.0), "1230", "5"], ["Итого", "I1О0", "5"]]) + lines, _ = statement_lines(cand, rules) + assert (lines[0].line_name, lines[0].line_name_confidence) == ("Дебиторская задолжен-ность", NAME_CONFIDENCE) + assert (lines[1].line_code, lines[1].line_code_text, lines[1].line_code_confidence) == ( + "1100", + "I1О0", + CODE_CONFIDENCE, + ) + assert [v.column_index for v in lines[0].values] == [2] + + +def test_value_out_of_range_warning(rules: Rules) -> None: + cand = excel([("Баланс", [["Наименование", "Код", "На 30.09.2025"], ["Запасы", "1210", D("123456789012345678")]])]) + lines, warnings = statement_lines(cand, rules) + assert lines[0].values[0].amount is None + assert [(w.code, w.severity, w.line_refs, w.column_index) for w in warnings] == [ + ("VALUE_OUT_OF_RANGE", Severity.INFO, ("Баланс!C2",), 2) + ] + + +def test_excel_code_ref_and_clean_confidence(rules: Rules) -> None: + cand = excel([("Баланс", [["Наименование", "Код", "На 30.09.2025"], ["Запасы", "1210", D("5")]])]) + lines, warnings = statement_lines(cand, rules) + assert (lines[0].ref, lines[0].line_code, lines[0].line_code_text) == ("Баланс!B2", "1210", "1210") + assert (lines[0].line_code_confidence, lines[0].needs_review, warnings) == (100.0, False, []) + + +def test_fractional_in_thousands(rules: Rules) -> None: + pairs = [("1210", "1.5"), ("1220", "2.25"), ("1230", "3"), ("1240", "4"), ("1200", "10.75")] + title: list[str | D | None] = ["Наименование", "Код", "На 30.09.2025"] + body: list[list[str | D | None]] = [[f"Строка {i}", c, D(v)] for i, (c, v) in enumerate(pairs)] + rows = [title, *body] + cand = excel([("Баланс", rows)]) + lines, _ = statement_lines(cand, rules) + head = parse_header(hdr(units="тыс. руб."), rules) + assert [w.code for w in units_warnings(lines, head, (), rules)] == ["FRACTIONAL_IN_THOUSANDS"] + xlsb = make_warning("XLSB_NO_FORMATS", "формат чисел xlsb не читается", rules) + (scale,) = units_warnings(lines, head, (xlsb,), rules) + assert (scale.code, scale.severity) == ("XLSB_SCALE_SUSPECT", Severity.ERROR) + assert units_warnings(lines, parse_header(hdr(units="руб."), rules), (), rules) == [] -- 2.47.2 From 3df179b7a3bd7b0b816157fe5d920454310f15c8 Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Sun, 18 Oct 2026 13:50:17 +0400 Subject: [PATCH 15/28] =?UTF-8?q?feat:=20=D1=88=D0=B0=D0=B3=207=20?= =?UTF-8?q?=E2=80=94=20=D0=BF=D0=BE=D1=81=D1=87=D0=B8=D1=82=D0=B0=D0=BD?= =?UTF-8?q?=D0=BD=D1=8B=D0=B5=20=D1=81=D0=BE=D0=BE=D1=82=D0=BD=D0=BE=D1=88?= =?UTF-8?q?=D0=B5=D0=BD=D0=B8=D1=8F=20=C2=A77,=20abs=5Fterms,=20=D0=B4?= =?UTF-8?q?=D0=BE=D0=BF=D1=83=D1=81=D0=BA=20=D0=B8=20=D0=BE=D0=BA=D1=80?= =?UTF-8?q?=D1=83=D0=B3=D0=BB=D0=B5=D0=BD=D0=B8=D0=B5,=20=D1=83=D0=BF?= =?UTF-8?q?=D1=80=D0=BE=D1=89=D1=91=D0=BD=D0=BD=D1=8B=D0=B9=20=D0=BD=D0=B0?= =?UTF-8?q?=D0=B1=D0=BE=D1=80,=20=D0=BF=D0=BE=D0=B4=D1=81=D1=82=D1=80?= =?UTF-8?q?=D0=BE=D0=BA=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Добавлены evaluation.py (EvalLine/EvalValue/is_check_line) и checks.py (form_variant, subline_parents, evaluate_relations, checks_from). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W --- arm_statements/checks.py | 184 ++++++++++++++++++++++++++++ arm_statements/evaluation.py | 36 ++++++ arm_statements/tests/test_checks.py | 107 ++++++++++++++++ 3 files changed, 327 insertions(+) create mode 100644 arm_statements/checks.py create mode 100644 arm_statements/evaluation.py create mode 100644 arm_statements/tests/test_checks.py diff --git a/arm_statements/checks.py b/arm_statements/checks.py new file mode 100644 index 0000000..16a3cf2 --- /dev/null +++ b/arm_statements/checks.py @@ -0,0 +1,184 @@ +"""Шаг 7: самопроверки §7 и вариант формы.""" + +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from decimal import Decimal + +from arm_statements.codes import document_type_of, section_of +from arm_statements.evaluation import EvalLine, is_check_line +from arm_statements.loader import RelationRule, Rules +from arm_statements.model import DocumentType, FailedRelation, FormVariant, StatementChecks + +_ZERO = Decimal("0") +_HUNDRED = Decimal("100") +_PARENT_LEN = 4 +_SUBLINE_LEN = 5 +_GROUP_LEN = 3 # первые цифры, общие у строки и её подстрок (1230 → 12301…12314) + + +def form_variant(document_type: DocumentType, ok_codes: Iterable[str]) -> FormVariant | None: + """Вариант формы по структуре кодов (спека «Шаг 6»); ОКУД признаком не служит.""" + codes = set(ok_codes) + if document_type is DocumentType.BALANCE: + simplified = "1600" in codes and not {"1100", "1200"} & codes + elif document_type is DocumentType.INCOME: + simplified = bool({"2400", "2110"} & codes) and not {"2100", "2200", "2300"} & codes + else: + return None + return FormVariant.SIMPLIFIED if simplified else FormVariant.FULL + + +def subline_parents(codes: Sequence[str]) -> list[str | None]: + """Родитель каждой позиции: у пятизначного кода — ближайший четырёхзначный выше с теми же тремя цифрами, + иначе первые четыре цифры; у прочих — `None`.""" + result: list[str | None] = [] + last_parent: str | None = None + for code in codes: + if len(code) == _PARENT_LEN: + last_parent = code + result.append(None) + elif len(code) == _SUBLINE_LEN: + if last_parent is not None and last_parent[:_GROUP_LEN] == code[:_GROUP_LEN]: + result.append(last_parent) + else: + result.append(code[:_PARENT_LEN]) + else: + result.append(None) + return result + + +@dataclass(frozen=True, slots=True) +class RelationOutcome: + column: str + rule: RelationRule + codes: tuple[str, ...] + computed: bool + expected: Decimal | None # None — цель пуста + actual: Decimal | None # None — пусты все члены + within_tolerance: bool + within_rounding: bool + + @property + def hard_failed(self) -> bool: + return self.rule.strict and self.computed and not self.within_tolerance and not self.within_rounding + + +def _column_sums(lines: Sequence[EvalLine], column: str) -> tuple[dict[str, Decimal], set[str]]: + """Код → сумма `amount` по строкам `is_check_line` в колонке; второй результат — коды с непустой суммой.""" + sums: dict[str, Decimal] = {} + for item in lines: + if not is_check_line(item) or item.line_code is None: + continue + for value in item.values: + if value.column == column and value.amount is not None: + sums[item.line_code] = sums.get(item.line_code, _ZERO) + value.amount + return sums, set(sums) + + +def _rhs(sums: Mapping[str, Decimal], rule: RelationRule, *, absolute: bool) -> Decimal: + total = _ZERO + for sign, code in rule.terms: + value = sums.get(code, _ZERO) + if absolute and code in rule.abs_terms: + value = abs(value) + total += sign * value + return total + + +def _actual(sums: Mapping[str, Decimal], rule: RelationRule, expected: Decimal, tolerance: Decimal) -> Decimal: + """Правая часть: сошедшийся вариант (как записано, затем по модулю), иначе ближайший к цели.""" + plain = _rhs(sums, rule, absolute=False) + if abs(expected - plain) <= tolerance or not rule.abs_terms: + return plain + with_abs = _rhs(sums, rule, absolute=True) + if abs(expected - with_abs) <= tolerance or abs(expected - with_abs) < abs(expected - plain): + return with_abs + return plain + + +def _outcome( + column: str, rule: RelationRule, sums: Mapping[str, Decimal], filled: set[str], rules: Rules +) -> RelationOutcome: + codes = (rule.target, *(code for _, code in rule.terms)) + target_filled = rule.target in filled + members_filled = any(code in filled for _, code in rule.terms) + if not target_filled and not members_filled: + return RelationOutcome(column, rule, codes, False, None, None, True, True) + expected = sums[rule.target] if target_filled else None + if expected is None or not members_filled: + actual = _rhs(sums, rule, absolute=False) if members_filled else None + return RelationOutcome(column, rule, codes, True, expected, actual, False, False) + actual = _actual(sums, rule, expected, rules.checks.tolerance) + diff = abs(expected - actual) + scale = max(abs(expected), *(abs(sums.get(code, _ZERO)) for _, code in rule.terms)) + within_rounding = diff <= scale * rules.checks.rounding_tolerance_percent / _HUNDRED + return RelationOutcome(column, rule, codes, True, expected, actual, diff <= rules.checks.tolerance, within_rounding) + + +def _subline_rules( + lines: Sequence[EvalLine], document_type: DocumentType, variant: FormVariant, rules: Rules +) -> list[RelationRule]: + """«родитель = Σ подстрок» для родителей из отрезка формы (мягкие), в порядке строк документа.""" + codes = [item.line_code for item in lines if is_check_line(item) and item.line_code is not None] + children: dict[str, list[str]] = {} + for code, parent in zip(codes, subline_parents(codes), strict=True): + if parent is None or code in children.get(parent, []): + continue + section = section_of(parent, rules) + if section is None or document_type_of(section) is not document_type: + continue + children.setdefault(parent, []).append(code) + return [ + RelationRule( + form=document_type, + variant=variant, + relation=f"{parent} = {' + '.join(kids)}", + target=parent, + terms=tuple((1, kid) for kid in kids), + strict=False, + abs_terms=frozenset(), + ) + for parent, kids in children.items() + ] + + +def evaluate_relations( + lines: Sequence[EvalLine], + columns: Sequence[str], + document_type: DocumentType, + variant: FormVariant | None, + rules: Rules, +) -> list[RelationOutcome]: + """Все соотношения набора формы и подстрок по каждой колонке. `lines` — в порядке документа.""" + effective = variant or FormVariant.FULL + static = list(rules.checks.relations_for(document_type, effective)) + generated = _subline_rules(lines, document_type, effective, rules) if rules.checks.subline_check else [] + outcomes: list[RelationOutcome] = [] + for column in columns: + sums, filled = _column_sums(lines, column) + outcomes.extend(_outcome(column, rule, sums, filled, rules) for rule in static) + # подстроки — только если хоть одна подстрока в колонке непуста (AURORA-1527) + outcomes.extend( + _outcome(column, rule, sums, filled, rules) + for rule in generated + if any(code in filled for _, code in rule.terms) + ) + return outcomes + + +def checks_from(outcomes: Sequence[RelationOutcome], column_index_of: Mapping[str, int | None]) -> StatementChecks: + hard: list[FailedRelation] = [] + soft: list[FailedRelation] = [] + for item in outcomes: + if not item.computed or item.within_tolerance: + continue + failure = FailedRelation( + column_index=column_index_of.get(item.column), + column_key=item.column, + relation=item.rule.relation, + codes=item.codes, + expected=item.expected, + actual=item.actual, + ) + (hard if item.hard_failed else soft).append(failure) + return StatementChecks(relations_failed=tuple(hard), soft_relations_failed=tuple(soft)) diff --git a/arm_statements/evaluation.py b/arm_statements/evaluation.py new file mode 100644 index 0000000..c75e4f7 --- /dev/null +++ b/arm_statements/evaluation.py @@ -0,0 +1,36 @@ +"""Общий вид строк шагов 6–8 у сервиса и ядра.""" + +from dataclasses import dataclass +from decimal import Decimal + +from arm_statements.model import LineStatus + + +@dataclass(frozen=True, slots=True) +class EvalValue: + column: str # ключ колонки: str(column_index) у сервиса, ключ хоста у ядра + amount: Decimal | None + text: str | None + confidence: float | None + is_recognized: bool + fixed: bool = False # правлено или подтверждено аналитиком + + +@dataclass(frozen=True, slots=True) +class EvalLine: + key: str # ref у сервиса, ключ хоста у ядра + order: int | None # row_index; у ядра None — строка правки + line_status: LineStatus + line_code: str | None + code_confidence: float | None + code_fixed: bool + values: tuple[EvalValue, ...] + + +def is_check_line(line: EvalLine) -> bool: + """Строка участвует в §7: `ok` с кодом; строка правки (`order is None`) — только с суммой (P1-8).""" + if line.line_status is not LineStatus.OK or line.line_code is None: + return False + if line.order is None: + return any(value.amount is not None for value in line.values) + return True diff --git a/arm_statements/tests/test_checks.py b/arm_statements/tests/test_checks.py new file mode 100644 index 0000000..d7ab03c --- /dev/null +++ b/arm_statements/tests/test_checks.py @@ -0,0 +1,107 @@ +from decimal import Decimal as D + +from arm_statements.checks import checks_from, evaluate_relations, form_variant, subline_parents +from arm_statements.evaluation import EvalLine, EvalValue, is_check_line +from arm_statements.loader import Rules +from arm_statements.model import DocumentType, FormVariant, LineStatus, StatementChecks + +BAL, INC = DocumentType.BALANCE, DocumentType.INCOME + + +def line(code: str | None, *amounts: str | None, status: LineStatus = LineStatus.OK, order: int = 0) -> EvalLine: + values = tuple(EvalValue(str(i), None if a is None else D(a), a, 100.0, True) for i, a in enumerate(amounts)) + return EvalLine(f"k{order}{code}", order, status, code, 100.0, False, values) + + +def failed(lines: list[EvalLine], doc: DocumentType, variant: FormVariant | None, rules: Rules) -> StatementChecks: + outcomes = evaluate_relations(lines, ["0"], doc, variant, rules) + return checks_from(outcomes, {"0": 0}) + + +def test_form_variant() -> None: + assert form_variant(BAL, ["1150", "1210", "1600", "1310", "1700"]) is FormVariant.SIMPLIFIED + assert form_variant(BAL, ["1100", "1600"]) is FormVariant.FULL + assert form_variant(INC, ["2110", "2400"]) is FormVariant.SIMPLIFIED + assert form_variant(INC, ["2110", "2100", "2400"]) is FormVariant.FULL + assert form_variant(DocumentType.DISCLOSURE, ["5101"]) is None + + +def test_empty_target_is_violation(rules: Rules) -> None: + lines = [ + line("1110", "30"), + line("1100", "30"), + line("1210", "70"), + line("1200", "70"), + line(None, "100", status=LineStatus.DETAIL), + ] + checks = failed(lines, BAL, FormVariant.FULL, rules) + assert [f.relation for f in checks.relations_failed] == ["1600 = 1100 + 1200"] + assert (checks.relations_failed[0].expected, checks.relations_failed[0].actual) == (None, D("100")) + + +def test_target_without_members_is_violation(rules: Rules) -> None: + lines = [line("1110", "30"), line("1100", "30"), line("1210", "70"), line("1200", "70"), line("1600", "100")] + checks = failed(lines, BAL, FormVariant.FULL, rules) + assert [f.relation for f in checks.relations_failed] == ["1600 = 1700"] + assert checks.relations_failed[0].codes == ("1600", "1700") + assert (checks.relations_failed[0].expected, checks.relations_failed[0].actual) == (D("100"), None) + + +def test_zero_target_with_empty_members_is_violation(rules: Rules) -> None: + lines = [line("1600", "0"), line("1110", "0"), line("1100", "0")] + hard = failed(lines, BAL, FormVariant.FULL, rules).relations_failed + assert [(f.relation, f.expected, f.actual) for f in hard] == [("1600 = 1700", D("0"), None)] + + +def test_duplicate_code_is_summed(rules: Rules) -> None: + lines = [line("2300", "849"), line("2410", "100"), line("2410", "649"), line("2400", "100")] + outcomes = evaluate_relations(lines, ["0"], INC, FormVariant.FULL, rules) + target = next(o for o in outcomes if o.rule.relation == "2400 = 2300 - 2410 + 2420") + assert target.within_tolerance + + +def test_simplified_income_soft(rules: Rules) -> None: + lines = [line("2110", "1000"), line("2120", "-600"), line("2410", "50"), line("2400", "500")] + checks = failed(lines, INC, FormVariant.SIMPLIFIED, rules) + assert checks.relations_failed == () + assert [f.relation for f in checks.soft_relations_failed] == ["2400 = 2110 - 2120 - 2330 + 2340 - 2350 - 2410"] + + +def test_rounding_goes_soft(rules: Rules) -> None: + lines = [ + line("1600", "17000000009.02"), + line("1700", "17000000000"), + line("1100", "17000000009.02"), + line("1300", "17000000000"), + ] + checks = failed(lines, BAL, FormVariant.FULL, rules) + assert "1600 = 1700" in [f.relation for f in checks.soft_relations_failed] + assert "1600 = 1700" not in [f.relation for f in checks.relations_failed] + + +def test_abs_terms_hold_with_minus(rules: Rules) -> None: + lines = [line("2110", "100"), line("2120", "-60"), line("2100", "40")] + hard = failed(lines, INC, FormVariant.FULL, rules).relations_failed + assert "2100 = 2110 - 2120" not in [f.relation for f in hard] + + +def test_sublines(rules: Rules) -> None: + assert subline_parents(["1230", "12301", "12302", "1240", "11101"]) == [None, "1230", "1230", None, "1110"] + # расхождение 2 (>1): в плане было 10 против 4 + 5 = 9, это в пределах допуска ±1 — спека «Шаг 7» (T13.1) + lines = [line("1230", "10"), line("12301", "4"), line("12302", "4")] + soft = failed(lines, BAL, FormVariant.FULL, rules).soft_relations_failed + assert [f.relation for f in soft] == ["1230 = 12301 + 12302"] + within = [line("1230", "10"), line("12301", "4"), line("12302", "5")] + assert failed(within, BAL, FormVariant.FULL, rules).soft_relations_failed == () + empty = [line("1230", "10"), line("12301", None), line("12302", None)] + assert failed(empty, BAL, FormVariant.FULL, rules).soft_relations_failed == () + + +def test_is_check_line() -> None: + assert is_check_line(line("1600", "1")) + assert not is_check_line(line("1600", "1", status=LineStatus.DETAIL)) + assert not is_check_line(line(None, "1")) + correction = EvalLine("c", None, LineStatus.OK, "1600", 100.0, False, line("1600", None).values) + assert not is_check_line(correction) + with_amount = EvalLine("c", None, LineStatus.OK, "1600", 100.0, False, line("1600", "5").values) + assert is_check_line(with_amount) -- 2.47.2 From e03040d16d8640e9aca2bce47d9133ebfcb4d7b2 Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Sun, 18 Oct 2026 13:53:05 +0400 Subject: [PATCH 16/28] =?UTF-8?q?feat:=20=D1=88=D0=B0=D0=B3=208=20?= =?UTF-8?q?=E2=80=94=20=D0=BF=D0=BE=D0=B4=D1=82=D0=B2=D0=B5=D1=80=D0=B6?= =?UTF-8?q?=D0=B4=D0=B5=D0=BD=D0=B8=D0=B5=20=D0=BD=D0=B5=D1=83=D0=B2=D0=B5?= =?UTF-8?q?=D1=80=D0=B5=D0=BD=D0=BD=D0=BE=D0=B3=D0=BE=20=D0=BA=D0=BE=D0=B4?= =?UTF-8?q?=D0=B0=20=D0=BF=D0=B5=D1=80=D0=B5=D1=87=D0=BD=D0=B5=D0=BC=20?= =?UTF-8?q?=D0=B1=D0=BB=D0=B0=D0=BD=D0=BA=D0=B0,=20=D1=8F=D0=BA=D0=BE?= =?UTF-8?q?=D1=80=D1=8F,=20=D0=B7=D0=B0=D0=BA=D0=BE=D0=BD=D0=BD=D1=8B?= =?UTF-8?q?=D0=B5=20=D0=BF=D0=BE=D0=B2=D1=82=D0=BE=D1=80=D1=8B,=20=D0=B2?= =?UTF-8?q?=D1=8B=D0=B1=D0=BE=D1=80=20=D1=80=D0=B5=D0=B4=D0=B0=D0=BA=D1=86?= =?UTF-8?q?=D0=B8=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W --- arm_statements/structure.py | 177 +++++++++++++++++++++++++ arm_statements/tests/test_structure.py | 106 +++++++++++++++ 2 files changed, 283 insertions(+) create mode 100644 arm_statements/structure.py create mode 100644 arm_statements/tests/test_structure.py diff --git a/arm_statements/structure.py b/arm_statements/structure.py new file mode 100644 index 0000000..c24ccb6 --- /dev/null +++ b/arm_statements/structure.py @@ -0,0 +1,177 @@ +"""Шаг 8 — подтверждение неуверенного кода структурой бланка и выбор редакции (спека «Шаг 8»).""" + +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import date + +from arm_statements.checks import form_variant, subline_parents +from arm_statements.evaluation import EvalLine, is_check_line +from arm_statements.loader import Rules +from arm_statements.model import DocumentType, Edition, FormVariant + +FSBU4_FROM = date(2025, 1, 1) # эффективная дата после 2024-12-31 → fsbu4 + +_CODE_LEN = 4 +_SUBLINE_LEN = 5 +_CONFIDENT = 100 + + +def edition_by_date(effective: date | None) -> Edition | None: + if effective is None: + return None + return Edition.FSBU4 if effective >= FSBU4_FROM else Edition.RAS_66N + + +@dataclass(frozen=True, slots=True) +class StructureResult: + edition: Edition | None + unconfirmed: frozenset[str] # ключи строк с неуверенным кодом, не подтверждённым структурой + + +# Элемент последовательности: (код, уверенный). +_Item = tuple[str, bool] + + +def _optimum(items: Sequence[_Item], order: Sequence[str], scale: int, skip: int | None = None) -> int: + """Взвешенная LCS `items` × `order` по равенству кода; вес позиции `scale * уверенный + 1`. + + `skip` — индекс позиции `items`, которой запрещено входить. Сложность O(|items|·|order|).""" + if not items or not order: + return 0 + previous = [0] * (len(order) + 1) + for index, (code, confident) in enumerate(items): + weight = None if index == skip else scale * int(confident) + 1 + current = [0] * (len(order) + 1) + for j, order_code in enumerate(order, start=1): + best = max(previous[j], current[j - 1]) + if weight is not None and code == order_code: + best = max(best, previous[j - 1] + weight) + current[j] = best + previous = current + return previous[-1] + + +def _editions(rules: Rules) -> tuple[Edition, ...]: + return rules.code_orders.editions or tuple(Edition) + + +def _pick(optima: dict[Edition, int], effective: date | None) -> tuple[Edition | None, tuple[Edition, ...]]: + """Редакция с большим оптимумом; равенство — по дате. Второе значение — редакции, чьи оптимумы лучшие.""" + top = max(optima.values()) + tied = tuple(edition for edition, value in optima.items() if value == top) + if len(tied) == 1: + return tied[0], tied + by_date = edition_by_date(effective) + return (by_date if by_date in tied else None), tied + + +def _filtered(items: Sequence[_Item], order: Sequence[str]) -> tuple[list[_Item], list[int]]: + allowed = set(order) + kept = [(item, index) for index, item in enumerate(items) if item[0] in allowed] + return [item for item, _ in kept], [index for _, index in kept] + + +def _select( + document_type: DocumentType, + items: Sequence[_Item], + variant: FormVariant, + effective: date | None, + rules: Rules, +) -> tuple[Edition | None, tuple[Edition, ...], dict[Edition, int]]: + scale = len(items) + 1 + optima = { + edition: _optimum( + _filtered(items, order := rules.code_orders.sequence(edition, document_type, variant))[0], + order, + scale, + ) + for edition in _editions(rules) + } + edition, tied = _pick(optima, effective) + return edition, tied, optima + + +def choose_edition( + document_type: DocumentType, + codes: Sequence[tuple[str, bool]], + effective: date | None, + rules: Rules, + *, + variant: FormVariant | None = None, +) -> Edition | None: + """Шаги 2–3 и 6 без подтверждения кодов; `codes` — пары `(код, уверенный)` строк `ok` по порядку.""" + if document_type is DocumentType.DISCLOSURE: + return edition_by_date(effective) + if variant is None: + variant = form_variant(document_type, [code for code, _ in codes]) or FormVariant.FULL + items = [(code, confident) for code, confident in codes if len(code) == _CODE_LEN] + return _select(document_type, items, variant, effective, rules)[0] + + +def _is_uncertain(line: EvalLine) -> bool: + return (line.code_confidence is None or line.code_confidence < _CONFIDENT) and not line.code_fixed + + +def _confirm_sublines(checked: Sequence[EvalLine], codes: Sequence[str], confirmed_keys: set[str]) -> set[str]: + """Пятизначные неуверенные коды: подтверждены родителем, стоящим раньше (§11.9); возвращает неподтверждённые.""" + unconfirmed: set[str] = set() + parents = subline_parents(codes) + for position, line in enumerate(checked): + code = line.line_code or "" + if len(code) == _CODE_LEN or not _is_uncertain(line): + continue + parent = parents[position] + found = None + if len(code) == _SUBLINE_LEN and parent is not None: + found = next((e for e in reversed(checked[:position]) if e.line_code == parent), None) + if found is not None and (not _is_uncertain(found) or found.key in confirmed_keys): + confirmed_keys.add(line.key) + else: + unconfirmed.add(line.key) + return unconfirmed + + +def confirm_codes( + lines: Sequence[EvalLine], + document_type: DocumentType, + variant: FormVariant | None, + effective: date | None, + rules: Rules, +) -> StructureResult: + if document_type is DocumentType.DISCLOSURE: + return StructureResult(edition_by_date(effective), frozenset()) + checked = sorted( + (line for line in lines if is_check_line(line) and line.order is not None), + key=lambda line: line.order or 0, + ) + codes = [line.line_code or "" for line in checked] + if variant is None: + variant = form_variant(document_type, codes) or FormVariant.FULL + four = [line for line in checked if len(line.line_code or "") == _CODE_LEN] + items: list[_Item] = [(line.line_code or "", not _is_uncertain(line)) for line in four] + edition, tied, _ = _select(document_type, items, variant, effective, rules) + + # Четырёхзначные неуверенные коды: входят во все лучшие подпоследовательности выбранной редакции + # (при равенстве без даты — во все лучшие обеих редакций, P1-4). + unconfirmed: set[str] = set() + confirmed_keys: set[str] = set() + scale = len(items) + 1 + for index, line in enumerate(four): + if not _is_uncertain(line): + confirmed_keys.add(line.key) + continue + confirmed = True + for candidate in tied if edition is None else (edition,): + order = rules.code_orders.sequence(candidate, document_type, variant) + if items[index][0] not in order: + confirmed = False + break + kept, indices = _filtered(items, order) + total = _optimum(kept, order, scale) + if _optimum(kept, order, scale, skip=indices.index(index)) >= total: + confirmed = False + break + (confirmed_keys if confirmed else unconfirmed).add(line.key) + + unconfirmed |= _confirm_sublines(checked, codes, confirmed_keys) + return StructureResult(edition, frozenset(unconfirmed)) diff --git a/arm_statements/tests/test_structure.py b/arm_statements/tests/test_structure.py new file mode 100644 index 0000000..3fe0f26 --- /dev/null +++ b/arm_statements/tests/test_structure.py @@ -0,0 +1,106 @@ +from datetime import date + +from arm_statements.evaluation import EvalLine +from arm_statements.loader import Rules +from arm_statements.model import DocumentType, Edition, FormVariant, LineStatus +from arm_statements.structure import choose_edition, confirm_codes + +BAL = DocumentType.BALANCE +Q3_2025 = date(2025, 9, 30) +FULL_CONF = 100.0 + + +def seq(*items: tuple[str, float]) -> list[EvalLine]: + return [EvalLine(f"r{i}", i, LineStatus.OK, code, conf, False, ()) for i, (code, conf) in enumerate(items)] + + +def unconfirmed(lines: list[EvalLine], rules: Rules, effective: date | None = Q3_2025) -> set[str]: + return set(confirm_codes(lines, BAL, FormVariant.FULL, effective, rules).unconfirmed) + + +def test_uncertain_code_in_place(rules: Rules) -> None: + assert unconfirmed(seq(("1220", 100.0), ("1230", 50.0), ("1240", 100.0)), rules) == set() + + +def test_swapped_uncertain_neighbours(rules: Rules) -> None: + lines = seq(("1210", 100.0), ("1230", 50.0), ("1220", 50.0), ("1240", 100.0)) + assert unconfirmed(lines, rules) == {"r1", "r2"} + + +def test_conflicting_anchors(rules: Rules) -> None: + lines = seq(("1210", 100.0), ("1240", 100.0), ("1230", 100.0), ("1250", 50.0)) + assert unconfirmed(lines, rules) == set() + + +def test_legal_repeat_1220_fsbu4(rules: Rules) -> None: + lines = seq(("1215", 100.0), ("1220", 100.0), ("1220", 50.0), ("1230", 100.0)) + assert unconfirmed(lines, rules) == set() + + +def test_repeat_outside_orders(rules: Rules) -> None: + lines = seq(("1220", 100.0), ("1230", 100.0), ("1230", 50.0), ("1240", 100.0)) + assert unconfirmed(lines, rules) == {"r2"} + + +def test_66n_blank_in_2025(rules: Rules) -> None: + lines = seq(("1110", 100.0), ("1120", 60.0), ("1130", 100.0)) + result = confirm_codes(lines, BAL, FormVariant.FULL, Q3_2025, rules) + assert (result.edition, result.unconfirmed) == (Edition.RAS_66N, frozenset()) + + +def test_tie_resolved_by_date(rules: Rules) -> None: + lines = seq(("1210", 100.0), ("1230", 100.0)) + assert confirm_codes(lines, BAL, FormVariant.FULL, Q3_2025, rules).edition is Edition.FSBU4 + assert confirm_codes(lines, BAL, FormVariant.FULL, date(2024, 12, 31), rules).edition is Edition.RAS_66N + + +def test_choose_edition_public(rules: Rules) -> None: + blank_66n = [("1110", True), ("1120", False), ("1130", True)] + assert choose_edition(BAL, blank_66n, Q3_2025, rules) is Edition.RAS_66N + tie = [("1210", True), ("1230", True)] + assert choose_edition(BAL, tie, Q3_2025, rules) is Edition.FSBU4 + assert choose_edition(BAL, tie, date(2024, 12, 31), rules) is Edition.RAS_66N + assert choose_edition(BAL, tie, None, rules) is None + assert choose_edition(BAL, [], Q3_2025, rules) is Edition.FSBU4 + assert choose_edition(DocumentType.DISCLOSURE, [("5101", True)], date(2024, 12, 31), rules) is Edition.RAS_66N + + +def test_choose_edition_equals_confirm_codes(rules: Rules) -> None: + lines = seq(("1110", 100.0), ("1120", 60.0), ("1130", 100.0)) + pairs = [(ln.line_code or "", (ln.code_confidence or 0.0) >= FULL_CONF) for ln in lines] + expected = confirm_codes(lines, BAL, FormVariant.FULL, Q3_2025, rules).edition + assert choose_edition(BAL, pairs, Q3_2025, rules, variant=FormVariant.FULL) is expected + + +def test_confident_nonstandard_code_not_flagged(rules: Rules) -> None: + assert unconfirmed(seq(("1230", 100.0), ("1231", 100.0), ("11101", 100.0)), rules) == set() + + +def test_subline_follows_confirmed_parent(rules: Rules) -> None: + lines = seq(("1220", 100.0), ("1230", 50.0), ("12301", 60.0), ("1240", 100.0)) + assert unconfirmed(lines, rules) == set() + orphan = seq(("1220", 100.0), ("12301", 60.0), ("1240", 100.0)) + assert unconfirmed(orphan, rules) == {"r1"} + + +def test_tie_without_date_union_of_editions(rules: Rules) -> None: + lines = seq(("1210", 100.0), ("1230", 50.0)) + result = confirm_codes(lines, BAL, FormVariant.FULL, None, rules) + assert result.edition is None + assert result.unconfirmed == frozenset() + + +def test_disclosure_has_no_unconfirmed(rules: Rules) -> None: + lines = seq(("5101", 50.0)) + result = confirm_codes(lines, DocumentType.DISCLOSURE, None, Q3_2025, rules) + assert (result.edition, result.unconfirmed) == (Edition.FSBU4, frozenset()) + + +def test_uncertain_code_outside_order_unconfirmed(rules: Rules) -> None: + assert unconfirmed(seq(("1230", 100.0), ("9999", 50.0), ("1240", 100.0)), rules) == {"r1"} + + +def test_performance_realistic_size(rules: Rules) -> None: + order = rules.code_orders.sequence(Edition.FSBU4, BAL, FormVariant.FULL) + lines = seq(*[(code, 50.0) for code in order] * 1) + assert confirm_codes(lines, BAL, FormVariant.FULL, Q3_2025, rules).unconfirmed == frozenset() -- 2.47.2 From 6067893db83f279a1931870a184621baae1fe71b Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Sun, 18 Oct 2026 13:56:08 +0400 Subject: [PATCH 17/28] =?UTF-8?q?feat:=20=D1=88=D0=B0=D0=B3=208=20?= =?UTF-8?q?=E2=80=94=20needs=5Freview=20=D0=BF=D0=BE=20=D0=BF=D0=BE=D0=BA?= =?UTF-8?q?=D1=80=D1=8B=D1=82=D0=B8=D1=8E=20=C2=A77=20=D0=B8=20=D0=BA?= =?UTF-8?q?=D0=BE=D0=BD=D0=B2=D0=B5=D0=BD=D1=86=D0=B8=D0=B8=20=D0=B7=D0=BD?= =?UTF-8?q?=D0=B0=D0=BA=D0=B0,=20=D0=BF=D1=80=D0=BE=D0=B2=D0=B5=D1=80?= =?UTF-8?q?=D0=BA=D0=B8=20=D0=BF=D0=BE=D0=BB=D0=B5=D0=B9=20=D1=88=D0=B0?= =?UTF-8?q?=D0=BF=D0=BA=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W --- arm_statements/review.py | 171 ++++++++++++++++++++++++++++ arm_statements/tests/test_review.py | 139 ++++++++++++++++++++++ 2 files changed, 310 insertions(+) create mode 100644 arm_statements/review.py create mode 100644 arm_statements/tests/test_review.py diff --git a/arm_statements/review.py b/arm_statements/review.py new file mode 100644 index 0000000..3b3ad7d --- /dev/null +++ b/arm_statements/review.py @@ -0,0 +1,171 @@ +"""Шаг 8: `needs_review` значений и строк, проверки полей шапки, сводная `evaluate`.""" + +from collections import Counter +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, replace +from datetime import date + +from arm_statements.amounts import sign_form_of +from arm_statements.checks import RelationOutcome, evaluate_relations, form_variant +from arm_statements.columns import DocumentDates +from arm_statements.evaluation import EvalLine, EvalValue, is_check_line +from arm_statements.issues import WarningCode, make_warning +from arm_statements.loader import Rules +from arm_statements.model import ( + DocumentType, + Edition, + FormVariant, + HeaderField, + LineStatus, + SignForm, + StatementHeader, + StatementWarning, +) +from arm_statements.structure import StructureResult, confirm_codes + +_CONFIDENT = 100 + + +@dataclass(frozen=True, slots=True) +class Evaluation: + variant: FormVariant | None + edition: Edition | None + outcomes: tuple[RelationOutcome, ...] + line_needs_review: Mapping[str, bool] + value_needs_review: Mapping[tuple[str, str], bool] + hard_failures: int + + +def _needs_coverage(value: EvalValue) -> bool: + return (value.confidence is not None and value.confidence < _CONFIDENT) or not value.is_recognized + + +def _is_covered(line: EvalLine, value: EvalValue, outcomes: Sequence[RelationOutcome]) -> bool: + if value.amount is None or line.line_code is None: + return False + for item in outcomes: + if item.column != value.column or line.line_code not in item.codes or not item.computed: + continue + if item.within_tolerance or (item.rule.strict and item.within_rounding): + return True + return False + + +def _sign_confirmed( + line: EvalLine, value: EvalValue, lines: Sequence[EvalLine], abs_codes: frozenset[str], rules: Rules +) -> bool: + """Форма знака ячейки совпадает с формой строгого большинства прочих членов `abs_terms` колонки.""" + forms: Counter[SignForm] = Counter() + for other in lines: + if other is line or not is_check_line(other) or other.line_code not in abs_codes: + continue + for item in other.values: + if item.column != value.column or item.amount is None or not item.is_recognized or item.fixed: + continue + form = sign_form_of(item.text, rules) + if form is not None: + forms[form] += 1 + total = sum(forms.values()) + own = sign_form_of(value.text, rules) + return own is not None and forms[own] * 2 > total + + +def value_needs_review( + lines: Sequence[EvalLine], + outcomes: Sequence[RelationOutcome], + abs_codes: frozenset[str], + rules: Rules, + *, + accepted: bool = False, +) -> dict[tuple[str, str], bool]: + hard_codes: set[tuple[str, str]] = set() + if not accepted: + for item in outcomes: + if item.hard_failed: + hard_codes |= {(item.column, code) for code in item.codes} + result: dict[tuple[str, str], bool] = {} + for line in lines: + for value in line.values: + key = (line.key, value.column) + if line.line_status is LineStatus.DETAIL or value.fixed: + result[key] = False + elif is_check_line(line) and (value.column, line.line_code or "") in hard_codes: + result[key] = True + elif not _needs_coverage(value): + result[key] = False + else: + covered = _is_covered(line, value, outcomes) + signed = line.line_code not in abs_codes or _sign_confirmed(line, value, lines, abs_codes, rules) + result[key] = not (covered and signed) + return result + + +def line_needs_review(lines: Sequence[EvalLine], structure: StructureResult) -> dict[str, bool]: + return { + line.key: line.key in structure.unconfirmed and not line.code_fixed and line.line_status is LineStatus.OK + for line in lines + } + + +def evaluate( # noqa: PLR0913 -- сигнатура задана публичным API пакета + lines: Sequence[EvalLine], + columns: Sequence[str], + document_type: DocumentType, + effective: date | None, + rules: Rules, + *, + accepted: bool = False, +) -> Evaluation: + codes = [item.line_code for item in lines if is_check_line(item) and item.line_code is not None] + variant = form_variant(document_type, codes) + outcomes = evaluate_relations(lines, columns, document_type, variant, rules) + structure = confirm_codes(lines, document_type, variant, effective, rules) + abs_codes = rules.checks.abs_codes(document_type, variant or FormVariant.FULL) + return Evaluation( + variant=variant, + edition=structure.edition, + outcomes=tuple(outcomes), + line_needs_review=line_needs_review(lines, structure), + value_needs_review=value_needs_review(lines, outcomes, abs_codes, rules, accepted=accepted), + hard_failures=sum(1 for item in outcomes if item.hard_failed), + ) + + +def _uncertain(field: HeaderField) -> bool: + return field.confidence is not None and field.confidence < _CONFIDENT + + +def review_header( + header: StatementHeader, dates: DocumentDates, rules: Rules +) -> tuple[StatementHeader, list[StatementWarning]]: + """Поля шапки (спека «Шаг 8 — поля шапки»). Для `DISCLOSURE` не вызывается: предупреждений шапки там нет (P1-6).""" + warnings: list[StatementWarning] = [] + review_fields: list[str] = [] + org, inn, report = header.organization_name, header.inn, header.report_date + + def warn(code: WarningCode, message: str, field: str | None = None) -> None: + warnings.append(make_warning(code, message, rules, field=field)) + + if org.value is None and inn.value is None: + warn(WarningCode.NO_ORGANIZATION_NO_INN, "В шапке нет ни организации, ни ИНН") + elif inn.value is None: + warn(WarningCode.INN_NOT_FOUND, "ИНН не найден", "inn") + elif org.value is None: + warn(WarningCode.ORGANIZATION_NOT_FOUND, "Организация не найдена", "organization_name") + if org.value is not None and _uncertain(org): + warn(WarningCode.ORGANIZATION_UNCERTAIN, "Название организации распознано неуверенно", "organization_name") + if inn.value is not None and _uncertain(inn): + warn(WarningCode.INN_UNCERTAIN, "ИНН распознан неуверенно", "inn") + + if report.value is not None: + confidence = report.confidence + if confidence is not None and confidence < _CONFIDENT and report.value != dates.confident_latest_absolute: + review_fields.append("report_date") + warn(WarningCode.REPORT_DATE_UNCERTAIN, "Дата отчёта неуверенна и не подтверждена колонками", "report_date") + elif report.text is not None and report.text.strip() and dates.latest_absolute is not None: + warn(WarningCode.REPORT_DATE_UNPARSED, "Дата отчёта не разобрана", "report_date") + if report.value is None and dates.latest_absolute is None: + warn(WarningCode.PERIOD_UNKNOWN, "Отчётный период не определён: нет ни даты отчёта, ни дат колонок") + if header.units.value is None: + warn(WarningCode.UNITS_NOT_FOUND, "Единицы измерения не найдены", "units") + return replace(header, needs_review_fields=tuple(review_fields)), warnings diff --git a/arm_statements/tests/test_review.py b/arm_statements/tests/test_review.py new file mode 100644 index 0000000..96de895 --- /dev/null +++ b/arm_statements/tests/test_review.py @@ -0,0 +1,139 @@ +from datetime import date +from decimal import Decimal as D + +from arm_statements.columns import DocumentDates +from arm_statements.evaluation import EvalLine, EvalValue +from arm_statements.header import parse_header +from arm_statements.loader import Rules +from arm_statements.model import DocumentType, LineStatus +from arm_statements.review import evaluate, review_header +from arm_statements.tests.builders import hdr + +OK, DETAIL, UNREADABLE = LineStatus.OK, LineStatus.DETAIL, LineStatus.UNREADABLE +NO_DATES = DocumentDates(None, None, None, None) + + +def ln( # noqa: PLR0913, PLR0917 -- сигнатура задана планом (Task 15) + code: str | None, + text: str, + amount: str | None, + conf: float = 100.0, + recognized: bool = True, + status: LineStatus = OK, + order: int = 0, +) -> EvalLine: + value = EvalValue("0", None if amount is None else D(amount), text, conf, recognized) + return EvalLine(f"{code}-{order}", order, status, code, 100.0, False, (value,)) + + +def flags(lines: list[EvalLine], doc: DocumentType, rules: Rules) -> dict[str, bool]: + result = evaluate(lines, ["0"], doc, date(2025, 9, 30), rules) + return {key: flag for (key, _), flag in result.value_needs_review.items()} + + +def test_guess_confirmed_by_relation(rules: Rules) -> None: + lines = [ + ln("1210", "100", "100", order=0), + ln("1230", "42831 1", "428311", recognized=False, order=1), + ln("1200", "428411", "428411", order=2), + ] + assert flags(lines, DocumentType.BALANCE, rules)["1230-1"] is False + + +def test_sign_in_curly_bracket_confirmed(rules: Rules) -> None: + lines = [ + ln("2110", "10000000", "10000000", order=0), + ln("2120", "(7744536}", "-7744536", recognized=False, order=1), + ln("2100", "2255464", "2255464", order=2), + ln("2210", "(100)", "-100", order=3), + ln("2220", "(200)", "-200", order=4), + ln("2200", "2255164", "2255164", order=5), + ] + assert flags(lines, DocumentType.INCOME, rules)["2120-1"] is False + + +def test_sign_outside_convention(rules: Rules) -> None: + lines = [ + ln("2110", "100000", "100000", order=0), + ln("2120", "38378", "38378", conf=60.0, order=1), + ln("2100", "61622", "61622", order=2), + ln("2210", "(100)", "-100", order=3), + ln("2220", "(200)", "-200", order=4), + ] + assert flags(lines, DocumentType.INCOME, rules)["2120-1"] is True + + +def test_null_amount_and_unchecked_code(rules: Rules) -> None: + lines = [ln("2120", "{38378)", None, recognized=False, order=0), ln("2900", "15", "15", conf=80.0, order=1)] + result = flags(lines, DocumentType.INCOME, rules) + assert (result["2120-0"], result["2900-1"]) == (True, True) + + +def test_detail_values_never_flagged(rules: Rules) -> None: + lines = [ln("2110", "100", "100", order=0), ln(None, "1 425 130", "1425130", conf=78.0, status=DETAIL, order=1)] + assert flags(lines, DocumentType.INCOME, rules)["None-1"] is False + + +def test_soft_relation_covers_only_exactly(rules: Rules) -> None: + exact = [ + ln("2300", "1000", "1000", order=0), + ln("2410", "200", "200", order=1), + ln("2400", "800", "800", conf=66.0, order=2), + ] + assert flags(exact, DocumentType.INCOME, rules)["2400-2"] is False + rounding = [ + ln("2300", "17000000000", "17000000000", order=0), + ln("2410", "0", "0", order=1), + ln("2400", "17000000009", "17000000009", conf=66.0, order=2), + ] + assert flags(rounding, DocumentType.INCOME, rules)["2400-2"] is True + + +def test_hard_failure_flags_clean_cells(rules: Rules) -> None: + lines = [ln("1210", "100", "100", order=0), ln("1230", "50", "50", order=1), ln("1200", "999", "999", order=2)] + result = evaluate(lines, ["0"], DocumentType.BALANCE, None, rules) + assert result.hard_failures >= 1 + assert all(result.value_needs_review[(f"{c}-{i}", "0")] for i, c in enumerate(["1210", "1230", "1200"])) + accepted = evaluate(lines, ["0"], DocumentType.BALANCE, None, rules, accepted=True) + assert not any(accepted.value_needs_review.values()) + + +def test_unreadable_values(rules: Rules) -> None: + lines = [ln(None, "7", "7", conf=60.0, status=UNREADABLE), ln(None, "5", "5", status=UNREADABLE, order=1)] + result = flags(lines, DocumentType.BALANCE, rules) + assert (result["None-0"], result["None-1"]) == (True, False) + + +def test_report_date_confirmed_or_not(rules: Rules) -> None: + head = parse_header( + hdr(report_date=("30.09.2025", 60.0), organization_name="ООО «Пример»", inn="7700000000", units="тыс. руб."), + rules, + ) + ok_dates = DocumentDates(date(2025, 9, 30), date(2025, 9, 30), date(2025, 9, 30), date(2025, 9, 30)) + reviewed, warnings = review_header(head, ok_dates, rules) + assert (reviewed.needs_review_fields, warnings) == ((), []) + relative_only = DocumentDates(date(2025, 9, 30), None, None, None) + reviewed, warnings = review_header(head, relative_only, rules) + assert reviewed.needs_review_fields == ("report_date",) + found = [(w.code, w.severity.value, w.field) for w in warnings] + assert found == [("REPORT_DATE_UNCERTAIN", "error", "report_date")] + + +def test_header_presence_warnings(rules: Rules) -> None: + cells = hdr(organization_name="ООО «Пример»", report_date="30.09.2025", units="тыс. руб.") + _, no_inn = review_header(parse_header(cells, rules), NO_DATES, rules) + assert [w.code for w in no_inn] == ["INN_NOT_FOUND"] + _, nothing = review_header(parse_header(hdr(), rules), NO_DATES, rules) + assert {w.code for w in nothing} == {"NO_ORGANIZATION_NO_INN", "PERIOD_UNKNOWN", "UNITS_NOT_FOUND"} + + +def test_uncertain_inn_and_unparsed_date(rules: Rules) -> None: + cells = hdr(organization_name=("ООО «Пример»", 70.0), inn=("7700000000", 80.0), report_date="дата?", units="руб.") + dates = DocumentDates(None, None, date(2025, 9, 30), None) + reviewed, warnings = review_header(parse_header(cells, rules), dates, rules) + assert reviewed.needs_review_fields == () + assert [(w.code, w.field) for w in warnings] == [ + ("ORGANIZATION_UNCERTAIN", "organization_name"), + ("INN_UNCERTAIN", "inn"), + ("REPORT_DATE_UNPARSED", "report_date"), + ] -- 2.47.2 From a5d10a6b0e98825d872984b76889f4932b7fd8ee Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Sun, 18 Oct 2026 13:58:27 +0400 Subject: [PATCH 18/28] =?UTF-8?q?feat:=20=D0=BE=D0=B1=D1=89=D0=B0=D1=8F=20?= =?UTF-8?q?=D1=84=D1=83=D0=BD=D0=BA=D1=86=D0=B8=D1=8F=20=D1=81=D1=82=D0=B0?= =?UTF-8?q?=D1=82=D1=83=D1=81=D0=B0=20=D0=B4=D0=BE=D0=BA=D1=83=D0=BC=D0=B5?= =?UTF-8?q?=D0=BD=D1=82=D0=B0=20(detail=20=D0=BD=D0=B5=20=D0=B4=D0=B5?= =?UTF-8?q?=D1=80=D0=B6=D0=B8=D1=82)=20=D0=B8=20=D1=81=D1=82=D0=B0=D1=82?= =?UTF-8?q?=D1=83=D1=81=D0=B0=20=D0=BA=D0=BE=D0=BD=D0=B2=D0=B5=D1=80=D1=82?= =?UTF-8?q?=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W --- arm_statements/status.py | 97 +++++++++++++++++++++++ arm_statements/tests/test_status.py | 117 ++++++++++++++++++++++++++++ 2 files changed, 214 insertions(+) create mode 100644 arm_statements/status.py create mode 100644 arm_statements/tests/test_status.py diff --git a/arm_statements/status.py b/arm_statements/status.py new file mode 100644 index 0000000..02be8f8 --- /dev/null +++ b/arm_statements/status.py @@ -0,0 +1,97 @@ +"""Статус документа и конверта (спека «Шаг 8 — статус документа и конверта»). + +Одна функция на сервис и ядро: вход не зависит от источника строк. +""" + +from collections.abc import Sequence +from dataclasses import dataclass + +from arm_statements.model import ( + DocStatus, + DocumentType, + EnvelopeStatus, + LineStatus, + PeriodKind, + Severity, + StatementDocument, + StatementWarning, +) + +_ACCEPTABLE_LINES = frozenset({LineStatus.OK, LineStatus.DETAIL}) + + +@dataclass(frozen=True, slots=True) +class StatusLine: + line_status: LineStatus + needs_review: bool + + +@dataclass(frozen=True, slots=True) +class StatusValue: + line_status: LineStatus + needs_review: bool + + +@dataclass(frozen=True, slots=True) +class StatusInput: + document_type: DocumentType + layout_found: bool + has_organization_or_inn: bool + period_known: bool + needs_review_fields: tuple[str, ...] + lines: tuple[StatusLine, ...] + values: tuple[StatusValue, ...] + hard_failures: int + accepted: bool + warning_severities: tuple[Severity, ...] + + +def document_status(data: StatusInput) -> DocStatus: + """`DISCLOSURE` — всегда `done`; иначе `done` только при выполнении всех условий спеки.""" + if data.document_type is DocumentType.DISCLOSURE: + return DocStatus.DONE + done = ( + data.layout_found + and data.has_organization_or_inn + and data.period_known + and all(line.line_status in _ACCEPTABLE_LINES for line in data.lines) + and not any(line.needs_review for line in data.lines) + # значения строк detail в статус не входят (Р-23) + and not any(v.needs_review for v in data.values if v.line_status is not LineStatus.DETAIL) + and not data.needs_review_fields + and (data.hard_failures == 0 or data.accepted) + and Severity.ERROR not in data.warning_severities + ) + return DocStatus.DONE if done else DocStatus.NEEDS_CHECK + + +def envelope_status( + document_statuses: Sequence[DocStatus], file_warnings: Sequence[StatementWarning] +) -> EnvelopeStatus: + if not document_statuses: + return EnvelopeStatus.ERROR + if all(s is DocStatus.DONE for s in document_statuses) and all( + w.severity is not Severity.ERROR for w in file_warnings + ): + return EnvelopeStatus.DONE + return EnvelopeStatus.NEEDS_CHECK + + +def status_input(document: StatementDocument) -> StatusInput: + """Вход функции статуса из документа шагов 2–8.""" + header = document.header + period_known = header.report_date.value is not None or any( + c.period_kind is PeriodKind.ABSOLUTE and c.period_date is not None for c in document.columns + ) + return StatusInput( + document_type=document.document_type, + layout_found=document.lines != (), + has_organization_or_inn=header.organization_name.value is not None or header.inn.value is not None, + period_known=period_known, + needs_review_fields=header.needs_review_fields, + lines=tuple(StatusLine(line.line_status, line.needs_review) for line in document.lines), + values=tuple(StatusValue(line.line_status, v.needs_review) for line in document.lines for v in line.values), + hard_failures=0 if document.checks is None else len(document.checks.relations_failed), + accepted=False, + warning_severities=tuple(w.severity for w in document.warnings), + ) diff --git a/arm_statements/tests/test_status.py b/arm_statements/tests/test_status.py new file mode 100644 index 0000000..ec5c9b2 --- /dev/null +++ b/arm_statements/tests/test_status.py @@ -0,0 +1,117 @@ +from dataclasses import replace +from datetime import date +from decimal import Decimal as D + +from arm_statements.header import parse_header +from arm_statements.issues import make_warning +from arm_statements.loader import Rules +from arm_statements.model import ( + DocStatus, + DocumentType, + EnvelopeStatus, + FormVariant, + LineStatus, + PeriodKind, + Severity, + StatementChecks, + StatementColumn, + StatementDocument, + StatementLine, + StatementValue, +) +from arm_statements.status import ( + StatusInput, + StatusLine, + StatusValue, + document_status, + envelope_status, + status_input, +) +from arm_statements.tests.builders import hdr + +OK, DETAIL, UNREADABLE = LineStatus.OK, LineStatus.DETAIL, LineStatus.UNREADABLE +DONE, CHECK = DocStatus.DONE, DocStatus.NEEDS_CHECK +BASE = StatusInput( + document_type=DocumentType.BALANCE, + layout_found=True, + has_organization_or_inn=True, + period_known=True, + needs_review_fields=(), + lines=(StatusLine(OK, False),), + values=(StatusValue(OK, False),), + hard_failures=0, + accepted=False, + warning_severities=(Severity.INFO,), +) + + +def test_clean_form_done() -> None: + assert document_status(BASE) is DONE + + +def test_unreadable_line_blocks() -> None: + assert document_status(replace(BASE, lines=(StatusLine(OK, False), StatusLine(UNREADABLE, False)))) is CHECK + + +def test_detail_value_does_not_block() -> None: + lines = (StatusLine(OK, False), StatusLine(DETAIL, False)) + values = (StatusValue(OK, False), StatusValue(DETAIL, True)) + assert document_status(replace(BASE, lines=lines, values=values)) is DONE + + +def test_blocking_conditions() -> None: + hard = 2 + assert document_status(replace(BASE, values=(StatusValue(OK, True),))) is CHECK + assert document_status(replace(BASE, lines=(StatusLine(OK, True),))) is CHECK + assert document_status(replace(BASE, needs_review_fields=("report_date",))) is CHECK + assert document_status(replace(BASE, hard_failures=hard)) is CHECK + assert document_status(replace(BASE, hard_failures=hard, accepted=True)) is DONE + assert document_status(replace(BASE, warning_severities=(Severity.ERROR,))) is CHECK + assert document_status(replace(BASE, layout_found=False)) is CHECK + assert document_status(replace(BASE, has_organization_or_inn=False)) is CHECK + assert document_status(replace(BASE, period_known=False)) is CHECK + + +def test_disclosure_always_done() -> None: + disclosure = replace(BASE, document_type=DocumentType.DISCLOSURE, layout_found=False, lines=(), values=()) + assert document_status(disclosure) is DONE + + +def test_envelope(rules: Rules) -> None: + assert envelope_status([DONE, CHECK], []) is EnvelopeStatus.NEEDS_CHECK + assert envelope_status([DONE], []) is EnvelopeStatus.DONE + lost = make_warning("TABLE_LOST", "таблица без колонки кодов", rules, locator="file:a.json;table:1") + assert envelope_status([DONE], [lost]) is EnvelopeStatus.NEEDS_CHECK + assert envelope_status([], []) is EnvelopeStatus.ERROR + + +def test_status_input_from_document(rules: Rules) -> None: + head = parse_header(hdr(organization_name="ООО «Пример»", units="тыс. руб."), rules) + column = StatementColumn(2, "На 30.09.2025", 100.0, date(2025, 9, 30), PeriodKind.ABSOLUTE) + value = StatementValue(2, D("5"), None, "5", 60.0, True, needs_review=True) + detail = StatementLine(1, "row:1", DETAIL, None, None, None, "в т. ч.", 100.0, "row:0", False, (value,)) + line = StatementLine( + 0, "row:0", OK, "1210", "1210", 100.0, "Запасы", 100.0, None, False, (replace(value, needs_review=False),) + ) + doc = StatementDocument( + DocumentType.BALANCE, + FormVariant.FULL, + "file:t.json;table:0", + head, + (column,), + (line, detail), + StatementChecks((), ()), + DocStatus.DONE, + (), + ) + data = status_input(doc) + assert (data.layout_found, data.has_organization_or_inn, data.period_known, data.hard_failures) == ( + True, + True, + True, + 0, + ) + assert data.values == (StatusValue(OK, False), StatusValue(DETAIL, True)) + assert document_status(data) is DONE + empty = replace(doc, columns=(), lines=(), checks=None) + assert (status_input(empty).layout_found, status_input(empty).period_known) == (False, False) -- 2.47.2 From 29198ddc447e60938405a9d9484a43da46f48dd7 Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Sun, 18 Oct 2026 14:01:56 +0400 Subject: [PATCH 19/28] =?UTF-8?q?feat:=20=D1=88=D0=B0=D0=B3=206=20=D0=B8?= =?UTF-8?q?=20=D0=BA=D0=BE=D0=BD=D0=B2=D0=B5=D0=B9=D0=B5=D1=80=20=D0=BA?= =?UTF-8?q?=D0=B0=D0=BD=D0=B4=D0=B8=D0=B4=D0=B0=D1=82=D0=B0=20=E2=80=94=20?= =?UTF-8?q?=D1=82=D0=B8=D0=BF=20=D0=BF=D0=BE=20=D0=BA=D0=BE=D0=B4=D0=B0?= =?UTF-8?q?=D0=BC,=20=D0=B2=D0=B0=D1=80=D0=B8=D0=B0=D0=BD=D1=82=20=D1=84?= =?UTF-8?q?=D0=BE=D1=80=D0=BC=D1=8B,=20=D0=BF=D0=BE=D1=8F=D1=81=D0=BD?= =?UTF-8?q?=D0=B5=D0=BD=D0=B8=D1=8F,=20=D0=BF=D1=80=D0=B5=D0=B4=D1=83?= =?UTF-8?q?=D0=BF=D1=80=D0=B5=D0=B6=D0=B4=D0=B5=D0=BD=D0=B8=D1=8F=20=D1=84?= =?UTF-8?q?=D0=B0=D0=B9=D0=BB=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit normalize_candidate / normalize; __init__ экспортирует уже реализованный публичный API. DISCLOSURE: шапка не проверяется, предупреждений шапки нет, статус done (P1-6). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W --- arm_statements/__init__.py | 118 +++++++++++++- arm_statements/pipeline.py | 215 ++++++++++++++++++++++++++ arm_statements/tests/test_pipeline.py | 196 +++++++++++++++++++++++ 3 files changed, 527 insertions(+), 2 deletions(-) create mode 100644 arm_statements/pipeline.py create mode 100644 arm_statements/tests/test_pipeline.py diff --git a/arm_statements/__init__.py b/arm_statements/__init__.py index 6339e26..5b2bcf4 100644 --- a/arm_statements/__init__.py +++ b/arm_statements/__init__.py @@ -1,5 +1,119 @@ -"""Общий код нормализации форм ББ/ОФР.""" +"""Общий код нормализации форм ББ/ОФР. +Публичный API — только имена `__all__` (раздел «Публичный API пакета» плана 1). Имена задач 18–19 +(`recompute`, `Stored*`, `to_contract`, `warning_to_contract`, `status_input_from_contract`) добавляются ими. +""" + +from arm_statements.amounts import parse_amount +from arm_statements.codes import parse_code, sheet_joins +from arm_statements.columns import amount_columns +from arm_statements.dates import parse_date, resolve_relative +from arm_statements.header import continuation_type, is_continuation, parse_header, type_hint +from arm_statements.issues import WarningCode, make_warning +from arm_statements.layout import code_column +from arm_statements.loader import Rules, default_rules, load_rules +from arm_statements.model import ( + HEADER_FIELDS, + CandidateResult, + Cell, + CellValue, + DateField, + DocStatus, + DocumentType, + Edition, + EnvelopeStatus, + Evidence, + FailedRelation, + FormCandidate, + FormVariant, + HeaderCells, + HeaderField, + LineStatus, + NormalizationResult, + PeriodKind, + Severity, + SignForm, + Source, + StatementChecks, + StatementColumn, + StatementDocument, + StatementHeader, + StatementLine, + StatementValue, + StatementWarning, +) +from arm_statements.pipeline import normalize, normalize_candidate +from arm_statements.serialize import candidate_from_dict, to_jsonable +from arm_statements.status import ( + StatusInput, + StatusLine, + StatusValue, + document_status, + envelope_status, + status_input, +) +from arm_statements.structure import choose_edition +from arm_statements.text import fix_lookalikes, normalize_caption, normalize_label from arm_statements.version import statements_version -__all__ = ["statements_version"] +__all__ = [ + "HEADER_FIELDS", + "CandidateResult", + "Cell", + "CellValue", + "DateField", + "DocStatus", + "DocumentType", + "Edition", + "EnvelopeStatus", + "Evidence", + "FailedRelation", + "FormCandidate", + "FormVariant", + "HeaderCells", + "HeaderField", + "LineStatus", + "NormalizationResult", + "PeriodKind", + "Rules", + "Severity", + "SignForm", + "Source", + "StatementChecks", + "StatementColumn", + "StatementDocument", + "StatementHeader", + "StatementLine", + "StatementValue", + "StatementWarning", + "StatusInput", + "StatusLine", + "StatusValue", + "WarningCode", + "amount_columns", + "candidate_from_dict", + "choose_edition", + "code_column", + "continuation_type", + "default_rules", + "document_status", + "envelope_status", + "fix_lookalikes", + "is_continuation", + "load_rules", + "make_warning", + "normalize", + "normalize_candidate", + "normalize_caption", + "normalize_label", + "parse_amount", + "parse_code", + "parse_date", + "parse_header", + "resolve_relative", + "sheet_joins", + "statements_version", + "status_input", + "to_jsonable", + "type_hint", +] diff --git a/arm_statements/pipeline.py b/arm_statements/pipeline.py new file mode 100644 index 0000000..aadcd0a --- /dev/null +++ b/arm_statements/pipeline.py @@ -0,0 +1,215 @@ +"""Шаг 6 и конвейер кандидата: шаги 2–8 одного кандидата и нормализация набора кандидатов.""" + +from collections.abc import Sequence +from dataclasses import replace + +from arm_statements.checks import checks_from +from arm_statements.codes import document_type_of, majority_section +from arm_statements.columns import DocumentDates +from arm_statements.evaluation import EvalLine, EvalValue +from arm_statements.header import parse_header +from arm_statements.issues import WarningCode, make_warning +from arm_statements.lines import build_lines, units_warnings +from arm_statements.loader import Rules, default_rules +from arm_statements.model import ( + CandidateResult, + DocStatus, + DocumentType, + FormCandidate, + FormVariant, + LineStatus, + NormalizationResult, + Source, + StatementDocument, + StatementHeader, + StatementLine, + StatementWarning, +) +from arm_statements.review import evaluate, review_header +from arm_statements.rows import RawLine, collect_raw_lines +from arm_statements.segments import Layout, find_layout +from arm_statements.status import document_status, status_input +from arm_statements.text import has_digit, is_disguised_digit + +_SHEET_PREFIX = "sheet:" + + +def _eval_lines(lines: Sequence[StatementLine]) -> list[EvalLine]: + """Строки сервиса в общий вид шагов 6–8: ключ — `ref`, колонка — `str(column_index)`.""" + return [ + EvalLine( + key=line.ref, + order=line.row_index, + line_status=line.line_status, + line_code=line.line_code, + code_confidence=line.line_code_confidence, + code_fixed=False, + values=tuple( + EvalValue(str(v.column_index), v.amount, v.text, v.confidence, v.is_recognized, fixed=False) + for v in line.values + ), + ) + for line in lines + ] + + +def _form_name(candidate: FormCandidate) -> str | None: + title = candidate.evidence.title + if title is None or not title.text.strip(): + return None + return title.text + + +def _has_content(candidate: FormCandidate, rules: Rules) -> bool: + """Содержимое таблицы: ячейка с цифрой или «цифрой под видом буквы».""" + return any( + cell is not None and (has_digit(cell.text) or is_disguised_digit(cell.text, rules.lookalikes)) + for row in candidate.rows + for cell in row + ) + + +def _lost_table(candidate: FormCandidate, rules: Rules) -> StatementWarning: + """Документа нет (нет кодов и `type_hint`): предупреждение файла по источнику и содержимому.""" + if candidate.source is Source.EXCEL: + sheet = candidate.locator.removeprefix(_SHEET_PREFIX).split(",")[0] + message = f"лист «{sheet}» не содержит формы ББ/ОФР" + return make_warning(WarningCode.SHEET_OTHER, message, rules, locator=f"{_SHEET_PREFIX}{sheet}") + if _has_content(candidate, rules): + message = "таблица с данными не распознана как форма: нет столбца кодов и типа формы" + return make_warning(WarningCode.TABLE_LOST, message, rules, locator=candidate.locator) + return make_warning(WarningCode.TABLE_SKIPPED, "таблица без данных пропущена", rules, locator=candidate.locator) + + +def _without_lines( # noqa: PLR0913 -- сборка документа без строк: поля заданы схемой + candidate: FormCandidate, + document_type: DocumentType, + header: StatementHeader, + warnings: Sequence[StatementWarning], + *, + dates: DocumentDates | None, + rules: Rules, +) -> StatementDocument: + """Документ без строк: `DISCLOSURE` (`dates=None` — шапку не проверяем, P1-6) или без столбца кодов (P1-19).""" + collected = list(warnings) + if dates is not None: + header, header_warnings = review_header(header, dates, rules) + collected.extend(header_warnings) + document = StatementDocument( + document_type=document_type, + form_variant=None if document_type is DocumentType.DISCLOSURE else FormVariant.FULL, + locator=candidate.locator, + header=header, + columns=(), + lines=(), + checks=None, + status=DocStatus.DONE, + warnings=tuple(collected), + ) + return replace(document, status=document_status(status_input(document))) + + +def _lines_document( # noqa: PLR0913 -- сборка документа со строками: поля заданы схемой + candidate: FormCandidate, + document_type: DocumentType, + header: StatementHeader, + layout: Layout, + *, + raw: Sequence[RawLine], + row_warnings: Sequence[StatementWarning], + conflict: Sequence[StatementWarning], + rules: Rules, +) -> StatementDocument: + lines, range_warnings = build_lines(raw, candidate.source, rules) + units = units_warnings(lines, header, candidate.warnings, rules) + column_keys = [str(column.column_index) for column in layout.columns] + evaluation = evaluate(_eval_lines(lines), column_keys, document_type, layout.dates.effective, rules, accepted=False) + checks = checks_from(evaluation.outcomes, {key: int(key) for key in column_keys}) + header, header_warnings = review_header(header, dates=layout.dates, rules=rules) + reviewed = tuple( + replace( + line, + needs_review=evaluation.line_needs_review.get(line.ref, False), + values=tuple( + replace(v, needs_review=evaluation.value_needs_review.get((line.ref, str(v.column_index)), False)) + for v in line.values + ), + ) + for line in lines + ) + document = StatementDocument( + document_type=document_type, + form_variant=evaluation.variant, + locator=candidate.locator, + header=header, + columns=layout.columns, + lines=reviewed, + checks=checks, + status=DocStatus.DONE, + warnings=( + *candidate.warnings, + *layout.warnings, + *row_warnings, + *range_warnings, + *units, + *conflict, + *header_warnings, + ), + ) + return replace(document, status=document_status(status_input(document))) + + +def normalize_candidate(candidate: FormCandidate, rules: Rules | None = None) -> CandidateResult: + """Шаги 2–8 кандидата; документа нет — предупреждение файла (`TABLE_LOST`, `TABLE_SKIPPED`, `SHEET_OTHER`).""" + rules = rules if rules is not None else default_rules() + header = replace(parse_header(candidate.header, rules), form_name=_form_name(candidate)) + layout = find_layout(candidate, header, rules) + hint = candidate.type_hint + + if not layout.code_found: + if hint is None: + return CandidateResult(None, (_lost_table(candidate, rules),)) + base = [*candidate.warnings, *layout.warnings] + if hint is DocumentType.DISCLOSURE: + return CandidateResult( + _without_lines(candidate, hint, header, candidate.warnings, dates=None, rules=rules), () + ) + message = "столбец кодов строк не найден" + base.append(make_warning(WarningCode.CODE_COLUMN_NOT_FOUND, message, rules)) + return CandidateResult(_without_lines(candidate, hint, header, base, dates=layout.dates, rules=rules), ()) + + raw, row_warnings = collect_raw_lines(candidate, layout, rules) + ok_codes = [item.code for item in raw if item.line_status is LineStatus.OK and item.code is not None] + section = majority_section(ok_codes, rules) + if section is not None: + document_type = document_type_of(section) + elif hint is not None: + document_type = hint + else: + return CandidateResult(None, (_lost_table(candidate, rules),)) + + conflict: list[StatementWarning] = [] + if hint is not None and hint is not document_type: + message = f"тип по кодам строк ({document_type}) противоречит типу по заголовку ({hint})" + conflict.append(make_warning(WarningCode.TYPE_CONFLICT, message, rules)) + + if document_type is DocumentType.DISCLOSURE: + warnings = [*candidate.warnings, *conflict] + return CandidateResult(_without_lines(candidate, document_type, header, warnings, dates=None, rules=rules), ()) + document = _lines_document( + candidate, document_type, header, layout, raw=raw, row_warnings=row_warnings, conflict=conflict, rules=rules + ) + return CandidateResult(document, ()) + + +def normalize(candidates: Sequence[FormCandidate], rules: Rules | None = None) -> NormalizationResult: + """Кандидаты по порядку; исключения не перехватываются (изоляцию кандидатов делает хост).""" + rules = rules if rules is not None else default_rules() + documents: list[StatementDocument] = [] + file_warnings: list[StatementWarning] = [] + for candidate in candidates: + result = normalize_candidate(candidate, rules) + if result.document is not None: + documents.append(result.document) + file_warnings.extend(result.file_warnings) + return NormalizationResult(tuple(documents), tuple(file_warnings)) diff --git a/arm_statements/tests/test_pipeline.py b/arm_statements/tests/test_pipeline.py new file mode 100644 index 0000000..45af8d9 --- /dev/null +++ b/arm_statements/tests/test_pipeline.py @@ -0,0 +1,196 @@ +from decimal import Decimal as D + +from arm_statements import document_status, normalize, normalize_candidate, status_input +from arm_statements.issues import make_warning +from arm_statements.loader import Rules +from arm_statements.model import DocStatus, DocumentType, FormVariant, Severity +from arm_statements.tests.builders import excel, hdr, ocr + +HEAD = ["Наименование показателя", "Код", "На 30.09.2025"] +FULL_CONFIDENCE = 100.0 +FULL_HEADER = hdr(organization_name="ООО «Пример»", inn="7700000000", report_date="30.09.2025", units="тыс. руб.") + + +def bal_rows() -> list[list[str]]: + return [ + HEAD, + ["Запасы", "1210", "60"], + ["Дебиторы", "1230", "40"], + ["Итого II", "1200", "100"], + ["НМА", "1110", "50"], + ["Итого I", "1100", "50"], + ["БАЛАНС", "1600", "150"], + ["Капитал", "1310", "150"], + ["Итого III", "1300", "150"], + ["БАЛАНС", "1700", "150"], + ] + + +def test_clean_ocr_balance_done(rules: Rules) -> None: + cand = ocr(bal_rows(), header=FULL_HEADER, type_hint=DocumentType.BALANCE, title="Бухгалтерский баланс") + result = normalize_candidate(cand, rules) + doc = result.document + assert doc is not None and result.file_warnings == () + assert doc.header.form_name == "Бухгалтерский баланс" + assert document_status(status_input(doc)) is doc.status + assert (doc.document_type, doc.form_variant, doc.status) == (DocumentType.BALANCE, FormVariant.FULL, DocStatus.DONE) + assert doc.checks is not None and doc.checks.relations_failed == () + assert doc.header.inn.value == "7700000000" + + +def test_disclosure_hint_with_balance_codes(rules: Rules) -> None: + doc = normalize_candidate(ocr(bal_rows(), header=FULL_HEADER, type_hint=DocumentType.DISCLOSURE), rules).document + assert doc is not None and doc.document_type is DocumentType.BALANCE + assert [(w.code, w.severity) for w in doc.warnings if w.code == "TYPE_CONFLICT"] == [ + ("TYPE_CONFLICT", Severity.ERROR) + ] + + +def test_balance_hint_with_income_codes(rules: Rules) -> None: + rows = [ + HEAD, + ["Выручка", "2110", "100"], + ["Себестоимость", "2120", "(60)"], + ["Валовая", "2100", "40"], + ["Коммерческие", "2210", "(10)"], + ["Прибыль от продаж", "2200", "30"], + ] + doc = normalize_candidate(ocr(rows, header=FULL_HEADER, type_hint=DocumentType.BALANCE), rules).document + assert doc is not None and doc.document_type is DocumentType.INCOME + assert "TYPE_CONFLICT" in [w.code for w in doc.warnings] + + +def test_excel_other_sheet(rules: Rules) -> None: + result = normalize_candidate(excel([("Примечания", [["Пояснительный текст без цифр"]])]), rules) + assert result.document is None + assert [(w.code, w.severity, w.locator) for w in result.file_warnings] == [ + ("SHEET_OTHER", Severity.INFO, "sheet:Примечания") + ] + + +def test_ocr_lost_and_skipped_tables(rules: Rules) -> None: + lost = normalize_candidate(ocr([["Итого", "", "15 000"]], locator="file:a.json;table:1"), rules) + assert [(w.code, w.locator) for w in lost.file_warnings] == [("TABLE_LOST", "file:a.json;table:1")] + skipped = normalize_candidate(ocr([["Руководитель", "Иванов"]]), rules) + assert [w.code for w in skipped.file_warnings] == ["TABLE_SKIPPED"] and skipped.document is None + + +def test_hint_without_code_column(rules: Rules) -> None: + doc = normalize_candidate( + ocr([["Итого", "15 000"]], type_hint=DocumentType.BALANCE, header=FULL_HEADER), rules + ).document + assert doc is not None and (doc.lines, doc.columns, doc.checks) == ((), (), None) + assert doc.form_variant is FormVariant.FULL + assert doc.status is DocStatus.NEEDS_CHECK and "CODE_COLUMN_NOT_FOUND" in [w.code for w in doc.warnings] + disc = normalize_candidate(ocr([["Текст"]], type_hint=DocumentType.DISCLOSURE), rules).document + assert disc is not None and (disc.status, disc.form_variant, disc.lines) == (DocStatus.DONE, None, ()) + assert disc.header.form_name is None + + +def test_disclosure_by_codes(rules: Rules) -> None: + rows = [HEAD, ["Пояснение", "5101", "1"], ["Пояснение", "5102", "2"], ["Пояснение", "5100", "3"]] + doc = normalize_candidate(ocr(rows, header=FULL_HEADER), rules).document + assert doc is not None and (doc.document_type, doc.lines, doc.status) == ( + DocumentType.DISCLOSURE, + (), + DocStatus.DONE, + ) + + +def test_simplified_balance(rules: Rules) -> None: + rows = [ + HEAD, + ["Матактивы", "1150", "50"], + ["Запасы", "1210", "50"], + ["БАЛАНС", "1600", "100"], + ["Капитал", "1300", "100"], + ["БАЛАНС", "1700", "100"], + ] + doc = normalize_candidate(ocr(rows, header=FULL_HEADER), rules).document + assert doc is not None and doc.form_variant is FormVariant.SIMPLIFIED + assert doc.checks is not None and doc.checks.relations_failed == () + + +def test_income_without_inn_still_done(rules: Rules) -> None: + rows: list[list[str | D]] = [ + ["Наименование", "Код", "За 9 месяцев 2025 г."], + ["Выручка", "2110", D("100")], + ["Себестоимость", "2120", D("-60")], + ["Валовая прибыль", "2100", D("40")], + ["Прибыль от продаж", "2200", D("40")], + ["Прибыль до налогообложения", "2300", D("40")], + ["Чистая прибыль", "2400", D("40")], + ] + head = hdr(organization_name="ООО «Пример»", units="тыс. руб.") + doc = normalize_candidate(excel([("ОФР", rows)], header=head), rules).document + assert doc is not None and doc.status is DocStatus.DONE + assert [(w.code, w.severity) for w in doc.warnings] == [("INN_NOT_FOUND", Severity.INFO)] + + +def test_normalize_collects(rules: Rules) -> None: + result = normalize([ocr(bal_rows(), header=FULL_HEADER), ocr([["Руководитель", "Иванов"]])], rules) + assert len(result.documents) == 1 and [w.code for w in result.file_warnings] == ["TABLE_SKIPPED"] + + +# --- P1-6: у DISCLOSURE предупреждений шапки нет, статус DONE (требование ревью T15.1) --- + + +def test_disclosure_by_hint_empty_header_has_no_warnings(rules: Rules) -> None: + doc = normalize_candidate(ocr([["Текст"]], type_hint=DocumentType.DISCLOSURE), rules).document + assert doc is not None + assert (doc.warnings, doc.status) == ((), DocStatus.DONE) + + +def test_disclosure_by_codes_empty_header_has_no_warnings(rules: Rules) -> None: + rows = [HEAD, ["Пояснение", "5101", "1"], ["Пояснение", "5102", "2"], ["Пояснение", "5100", "3"]] + doc = normalize_candidate(ocr(rows), rules).document + assert doc is not None and doc.document_type is DocumentType.DISCLOSURE + assert (doc.warnings, doc.status) == ((), DocStatus.DONE) + + +# --- ROW_SPLIT: предупреждение документа без locator (место — locator документа, спека «Шаг 8 — предупреждения») --- + + +def test_row_split_has_no_locator(rules: Rules) -> None: + rows = bal_rows() + rows[1] = ["Запасы и дебиторы", "1210\n1230", "60\n40"] + doc = normalize_candidate(ocr(rows, header=FULL_HEADER, type_hint=DocumentType.BALANCE), rules).document + assert doc is not None + split = [w for w in doc.warnings if w.code == "ROW_SPLIT"] + assert len(split) == 1 and split[0].locator is None and split[0].severity is Severity.INFO + + +# --- строка «в тысячах рублей» над таблицей и уверенность даты отчёта у Excel --- + + +def test_units_row_above_table_is_not_header_and_excel_date_is_confident(rules: Rules) -> None: + rows = [["Бухгалтерский баланс"], ["в тысячах рублей"], *bal_rows()] + cand = excel([("Баланс", rows)], header=FULL_HEADER, type_hint=DocumentType.BALANCE) + doc = normalize_candidate(cand, rules).document + assert doc is not None + assert doc.header.report_date.confidence == FULL_CONFIDENCE and doc.header.report_date.value is not None + assert [c.column_index for c in doc.columns] == [2] + assert (doc.status, [w.code for w in doc.warnings]) == (DocStatus.DONE, []) + + +# --- порядок предупреждений документа: адаптера, раскладки, строк, единиц, TYPE_CONFLICT, шапки --- + + +def test_document_warnings_order(rules: Rules) -> None: + adapter = make_warning("NUMBER_FORMAT_UNPARSED", "формат", rules) + rows = [["Наименование", "Код", "Итого"], *bal_rows()[1:]] + rows[1] = ["Запасы и дебиторы", "1210\n1230", "60,5\n40,5"] + cand = excel([("Баланс", rows)], header=hdr(units="тыс. руб."), type_hint=DocumentType.INCOME, warnings=[adapter]) + doc = normalize_candidate(cand, rules).document + assert doc is not None + codes = [w.code for w in doc.warnings] + expected = [ + "NUMBER_FORMAT_UNPARSED", + "COLUMN_WITHOUT_DATE", + "ROW_SPLIT", + "FRACTIONAL_IN_THOUSANDS", + "TYPE_CONFLICT", + "NO_ORGANIZATION_NO_INN", + ] + assert [c for c in codes if c in expected] == [c for c in expected if c in codes] + assert codes[0] == "NUMBER_FORMAT_UNPARSED" and codes.index("TYPE_CONFLICT") < codes.index("NO_ORGANIZATION_NO_INN") -- 2.47.2 From 9e28213ce6ffe4fdc2efdea8125b433652ebe162 Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Sun, 18 Oct 2026 14:04:19 +0400 Subject: [PATCH 20/28] =?UTF-8?q?feat:=20=D1=82=D0=BE=D1=87=D0=BA=D0=B0=20?= =?UTF-8?q?=D0=B2=D1=85=D0=BE=D0=B4=D0=B0=20=D0=BF=D0=B5=D1=80=D0=B5=D1=81?= =?UTF-8?q?=D1=87=D1=91=D1=82=D0=B0=20=D1=84=D0=BE=D1=80=D0=BC=D1=8B=20?= =?UTF-8?q?=D0=BF=D0=BE=20=D1=81=D0=BE=D1=85=D1=80=D0=B0=D0=BD=D1=91=D0=BD?= =?UTF-8?q?=D0=BD=D1=8B=D0=BC=20=D0=B4=D0=B0=D0=BD=D0=BD=D1=8B=D0=BC=20(?= =?UTF-8?q?=D0=BF=D1=80=D0=B0=D0=B2=D0=BA=D0=B8,=20=D0=BF=D0=BE=D0=B4?= =?UTF-8?q?=D1=82=D0=B2=D0=B5=D1=80=D0=B6=D0=B4=D0=B5=D0=BD=D0=B8=D1=8F,?= =?UTF-8?q?=20=D1=81=D1=82=D1=80=D0=BE=D0=BA=D0=B8=20=D0=BF=D1=80=D0=B0?= =?UTF-8?q?=D0=B2=D0=BA=D0=B8)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W --- arm_statements/__init__.py | 20 ++- arm_statements/recompute.py | 177 +++++++++++++++++++++++++ arm_statements/tests/test_recompute.py | 87 ++++++++++++ 3 files changed, 282 insertions(+), 2 deletions(-) create mode 100644 arm_statements/recompute.py create mode 100644 arm_statements/tests/test_recompute.py diff --git a/arm_statements/__init__.py b/arm_statements/__init__.py index 5b2bcf4..d8027ce 100644 --- a/arm_statements/__init__.py +++ b/arm_statements/__init__.py @@ -1,7 +1,7 @@ """Общий код нормализации форм ББ/ОФР. -Публичный API — только имена `__all__` (раздел «Публичный API пакета» плана 1). Имена задач 18–19 -(`recompute`, `Stored*`, `to_contract`, `warning_to_contract`, `status_input_from_contract`) добавляются ими. +Публичный API — только имена `__all__` (раздел «Публичный API пакета» плана 1). Имена задачи 19 +(`to_contract`, `warning_to_contract`, `status_input_from_contract`) добавляются ими. """ from arm_statements.amounts import parse_amount @@ -43,6 +43,15 @@ from arm_statements.model import ( StatementWarning, ) from arm_statements.pipeline import normalize, normalize_candidate +from arm_statements.recompute import ( + RecomputeResult, + StoredColumn, + StoredForm, + StoredLine, + StoredValue, + StoredWarning, + recompute, +) from arm_statements.serialize import candidate_from_dict, to_jsonable from arm_statements.status import ( StatusInput, @@ -75,6 +84,7 @@ __all__ = [ "LineStatus", "NormalizationResult", "PeriodKind", + "RecomputeResult", "Rules", "Severity", "SignForm", @@ -86,6 +96,11 @@ __all__ = [ "StatementLine", "StatementValue", "StatementWarning", + "StoredColumn", + "StoredForm", + "StoredLine", + "StoredValue", + "StoredWarning", "StatusInput", "StatusLine", "StatusValue", @@ -110,6 +125,7 @@ __all__ = [ "parse_code", "parse_date", "parse_header", + "recompute", "resolve_relative", "sheet_joins", "statements_version", diff --git a/arm_statements/recompute.py b/arm_statements/recompute.py new file mode 100644 index 0000000..8e2b335 --- /dev/null +++ b/arm_statements/recompute.py @@ -0,0 +1,177 @@ +"""Точка входа пересчёта формы по сохранённым данным (спека «Точка входа пакета для пересчёта…»). + +Ядро собирает `StoredForm` из своих строк БД; сервисы функцией не пользуются. Пересчёт не меняет `line_status`, +суммы и коды — только выводит признаки (Р-34: подтверждение кода — `code_fixed`, уверенность кода не меняется). +""" + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from datetime import date +from decimal import Decimal + +from arm_statements.checks import checks_from +from arm_statements.evaluation import EvalLine, EvalValue +from arm_statements.loader import Rules, default_rules +from arm_statements.model import ( + DocStatus, + DocumentType, + Edition, + FormVariant, + LineStatus, + PeriodKind, + Severity, + StatementChecks, +) +from arm_statements.review import evaluate +from arm_statements.status import StatusInput, StatusLine, StatusValue, document_status +from arm_statements.structure import edition_by_date + + +@dataclass(frozen=True, slots=True) +class StoredColumn: + key: str + column_index: int | None + period_date: date | None + period_kind: PeriodKind + + +@dataclass(frozen=True, slots=True) +class StoredValue: + column_key: str + amount: Decimal | None + text: str | None # source_text; у значения правки None + confidence: float | None + is_recognized: bool + value_fixed: bool # is_modified или is_confirmed + + +@dataclass(frozen=True, slots=True) +class StoredLine: + key: str + row_index: int | None # None — строка правки + line_status: LineStatus + line_code: str | None + line_code_confidence: float | None + line_name: str | None + code_fixed: bool # line_code_modified или is_confirmed строки + parent_key: str | None + values: tuple[StoredValue, ...] + + +@dataclass(frozen=True, slots=True) +class StoredWarning: + code: str + severity: Severity + + +@dataclass(frozen=True, slots=True) +class StoredForm: + document_type: DocumentType + effective_date: date | None + has_organization_or_inn: bool + period_known: bool + needs_review_fields: tuple[str, ...] + accepted: bool + warnings: tuple[StoredWarning, ...] + columns: tuple[StoredColumn, ...] + lines: tuple[StoredLine, ...] + + +@dataclass(frozen=True, slots=True) +class RecomputeResult: + form_variant: FormVariant | None + edition: Edition | None + checks: StatementChecks + line_needs_review: Mapping[str, bool] + value_needs_review: Mapping[tuple[str, str], bool] + detail_parents: Mapping[str, str | None] + status: DocStatus + + +def _sort_key(line: StoredLine) -> tuple[bool, int]: + return (line.row_index is None, line.row_index or 0) + + +def _eval_line(line: StoredLine) -> EvalLine: + return EvalLine( + key=line.key, + order=line.row_index, + line_status=line.line_status, + line_code=line.line_code, + code_confidence=line.line_code_confidence, + code_fixed=line.code_fixed, + values=tuple( + EvalValue(v.column_key, v.amount, v.text, v.confidence, v.is_recognized, fixed=v.value_fixed) + for v in line.values + ), + ) + + +def _detail_parents(lines: Sequence[StoredLine]) -> dict[str, str | None]: + """Ближайшая выше по `row_index` строка `ok` с `row_index`; у `detail` без `row_index` и без такой — `None`.""" + parents: dict[str, str | None] = {} + last_ok: str | None = None + for line in lines: # уже отсортированы: строки правки в конце + if line.row_index is None: + if line.line_status is LineStatus.DETAIL: + parents[line.key] = None + continue + if line.line_status is LineStatus.DETAIL: + parents[line.key] = last_ok + elif line.line_status is LineStatus.OK: + last_ok = line.key + return parents + + +def recompute(form: StoredForm, rules: Rules | None = None) -> RecomputeResult: + """Шаги 6–8 по сохранённым данным: вариант, редакция, §7, `needs_review`, родители `detail`, статус.""" + rules = rules if rules is not None else default_rules() + if form.document_type is DocumentType.DISCLOSURE: + return RecomputeResult( + form_variant=None, + edition=edition_by_date(form.effective_date), + checks=StatementChecks((), ()), + line_needs_review={}, + value_needs_review={}, + detail_parents={}, + status=DocStatus.DONE, + ) + lines = sorted(form.lines, key=_sort_key) + evaluation = evaluate( + [_eval_line(line) for line in lines], + [column.key for column in form.columns], + form.document_type, + form.effective_date, + rules, + accepted=form.accepted, + ) + checks = checks_from(evaluation.outcomes, {column.key: column.column_index for column in form.columns}) + status = document_status( + StatusInput( + document_type=form.document_type, + layout_found=any(line.row_index is not None for line in lines), + has_organization_or_inn=form.has_organization_or_inn, + period_known=form.period_known, + needs_review_fields=form.needs_review_fields, + lines=tuple( + StatusLine(line.line_status, evaluation.line_needs_review.get(line.key, False)) for line in lines + ), + values=tuple( + StatusValue(line.line_status, evaluation.value_needs_review.get((line.key, v.column_key), False)) + for line in lines + for v in line.values + ), + hard_failures=evaluation.hard_failures, + accepted=form.accepted, + warning_severities=tuple(w.severity for w in form.warnings), + ) + ) + return RecomputeResult( + form_variant=evaluation.variant, + edition=evaluation.edition, + checks=checks, + line_needs_review=evaluation.line_needs_review, + value_needs_review=evaluation.value_needs_review, + detail_parents=_detail_parents(lines), + status=status, + ) diff --git a/arm_statements/tests/test_recompute.py b/arm_statements/tests/test_recompute.py new file mode 100644 index 0000000..9564b8a --- /dev/null +++ b/arm_statements/tests/test_recompute.py @@ -0,0 +1,87 @@ +from dataclasses import replace +from datetime import date +from decimal import Decimal as D + +from arm_statements import StoredColumn, StoredForm, StoredLine, StoredValue, choose_edition, recompute +from arm_statements.loader import Rules +from arm_statements.model import DocStatus, DocumentType, Edition, FormVariant, LineStatus, PeriodKind + +OK, DETAIL = LineStatus.OK, LineStatus.DETAIL +COL = StoredColumn("c1", 2, date(2025, 9, 30), PeriodKind.ABSOLUTE) + + +def sl( # noqa: PLR0913 -- конструктор тестовых строк + key: str, + row: int | None, + code: str | None, + amount: str | None, + *, + status: LineStatus = OK, + conf: float = 100.0, + code_fixed: bool = False, + value_conf: float = 100.0, + value_fixed: bool = False, + recognized: bool = True, +) -> StoredLine: + value = StoredValue("c1", None if amount is None else D(amount), amount, value_conf, recognized, value_fixed) + return StoredLine(key, row, status, code, conf, None, code_fixed, None, (value,)) + + +def form(*lines: StoredLine, document_type: DocumentType = DocumentType.BALANCE) -> StoredForm: + return StoredForm(document_type, date(2025, 9, 30), True, True, (), False, (), (COL,), tuple(lines)) + + +def test_recompute_after_edit(rules: Rules) -> None: + result = recompute( + form( + sl("l1210", 0, "1210", "100"), + sl("l1230", 1, "1230", "300", value_conf=60.0, value_fixed=True), + sl("l1250", 2, "1250", "50", conf=40.0, code_fixed=True), + sl("l1260", None, "1260", "50"), + sl("ldet", 3, None, "7", status=DETAIL, value_conf=60.0), + sl("l1200", 4, "1200", "500"), + ), + rules, + ) + # 1200 = 100 + 300 + 50 + 50 (с 1250 и строкой правки 1260) — сходится; 1600 нет — «пустая цель» + assert [(f.relation, f.column_key, f.column_index) for f in result.checks.relations_failed] == [ + ("1600 = 1100 + 1200", "c1", 2) + ] + assert result.value_needs_review[("l1230", "c1")] is False + assert result.value_needs_review[("ldet", "c1")] is False + assert result.line_needs_review["l1250"] is False + assert result.detail_parents == {"ldet": "l1250"} + assert (result.edition, result.form_variant, result.status) == ( + Edition.FSBU4, + FormVariant.FULL, + DocStatus.NEEDS_CHECK, + ) + + +def test_code_change_switches_variant(rules: Rules) -> None: + simplified = form( + sl("a", 0, "1150", "50"), sl("b", 1, "1600", "50"), sl("c", 2, "1300", "50"), sl("d", 3, "1700", "50") + ) + assert recompute(simplified, rules).form_variant is FormVariant.SIMPLIFIED + edited = form( + sl("a", 0, "1100", "50", code_fixed=True), + sl("b", 1, "1600", "50"), + sl("c", 2, "1300", "50"), + sl("d", 3, "1700", "50"), + ) + result = recompute(edited, rules) + assert result.form_variant is FormVariant.FULL + assert result.status is DocStatus.NEEDS_CHECK + + +def test_edition_equals_choose_edition(rules: Rules) -> None: + stored = form(sl("a", 0, "1110", "10"), sl("b", 1, "1120", "5", conf=60.0), sl("c", 2, "1130", "5")) + pairs = [("1110", True), ("1120", False), ("1130", True)] + expected = choose_edition(DocumentType.BALANCE, pairs, date(2025, 9, 30), rules) + assert recompute(stored, rules).edition is expected is Edition.RAS_66N + + +def test_accepted_and_empty_edit_line(rules: Rules) -> None: + broken = form(sl("a", 0, "1600", "100"), sl("b", 1, "1700", "90"), sl("e", None, "1220", None)) + assert recompute(broken, rules).status is DocStatus.NEEDS_CHECK + assert recompute(replace(broken, accepted=True), rules).status is DocStatus.DONE -- 2.47.2 From 3ee74c6907542e3a3a369d107d82430338b69756 Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Sun, 18 Oct 2026 14:06:55 +0400 Subject: [PATCH 21/28] =?UTF-8?q?feat:=20=D0=BF=D0=B5=D1=80=D0=B5=D0=B2?= =?UTF-8?q?=D0=BE=D0=B4=20=D0=B4=D0=BE=D0=BA=D1=83=D0=BC=D0=B5=D0=BD=D1=82?= =?UTF-8?q?=D0=B0=20=D0=B8=20=D0=BF=D1=80=D0=B5=D0=B4=D1=83=D0=BF=D1=80?= =?UTF-8?q?=D0=B5=D0=B6=D0=B4=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=B2=20=D1=84?= =?UTF-8?q?=D0=BE=D1=80=D0=BC=D1=83=20=D0=BA=D0=BE=D0=BD=D1=82=D1=80=D0=B0?= =?UTF-8?q?=D0=BA=D1=82=D0=B0=20v2.0,=20=D0=B2=D1=85=D0=BE=D0=B4=20=D1=81?= =?UTF-8?q?=D1=82=D0=B0=D1=82=D1=83=D1=81=D0=B0=20=D0=B8=D0=B7=20=D0=B4?= =?UTF-8?q?=D0=BE=D0=BA=D1=83=D0=BC=D0=B5=D0=BD=D1=82=D0=B0=20=D0=BA=D0=BE?= =?UTF-8?q?=D0=BD=D1=82=D1=80=D0=B0=D0=BA=D1=82=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W --- arm_statements/__init__.py | 7 +- arm_statements/contract.py | 151 ++++++++++++++++++++++++++ arm_statements/tests/test_contract.py | 99 +++++++++++++++++ 3 files changed, 255 insertions(+), 2 deletions(-) create mode 100644 arm_statements/contract.py create mode 100644 arm_statements/tests/test_contract.py diff --git a/arm_statements/__init__.py b/arm_statements/__init__.py index d8027ce..d75b97f 100644 --- a/arm_statements/__init__.py +++ b/arm_statements/__init__.py @@ -1,12 +1,12 @@ """Общий код нормализации форм ББ/ОФР. -Публичный API — только имена `__all__` (раздел «Публичный API пакета» плана 1). Имена задачи 19 -(`to_contract`, `warning_to_contract`, `status_input_from_contract`) добавляются ими. +Публичный API — только имена `__all__` (раздел «Публичный API пакета» плана 1). Контрактные имена — `contract.py`. """ from arm_statements.amounts import parse_amount from arm_statements.codes import parse_code, sheet_joins from arm_statements.columns import amount_columns +from arm_statements.contract import status_input_from_contract, to_contract, warning_to_contract from arm_statements.dates import parse_date, resolve_relative from arm_statements.header import continuation_type, is_continuation, parse_header, type_hint from arm_statements.issues import WarningCode, make_warning @@ -130,6 +130,9 @@ __all__ = [ "sheet_joins", "statements_version", "status_input", + "status_input_from_contract", + "to_contract", "to_jsonable", "type_hint", + "warning_to_contract", ] diff --git a/arm_statements/contract.py b/arm_statements/contract.py new file mode 100644 index 0000000..10e7d80 --- /dev/null +++ b/arm_statements/contract.py @@ -0,0 +1,151 @@ +"""Перевод выхода пакета в форму контракта v2.0 и вход статуса из документа контракта. + +Один перевод на оба сервиса (P1-26): хосты свой `contract_map` не пишут. Схему пакет не читает — +форма держится тестами и разовой сверкой со `statementDocument` схемы v2.0. +""" + +from collections.abc import Mapping +from decimal import Decimal +from typing import Any + +from arm_statements.model import ( + HEADER_FIELDS, + DocumentType, + FailedRelation, + LineStatus, + PeriodKind, + Severity, + StatementDocument, + StatementHeader, + StatementWarning, +) +from arm_statements.status import StatusInput, StatusLine, StatusValue + + +def _money(value: Decimal | None) -> str | None: + return None if value is None else format(value, "f") + + +def warning_to_contract(warning: StatementWarning, *, file_level: bool = False) -> dict[str, Any]: + """`warning` схемы; при `file_level` — `fileWarning` (с `locator`). Ключи без значения опускаются.""" + data: dict[str, Any] = { + "code": warning.code, + "severity": str(warning.severity), + "message": warning.message, + } + if warning.line_refs: + data["line_refs"] = list(warning.line_refs) + if warning.column_index is not None: + data["column_index"] = warning.column_index + if warning.field is not None: + data["field"] = warning.field + if file_level and warning.locator is not None: + data["locator"] = warning.locator + return data + + +def _header(header: StatementHeader) -> dict[str, Any]: + data: dict[str, Any] = {"form_name": header.form_name} + for name in HEADER_FIELDS: + field = getattr(header, name) + value = field.value + data[name] = value.isoformat() if name == "report_date" and value is not None else value + data[f"{name}_text"] = field.text + data[f"{name}_confidence"] = field.confidence + data["needs_review_fields"] = list(header.needs_review_fields) + return data + + +def _relation(relation: FailedRelation) -> dict[str, Any]: + if relation.column_index is None: + raise ValueError(f"у соотношения {relation.relation!r} нет column_index: у сервиса номер колонки есть всегда") + return { + "column_index": relation.column_index, + "relation": relation.relation, + "codes": list(relation.codes), + "expected": _money(relation.expected), + "actual": _money(relation.actual), + } + + +def to_contract(document: StatementDocument) -> dict[str, Any]: + """`statementDocument` схемы v2.0 как json-совместимый `dict`.""" + data: dict[str, Any] = { + "document_type": str(document.document_type), + "form_variant": None if document.form_variant is None else str(document.form_variant), + "locator": document.locator, + "status": str(document.status), + "warnings": [warning_to_contract(w) for w in document.warnings], + "header": _header(document.header), + "columns": [ + { + "column_index": c.column_index, + "label": c.label, + "label_confidence": c.label_confidence, + "period_date": None if c.period_date is None else c.period_date.isoformat(), + "period_kind": str(c.period_kind), + } + for c in document.columns + ], + "lines": [ + { + "row_index": ln.row_index, + "ref": ln.ref, + "line_status": str(ln.line_status), + "line_code": ln.line_code, + "line_code_text": ln.line_code_text, + "line_code_confidence": ln.line_code_confidence, + "line_name": ln.line_name, + "line_name_confidence": ln.line_name_confidence, + "parent_ref": ln.parent_ref, + "needs_review": ln.needs_review, + "values": [ + { + "column_index": v.column_index, + "amount": _money(v.amount), + "string_value": v.string_value, + "text": v.text, + "confidence": v.confidence, + "is_recognized": v.is_recognized, + "needs_review": v.needs_review, + } + for v in ln.values + ], + } + for ln in document.lines + ], + } + if document.checks is not None: + data["checks"] = { + "relations_failed": [_relation(r) for r in document.checks.relations_failed], + "soft_relations_failed": [_relation(r) for r in document.checks.soft_relations_failed], + } + return data + + +def status_input_from_contract(document: Mapping[str, Any]) -> StatusInput: + """Вход функции статуса из `statementDocument` контракта v2.0 (то же правило, что `status_input`). + + Схему не проверяет: битый документ — `KeyError` / `ValueError`. Отсутствующий необязательный ключ и `null` равны. + """ + header = document["header"] + lines = document["lines"] + columns = document["columns"] + period_known = header.get("report_date") is not None or any( + c["period_kind"] == PeriodKind.ABSOLUTE and c.get("period_date") for c in columns + ) + checks = document.get("checks") + return StatusInput( + document_type=DocumentType(document["document_type"]), + layout_found=len(lines) > 0, + has_organization_or_inn=bool(header.get("organization_name") or header.get("inn")), + period_known=period_known, + needs_review_fields=tuple(header.get("needs_review_fields") or ()), + lines=tuple(StatusLine(LineStatus(ln["line_status"]), bool(ln["needs_review"])) for ln in lines), + values=tuple( + StatusValue(LineStatus(ln["line_status"]), bool(v["needs_review"])) for ln in lines for v in ln["values"] + ), + hard_failures=0 if checks is None else len(checks["relations_failed"]), + accepted=False, + warning_severities=tuple(Severity(w["severity"]) for w in document["warnings"]), + ) diff --git a/arm_statements/tests/test_contract.py b/arm_statements/tests/test_contract.py new file mode 100644 index 0000000..fe63ba8 --- /dev/null +++ b/arm_statements/tests/test_contract.py @@ -0,0 +1,99 @@ +import json +from dataclasses import replace +from decimal import Decimal as D + +import pytest + +from arm_statements import ( + DocStatus, + DocumentType, + document_status, + make_warning, + normalize_candidate, + status_input, + status_input_from_contract, + to_contract, + warning_to_contract, +) +from arm_statements.loader import Rules +from arm_statements.model import FailedRelation, StatementChecks, StatementDocument +from arm_statements.tests.builders import hdr, ocr + +HEAD = ["Наименование показателя", "Код", "На 30.09.2025"] +FULL_HEADER = hdr( + organization_name="ООО «Пример»", inn=("7700000000", 80.0), report_date="30.09.2025", units="тыс. руб." +) +HEADER_KEYS = {"form_name", "needs_review_fields"} | { + f"{name}{suffix}" + for name in ("organization_name", "inn", "report_date", "units", "okud", "okpo", "okopf", "okfs", "okei", "address") + for suffix in ("", "_text", "_confidence") +} + + +def balance(rules: Rules, *rows: list[str]) -> StatementDocument: + cand = ocr([HEAD, *rows], header=FULL_HEADER, type_hint=DocumentType.BALANCE, title="Бухгалтерский баланс") + doc = normalize_candidate(cand, rules).document + assert doc is not None + return doc + + +def test_document_in_contract_form(rules: Rules) -> None: + doc = balance(rules, ["Запасы", "1210", "60"], ["Итого II", "1200", "60"], ["НИ", "", "7"]) + data = json.loads(json.dumps(to_contract(doc), ensure_ascii=False)) + assert set(data["header"]) == HEADER_KEYS + assert (data["header"]["form_name"], data["header"]["inn"], data["header"]["inn_confidence"]) == ( + "Бухгалтерский баланс", + "7700000000", + 80.0, + ) + assert data["header"]["report_date"] == "2025-09-30" and data["header"]["okei"] is None + assert data["lines"][0]["values"][0]["amount"] == "60.00" + assert {ln["line_status"] for ln in data["lines"]} >= {"ok"} + assert "column_key" not in json.dumps(data) and "row:" not in json.dumps(data["header"]) + assert "locator" not in json.dumps(data["warnings"]) + assert set(data["checks"]) == {"relations_failed", "soft_relations_failed"} + + +def test_empty_target_is_null_in_contract(rules: Rules) -> None: + doc = balance(rules, ["Запасы", "1210", "60"], ["Итого II", "1200", "60"], ["Итого I", "1100", "40"]) + failed = to_contract(doc)["checks"]["relations_failed"] + target = next(f for f in failed if f["relation"] == "1600 = 1100 + 1200") + assert (target["expected"], target["actual"], target["column_index"]) == (None, "100.00", 2) + + +def test_no_lines_no_checks_key(rules: Rules) -> None: + doc = normalize_candidate( + ocr([["Итого", "15 000"]], type_hint=DocumentType.BALANCE, header=FULL_HEADER), rules + ).document + assert doc is not None and "checks" not in to_contract(doc) and to_contract(doc)["lines"] == [] + + +def test_relation_without_column_index_is_error(rules: Rules) -> None: + doc = balance(rules, ["Запасы", "1210", "60"], ["Итого II", "1200", "60"]) + bad = FailedRelation(None, "c1", "1600 = 1700", ("1600", "1700"), D("1"), None) + with pytest.raises(ValueError, match="column_index"): + to_contract(replace(doc, checks=StatementChecks((bad,), ()))) + + +def test_warning_keys(rules: Rules) -> None: + w = make_warning("ROW_SPLIT", "разлеплено", rules, line_refs=("row:3#1", "row:3#2"), locator="file:a.json") + assert warning_to_contract(w) == { + "code": "ROW_SPLIT", + "severity": "info", + "message": "разлеплено", + "line_refs": ["row:3#1", "row:3#2"], + } + lost = make_warning("TABLE_LOST", "таблица", rules, locator="file:a.json;table:1") + assert warning_to_contract(lost, file_level=True) == { + "code": "TABLE_LOST", + "severity": "error", + "message": "таблица", + "locator": "file:a.json;table:1", + } + + +def test_status_input_roundtrip(rules: Rules) -> None: + doc = balance(rules, ["Запасы", "1210", "60"], ["Итого II", "1200", "60"], ["НИ", "", "7"]) + data = status_input_from_contract(json.loads(json.dumps(to_contract(doc), ensure_ascii=False))) + assert data == status_input(doc) + assert document_status(data) is doc.status is DocStatus.NEEDS_CHECK -- 2.47.2 From 2da7bff71a30c3b859aa4df0d6a3190e30399c7c Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Sun, 18 Oct 2026 14:12:29 +0400 Subject: [PATCH 22/28] =?UTF-8?q?test:=20golden=20=D0=BD=D0=B0=20=D0=BE?= =?UTF-8?q?=D0=B1=D0=B5=D0=B7=D0=BB=D0=B8=D1=87=D0=B5=D0=BD=D0=BD=D1=8B?= =?UTF-8?q?=D1=85=20=D0=B2=D1=85=D0=BE=D0=B4=D0=B0=D1=85=20aep=5F3,=20?= =?UTF-8?q?=D1=81=D0=BA=D0=BB=D0=B5=D0=B9=D0=BA=D0=B8,=20=D0=B1=D0=BB?= =?UTF-8?q?=D0=BE=D0=BA=D0=B0=20=C2=AB=D0=9A=D0=BE=D0=B4=D1=8B=C2=BB=201?= =?UTF-8?q?=D0=A1=20=D0=B8=20=D0=BB=D0=B8=D1=81=D1=82=D0=B0-=D0=BF=D1=80?= =?UTF-8?q?=D0=BE=D0=B4=D0=BE=D0=BB=D0=B6=D0=B5=D0=BD=D0=B8=D1=8F=20=D0=9D?= =?UTF-8?q?=D0=9F=D0=97;=20=D0=BF=D1=83=D0=B1=D0=BB=D0=B8=D1=87=D0=BD?= =?UTF-8?q?=D1=8B=D0=B9=20API?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Пять golden-входов по Review Focus 1–5, эталоны, свойство status_input_from_contract(to_contract(d)) == status_input(d) на golden и граничных кейсах (DISCLOSURE, без строк, абсолютная колонка без даты), test_public_api, RulesError в __all__, README: публичный API, словари, golden-update. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W --- README.md | 252 ++++- arm_statements/__init__.py | 3 +- .../tests/golden/aep3_balance.candidate.json | 750 +++++++++++++++ .../tests/golden/aep3_balance.expected.json | 868 ++++++++++++++++++ .../tests/golden/aep3_income.candidate.json | 290 ++++++ .../tests/golden/aep3_income.expected.json | 356 +++++++ .../tests/golden/glued_balance.candidate.json | 330 +++++++ .../tests/golden/glued_balance.expected.json | 411 +++++++++ .../golden/kody_title_balance.candidate.json | 538 +++++++++++ .../golden/kody_title_balance.expected.json | 371 ++++++++ .../tests/golden/npz_income.candidate.json | 433 +++++++++ .../tests/golden/npz_income.expected.json | 488 ++++++++++ arm_statements/tests/test_golden.py | 159 ++++ arm_statements/tests/test_model.py | 81 ++ 14 files changed, 5328 insertions(+), 2 deletions(-) create mode 100644 arm_statements/tests/golden/aep3_balance.candidate.json create mode 100644 arm_statements/tests/golden/aep3_balance.expected.json create mode 100644 arm_statements/tests/golden/aep3_income.candidate.json create mode 100644 arm_statements/tests/golden/aep3_income.expected.json create mode 100644 arm_statements/tests/golden/glued_balance.candidate.json create mode 100644 arm_statements/tests/golden/glued_balance.expected.json create mode 100644 arm_statements/tests/golden/kody_title_balance.candidate.json create mode 100644 arm_statements/tests/golden/kody_title_balance.expected.json create mode 100644 arm_statements/tests/golden/npz_income.candidate.json create mode 100644 arm_statements/tests/golden/npz_income.expected.json create mode 100644 arm_statements/tests/test_golden.py diff --git a/README.md b/README.md index b5b0bbe..76101d9 100644 --- a/README.md +++ b/README.md @@ -12,4 +12,254 @@ make venv && make check ``` -Раздел «Публичный API» дописывается в задаче 20. +## Публичный API + +Импорт — `from arm_statements import …`. Всё, что не перечислено, — внутреннее (может меняться без координации с +хостами). Все dataclass-ы — `frozen=True, slots=True`; последовательности — `tuple`. Раздел сведён с запросами планов +3, 4, 5 (18.10.2026, P1-21…P1-28): хосты не повторяют правил пакета у себя — ни столбца кодов, ни нормализации меток, ни +перевода в контракт. + +### Перечисления (`StrEnum`, значение — строка контракта) + +| Имя | Значения | +|---|---| +| `Source` | `OCR = "ocr"`, `EXCEL = "excel"` | +| `DocumentType` | `BALANCE`, `INCOME`, `DISCLOSURE` | +| `FormVariant` | `FULL = "full"`, `SIMPLIFIED = "simplified"` | +| `Edition` | `RAS_66N = "66n"`, `FSBU4 = "fsbu4"` | +| `PeriodKind` | `ABSOLUTE = "absolute"`, `CURRENT = "relative:current"`, `PREV_YEAR_END = "relative:prev_year_end"`, `PREV_PREV_YEAR_END = "relative:prev_prev_year_end"`, `PREV_PERIOD = "relative:prev_period"`, `NONE = "none"` | +| `LineStatus` | `OK = "ok"`, `UNREADABLE = "unreadable"`, `MISSING = "missing"`, `DETAIL = "detail"` | +| `Severity` | `ERROR = "error"`, `INFO = "info"` | +| `DocStatus` | `DONE = "done"`, `NEEDS_CHECK = "needs_check"` | +| `EnvelopeStatus` | `DONE = "done"`, `NEEDS_CHECK = "needs_check"`, `ERROR = "error"` | +| `SignForm` | `PARENTHESES = "parentheses"`, `MINUS = "minus"`, `UNSIGNED = "unsigned"` | +| `WarningCode` | по члену на код словаря `statement_warnings.yaml`, имя = значение: `JSON_FAILED`, `TABLE_PARSE_ERROR`, `TABLE_LOST`, `TABLE_SKIPPED`, `CODE_COLUMN_NOT_FOUND`, `SEGMENT_WITHOUT_CODES`, `SHEET_NOT_JOINED`, `CONTINUATION_TYPE_CONFLICT`, `SEGMENT_COLUMNS_MISMATCH`, `COLUMN_WITHOUT_DATE`, `COLUMN_DATE_UNCERTAIN`, `HEADER_ROW_NOT_FOUND`, `TYPE_CONFLICT`, `NO_ORGANIZATION_NO_INN`, `PERIOD_UNKNOWN`, `REPORT_DATE_UNCERTAIN`, `XLSB_SCALE_SUSPECT`, `INN_NOT_FOUND`, `INN_UNCERTAIN`, `ORGANIZATION_NOT_FOUND`, `ORGANIZATION_UNCERTAIN`, `REPORT_DATE_UNPARSED`, `UNITS_NOT_FOUND`, `FILE_TYPE_MISMATCH`, `COLUMN_SKIPPED`, `FRACTIONAL_IN_THOUSANDS`, `XLSB_NO_FORMATS`, `NUMBER_FORMAT_UNPARSED`, `VALUE_OUT_OF_RANGE`, `SHEET_OTHER`, `ROW_SPLIT`, `XLS_NO_CODEPAGE`. Коды `OSV_*` пути ОСВ сюда не входят (их создаёт tabular-parser, спека «Шаг 8 — предупреждения») | + +### Вход шага 1 (строит адаптер хоста) + +```python +CellValue = Decimal | date | bool | None + +Cell(text: str, value: CellValue, confidence: float, ref: str) + # text — как показывает источник; value — у OCR всегда None, у Excel — по числовому формату (Decimal), дата или bool; + # confidence 0–100 (Excel — 100); ref — OCR "row:", Excel "<лист>!" по правилу Excel. + # ref ячеек шапки (HeaderCells) и evidence пакет только читает: в выход шагов 2–8 он не переносится (у HeaderField + # поля ref нет), формат — на усмотрение адаптера (scan-gateway — "data_ocr.<поле ИС>", P5-5) +Evidence(okud: Cell | None = None, title: Cell | None = None) + # title.text без изменений становится StatementHeader.form_name (пустой после strip — None) +HeaderCells(organization_name, inn, report_date, units, okud, okpo, okopf, okfs, okei, address: Cell | None = None) +HEADER_FIELDS: tuple[str, ...] # имена полей HeaderCells по порядку +StatementWarning(code: str, severity: Severity, message: str, line_refs: tuple[str, ...] = (), + column_index: int | None = None, field: str | None = None, locator: str | None = None) +FormCandidate(source: Source, locator: str, type_hint: DocumentType | None, evidence: Evidence, header: HeaderCells, + rows: tuple[tuple[Cell | None, ...], ...], segment_breaks: tuple[int, ...] = (), + warnings: tuple[StatementWarning, ...] = ()) + # rows: позиция ячейки в строке = номер столбца сетки (OCR — column_index ИС, Excel — столбец листа с 0); + # locator: "file:<имя json>;table:" | "sheet:[,…]"; warnings — предупреждения адаптера документу + # (XLSB_NO_FORMATS, NUMBER_FORMAT_UNPARSED, CONTINUATION_TYPE_CONFLICT, SHEET_NOT_JOINED …) +``` + +### Выход шагов 2–8 + +```python +HeaderField(text: str | None, confidence: float | None, value: str | None) +DateField(text: str | None, confidence: float | None, value: date | None) +StatementHeader(organization_name: HeaderField, inn: HeaderField, report_date: DateField, units: HeaderField, + okud: HeaderField, okpo: HeaderField, okopf: HeaderField, okfs: HeaderField, okei: HeaderField, + address: HeaderField, form_name: str | None = None, needs_review_fields: tuple[str, ...] = ()) + # контракт: <поле> = value, <поле>_text = text, <поле>_confidence = confidence; form_name — header.form_name + # (заголовок формы как в источнике, из evidence.title кандидата; не теряется при normalize) +StatementColumn(column_index: int, label: str, label_confidence: float | None, period_date: date | None, + period_kind: PeriodKind) +StatementValue(column_index: int, amount: Decimal | None, string_value: str | None, text: str | None, + confidence: float, is_recognized: bool, needs_review: bool = False) +StatementLine(row_index: int, ref: str, line_status: LineStatus, line_code: str | None, line_code_text: str | None, + line_code_confidence: float | None, line_name: str | None, line_name_confidence: float | None, + parent_ref: str | None, needs_review: bool, values: tuple[StatementValue, ...]) + # values — по всем columns документа, в их порядке +FailedRelation(column_index: int | None, column_key: str, relation: str, codes: tuple[str, ...], + expected: Decimal | None, actual: Decimal | None) + # сервис: column_key = str(column_index); ядро: column_key — StoredColumn.key. + # expected is None ⇔ цель пуста, actual is None ⇔ пусты все члены (контракт: money, null — «цели нет»); такое + # посчитанное соотношение — всегда нарушение (спека «Шаг 7»), P1-21 +StatementChecks(relations_failed: tuple[FailedRelation, ...], soft_relations_failed: tuple[FailedRelation, ...]) +StatementDocument(document_type: DocumentType, form_variant: FormVariant | None, locator: str, + header: StatementHeader, columns: tuple[StatementColumn, ...], lines: tuple[StatementLine, ...], + checks: StatementChecks | None, status: DocStatus, warnings: tuple[StatementWarning, ...]) + # checks is None ⇔ lines == (); form_variant is None ⇔ DISCLOSURE (у BALANCE/INCOME без строк — FULL, P1-19); + # status == document_status(status_input(document)) всегда +CandidateResult(document: StatementDocument | None, file_warnings: tuple[StatementWarning, ...]) +NormalizationResult(documents: tuple[StatementDocument, ...], file_warnings: tuple[StatementWarning, ...]) +``` + +### Функции конвейера и разборов + +```python +normalize_candidate(candidate: FormCandidate, rules: Rules | None = None) -> CandidateResult + # шаги 2–8 одного кандидата; документа нет — file_warnings (TABLE_LOST | TABLE_SKIPPED | SHEET_OTHER) +normalize(candidates: Sequence[FormCandidate], rules: Rules | None = None) -> NormalizationResult + # по порядку; исключение кандидата не перехватывается — хост, которому нужна изоляция кандидатов + # (TABLE_PARSE_ERROR), зовёт normalize_candidate в цикле +recompute(form: StoredForm, rules: Rules | None = None) -> RecomputeResult +parse_header(cells: HeaderCells, rules: Rules) -> StatementHeader # form_name = None (ставит конвейер) +type_hint(okud_text: str | None, title_text: str | None, rules: Rules) -> DocumentType | None +is_continuation(text: str | None, rules: Rules) -> bool # маркер «Форма 0710001 с.2» (любой ОКУД) +continuation_type(text: str | None, rules: Rules) -> DocumentType | None # тип маркера; не маркер или ОКУД не из словаря — None +sheet_joins(form_type: DocumentType, sheet_codes: Sequence[str], sheet_amount_columns: int, + form_amount_columns: int, rules: Rules) -> bool # лист без маркера и шапки (адаптер Excel плана 4) +code_column(rows: Sequence[Sequence[Cell | None]], rules: Rules, *, relaxed: DocumentType | None = None) -> int | None + # столбец кодов сетки — правило (а)/(б) шага 2 (с защитой от блока «Коды»); relaxed=BALANCE|INCOME — ослабленный + # поиск в отрезке этой формы, минимум rules.code_ranges.min_codes_continuation; relaxed=DISCLOSURE — ValueError +amount_columns(rows: Sequence[Sequence[Cell | None]], code_col: int, rules: Rules, *, + source: Source = Source.EXCEL) -> tuple[int, ...] + # столбцы правее code_col, где в строках с кодом есть число — колонки сегмента без заголовка; «число колонок сумм» + # для sheet_joins — len(amount_columns(...)) +parse_code(text: str | None, rules: Rules) -> CodeParse # CodeParse(code: str | None, fixed: bool) +parse_amount(text: str | None, source: Source, rules: Rules) -> AmountParse + # AmountParse(amount: Decimal | None, string_value: str | None, is_recognized: bool, sign_form: SignForm | None) +parse_date(text: str | None, rules: Rules) -> date | None +resolve_relative(kind: PeriodKind, effective: date | None) -> date | None + # дата относительной колонки от эффективной даты (ядро — после правки даты отчёта); effective is None → None; + # ABSOLUTE и NONE → None, не исключение (P1-24) +choose_edition(document_type: DocumentType, codes: Sequence[tuple[str, bool]], effective: date | None, rules: Rules, + *, variant: FormVariant | None = None) -> Edition | None + # редакция формы без полного пересчёта (ядро — при приёме результата 2.0): codes — (код, уверенный) строк ok по + # порядку строк, уверенный ⇔ confidence == 100 или код зафиксирован аналитиком; variant None → по кодам; + # лучшая подпоследовательность перечня (больше уверенных, затем длиннее), ничья — по дате, ничья без даты — None; + # DISCLOSURE и пустые codes — по дате. recompute отдаёт ту же редакцию (P1-25) +fix_lookalikes(text: str, lookalikes: Mapping[str, str], *, anywhere: bool = False) -> str +normalize_caption(text: str | None) -> str # подписи колонок: нижний регистр, ё→е, всё кроме букв, цифр и % → пробел +normalize_label(text: str | None) -> str # метки шапки и заголовки: = normalize_text tabular-parser (P1-22) +make_warning(code: str, message: str, rules: Rules, *, line_refs: Sequence[str] = (), column_index: int | None = None, + field: str | None = None, locator: str | None = None) -> StatementWarning + # code — WarningCode (или строка того же значения); уровень — из словаря statement_warnings.yaml; неизвестный код — + # KeyError (предупреждение с выдуманным кодом не создаётся). Хосты создают JSON_FAILED, TABLE_PARSE_ERROR, + # FILE_TYPE_MISMATCH, XLS_NO_CODEPAGE, XLSB_NO_FORMATS, NUMBER_FORMAT_UNPARSED, CONTINUATION_TYPE_CONFLICT, + # SHEET_NOT_JOINED, SHEET_OTHER только через неё +to_jsonable(obj: object) -> Any # dataclass пакета → json-совместимое значение (golden) +candidate_from_dict(data: Mapping[str, Any]) -> FormCandidate # обратная для кандидата +statements_version(path: Path | None = None) -> str | None # содержимое api/arm_statements/VERSION копии +load_rules(directory: Path | None = None) -> Rules # RulesError при ошибке словаря +default_rules() -> Rules # кэш словарей пакета +``` + +`Rules` и `RulesError` входят в `__all__`: это тип результата `load_rules` / `default_rules` и ошибка словаря (хостам нужны для аннотаций и обработки `RulesError`); атрибуты `Rules`, доступные хостам, перечислены в разделе «Словари (`Rules`)» ниже. + +`CodeParse` и `AmountParse` в `__all__` не входят (возвращаются функциями; при необходимости аннотаций — +`arm_statements.codes.CodeParse`, `arm_statements.amounts.AmountParse`). + +### Форма контракта v2.0 (сервисы пишут, ядро читает) + +```python +to_contract(document: StatementDocument) -> dict[str, Any] + # statementDocument схемы v2.0: шапка плоская (form_name, <поле>, <поле>_text, <поле>_confidence для всех + # HEADER_FIELDS, needs_review_fields), деньги — строки format(x, "f") или null, у нарушений нет column_key, + # checks нет ⇔ строк нет, предупреждения — warning_to_contract без locator. Хост валидирует dict своей моделью +warning_to_contract(warning: StatementWarning, *, file_level: bool = False) -> dict[str, Any] + # {code, severity, message} + непустые line_refs / column_index / field; locator — только file_level=True + # (metrics.warnings); null у необязательных ключей не пишется — схема его не допускает +status_input_from_contract(document: Mapping[str, Any]) -> StatusInput + # вход функции статуса по statementDocument контракта (ядро: страховка document.status результата 2.0); + # status_input_from_contract(to_contract(d)) == status_input(d) +``` + +### Статус + +```python +StatusLine(line_status: LineStatus, needs_review: bool) +StatusValue(line_status: LineStatus, needs_review: bool) # line_status — статус строки значения +StatusInput(document_type: DocumentType, layout_found: bool, has_organization_or_inn: bool, period_known: bool, + needs_review_fields: tuple[str, ...], lines: tuple[StatusLine, ...], values: tuple[StatusValue, ...], + hard_failures: int, accepted: bool, warning_severities: tuple[Severity, ...]) + # needs_review на входе уже учитывает правки и подтверждения; значения строк DETAIL функция игнорирует +status_input(document: StatementDocument) -> StatusInput + # layout_found = строки есть; period_known = дата отчёта или колонка absolute с датой; hard_failures = + # len(checks.relations_failed); accepted = False — сервис и ядро собирают вход одним правилом +document_status(data: StatusInput) -> DocStatus +envelope_status(document_statuses: Sequence[DocStatus], file_warnings: Sequence[StatementWarning]) -> EnvelopeStatus + # ERROR при отсутствии документов; код ошибки (RECOGNITION_FAILED / INVALID_DOCUMENT) выбирает хост +``` + +### Пересчёт по сохранённым данным (ядро) + +```python +StoredColumn(key: str, column_index: int | None, period_date: date | None, period_kind: PeriodKind) +StoredValue(column_key: str, amount: Decimal | None, text: str | None, confidence: float | None, + is_recognized: bool, value_fixed: bool) + # text — source_text (у значения правки None); value_fixed — is_modified или is_confirmed +StoredLine(key: str, row_index: int | None, line_status: LineStatus, line_code: str | None, + line_code_confidence: float | None, line_name: str | None, code_fixed: bool, parent_key: str | None, + values: tuple[StoredValue, ...]) + # row_index None — строка правки (в §7 и варианте — пока есть сумма; в структуре и родителях detail — нет); + # code_fixed — line_code_modified или is_confirmed строки +StoredWarning(code: str, severity: Severity) +StoredForm(document_type: DocumentType, effective_date: date | None, has_organization_or_inn: bool, + period_known: bool, needs_review_fields: tuple[str, ...], accepted: bool, + warnings: tuple[StoredWarning, ...], columns: tuple[StoredColumn, ...], lines: tuple[StoredLine, ...]) +RecomputeResult(form_variant: FormVariant | None, edition: Edition | None, checks: StatementChecks, + line_needs_review: Mapping[str, bool], value_needs_review: Mapping[tuple[str, str], bool], + detail_parents: Mapping[str, str | None], status: DocStatus) + # value_needs_review: ключ (StoredLine.key, StoredValue.column_key); detail_parents — только строки DETAIL; + # checks.*.expected / actual — Decimal | None, как у сервиса +``` + +### Словари (`Rules`) — что нужно хостам + +Хосты читают словари только через эти атрибуты; прочие поля `Rules` — внутренние. + +```python +Rules.code_orders.sequence(edition: Edition, form: DocumentType, variant: FormVariant) -> tuple[str, ...] + # перечень кодов бланка с законными повторами (ядру — порядок и состав строк формы); DISCLOSURE → (); + # variant — FormVariant, не None: у формы без варианта хост передаёт FormVariant.FULL (как набор §7) +Rules.code_orders.editions -> tuple[Edition, ...] # порядок редакций перечня: (Edition.RAS_66N, Edition.FSBU4) +Rules.checks.tolerance -> Decimal # Decimal("1") — допуск ±1 (ядру — межформенная проверка) +Rules.code_ranges.min_codes -> int # 3 — «три и более кодов» (столбец кодов, sheet_joins) +Rules.code_ranges.min_codes_continuation -> int # 1 — минимум кодов слабого продолжения (code_column relaxed, sheet_joins) +Rules.headers.labels -> Mapping[str, tuple[re.Pattern[str], ...]] + # ключи ровно: organization, inn, okpo, okopf, okfs, okopf_okfs, okei, units, address, date (LABEL_KEYS) +Rules.headers.form_titles / okud -> Mapping[DocumentType, tuple[re.Pattern[str], ...]] +Rules.headers.okud_code -> re.Pattern[str] +Rules.headers.continuation / auditor / legal_form -> tuple[re.Pattern[str], ...] +Rules.headers.code_caption / name_caption / explanations_caption -> re.Pattern[str] +Rules.headers.column_exclusions -> tuple[re.Pattern[str], ...] +Rules.periods.relative -> tuple[tuple[PeriodKind, re.Pattern[str]], ...] +Rules.lookalikes -> Mapping[str, str] # буквы-двойники +Rules.warnings -> Mapping[str, WarningRule] # WarningRule(code, severity, scope: "document" | "file") +``` + +Какой текст подаётся шаблонам (`re.search`), — правило словаря, хост его не выбирает (P1-22): + +| Нормализация | Ключи | +|---|---| +| `normalize_label(text)` — пунктуация сохраняется | `headers.form_titles`, `okud`, `okud_code`, `continuation`, `labels.*`, `auditor`, `legal_form` | +| `normalize_caption(text)` — пунктуация → пробел | `headers.code_caption`, `name_caption`, `explanations_caption`, `column_exclusions`, `units.spellings`, `periods.relative` | + +## Как менять словари + +Словари пакета — данные в `arm_statements/rules/*.yaml`, правка идёт PR сюда, затем `make -C docs sync-statements` в +хостах («сначала пакет»). + +- `statement_code_orders.yaml` — **байтовая копия** `docs/contracts/statement-code-orders.yaml` (arm-ka-docs); руками не + правится. Перечень кодов меняется только PR в docs, затем копия переносится сюда без изменений. +- Новый код предупреждения — три правки: строка в `rules/statement_warnings.yaml` (уровень и область), член `WarningCode` + в `arm_statements/issues.py` (загрузчик требует равенства множеств), строка таблицы §11.9 `05-domain-rules.md` в docs и + таблица спеки «Шаг 8 — предупреждения». +- Новый синоним метки шапки, заголовка, подписи колонки — в соответствующий `statement_*.yaml` под нормализацию своего + ключа (таблица «ключ словаря → нормализация» в разделе «Публичный API»). +- Новое соотношение §7 — строка в `statement_checks.yaml`; прочая логика не меняется. +- После правки словаря: `make check`; golden-эталоны, которые изменились осознанно, пересобираются `make golden-update`. + +## golden-update + +Эталоны `arm_statements/tests/golden/*.expected.json` — `to_jsonable` результата `normalize_candidate` для пяти +обезличенных входов (`*.candidate.json`), закрепляющих сочетания, которые сценарии спеки описывают по одному. +Без переменной тест сравнивает с эталоном. Пересборка: + +```bash +make golden-update # GOLDEN_UPDATE=1 pytest arm_statements/tests/test_golden.py +``` + +Тест переписывает эталоны и падает с напоминанием. Дальше: просмотреть `git diff` эталонов глазами против Review Focus +плана 1 (каждое изменение — осознанное), запустить `make check` без переменной и закоммитить эталоны вместе с кодом. +Реальные названия организаций, ИНН и адреса во входах запрещены — только обезличенные. + diff --git a/arm_statements/__init__.py b/arm_statements/__init__.py index d75b97f..34ad4e5 100644 --- a/arm_statements/__init__.py +++ b/arm_statements/__init__.py @@ -11,7 +11,7 @@ from arm_statements.dates import parse_date, resolve_relative from arm_statements.header import continuation_type, is_continuation, parse_header, type_hint from arm_statements.issues import WarningCode, make_warning from arm_statements.layout import code_column -from arm_statements.loader import Rules, default_rules, load_rules +from arm_statements.loader import Rules, RulesError, default_rules, load_rules from arm_statements.model import ( HEADER_FIELDS, CandidateResult, @@ -86,6 +86,7 @@ __all__ = [ "PeriodKind", "RecomputeResult", "Rules", + "RulesError", "Severity", "SignForm", "Source", diff --git a/arm_statements/tests/golden/aep3_balance.candidate.json b/arm_statements/tests/golden/aep3_balance.candidate.json new file mode 100644 index 0000000..a3c898e --- /dev/null +++ b/arm_statements/tests/golden/aep3_balance.candidate.json @@ -0,0 +1,750 @@ +{ + "source": "ocr", + "locator": "file:aep3.json;table:0", + "type_hint": "BALANCE", + "evidence": { + "okud": null, + "title": { + "text": "Бухгалтерский баланс", + "value": null, + "confidence": 100.0, + "ref": "evidence:title" + } + }, + "header": { + "organization_name": { + "text": "НКО «Пример»", + "value": null, + "confidence": 100.0, + "ref": "hdr:organization_name" + }, + "inn": { + "text": "3|1 21 2|", + "value": null, + "confidence": 80.0, + "ref": "hdr:inn" + }, + "report_date": { + "text": "31.12.2024", + "value": null, + "confidence": 75.0, + "ref": "hdr:report_date" + }, + "units": { + "text": "(тыс. руб. - год по ОКЕИ)", + "value": null, + "confidence": 69.0, + "ref": "hdr:units" + }, + "okud": null, + "okpo": null, + "okopf": null, + "okfs": null, + "okei": null, + "address": null + }, + "rows": [ + [ + { + "text": "Наименование показателя", + "value": null, + "confidence": 100.0, + "ref": "row:0" + }, + { + "text": "Код", + "value": null, + "confidence": 100.0, + "ref": "row:0" + }, + { + "text": "На 31 декабря 2024 г.", + "value": null, + "confidence": 100.0, + "ref": "row:0" + } + ], + [ + { + "text": "Нематериальные активы", + "value": null, + "confidence": 100.0, + "ref": "row:1" + }, + { + "text": "1110", + "value": null, + "confidence": 100.0, + "ref": "row:1" + }, + { + "text": "30127", + "value": null, + "confidence": 60.0, + "ref": "row:1" + } + ], + [ + { + "text": "в том числе лицензии", + "value": null, + "confidence": 100.0, + "ref": "row:2" + }, + { + "text": "НИ", + "value": null, + "confidence": 0.0, + "ref": "row:2" + }, + { + "text": "100", + "value": null, + "confidence": 60.0, + "ref": "row:2" + } + ], + [ + { + "text": "Материальные поисковые\nактивы", + "value": null, + "confidence": 100.0, + "ref": "row:3" + }, + { + "text": "| 140", + "value": null, + "confidence": 40.0, + "ref": "row:3" + }, + { + "text": "500", + "value": null, + "confidence": 100.0, + "ref": "row:3" + } + ], + [ + { + "text": "Нематериальные поисковые\nактивы", + "value": null, + "confidence": 100.0, + "ref": "row:4" + }, + { + "text": "изо :", + "value": null, + "confidence": 80.0, + "ref": "row:4" + }, + { + "text": "3", + "value": null, + "confidence": 0.0, + "ref": "row:4" + } + ], + [ + { + "text": "Основные средства", + "value": null, + "confidence": 100.0, + "ref": "row:5" + }, + { + "text": "1150", + "value": null, + "confidence": 100.0, + "ref": "row:5" + }, + { + "text": "2875532", + "value": null, + "confidence": 100.0, + "ref": "row:5" + } + ], + [ + { + "text": "Итого по разделу I", + "value": null, + "confidence": 100.0, + "ref": "row:6" + }, + { + "text": "1100", + "value": null, + "confidence": 80.0, + "ref": "row:6" + }, + { + "text": "2906159", + "value": null, + "confidence": 100.0, + "ref": "row:6" + } + ], + [ + { + "text": "Запасы", + "value": null, + "confidence": 100.0, + "ref": "row:7" + }, + { + "text": "1210", + "value": null, + "confidence": 100.0, + "ref": "row:7" + }, + { + "text": "1108817", + "value": null, + "confidence": 100.0, + "ref": "row:7" + } + ], + [ + { + "text": "Налог на добавленную стоимость\nпо приобретённым ценностям", + "value": null, + "confidence": 100.0, + "ref": "row:8" + }, + { + "text": "1220", + "value": null, + "confidence": 100.0, + "ref": "row:8" + }, + { + "text": "157", + "value": null, + "confidence": 100.0, + "ref": "row:8" + } + ], + [ + { + "text": "Дебиторская задолженность", + "value": null, + "confidence": 100.0, + "ref": "row:9" + }, + { + "text": "1230", + "value": null, + "confidence": 100.0, + "ref": "row:9" + }, + { + "text": "1285022", + "value": null, + "confidence": 100.0, + "ref": "row:9" + } + ], + [ + { + "text": "Финансовые вложения", + "value": null, + "confidence": 100.0, + "ref": "row:10" + }, + { + "text": "1240", + "value": null, + "confidence": 100.0, + "ref": "row:10" + }, + { + "text": "183600", + "value": null, + "confidence": 100.0, + "ref": "row:10" + } + ], + [ + { + "text": "Денежные средства", + "value": null, + "confidence": 100.0, + "ref": "row:11" + }, + { + "text": "1250", + "value": null, + "confidence": 100.0, + "ref": "row:11" + }, + { + "text": "4447", + "value": null, + "confidence": 100.0, + "ref": "row:11" + } + ], + [ + { + "text": "Прочие оборотные активы", + "value": null, + "confidence": 100.0, + "ref": "row:12" + }, + { + "text": "1260", + "value": null, + "confidence": 100.0, + "ref": "row:12" + }, + { + "text": "30524", + "value": null, + "confidence": 100.0, + "ref": "row:12" + } + ], + [ + { + "text": "Итого по разделу II", + "value": null, + "confidence": 100.0, + "ref": "row:13" + }, + { + "text": "1200", + "value": null, + "confidence": 100.0, + "ref": "row:13" + }, + { + "text": "2612567", + "value": null, + "confidence": 100.0, + "ref": "row:13" + } + ], + [ + { + "text": "БАЛАНС", + "value": null, + "confidence": 100.0, + "ref": "row:14" + }, + { + "text": "1600", + "value": null, + "confidence": 100.0, + "ref": "row:14" + }, + { + "text": "5518726", + "value": null, + "confidence": 100.0, + "ref": "row:14" + } + ], + [ + { + "text": "Переоценка внеоборотных активов", + "value": null, + "confidence": 100.0, + "ref": "row:15" + }, + { + "text": "1340", + "value": null, + "confidence": 100.0, + "ref": "row:15" + }, + { + "text": "146927", + "value": null, + "confidence": 100.0, + "ref": "row:15" + } + ], + [ + { + "text": "Добавочный капитал", + "value": null, + "confidence": 100.0, + "ref": "row:16" + }, + { + "text": "1350", + "value": null, + "confidence": 100.0, + "ref": "row:16" + }, + { + "text": "24", + "value": null, + "confidence": 100.0, + "ref": "row:16" + } + ], + [ + { + "text": "Резервный капитал", + "value": null, + "confidence": 100.0, + "ref": "row:17" + }, + { + "text": "1360", + "value": null, + "confidence": 100.0, + "ref": "row:17" + }, + { + "text": "2416", + "value": null, + "confidence": 100.0, + "ref": "row:17" + } + ], + [ + { + "text": "Нераспределенная прибыль", + "value": null, + "confidence": 100.0, + "ref": "row:18" + }, + { + "text": "1370", + "value": null, + "confidence": 100.0, + "ref": "row:18" + }, + { + "text": "3262545", + "value": null, + "confidence": 100.0, + "ref": "row:18" + } + ], + [ + { + "text": "Итого по разделу III", + "value": null, + "confidence": 100.0, + "ref": "row:19" + }, + { + "text": "1300", + "value": null, + "confidence": 100.0, + "ref": "row:19" + }, + { + "text": "3411912", + "value": null, + "confidence": 100.0, + "ref": "row:19" + } + ], + [ + { + "text": "Паевой фонд", + "value": null, + "confidence": 100.0, + "ref": "row:20" + }, + { + "text": "1310", + "value": null, + "confidence": 100.0, + "ref": "row:20" + }, + { + "text": "", + "value": null, + "confidence": 100.0, + "ref": "row:20" + } + ], + [ + { + "text": "Целевой капитал", + "value": null, + "confidence": 100.0, + "ref": "row:21" + }, + { + "text": "1320", + "value": null, + "confidence": 100.0, + "ref": "row:21" + }, + { + "text": "", + "value": null, + "confidence": 100.0, + "ref": "row:21" + } + ], + [ + { + "text": "Целевые средства", + "value": null, + "confidence": 100.0, + "ref": "row:22" + }, + { + "text": "1350", + "value": null, + "confidence": 100.0, + "ref": "row:22" + }, + { + "text": "", + "value": null, + "confidence": 100.0, + "ref": "row:22" + } + ], + [ + { + "text": "Фонд недвижимого и особо\nценного движимого имущества", + "value": null, + "confidence": 100.0, + "ref": "row:23" + }, + { + "text": "1360", + "value": null, + "confidence": 100.0, + "ref": "row:23" + }, + { + "text": "", + "value": null, + "confidence": 100.0, + "ref": "row:23" + } + ], + [ + { + "text": "Резервный и иные целевые\nфонды", + "value": null, + "confidence": 100.0, + "ref": "row:24" + }, + { + "text": "1370", + "value": null, + "confidence": 100.0, + "ref": "row:24" + }, + { + "text": "", + "value": null, + "confidence": 100.0, + "ref": "row:24" + } + ], + [ + { + "text": "Итого по разделу III", + "value": null, + "confidence": 100.0, + "ref": "row:25" + }, + { + "text": "1300", + "value": null, + "confidence": 100.0, + "ref": "row:25" + }, + { + "text": "", + "value": null, + "confidence": 100.0, + "ref": "row:25" + } + ], + [ + { + "text": "Заемные средства", + "value": null, + "confidence": 100.0, + "ref": "row:26" + }, + { + "text": "1410", + "value": null, + "confidence": 100.0, + "ref": "row:26" + }, + { + "text": "628256", + "value": null, + "confidence": 100.0, + "ref": "row:26" + } + ], + [ + { + "text": "Отложенные налоговые\nобязательства", + "value": null, + "confidence": 100.0, + "ref": "row:27" + }, + { + "text": "1420", + "value": null, + "confidence": 100.0, + "ref": "row:27" + }, + { + "text": "43094", + "value": null, + "confidence": 100.0, + "ref": "row:27" + } + ], + [ + { + "text": "Итого по разделу IV", + "value": null, + "confidence": 100.0, + "ref": "row:28" + }, + { + "text": "1400", + "value": null, + "confidence": 100.0, + "ref": "row:28" + }, + { + "text": "671350", + "value": null, + "confidence": 100.0, + "ref": "row:28" + } + ], + [ + { + "text": "Заемные средства", + "value": null, + "confidence": 100.0, + "ref": "row:29" + }, + { + "text": "1510", + "value": null, + "confidence": 100.0, + "ref": "row:29" + }, + { + "text": "570663", + "value": null, + "confidence": 100.0, + "ref": "row:29" + } + ], + [ + { + "text": "Кредиторская задолженность", + "value": null, + "confidence": 100.0, + "ref": "row:30" + }, + { + "text": "1520", + "value": null, + "confidence": 100.0, + "ref": "row:30" + }, + { + "text": "817769", + "value": null, + "confidence": 100.0, + "ref": "row:30" + } + ], + [ + { + "text": "Оценочные обязательства", + "value": null, + "confidence": 100.0, + "ref": "row:31" + }, + { + "text": "1540", + "value": null, + "confidence": 100.0, + "ref": "row:31" + }, + { + "text": "47001", + "value": null, + "confidence": 100.0, + "ref": "row:31" + } + ], + [ + { + "text": "Прочие обязательства", + "value": null, + "confidence": 100.0, + "ref": "row:32" + }, + { + "text": "1550", + "value": null, + "confidence": 100.0, + "ref": "row:32" + }, + { + "text": "31", + "value": null, + "confidence": 100.0, + "ref": "row:32" + } + ], + [ + { + "text": "Итого по разделу V", + "value": null, + "confidence": 100.0, + "ref": "row:33" + }, + { + "text": "1500", + "value": null, + "confidence": 100.0, + "ref": "row:33" + }, + { + "text": "1435464", + "value": null, + "confidence": 100.0, + "ref": "row:33" + } + ], + [ + { + "text": "БАЛАНС", + "value": null, + "confidence": 100.0, + "ref": "row:34" + }, + { + "text": "1700", + "value": null, + "confidence": 100.0, + "ref": "row:34" + }, + { + "text": "5518726", + "value": null, + "confidence": 100.0, + "ref": "row:34" + } + ] + ], + "segment_breaks": [], + "warnings": [] +} diff --git a/arm_statements/tests/golden/aep3_balance.expected.json b/arm_statements/tests/golden/aep3_balance.expected.json new file mode 100644 index 0000000..a8366b6 --- /dev/null +++ b/arm_statements/tests/golden/aep3_balance.expected.json @@ -0,0 +1,868 @@ +{ + "document_type": "BALANCE", + "form_variant": "full", + "locator": "file:aep3.json;table:0", + "header": { + "organization_name": { + "text": "НКО «Пример»", + "confidence": 100.0, + "value": "НКО «Пример»" + }, + "inn": { + "text": "3|1 21 2|", + "confidence": 80.0, + "value": null + }, + "report_date": { + "text": "31.12.2024", + "confidence": 75.0, + "value": "2024-12-31" + }, + "units": { + "text": "(тыс. руб. - год по ОКЕИ)", + "confidence": 69.0, + "value": "тыс. руб." + }, + "okud": { + "text": null, + "confidence": null, + "value": null + }, + "okpo": { + "text": null, + "confidence": null, + "value": null + }, + "okopf": { + "text": null, + "confidence": null, + "value": null + }, + "okfs": { + "text": null, + "confidence": null, + "value": null + }, + "okei": { + "text": null, + "confidence": null, + "value": null + }, + "address": { + "text": null, + "confidence": null, + "value": null + }, + "form_name": "Бухгалтерский баланс", + "needs_review_fields": [] + }, + "columns": [ + { + "column_index": 2, + "label": "На 31 декабря 2024 г.", + "label_confidence": 100.0, + "period_date": "2024-12-31", + "period_kind": "absolute" + } + ], + "lines": [ + { + "row_index": 0, + "ref": "row:1", + "line_status": "ok", + "line_code": "1110", + "line_code_text": "1110", + "line_code_confidence": 100.0, + "line_name": "Нематериальные активы", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "30127.00", + "string_value": null, + "text": "30127", + "confidence": 60.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 1, + "ref": "row:2", + "line_status": "detail", + "line_code": null, + "line_code_text": "НИ", + "line_code_confidence": 0.0, + "line_name": "в том числе лицензии", + "line_name_confidence": 100.0, + "parent_ref": "row:1", + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "100.00", + "string_value": null, + "text": "100", + "confidence": 60.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 2, + "ref": "row:3", + "line_status": "ok", + "line_code": "1140", + "line_code_text": "| 140", + "line_code_confidence": 40.0, + "line_name": "Материальные поисковые\nактивы", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "500.00", + "string_value": null, + "text": "500", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 3, + "ref": "row:4", + "line_status": "detail", + "line_code": null, + "line_code_text": "изо :", + "line_code_confidence": 80.0, + "line_name": "Нематериальные поисковые\nактивы", + "line_name_confidence": 100.0, + "parent_ref": "row:3", + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "3.00", + "string_value": null, + "text": "3", + "confidence": 0.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 4, + "ref": "row:5", + "line_status": "ok", + "line_code": "1150", + "line_code_text": "1150", + "line_code_confidence": 100.0, + "line_name": "Основные средства", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "2875532.00", + "string_value": null, + "text": "2875532", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 5, + "ref": "row:6", + "line_status": "ok", + "line_code": "1100", + "line_code_text": "1100", + "line_code_confidence": 80.0, + "line_name": "Итого по разделу I", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "2906159.00", + "string_value": null, + "text": "2906159", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 6, + "ref": "row:7", + "line_status": "ok", + "line_code": "1210", + "line_code_text": "1210", + "line_code_confidence": 100.0, + "line_name": "Запасы", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "1108817.00", + "string_value": null, + "text": "1108817", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 7, + "ref": "row:8", + "line_status": "ok", + "line_code": "1220", + "line_code_text": "1220", + "line_code_confidence": 100.0, + "line_name": "Налог на добавленную стоимость\nпо приобретённым ценностям", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "157.00", + "string_value": null, + "text": "157", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 8, + "ref": "row:9", + "line_status": "ok", + "line_code": "1230", + "line_code_text": "1230", + "line_code_confidence": 100.0, + "line_name": "Дебиторская задолженность", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "1285022.00", + "string_value": null, + "text": "1285022", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 9, + "ref": "row:10", + "line_status": "ok", + "line_code": "1240", + "line_code_text": "1240", + "line_code_confidence": 100.0, + "line_name": "Финансовые вложения", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "183600.00", + "string_value": null, + "text": "183600", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 10, + "ref": "row:11", + "line_status": "ok", + "line_code": "1250", + "line_code_text": "1250", + "line_code_confidence": 100.0, + "line_name": "Денежные средства", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "4447.00", + "string_value": null, + "text": "4447", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 11, + "ref": "row:12", + "line_status": "ok", + "line_code": "1260", + "line_code_text": "1260", + "line_code_confidence": 100.0, + "line_name": "Прочие оборотные активы", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "30524.00", + "string_value": null, + "text": "30524", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 12, + "ref": "row:13", + "line_status": "ok", + "line_code": "1200", + "line_code_text": "1200", + "line_code_confidence": 100.0, + "line_name": "Итого по разделу II", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "2612567.00", + "string_value": null, + "text": "2612567", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 13, + "ref": "row:14", + "line_status": "ok", + "line_code": "1600", + "line_code_text": "1600", + "line_code_confidence": 100.0, + "line_name": "БАЛАНС", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "5518726.00", + "string_value": null, + "text": "5518726", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 14, + "ref": "row:15", + "line_status": "ok", + "line_code": "1340", + "line_code_text": "1340", + "line_code_confidence": 100.0, + "line_name": "Переоценка внеоборотных активов", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "146927.00", + "string_value": null, + "text": "146927", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 15, + "ref": "row:16", + "line_status": "ok", + "line_code": "1350", + "line_code_text": "1350", + "line_code_confidence": 100.0, + "line_name": "Добавочный капитал", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "24.00", + "string_value": null, + "text": "24", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 16, + "ref": "row:17", + "line_status": "ok", + "line_code": "1360", + "line_code_text": "1360", + "line_code_confidence": 100.0, + "line_name": "Резервный капитал", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "2416.00", + "string_value": null, + "text": "2416", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 17, + "ref": "row:18", + "line_status": "ok", + "line_code": "1370", + "line_code_text": "1370", + "line_code_confidence": 100.0, + "line_name": "Нераспределенная прибыль", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "3262545.00", + "string_value": null, + "text": "3262545", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 18, + "ref": "row:19", + "line_status": "ok", + "line_code": "1300", + "line_code_text": "1300", + "line_code_confidence": 100.0, + "line_name": "Итого по разделу III", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "3411912.00", + "string_value": null, + "text": "3411912", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 19, + "ref": "row:20", + "line_status": "ok", + "line_code": "1310", + "line_code_text": "1310", + "line_code_confidence": 100.0, + "line_name": "Паевой фонд", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": null, + "string_value": null, + "text": "", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 20, + "ref": "row:21", + "line_status": "ok", + "line_code": "1320", + "line_code_text": "1320", + "line_code_confidence": 100.0, + "line_name": "Целевой капитал", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": null, + "string_value": null, + "text": "", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 21, + "ref": "row:22", + "line_status": "ok", + "line_code": "1350", + "line_code_text": "1350", + "line_code_confidence": 100.0, + "line_name": "Целевые средства", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": null, + "string_value": null, + "text": "", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 22, + "ref": "row:23", + "line_status": "ok", + "line_code": "1360", + "line_code_text": "1360", + "line_code_confidence": 100.0, + "line_name": "Фонд недвижимого и особо\nценного движимого имущества", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": null, + "string_value": null, + "text": "", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 23, + "ref": "row:24", + "line_status": "ok", + "line_code": "1370", + "line_code_text": "1370", + "line_code_confidence": 100.0, + "line_name": "Резервный и иные целевые\nфонды", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": null, + "string_value": null, + "text": "", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 24, + "ref": "row:25", + "line_status": "ok", + "line_code": "1300", + "line_code_text": "1300", + "line_code_confidence": 100.0, + "line_name": "Итого по разделу III", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": null, + "string_value": null, + "text": "", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 25, + "ref": "row:26", + "line_status": "ok", + "line_code": "1410", + "line_code_text": "1410", + "line_code_confidence": 100.0, + "line_name": "Заемные средства", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "628256.00", + "string_value": null, + "text": "628256", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 26, + "ref": "row:27", + "line_status": "ok", + "line_code": "1420", + "line_code_text": "1420", + "line_code_confidence": 100.0, + "line_name": "Отложенные налоговые\nобязательства", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "43094.00", + "string_value": null, + "text": "43094", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 27, + "ref": "row:28", + "line_status": "ok", + "line_code": "1400", + "line_code_text": "1400", + "line_code_confidence": 100.0, + "line_name": "Итого по разделу IV", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "671350.00", + "string_value": null, + "text": "671350", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 28, + "ref": "row:29", + "line_status": "ok", + "line_code": "1510", + "line_code_text": "1510", + "line_code_confidence": 100.0, + "line_name": "Заемные средства", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "570663.00", + "string_value": null, + "text": "570663", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 29, + "ref": "row:30", + "line_status": "ok", + "line_code": "1520", + "line_code_text": "1520", + "line_code_confidence": 100.0, + "line_name": "Кредиторская задолженность", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "817769.00", + "string_value": null, + "text": "817769", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 30, + "ref": "row:31", + "line_status": "ok", + "line_code": "1540", + "line_code_text": "1540", + "line_code_confidence": 100.0, + "line_name": "Оценочные обязательства", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "47001.00", + "string_value": null, + "text": "47001", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 31, + "ref": "row:32", + "line_status": "ok", + "line_code": "1550", + "line_code_text": "1550", + "line_code_confidence": 100.0, + "line_name": "Прочие обязательства", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "31.00", + "string_value": null, + "text": "31", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 32, + "ref": "row:33", + "line_status": "ok", + "line_code": "1500", + "line_code_text": "1500", + "line_code_confidence": 100.0, + "line_name": "Итого по разделу V", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "1435464.00", + "string_value": null, + "text": "1435464", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 33, + "ref": "row:34", + "line_status": "ok", + "line_code": "1700", + "line_code_text": "1700", + "line_code_confidence": 100.0, + "line_name": "БАЛАНС", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "5518726.00", + "string_value": null, + "text": "5518726", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + } + ], + "checks": { + "relations_failed": [], + "soft_relations_failed": [] + }, + "status": "done", + "warnings": [ + { + "code": "INN_NOT_FOUND", + "severity": "info", + "message": "ИНН не найден", + "line_refs": [], + "column_index": null, + "field": "inn", + "locator": null + } + ] +} diff --git a/arm_statements/tests/golden/aep3_income.candidate.json b/arm_statements/tests/golden/aep3_income.candidate.json new file mode 100644 index 0000000..265443e --- /dev/null +++ b/arm_statements/tests/golden/aep3_income.candidate.json @@ -0,0 +1,290 @@ +{ + "source": "ocr", + "locator": "file:aep3_ofr.json;table:0", + "type_hint": "INCOME", + "evidence": { + "okud": null, + "title": { + "text": "Отчет о финансовых результатах", + "value": null, + "confidence": 100.0, + "ref": "evidence:title" + } + }, + "header": { + "organization_name": { + "text": "НКО «Пример»", + "value": null, + "confidence": 100.0, + "ref": "hdr:organization_name" + }, + "inn": { + "text": "7700000000", + "value": null, + "confidence": 100.0, + "ref": "hdr:inn" + }, + "report_date": { + "text": "31.12.2024", + "value": null, + "confidence": 75.0, + "ref": "hdr:report_date" + }, + "units": { + "text": "тыс. руб.", + "value": null, + "confidence": 100.0, + "ref": "hdr:units" + }, + "okud": null, + "okpo": null, + "okopf": null, + "okfs": null, + "okei": null, + "address": null + }, + "rows": [ + [ + { + "text": "Наименование показателя", + "value": null, + "confidence": 100.0, + "ref": "row:0" + }, + { + "text": "Код", + "value": null, + "confidence": 100.0, + "ref": "row:0" + }, + { + "text": "За 2024 г.", + "value": null, + "confidence": 100.0, + "ref": "row:0" + } + ], + [ + { + "text": "Выручка", + "value": null, + "confidence": 100.0, + "ref": "row:1" + }, + { + "text": "2110", + "value": null, + "confidence": 100.0, + "ref": "row:1" + }, + { + "text": "8294952", + "value": null, + "confidence": 100.0, + "ref": "row:1" + } + ], + [ + { + "text": "Себестоимость продаж", + "value": null, + "confidence": 100.0, + "ref": "row:2" + }, + { + "text": "2120", + "value": null, + "confidence": 100.0, + "ref": "row:2" + }, + { + "text": "(7744536}", + "value": null, + "confidence": 88.9, + "ref": "row:2" + } + ], + [ + { + "text": "Валовая прибыль (убыток)", + "value": null, + "confidence": 100.0, + "ref": "row:3" + }, + { + "text": "2100", + "value": null, + "confidence": 100.0, + "ref": "row:3" + }, + { + "text": "550416", + "value": null, + "confidence": 100.0, + "ref": "row:3" + } + ], + [ + { + "text": "Коммерческие расходы", + "value": null, + "confidence": 100.0, + "ref": "row:4" + }, + { + "text": "2210", + "value": null, + "confidence": 100.0, + "ref": "row:4" + }, + { + "text": "(100)", + "value": null, + "confidence": 100.0, + "ref": "row:4" + } + ], + [ + { + "text": "Управленческие расходы", + "value": null, + "confidence": 100.0, + "ref": "row:5" + }, + { + "text": "2220", + "value": null, + "confidence": 100.0, + "ref": "row:5" + }, + { + "text": "(200)", + "value": null, + "confidence": 100.0, + "ref": "row:5" + } + ], + [ + { + "text": "Прибыль (убыток) от продаж", + "value": null, + "confidence": 100.0, + "ref": "row:6" + }, + { + "text": "2200", + "value": null, + "confidence": 100.0, + "ref": "row:6" + }, + { + "text": "550116", + "value": null, + "confidence": 100.0, + "ref": "row:6" + } + ], + [ + { + "text": "Доходы от участия в других\nорганизациях", + "value": null, + "confidence": 100.0, + "ref": "row:7" + }, + { + "text": "2310", + "value": null, + "confidence": 100.0, + "ref": "row:7" + }, + { + "text": "211(1 |", + "value": null, + "confidence": 85.7, + "ref": "row:7" + } + ], + [ + { + "text": "Проценты к получению", + "value": null, + "confidence": 100.0, + "ref": "row:8" + }, + { + "text": "2320", + "value": null, + "confidence": 100.0, + "ref": "row:8" + }, + { + "text": "42831 1", + "value": null, + "confidence": 90.0, + "ref": "row:8" + } + ], + [ + { + "text": "Прибыль (убыток) до налогообложения", + "value": null, + "confidence": 100.0, + "ref": "row:9" + }, + { + "text": "2300", + "value": null, + "confidence": 100.0, + "ref": "row:9" + }, + { + "text": "978427", + "value": null, + "confidence": 100.0, + "ref": "row:9" + } + ], + [ + { + "text": "Налог на прибыль", + "value": null, + "confidence": 100.0, + "ref": "row:10" + }, + { + "text": "2410", + "value": null, + "confidence": 100.0, + "ref": "row:10" + }, + { + "text": "(108450)", + "value": null, + "confidence": 100.0, + "ref": "row:10" + } + ], + [ + { + "text": "Чистая прибыль (убыток)", + "value": null, + "confidence": 100.0, + "ref": "row:11" + }, + { + "text": "2400", + "value": null, + "confidence": 100.0, + "ref": "row:11" + }, + { + "text": "869977", + "value": null, + "confidence": 100.0, + "ref": "row:11" + } + ] + ], + "segment_breaks": [], + "warnings": [] +} diff --git a/arm_statements/tests/golden/aep3_income.expected.json b/arm_statements/tests/golden/aep3_income.expected.json new file mode 100644 index 0000000..4c8973e --- /dev/null +++ b/arm_statements/tests/golden/aep3_income.expected.json @@ -0,0 +1,356 @@ +{ + "document_type": "INCOME", + "form_variant": "full", + "locator": "file:aep3_ofr.json;table:0", + "header": { + "organization_name": { + "text": "НКО «Пример»", + "confidence": 100.0, + "value": "НКО «Пример»" + }, + "inn": { + "text": "7700000000", + "confidence": 100.0, + "value": "7700000000" + }, + "report_date": { + "text": "31.12.2024", + "confidence": 75.0, + "value": "2024-12-31" + }, + "units": { + "text": "тыс. руб.", + "confidence": 100.0, + "value": "тыс. руб." + }, + "okud": { + "text": null, + "confidence": null, + "value": null + }, + "okpo": { + "text": null, + "confidence": null, + "value": null + }, + "okopf": { + "text": null, + "confidence": null, + "value": null + }, + "okfs": { + "text": null, + "confidence": null, + "value": null + }, + "okei": { + "text": null, + "confidence": null, + "value": null + }, + "address": { + "text": null, + "confidence": null, + "value": null + }, + "form_name": "Отчет о финансовых результатах", + "needs_review_fields": [] + }, + "columns": [ + { + "column_index": 2, + "label": "За 2024 г.", + "label_confidence": 100.0, + "period_date": "2024-12-31", + "period_kind": "absolute" + } + ], + "lines": [ + { + "row_index": 0, + "ref": "row:1", + "line_status": "ok", + "line_code": "2110", + "line_code_text": "2110", + "line_code_confidence": 100.0, + "line_name": "Выручка", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "8294952.00", + "string_value": null, + "text": "8294952", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 1, + "ref": "row:2", + "line_status": "ok", + "line_code": "2120", + "line_code_text": "2120", + "line_code_confidence": 100.0, + "line_name": "Себестоимость продаж", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "-7744536.00", + "string_value": null, + "text": "(7744536}", + "confidence": 88.9, + "is_recognized": false, + "needs_review": false + } + ] + }, + { + "row_index": 2, + "ref": "row:3", + "line_status": "ok", + "line_code": "2100", + "line_code_text": "2100", + "line_code_confidence": 100.0, + "line_name": "Валовая прибыль (убыток)", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "550416.00", + "string_value": null, + "text": "550416", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 3, + "ref": "row:4", + "line_status": "ok", + "line_code": "2210", + "line_code_text": "2210", + "line_code_confidence": 100.0, + "line_name": "Коммерческие расходы", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "-100.00", + "string_value": null, + "text": "(100)", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 4, + "ref": "row:5", + "line_status": "ok", + "line_code": "2220", + "line_code_text": "2220", + "line_code_confidence": 100.0, + "line_name": "Управленческие расходы", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "-200.00", + "string_value": null, + "text": "(200)", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 5, + "ref": "row:6", + "line_status": "ok", + "line_code": "2200", + "line_code_text": "2200", + "line_code_confidence": 100.0, + "line_name": "Прибыль (убыток) от продаж", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "550116.00", + "string_value": null, + "text": "550116", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 6, + "ref": "row:7", + "line_status": "ok", + "line_code": "2310", + "line_code_text": "2310", + "line_code_confidence": 100.0, + "line_name": "Доходы от участия в других\nорганизациях", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": null, + "string_value": "211(1 |", + "text": "211(1 |", + "confidence": 85.7, + "is_recognized": false, + "needs_review": true + } + ] + }, + { + "row_index": 7, + "ref": "row:8", + "line_status": "ok", + "line_code": "2320", + "line_code_text": "2320", + "line_code_confidence": 100.0, + "line_name": "Проценты к получению", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "428311", + "string_value": null, + "text": "42831 1", + "confidence": 90.0, + "is_recognized": false, + "needs_review": false + } + ] + }, + { + "row_index": 8, + "ref": "row:9", + "line_status": "ok", + "line_code": "2300", + "line_code_text": "2300", + "line_code_confidence": 100.0, + "line_name": "Прибыль (убыток) до налогообложения", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "978427.00", + "string_value": null, + "text": "978427", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 9, + "ref": "row:10", + "line_status": "ok", + "line_code": "2410", + "line_code_text": "2410", + "line_code_confidence": 100.0, + "line_name": "Налог на прибыль", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "-108450.00", + "string_value": null, + "text": "(108450)", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 10, + "ref": "row:11", + "line_status": "ok", + "line_code": "2400", + "line_code_text": "2400", + "line_code_confidence": 100.0, + "line_name": "Чистая прибыль (убыток)", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "869977.00", + "string_value": null, + "text": "869977", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + } + ], + "checks": { + "relations_failed": [], + "soft_relations_failed": [ + { + "column_index": 2, + "column_key": "2", + "relation": "2410 = 2411 + 2412", + "codes": [ + "2410", + "2411", + "2412" + ], + "expected": "-108450.00", + "actual": null + }, + { + "column_index": 2, + "column_key": "2", + "relation": "2500 = 2400 + 2510 + 2520 + 2530", + "codes": [ + "2500", + "2400", + "2510", + "2520", + "2530" + ], + "expected": null, + "actual": "869977.00" + } + ] + }, + "status": "needs_check", + "warnings": [] +} diff --git a/arm_statements/tests/golden/glued_balance.candidate.json b/arm_statements/tests/golden/glued_balance.candidate.json new file mode 100644 index 0000000..b88ba09 --- /dev/null +++ b/arm_statements/tests/golden/glued_balance.candidate.json @@ -0,0 +1,330 @@ +{ + "source": "ocr", + "locator": "file:glued.json;table:0", + "type_hint": "BALANCE", + "evidence": { + "okud": null, + "title": { + "text": "Бухгалтерский баланс", + "value": null, + "confidence": 100.0, + "ref": "evidence:title" + } + }, + "header": { + "organization_name": { + "text": "ООО «Пример»", + "value": null, + "confidence": 100.0, + "ref": "hdr:organization_name" + }, + "inn": { + "text": "7700000000", + "value": null, + "confidence": 100.0, + "ref": "hdr:inn" + }, + "report_date": { + "text": "31.12.2024", + "value": null, + "confidence": 100.0, + "ref": "hdr:report_date" + }, + "units": { + "text": "тыс. руб.", + "value": null, + "confidence": 100.0, + "ref": "hdr:units" + }, + "okud": null, + "okpo": null, + "okopf": null, + "okfs": null, + "okei": null, + "address": null + }, + "rows": [ + [ + { + "text": "Наименование показателя", + "value": null, + "confidence": 100.0, + "ref": "row:0" + }, + { + "text": "Код", + "value": null, + "confidence": 100.0, + "ref": "row:0" + }, + { + "text": "На 31 декабря 2024 г.", + "value": null, + "confidence": 100.0, + "ref": "row:0" + } + ], + [ + { + "text": "Основные средства", + "value": null, + "confidence": 100.0, + "ref": "row:1" + }, + { + "text": "1150", + "value": null, + "confidence": 100.0, + "ref": "row:1" + }, + { + "text": "600000", + "value": null, + "confidence": 100.0, + "ref": "row:1" + } + ], + [ + { + "text": "Итого по разделу I", + "value": null, + "confidence": 100.0, + "ref": "row:2" + }, + { + "text": "1100", + "value": null, + "confidence": 100.0, + "ref": "row:2" + }, + { + "text": "600000", + "value": null, + "confidence": 100.0, + "ref": "row:2" + } + ], + [ + { + "text": "Запасы", + "value": null, + "confidence": 100.0, + "ref": "row:3" + }, + { + "text": "1210", + "value": null, + "confidence": 100.0, + "ref": "row:3" + }, + { + "text": "1771333", + "value": null, + "confidence": 100.0, + "ref": "row:3" + } + ], + [ + { + "text": "Итого по разделу II", + "value": null, + "confidence": 100.0, + "ref": "row:4" + }, + { + "text": "1200", + "value": null, + "confidence": 100.0, + "ref": "row:4" + }, + { + "text": "1771333", + "value": null, + "confidence": 100.0, + "ref": "row:4" + } + ], + [ + { + "text": "БАЛАНС", + "value": null, + "confidence": 100.0, + "ref": "row:5" + }, + { + "text": "1600", + "value": null, + "confidence": 100.0, + "ref": "row:5" + }, + { + "text": "2371333", + "value": null, + "confidence": 100.0, + "ref": "row:5" + } + ], + [ + { + "text": "Уставный капитал", + "value": null, + "confidence": 100.0, + "ref": "row:6" + }, + { + "text": "1310", + "value": null, + "confidence": 100.0, + "ref": "row:6" + }, + { + "text": "1000000", + "value": null, + "confidence": 100.0, + "ref": "row:6" + } + ], + [ + { + "text": "Нераспределенная прибыль", + "value": null, + "confidence": 100.0, + "ref": "row:7" + }, + { + "text": "1370", + "value": null, + "confidence": 100.0, + "ref": "row:7" + }, + { + "text": "253558", + "value": null, + "confidence": 100.0, + "ref": "row:7" + } + ], + [ + { + "text": "Итого по разделу III", + "value": null, + "confidence": 100.0, + "ref": "row:8" + }, + { + "text": "1300", + "value": null, + "confidence": 100.0, + "ref": "row:8" + }, + { + "text": "1253558", + "value": null, + "confidence": 100.0, + "ref": "row:8" + } + ], + [ + { + "text": "Заемные средства", + "value": null, + "confidence": 100.0, + "ref": "row:9" + }, + { + "text": "1410", + "value": null, + "confidence": 100.0, + "ref": "row:9" + }, + { + "text": "200000", + "value": null, + "confidence": 100.0, + "ref": "row:9" + } + ], + [ + { + "text": "Итого по разделу IV", + "value": null, + "confidence": 100.0, + "ref": "row:10" + }, + { + "text": "1400", + "value": null, + "confidence": 100.0, + "ref": "row:10" + }, + { + "text": "200000", + "value": null, + "confidence": 100.0, + "ref": "row:10" + } + ], + [ + { + "text": "Заемные средства", + "value": null, + "confidence": 100.0, + "ref": "row:11" + }, + { + "text": "1510", + "value": null, + "confidence": 100.0, + "ref": "row:11" + }, + { + "text": "400000", + "value": null, + "confidence": 100.0, + "ref": "row:11" + } + ], + [ + { + "text": "Кредиторская задолженность", + "value": null, + "confidence": 100.0, + "ref": "row:12" + }, + { + "text": "1520", + "value": null, + "confidence": 100.0, + "ref": "row:12" + }, + { + "text": "517775", + "value": null, + "confidence": 100.0, + "ref": "row:12" + } + ], + [ + { + "text": "Итого по разделу V\nБАЛАНС", + "value": null, + "confidence": 100.0, + "ref": "row:13" + }, + { + "text": "1500\n1700", + "value": null, + "confidence": 90.0, + "ref": "row:13" + }, + { + "text": "917 775\n2 371 333", + "value": null, + "confidence": 90.0, + "ref": "row:13" + } + ] + ], + "segment_breaks": [], + "warnings": [] +} diff --git a/arm_statements/tests/golden/glued_balance.expected.json b/arm_statements/tests/golden/glued_balance.expected.json new file mode 100644 index 0000000..ce2716c --- /dev/null +++ b/arm_statements/tests/golden/glued_balance.expected.json @@ -0,0 +1,411 @@ +{ + "document_type": "BALANCE", + "form_variant": "full", + "locator": "file:glued.json;table:0", + "header": { + "organization_name": { + "text": "ООО «Пример»", + "confidence": 100.0, + "value": "ООО «Пример»" + }, + "inn": { + "text": "7700000000", + "confidence": 100.0, + "value": "7700000000" + }, + "report_date": { + "text": "31.12.2024", + "confidence": 100.0, + "value": "2024-12-31" + }, + "units": { + "text": "тыс. руб.", + "confidence": 100.0, + "value": "тыс. руб." + }, + "okud": { + "text": null, + "confidence": null, + "value": null + }, + "okpo": { + "text": null, + "confidence": null, + "value": null + }, + "okopf": { + "text": null, + "confidence": null, + "value": null + }, + "okfs": { + "text": null, + "confidence": null, + "value": null + }, + "okei": { + "text": null, + "confidence": null, + "value": null + }, + "address": { + "text": null, + "confidence": null, + "value": null + }, + "form_name": "Бухгалтерский баланс", + "needs_review_fields": [] + }, + "columns": [ + { + "column_index": 2, + "label": "На 31 декабря 2024 г.", + "label_confidence": 100.0, + "period_date": "2024-12-31", + "period_kind": "absolute" + } + ], + "lines": [ + { + "row_index": 0, + "ref": "row:1", + "line_status": "ok", + "line_code": "1150", + "line_code_text": "1150", + "line_code_confidence": 100.0, + "line_name": "Основные средства", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "600000.00", + "string_value": null, + "text": "600000", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 1, + "ref": "row:2", + "line_status": "ok", + "line_code": "1100", + "line_code_text": "1100", + "line_code_confidence": 100.0, + "line_name": "Итого по разделу I", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "600000.00", + "string_value": null, + "text": "600000", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 2, + "ref": "row:3", + "line_status": "ok", + "line_code": "1210", + "line_code_text": "1210", + "line_code_confidence": 100.0, + "line_name": "Запасы", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "1771333.00", + "string_value": null, + "text": "1771333", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 3, + "ref": "row:4", + "line_status": "ok", + "line_code": "1200", + "line_code_text": "1200", + "line_code_confidence": 100.0, + "line_name": "Итого по разделу II", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "1771333.00", + "string_value": null, + "text": "1771333", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 4, + "ref": "row:5", + "line_status": "ok", + "line_code": "1600", + "line_code_text": "1600", + "line_code_confidence": 100.0, + "line_name": "БАЛАНС", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "2371333.00", + "string_value": null, + "text": "2371333", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 5, + "ref": "row:6", + "line_status": "ok", + "line_code": "1310", + "line_code_text": "1310", + "line_code_confidence": 100.0, + "line_name": "Уставный капитал", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "1000000.00", + "string_value": null, + "text": "1000000", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 6, + "ref": "row:7", + "line_status": "ok", + "line_code": "1370", + "line_code_text": "1370", + "line_code_confidence": 100.0, + "line_name": "Нераспределенная прибыль", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "253558.00", + "string_value": null, + "text": "253558", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 7, + "ref": "row:8", + "line_status": "ok", + "line_code": "1300", + "line_code_text": "1300", + "line_code_confidence": 100.0, + "line_name": "Итого по разделу III", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "1253558.00", + "string_value": null, + "text": "1253558", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 8, + "ref": "row:9", + "line_status": "ok", + "line_code": "1410", + "line_code_text": "1410", + "line_code_confidence": 100.0, + "line_name": "Заемные средства", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "200000.00", + "string_value": null, + "text": "200000", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 9, + "ref": "row:10", + "line_status": "ok", + "line_code": "1400", + "line_code_text": "1400", + "line_code_confidence": 100.0, + "line_name": "Итого по разделу IV", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "200000.00", + "string_value": null, + "text": "200000", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 10, + "ref": "row:11", + "line_status": "ok", + "line_code": "1510", + "line_code_text": "1510", + "line_code_confidence": 100.0, + "line_name": "Заемные средства", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "400000.00", + "string_value": null, + "text": "400000", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 11, + "ref": "row:12", + "line_status": "ok", + "line_code": "1520", + "line_code_text": "1520", + "line_code_confidence": 100.0, + "line_name": "Кредиторская задолженность", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "517775.00", + "string_value": null, + "text": "517775", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 12, + "ref": "row:13#1", + "line_status": "ok", + "line_code": "1500", + "line_code_text": "1500", + "line_code_confidence": 90.0, + "line_name": "Итого по разделу V", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "917775.00", + "string_value": null, + "text": "917 775", + "confidence": 90.0, + "is_recognized": false, + "needs_review": false + } + ] + }, + { + "row_index": 13, + "ref": "row:13#2", + "line_status": "ok", + "line_code": "1700", + "line_code_text": "1700", + "line_code_confidence": 90.0, + "line_name": "БАЛАНС", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 2, + "amount": "2371333.00", + "string_value": null, + "text": "2 371 333", + "confidence": 90.0, + "is_recognized": false, + "needs_review": false + } + ] + } + ], + "checks": { + "relations_failed": [], + "soft_relations_failed": [] + }, + "status": "done", + "warnings": [ + { + "code": "ROW_SPLIT", + "severity": "info", + "message": "склеенная строка сетки 13 разделена на 2 строк", + "line_refs": [ + "row:13#1", + "row:13#2" + ], + "column_index": null, + "field": null, + "locator": null + } + ] +} diff --git a/arm_statements/tests/golden/kody_title_balance.candidate.json b/arm_statements/tests/golden/kody_title_balance.candidate.json new file mode 100644 index 0000000..e9a164b --- /dev/null +++ b/arm_statements/tests/golden/kody_title_balance.candidate.json @@ -0,0 +1,538 @@ +{ + "source": "excel", + "locator": "sheet:Баланс", + "type_hint": "BALANCE", + "evidence": { + "okud": { + "text": "0710001", + "value": null, + "confidence": 100.0, + "ref": "evidence:okud" + }, + "title": { + "text": "Бухгалтерский баланс", + "value": null, + "confidence": 100.0, + "ref": "evidence:title" + } + }, + "header": { + "organization_name": { + "text": "ООО «Пример»", + "value": null, + "confidence": 100.0, + "ref": "hdr:organization_name" + }, + "inn": { + "text": "7700000000", + "value": null, + "confidence": 100.0, + "ref": "hdr:inn" + }, + "report_date": { + "text": "30.09.2025", + "value": null, + "confidence": 100.0, + "ref": "hdr:report_date" + }, + "units": null, + "okud": { + "text": "0710001", + "value": null, + "confidence": 100.0, + "ref": "hdr:okud" + }, + "okpo": null, + "okopf": { + "text": "12267", + "value": null, + "confidence": 100.0, + "ref": "hdr:okopf" + }, + "okfs": null, + "okei": { + "text": "384", + "value": null, + "confidence": 100.0, + "ref": "hdr:okei" + }, + "address": null + }, + "rows": [ + [ + null, + null, + null, + null, + null, + { + "text": "Коды", + "value": null, + "confidence": 100.0, + "ref": "Баланс!F1" + } + ], + [ + { + "text": "Бухгалтерский баланс", + "value": null, + "confidence": 100.0, + "ref": "Баланс!A2" + }, + null, + null, + null, + { + "text": "Форма по ОКУД", + "value": null, + "confidence": 100.0, + "ref": "Баланс!E2" + }, + { + "text": "0710001", + "value": null, + "confidence": 100.0, + "ref": "Баланс!F2" + } + ], + [ + { + "text": "на 30 сентября 2025 г.", + "value": null, + "confidence": 100.0, + "ref": "Баланс!A3" + }, + null, + null, + null, + { + "text": "Дата (число, месяц, год)", + "value": null, + "confidence": 100.0, + "ref": "Баланс!E3" + }, + { + "text": "30 | 09 | 2025", + "value": null, + "confidence": 100.0, + "ref": "Баланс!F3" + } + ], + [ + { + "text": "Организация ООО «Пример»", + "value": null, + "confidence": 100.0, + "ref": "Баланс!A4" + }, + null, + null, + null, + { + "text": "по ОКПО", + "value": null, + "confidence": 100.0, + "ref": "Баланс!E4" + }, + { + "text": "12 345 678", + "value": { + "decimal": "12345678" + }, + "confidence": 100.0, + "ref": "Баланс!F4" + } + ], + [ + { + "text": "Идентификационный номер налогоплательщика", + "value": null, + "confidence": 100.0, + "ref": "Баланс!A5" + }, + null, + null, + null, + { + "text": "ИНН", + "value": null, + "confidence": 100.0, + "ref": "Баланс!E5" + }, + { + "text": "7700000000", + "value": null, + "confidence": 100.0, + "ref": "Баланс!F5" + } + ], + [ + { + "text": "Организационно-правовая форма / форма собственности", + "value": null, + "confidence": 100.0, + "ref": "Баланс!A6" + }, + null, + null, + null, + { + "text": "по ОКОПФ / ОКФС", + "value": null, + "confidence": 100.0, + "ref": "Баланс!E6" + }, + { + "text": "12 267", + "value": { + "decimal": "12267" + }, + "confidence": 100.0, + "ref": "Баланс!F6" + } + ], + [ + { + "text": "Единица измерения: тыс. руб.", + "value": null, + "confidence": 100.0, + "ref": "Баланс!A7" + }, + null, + null, + null, + { + "text": "по ОКЕИ", + "value": null, + "confidence": 100.0, + "ref": "Баланс!E7" + }, + { + "text": "384", + "value": { + "decimal": "384" + }, + "confidence": 100.0, + "ref": "Баланс!F7" + } + ], + [ + null, + null, + null, + null, + null, + { + "text": "16", + "value": { + "decimal": "16" + }, + "confidence": 100.0, + "ref": "Баланс!F8" + } + ], + [ + { + "text": "Наименование показателя", + "value": null, + "confidence": 100.0, + "ref": "Баланс!A9" + }, + null, + null, + { + "text": "На 30 сентября 2025 г.", + "value": null, + "confidence": 100.0, + "ref": "Баланс!D9" + }, + { + "text": "На 31 декабря 2024 г.", + "value": null, + "confidence": 100.0, + "ref": "Баланс!E9" + } + ], + [ + { + "text": "Основные средства", + "value": null, + "confidence": 100.0, + "ref": "Баланс!A10" + }, + null, + { + "text": "1150", + "value": null, + "confidence": 100.0, + "ref": "Баланс!C10" + }, + { + "text": "50 000", + "value": { + "decimal": "50000" + }, + "confidence": 100.0, + "ref": "Баланс!D10" + }, + { + "text": "40 000", + "value": { + "decimal": "40000" + }, + "confidence": 100.0, + "ref": "Баланс!E10" + } + ], + [ + { + "text": "Итого по разделу I", + "value": null, + "confidence": 100.0, + "ref": "Баланс!A11" + }, + null, + { + "text": "1100", + "value": null, + "confidence": 100.0, + "ref": "Баланс!C11" + }, + { + "text": "50 000", + "value": { + "decimal": "50000" + }, + "confidence": 100.0, + "ref": "Баланс!D11" + }, + { + "text": "40 000", + "value": { + "decimal": "40000" + }, + "confidence": 100.0, + "ref": "Баланс!E11" + } + ], + [ + { + "text": "Запасы", + "value": null, + "confidence": 100.0, + "ref": "Баланс!A12" + }, + null, + { + "text": "1210", + "value": null, + "confidence": 100.0, + "ref": "Баланс!C12" + }, + { + "text": "60 000", + "value": { + "decimal": "60000" + }, + "confidence": 100.0, + "ref": "Баланс!D12" + }, + { + "text": "50 000", + "value": { + "decimal": "50000" + }, + "confidence": 100.0, + "ref": "Баланс!E12" + } + ], + [ + { + "text": "Дебиторская задолженность", + "value": null, + "confidence": 100.0, + "ref": "Баланс!A13" + }, + null, + { + "text": "1230", + "value": null, + "confidence": 100.0, + "ref": "Баланс!C13" + }, + { + "text": "40 000", + "value": { + "decimal": "40000" + }, + "confidence": 100.0, + "ref": "Баланс!D13" + }, + { + "text": "30 000", + "value": { + "decimal": "30000" + }, + "confidence": 100.0, + "ref": "Баланс!E13" + } + ], + [ + { + "text": "Итого по разделу II", + "value": null, + "confidence": 100.0, + "ref": "Баланс!A14" + }, + null, + { + "text": "1200", + "value": null, + "confidence": 100.0, + "ref": "Баланс!C14" + }, + { + "text": "100 000", + "value": { + "decimal": "100000" + }, + "confidence": 100.0, + "ref": "Баланс!D14" + }, + { + "text": "80 000", + "value": { + "decimal": "80000" + }, + "confidence": 100.0, + "ref": "Баланс!E14" + } + ], + [ + { + "text": "БАЛАНС", + "value": null, + "confidence": 100.0, + "ref": "Баланс!A15" + }, + null, + { + "text": "1600", + "value": null, + "confidence": 100.0, + "ref": "Баланс!C15" + }, + { + "text": "150 000", + "value": { + "decimal": "150000" + }, + "confidence": 100.0, + "ref": "Баланс!D15" + }, + { + "text": "120 000", + "value": { + "decimal": "120000" + }, + "confidence": 100.0, + "ref": "Баланс!E15" + } + ], + [ + { + "text": "Нераспределенная прибыль", + "value": null, + "confidence": 100.0, + "ref": "Баланс!A16" + }, + null, + { + "text": "1370", + "value": null, + "confidence": 100.0, + "ref": "Баланс!C16" + }, + { + "text": "150 000", + "value": { + "decimal": "150000" + }, + "confidence": 100.0, + "ref": "Баланс!D16" + }, + { + "text": "120 000", + "value": { + "decimal": "120000" + }, + "confidence": 100.0, + "ref": "Баланс!E16" + } + ], + [ + { + "text": "Итого по разделу III", + "value": null, + "confidence": 100.0, + "ref": "Баланс!A17" + }, + null, + { + "text": "1300", + "value": null, + "confidence": 100.0, + "ref": "Баланс!C17" + }, + { + "text": "150 000", + "value": { + "decimal": "150000" + }, + "confidence": 100.0, + "ref": "Баланс!D17" + }, + { + "text": "120 000", + "value": { + "decimal": "120000" + }, + "confidence": 100.0, + "ref": "Баланс!E17" + } + ], + [ + { + "text": "БАЛАНС", + "value": null, + "confidence": 100.0, + "ref": "Баланс!A18" + }, + null, + { + "text": "1700", + "value": null, + "confidence": 100.0, + "ref": "Баланс!C18" + }, + { + "text": "150 000", + "value": { + "decimal": "150000" + }, + "confidence": 100.0, + "ref": "Баланс!D18" + }, + { + "text": "120 000", + "value": { + "decimal": "120000" + }, + "confidence": 100.0, + "ref": "Баланс!E18" + } + ] + ], + "segment_breaks": [], + "warnings": [] +} diff --git a/arm_statements/tests/golden/kody_title_balance.expected.json b/arm_statements/tests/golden/kody_title_balance.expected.json new file mode 100644 index 0000000..b24d123 --- /dev/null +++ b/arm_statements/tests/golden/kody_title_balance.expected.json @@ -0,0 +1,371 @@ +{ + "document_type": "BALANCE", + "form_variant": "full", + "locator": "sheet:Баланс", + "header": { + "organization_name": { + "text": "ООО «Пример»", + "confidence": 100.0, + "value": "ООО «Пример»" + }, + "inn": { + "text": "7700000000", + "confidence": 100.0, + "value": "7700000000" + }, + "report_date": { + "text": "30.09.2025", + "confidence": 100.0, + "value": "2025-09-30" + }, + "units": { + "text": null, + "confidence": null, + "value": "тыс. руб." + }, + "okud": { + "text": "0710001", + "confidence": 100.0, + "value": "0710001" + }, + "okpo": { + "text": null, + "confidence": null, + "value": null + }, + "okopf": { + "text": "12267", + "confidence": 100.0, + "value": "12267" + }, + "okfs": { + "text": null, + "confidence": null, + "value": null + }, + "okei": { + "text": "384", + "confidence": 100.0, + "value": "384" + }, + "address": { + "text": null, + "confidence": null, + "value": null + }, + "form_name": "Бухгалтерский баланс", + "needs_review_fields": [] + }, + "columns": [ + { + "column_index": 3, + "label": "На 30 сентября 2025 г.", + "label_confidence": 100.0, + "period_date": "2025-09-30", + "period_kind": "absolute" + }, + { + "column_index": 4, + "label": "На 31 декабря 2024 г.", + "label_confidence": 100.0, + "period_date": "2024-12-31", + "period_kind": "absolute" + } + ], + "lines": [ + { + "row_index": 0, + "ref": "Баланс!C10", + "line_status": "ok", + "line_code": "1150", + "line_code_text": "1150", + "line_code_confidence": 100.0, + "line_name": "Основные средства", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 3, + "amount": "50000.00", + "string_value": null, + "text": "50 000", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + }, + { + "column_index": 4, + "amount": "40000.00", + "string_value": null, + "text": "40 000", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 1, + "ref": "Баланс!C11", + "line_status": "ok", + "line_code": "1100", + "line_code_text": "1100", + "line_code_confidence": 100.0, + "line_name": "Итого по разделу I", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 3, + "amount": "50000.00", + "string_value": null, + "text": "50 000", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + }, + { + "column_index": 4, + "amount": "40000.00", + "string_value": null, + "text": "40 000", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 2, + "ref": "Баланс!C12", + "line_status": "ok", + "line_code": "1210", + "line_code_text": "1210", + "line_code_confidence": 100.0, + "line_name": "Запасы", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 3, + "amount": "60000.00", + "string_value": null, + "text": "60 000", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + }, + { + "column_index": 4, + "amount": "50000.00", + "string_value": null, + "text": "50 000", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 3, + "ref": "Баланс!C13", + "line_status": "ok", + "line_code": "1230", + "line_code_text": "1230", + "line_code_confidence": 100.0, + "line_name": "Дебиторская задолженность", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 3, + "amount": "40000.00", + "string_value": null, + "text": "40 000", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + }, + { + "column_index": 4, + "amount": "30000.00", + "string_value": null, + "text": "30 000", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 4, + "ref": "Баланс!C14", + "line_status": "ok", + "line_code": "1200", + "line_code_text": "1200", + "line_code_confidence": 100.0, + "line_name": "Итого по разделу II", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 3, + "amount": "100000.00", + "string_value": null, + "text": "100 000", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + }, + { + "column_index": 4, + "amount": "80000.00", + "string_value": null, + "text": "80 000", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 5, + "ref": "Баланс!C15", + "line_status": "ok", + "line_code": "1600", + "line_code_text": "1600", + "line_code_confidence": 100.0, + "line_name": "БАЛАНС", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 3, + "amount": "150000.00", + "string_value": null, + "text": "150 000", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + }, + { + "column_index": 4, + "amount": "120000.00", + "string_value": null, + "text": "120 000", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 6, + "ref": "Баланс!C16", + "line_status": "ok", + "line_code": "1370", + "line_code_text": "1370", + "line_code_confidence": 100.0, + "line_name": "Нераспределенная прибыль", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 3, + "amount": "150000.00", + "string_value": null, + "text": "150 000", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + }, + { + "column_index": 4, + "amount": "120000.00", + "string_value": null, + "text": "120 000", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 7, + "ref": "Баланс!C17", + "line_status": "ok", + "line_code": "1300", + "line_code_text": "1300", + "line_code_confidence": 100.0, + "line_name": "Итого по разделу III", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 3, + "amount": "150000.00", + "string_value": null, + "text": "150 000", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + }, + { + "column_index": 4, + "amount": "120000.00", + "string_value": null, + "text": "120 000", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 8, + "ref": "Баланс!C18", + "line_status": "ok", + "line_code": "1700", + "line_code_text": "1700", + "line_code_confidence": 100.0, + "line_name": "БАЛАНС", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 3, + "amount": "150000.00", + "string_value": null, + "text": "150 000", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + }, + { + "column_index": 4, + "amount": "120000.00", + "string_value": null, + "text": "120 000", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + } + ], + "checks": { + "relations_failed": [], + "soft_relations_failed": [] + }, + "status": "done", + "warnings": [] +} diff --git a/arm_statements/tests/golden/npz_income.candidate.json b/arm_statements/tests/golden/npz_income.candidate.json new file mode 100644 index 0000000..03c177e --- /dev/null +++ b/arm_statements/tests/golden/npz_income.candidate.json @@ -0,0 +1,433 @@ +{ + "source": "excel", + "locator": "sheet:ОФР,ОФР_2", + "type_hint": "INCOME", + "evidence": { + "okud": null, + "title": { + "text": "Отчет о финансовых результатах", + "value": null, + "confidence": 100.0, + "ref": "evidence:title" + } + }, + "header": { + "organization_name": { + "text": "ООО «Пример»", + "value": null, + "confidence": 100.0, + "ref": "hdr:organization_name" + }, + "inn": { + "text": "7700000000", + "value": null, + "confidence": 100.0, + "ref": "hdr:inn" + }, + "report_date": { + "text": "30.09.2025", + "value": null, + "confidence": 100.0, + "ref": "hdr:report_date" + }, + "units": { + "text": "тыс. руб.", + "value": null, + "confidence": 100.0, + "ref": "hdr:units" + }, + "okud": null, + "okpo": null, + "okopf": null, + "okfs": null, + "okei": null, + "address": null + }, + "rows": [ + [ + { + "text": "Отчет о финансовых результатах", + "value": null, + "confidence": 100.0, + "ref": "ОФР!A1" + } + ], + [ + { + "text": "Наименование показателя", + "value": null, + "confidence": 100.0, + "ref": "ОФР!A2" + }, + null, + null, + { + "text": "Код", + "value": null, + "confidence": 100.0, + "ref": "ОФР!D2" + }, + { + "text": "За 9 месяцев 2025 г.", + "value": null, + "confidence": 100.0, + "ref": "ОФР!E2" + }, + { + "text": "За 9 месяцев 2024 г.", + "value": null, + "confidence": 100.0, + "ref": "ОФР!F2" + } + ], + [ + { + "text": "Выручка", + "value": null, + "confidence": 100.0, + "ref": "ОФР!A3" + }, + null, + null, + { + "text": "2110", + "value": null, + "confidence": 100.0, + "ref": "ОФР!D3" + }, + { + "text": "1 000", + "value": { + "decimal": "1000" + }, + "confidence": 100.0, + "ref": "ОФР!E3" + }, + { + "text": "900", + "value": { + "decimal": "900" + }, + "confidence": 100.0, + "ref": "ОФР!F3" + } + ], + [ + { + "text": "Себестоимость продаж", + "value": null, + "confidence": 100.0, + "ref": "ОФР!A4" + }, + null, + null, + { + "text": "2120", + "value": null, + "confidence": 100.0, + "ref": "ОФР!D4" + }, + { + "text": "-600", + "value": { + "decimal": "-600" + }, + "confidence": 100.0, + "ref": "ОФР!E4" + }, + { + "text": "-540", + "value": { + "decimal": "-540" + }, + "confidence": 100.0, + "ref": "ОФР!F4" + } + ], + [ + { + "text": "Валовая прибыль (убыток)", + "value": null, + "confidence": 100.0, + "ref": "ОФР!A5" + }, + null, + null, + { + "text": "2100", + "value": null, + "confidence": 100.0, + "ref": "ОФР!D5" + }, + { + "text": "400", + "value": { + "decimal": "400" + }, + "confidence": 100.0, + "ref": "ОФР!E5" + }, + { + "text": "360", + "value": { + "decimal": "360" + }, + "confidence": 100.0, + "ref": "ОФР!F5" + } + ], + [ + { + "text": "Коммерческие расходы", + "value": null, + "confidence": 100.0, + "ref": "ОФР!A6" + }, + null, + null, + { + "text": "2210", + "value": null, + "confidence": 100.0, + "ref": "ОФР!D6" + }, + { + "text": "-50", + "value": { + "decimal": "-50" + }, + "confidence": 100.0, + "ref": "ОФР!E6" + }, + { + "text": "-45", + "value": { + "decimal": "-45" + }, + "confidence": 100.0, + "ref": "ОФР!F6" + } + ], + [ + { + "text": "Управленческие расходы", + "value": null, + "confidence": 100.0, + "ref": "ОФР!A7" + }, + null, + null, + { + "text": "2220", + "value": null, + "confidence": 100.0, + "ref": "ОФР!D7" + }, + { + "text": "-30", + "value": { + "decimal": "-30" + }, + "confidence": 100.0, + "ref": "ОФР!E7" + }, + { + "text": "-27", + "value": { + "decimal": "-27" + }, + "confidence": 100.0, + "ref": "ОФР!F7" + } + ], + [ + { + "text": "Прибыль (убыток) от продаж", + "value": null, + "confidence": 100.0, + "ref": "ОФР!A8" + }, + null, + null, + { + "text": "2200", + "value": null, + "confidence": 100.0, + "ref": "ОФР!D8" + }, + { + "text": "320", + "value": { + "decimal": "320" + }, + "confidence": 100.0, + "ref": "ОФР!E8" + }, + { + "text": "288", + "value": { + "decimal": "288" + }, + "confidence": 100.0, + "ref": "ОФР!F8" + } + ], + [ + { + "text": "Прочие доходы", + "value": null, + "confidence": 100.0, + "ref": "ОФР!A9" + }, + null, + null, + { + "text": "2340", + "value": null, + "confidence": 100.0, + "ref": "ОФР!D9" + }, + { + "text": "10", + "value": { + "decimal": "10" + }, + "confidence": 100.0, + "ref": "ОФР!E9" + }, + { + "text": "9", + "value": { + "decimal": "9" + }, + "confidence": 100.0, + "ref": "ОФР!F9" + } + ], + [ + { + "text": "Прибыль (убыток) до налогообложения", + "value": null, + "confidence": 100.0, + "ref": "ОФР!A10" + }, + null, + null, + { + "text": "2300", + "value": null, + "confidence": 100.0, + "ref": "ОФР!D10" + }, + { + "text": "330", + "value": { + "decimal": "330" + }, + "confidence": 100.0, + "ref": "ОФР!E10" + }, + { + "text": "297", + "value": { + "decimal": "297" + }, + "confidence": 100.0, + "ref": "ОФР!F10" + } + ], + [ + { + "text": "Налог на прибыль", + "value": null, + "confidence": 100.0, + "ref": "ОФР!A11" + }, + null, + null, + { + "text": "2410", + "value": null, + "confidence": 100.0, + "ref": "ОФР!D11" + }, + { + "text": "-66", + "value": { + "decimal": "-66" + }, + "confidence": 100.0, + "ref": "ОФР!E11" + }, + { + "text": "-59", + "value": { + "decimal": "-59" + }, + "confidence": 100.0, + "ref": "ОФР!F11" + } + ], + [ + { + "text": "Прочее", + "value": null, + "confidence": 100.0, + "ref": "ОФР!A12" + }, + null, + null, + { + "text": "2460", + "value": null, + "confidence": 100.0, + "ref": "ОФР!D12" + }, + { + "text": "0", + "value": { + "decimal": "0" + }, + "confidence": 100.0, + "ref": "ОФР!E12" + }, + { + "text": "0", + "value": { + "decimal": "0" + }, + "confidence": 100.0, + "ref": "ОФР!F12" + } + ], + [ + null, + { + "text": "2400", + "value": null, + "confidence": 100.0, + "ref": "ОФР_2!B1" + }, + { + "text": "264", + "value": { + "decimal": "264" + }, + "confidence": 100.0, + "ref": "ОФР_2!C1" + }, + { + "text": "238", + "value": { + "decimal": "238" + }, + "confidence": 100.0, + "ref": "ОФР_2!D1" + } + ] + ], + "segment_breaks": [ + 12 + ], + "warnings": [] +} diff --git a/arm_statements/tests/golden/npz_income.expected.json b/arm_statements/tests/golden/npz_income.expected.json new file mode 100644 index 0000000..d0e41cc --- /dev/null +++ b/arm_statements/tests/golden/npz_income.expected.json @@ -0,0 +1,488 @@ +{ + "document_type": "INCOME", + "form_variant": "full", + "locator": "sheet:ОФР,ОФР_2", + "header": { + "organization_name": { + "text": "ООО «Пример»", + "confidence": 100.0, + "value": "ООО «Пример»" + }, + "inn": { + "text": "7700000000", + "confidence": 100.0, + "value": "7700000000" + }, + "report_date": { + "text": "30.09.2025", + "confidence": 100.0, + "value": "2025-09-30" + }, + "units": { + "text": "тыс. руб.", + "confidence": 100.0, + "value": "тыс. руб." + }, + "okud": { + "text": null, + "confidence": null, + "value": null + }, + "okpo": { + "text": null, + "confidence": null, + "value": null + }, + "okopf": { + "text": null, + "confidence": null, + "value": null + }, + "okfs": { + "text": null, + "confidence": null, + "value": null + }, + "okei": { + "text": null, + "confidence": null, + "value": null + }, + "address": { + "text": null, + "confidence": null, + "value": null + }, + "form_name": "Отчет о финансовых результатах", + "needs_review_fields": [] + }, + "columns": [ + { + "column_index": 4, + "label": "За 9 месяцев 2025 г.", + "label_confidence": 100.0, + "period_date": "2025-09-30", + "period_kind": "absolute" + }, + { + "column_index": 5, + "label": "За 9 месяцев 2024 г.", + "label_confidence": 100.0, + "period_date": "2024-09-30", + "period_kind": "absolute" + } + ], + "lines": [ + { + "row_index": 0, + "ref": "ОФР!D3", + "line_status": "ok", + "line_code": "2110", + "line_code_text": "2110", + "line_code_confidence": 100.0, + "line_name": "Выручка", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 4, + "amount": "1000.00", + "string_value": null, + "text": "1 000", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + }, + { + "column_index": 5, + "amount": "900.00", + "string_value": null, + "text": "900", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 1, + "ref": "ОФР!D4", + "line_status": "ok", + "line_code": "2120", + "line_code_text": "2120", + "line_code_confidence": 100.0, + "line_name": "Себестоимость продаж", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 4, + "amount": "-600.00", + "string_value": null, + "text": "-600", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + }, + { + "column_index": 5, + "amount": "-540.00", + "string_value": null, + "text": "-540", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 2, + "ref": "ОФР!D5", + "line_status": "ok", + "line_code": "2100", + "line_code_text": "2100", + "line_code_confidence": 100.0, + "line_name": "Валовая прибыль (убыток)", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 4, + "amount": "400.00", + "string_value": null, + "text": "400", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + }, + { + "column_index": 5, + "amount": "360.00", + "string_value": null, + "text": "360", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 3, + "ref": "ОФР!D6", + "line_status": "ok", + "line_code": "2210", + "line_code_text": "2210", + "line_code_confidence": 100.0, + "line_name": "Коммерческие расходы", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 4, + "amount": "-50.00", + "string_value": null, + "text": "-50", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + }, + { + "column_index": 5, + "amount": "-45.00", + "string_value": null, + "text": "-45", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 4, + "ref": "ОФР!D7", + "line_status": "ok", + "line_code": "2220", + "line_code_text": "2220", + "line_code_confidence": 100.0, + "line_name": "Управленческие расходы", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 4, + "amount": "-30.00", + "string_value": null, + "text": "-30", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + }, + { + "column_index": 5, + "amount": "-27.00", + "string_value": null, + "text": "-27", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 5, + "ref": "ОФР!D8", + "line_status": "ok", + "line_code": "2200", + "line_code_text": "2200", + "line_code_confidence": 100.0, + "line_name": "Прибыль (убыток) от продаж", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 4, + "amount": "320.00", + "string_value": null, + "text": "320", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + }, + { + "column_index": 5, + "amount": "288.00", + "string_value": null, + "text": "288", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 6, + "ref": "ОФР!D9", + "line_status": "ok", + "line_code": "2340", + "line_code_text": "2340", + "line_code_confidence": 100.0, + "line_name": "Прочие доходы", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 4, + "amount": "10.00", + "string_value": null, + "text": "10", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + }, + { + "column_index": 5, + "amount": "9.00", + "string_value": null, + "text": "9", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 7, + "ref": "ОФР!D10", + "line_status": "ok", + "line_code": "2300", + "line_code_text": "2300", + "line_code_confidence": 100.0, + "line_name": "Прибыль (убыток) до налогообложения", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 4, + "amount": "330.00", + "string_value": null, + "text": "330", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + }, + { + "column_index": 5, + "amount": "297.00", + "string_value": null, + "text": "297", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 8, + "ref": "ОФР!D11", + "line_status": "ok", + "line_code": "2410", + "line_code_text": "2410", + "line_code_confidence": 100.0, + "line_name": "Налог на прибыль", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 4, + "amount": "-66.00", + "string_value": null, + "text": "-66", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + }, + { + "column_index": 5, + "amount": "-59.00", + "string_value": null, + "text": "-59", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 9, + "ref": "ОФР!D12", + "line_status": "ok", + "line_code": "2460", + "line_code_text": "2460", + "line_code_confidence": 100.0, + "line_name": "Прочее", + "line_name_confidence": 100.0, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 4, + "amount": "0.00", + "string_value": null, + "text": "0", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + }, + { + "column_index": 5, + "amount": "0.00", + "string_value": null, + "text": "0", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + }, + { + "row_index": 10, + "ref": "ОФР_2!B1", + "line_status": "ok", + "line_code": "2400", + "line_code_text": "2400", + "line_code_confidence": 100.0, + "line_name": null, + "line_name_confidence": null, + "parent_ref": null, + "needs_review": false, + "values": [ + { + "column_index": 4, + "amount": "264.00", + "string_value": null, + "text": "264", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + }, + { + "column_index": 5, + "amount": "238.00", + "string_value": null, + "text": "238", + "confidence": 100.0, + "is_recognized": true, + "needs_review": false + } + ] + } + ], + "checks": { + "relations_failed": [], + "soft_relations_failed": [ + { + "column_index": 4, + "column_key": "4", + "relation": "2410 = 2411 + 2412", + "codes": [ + "2410", + "2411", + "2412" + ], + "expected": "-66.00", + "actual": null + }, + { + "column_index": 4, + "column_key": "4", + "relation": "2500 = 2400 + 2510 + 2520 + 2530", + "codes": [ + "2500", + "2400", + "2510", + "2520", + "2530" + ], + "expected": null, + "actual": "264.00" + }, + { + "column_index": 5, + "column_key": "5", + "relation": "2410 = 2411 + 2412", + "codes": [ + "2410", + "2411", + "2412" + ], + "expected": "-59.00", + "actual": null + }, + { + "column_index": 5, + "column_key": "5", + "relation": "2500 = 2400 + 2510 + 2520 + 2530", + "codes": [ + "2500", + "2400", + "2510", + "2520", + "2530" + ], + "expected": null, + "actual": "238.00" + } + ] + }, + "status": "done", + "warnings": [] +} diff --git a/arm_statements/tests/test_golden.py b/arm_statements/tests/test_golden.py new file mode 100644 index 0000000..d00c536 --- /dev/null +++ b/arm_statements/tests/test_golden.py @@ -0,0 +1,159 @@ +"""Golden на обезличенных входах Review Focus 1–5 (план 1, задача 20). + +Входы `golden/*.candidate.json` собраны из `builders` один раз и лежат файлами; мусорный текст ячеек — дословно из +реальных распознаваний, названия, ИНН и суммы — обезличенные и синтетические. Эталон `*.expected.json` — +`to_jsonable(normalize_candidate(...).document)`; пересборка — `make golden-update` (`GOLDEN_UPDATE=1`). +""" + +import json +import os +from collections.abc import Callable +from decimal import Decimal as D +from pathlib import Path + +import pytest + +from arm_statements import ( + document_status, + normalize_candidate, + status_input, + status_input_from_contract, + to_contract, +) +from arm_statements.loader import Rules +from arm_statements.model import DocStatus, DocumentType, FormCandidate, LineStatus, StatementDocument +from arm_statements.serialize import candidate_from_dict, to_jsonable +from arm_statements.tests.builders import hdr, ocr + +GOLDEN = Path(__file__).parent / "golden" +FULL_CONFIDENCE = 100.0 +NAMES = ["aep3_balance", "aep3_income", "glued_balance", "kody_title_balance", "npz_income"] +HEAD = ["Наименование показателя", "Код", "На 30.09.2025"] +FULL_HEADER = hdr(organization_name="ООО «Пример»", inn="7700000000", report_date="30.09.2025", units="тыс. руб.") + + +def run(name: str, rules: Rules) -> StatementDocument: + data = json.loads((GOLDEN / f"{name}.candidate.json").read_text(encoding="utf-8")) + result = normalize_candidate(candidate_from_dict(data), rules) + assert result.document is not None + return result.document + + +@pytest.mark.parametrize("name", NAMES) +def test_golden_snapshot(name: str, rules: Rules) -> None: + actual = to_jsonable(run(name, rules)) + target = GOLDEN / f"{name}.expected.json" + if os.environ.get("GOLDEN_UPDATE") == "1": + target.write_text(json.dumps(actual, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + pytest.fail(f"эталон {target.name} пересобран — просмотрите diff и запустите без GOLDEN_UPDATE") + assert actual == json.loads(target.read_text(encoding="utf-8")) + + +def assert_status_is_function_of_contract(doc: StatementDocument) -> None: + contract = json.loads(json.dumps(to_contract(doc), ensure_ascii=False)) + assert status_input_from_contract(contract) == status_input(doc) + assert document_status(status_input(doc)) is doc.status + + +@pytest.mark.parametrize("name", NAMES) +def test_status_is_function_of_contract_document(name: str, rules: Rules) -> None: + assert_status_is_function_of_contract(run(name, rules)) + + +def _disclosure() -> FormCandidate: + return ocr([["Текст"]], type_hint=DocumentType.DISCLOSURE) + + +def _without_lines() -> FormCandidate: + return ocr([["Итого", "15 000"]], type_hint=DocumentType.BALANCE, header=FULL_HEADER) + + +def _absolute_column_without_report_date() -> FormCandidate: + rows = [ + HEAD, + ["Запасы", "1210", "60"], + ["Дебиторы", "1230", "40"], + ["Итого II", "1200", "100"], + ] + header = hdr(organization_name="ООО «Пример»", inn="7700000000", units="тыс. руб.") + return ocr(rows, type_hint=DocumentType.BALANCE, header=header) + + +EXTRA: dict[str, Callable[[], FormCandidate]] = { + "disclosure": _disclosure, + "without_lines": _without_lines, + "absolute_column_without_report_date": _absolute_column_without_report_date, +} + + +@pytest.mark.parametrize("case", sorted(EXTRA)) +def test_status_is_function_of_contract_document_edge_cases(case: str, rules: Rules) -> None: + doc = normalize_candidate(EXTRA[case](), rules).document + assert doc is not None + assert_status_is_function_of_contract(doc) + + +def test_edge_cases_cover_their_branch(rules: Rules) -> None: + disclosure = normalize_candidate(_disclosure(), rules).document + assert disclosure is not None and disclosure.document_type is DocumentType.DISCLOSURE + assert disclosure.status is DocStatus.DONE + empty = normalize_candidate(_without_lines(), rules).document + assert empty is not None and empty.lines == () + assert [w.code for w in empty.warnings if w.code == "CODE_COLUMN_NOT_FOUND"] == ["CODE_COLUMN_NOT_FOUND"] + assert status_input(empty).layout_found is False and empty.status is DocStatus.NEEDS_CHECK + absolute = normalize_candidate(_absolute_column_without_report_date(), rules).document + assert absolute is not None and absolute.header.report_date.value is None + assert status_input(absolute).period_known is True + + +def test_aep3_junk_codes(rules: Rules) -> None: # Review Focus 1 + doc = run("aep3_balance", rules) + by_text = {ln.line_code_text: ln for ln in doc.lines} + assert (by_text["| 140"].line_status, by_text["| 140"].line_code) == (LineStatus.OK, "1140") + confidence = by_text["| 140"].line_code_confidence + assert confidence is not None and confidence < FULL_CONFIDENCE + for junk in ("НИ", "изо :"): + assert by_text[junk].line_status is LineStatus.DETAIL and by_text[junk].parent_ref is not None + refs = [ln.ref for ln in doc.lines] + assert len(refs) == len(set(refs)) and [ln.row_index for ln in doc.lines] == list(range(len(doc.lines))) + assert by_text["НИ"].parent_ref == by_text["1110"].ref and by_text["изо :"].parent_ref == by_text["| 140"].ref + assert len(doc.lines) == len(json.loads((GOLDEN / "aep3_balance.candidate.json").read_text("utf-8"))["rows"]) - 1 + + +def test_aep3_dirty_income_amounts(rules: Rules) -> None: # Review Focus 2 + doc = run("aep3_income", rules) + values = {v.text: v for ln in doc.lines for v in ln.values if v.text} + assert (values["(7744536}"].amount, values["(7744536}"].needs_review) == (D("-7744536"), False) + assert (values["211(1 |"].amount, values["211(1 |"].needs_review) == (None, True) + assert (values["42831 1"].amount, values["42831 1"].needs_review) == (D("428311"), False) + assert doc.checks is not None and doc.checks.relations_failed == () + assert doc.status is DocStatus.NEEDS_CHECK + + +def test_glued_rows(rules: Rules) -> None: # Review Focus 3 + doc = run("glued_balance", rules) + split = [ln for ln in doc.lines if "#" in ln.ref] + assert [(ln.line_code, ln.values[0].amount) for ln in split] == [("1500", D("917775")), ("1700", D("2371333"))] + assert [ln.ref for ln in split] == ["row:13#1", "row:13#2"] + assert len({ln.ref for ln in doc.lines}) == len(doc.lines) + assert [w.code for w in doc.warnings if w.code == "ROW_SPLIT"] == ["ROW_SPLIT"] + assert not [v for ln in doc.lines for v in ln.values if v.needs_review] + assert not [ln for ln in doc.lines if ln.needs_review] + + +def test_kody_title_block(rules: Rules) -> None: # Review Focus 4 + doc = run("kody_title_balance", rules) + assert "12267" not in [ln.line_code for ln in doc.lines] + assert doc.header.okopf.value == "12267" and doc.status is DocStatus.DONE + assert not [w for w in doc.warnings if w.code == "CODE_COLUMN_NOT_FOUND"] + assert all(ln.ref.startswith("Баланс!C") for ln in doc.lines) + + +def test_npz_continuation_single_code(rules: Rules) -> None: # Review Focus 5 + doc = run("npz_income", rules) + (line_2400,) = [ln for ln in doc.lines if ln.line_code == "2400"] + assert [v.column_index for v in line_2400.values] == [c.column_index for c in doc.columns] + assert all(v.amount is not None for v in line_2400.values) + assert [v.amount for v in line_2400.values] == [D("264"), D("238")] + assert "," in doc.locator and not [w for w in doc.warnings if w.code.startswith("SEGMENT_")] + assert doc.status is DocStatus.DONE diff --git a/arm_statements/tests/test_model.py b/arm_statements/tests/test_model.py index 945eeb1..6885794 100644 --- a/arm_statements/tests/test_model.py +++ b/arm_statements/tests/test_model.py @@ -2,6 +2,7 @@ import json from datetime import date from decimal import Decimal as D +import arm_statements from arm_statements.model import Cell, DocumentType, Evidence, FormCandidate, HeaderCells, Source from arm_statements.serialize import candidate_from_dict, to_jsonable from arm_statements.tests.builders import excel, ocr @@ -35,3 +36,83 @@ def test_ocr_builder_cells() -> None: cell = cand.rows[0][1] assert cell == Cell("[190 :", None, 25.0, "row:0") assert cand.rows[0][2] is None and cand.source is Source.OCR + + +PUBLIC = { + "Source", + "DocumentType", + "FormVariant", + "Edition", + "PeriodKind", + "LineStatus", + "Severity", + "DocStatus", + "EnvelopeStatus", + "SignForm", + "CellValue", + "Cell", + "Evidence", + "HeaderCells", + "HEADER_FIELDS", + "StatementWarning", + "FormCandidate", + "HeaderField", + "DateField", + "StatementHeader", + "StatementColumn", + "StatementValue", + "StatementLine", + "FailedRelation", + "StatementChecks", + "StatementDocument", + "CandidateResult", + "NormalizationResult", + "Rules", + "RulesError", + "load_rules", + "default_rules", + "normalize_candidate", + "normalize", + "parse_header", + "type_hint", + "continuation_type", + "sheet_joins", + "parse_code", + "parse_amount", + "parse_date", + "resolve_relative", + "fix_lookalikes", + "make_warning", + "StatusInput", + "StatusLine", + "StatusValue", + "document_status", + "envelope_status", + "StoredForm", + "StoredColumn", + "StoredLine", + "StoredValue", + "StoredWarning", + "RecomputeResult", + "recompute", + "to_jsonable", + "candidate_from_dict", + "statements_version", + "WarningCode", + "choose_edition", + "status_input", + "status_input_from_contract", + "to_contract", + "warning_to_contract", + "code_column", + "amount_columns", + "normalize_label", + "normalize_caption", + "is_continuation", +} + + +def test_public_api() -> None: + assert set(arm_statements.__all__) == PUBLIC + assert len(arm_statements.__all__) == len(set(arm_statements.__all__)) + assert all(hasattr(arm_statements, name) for name in PUBLIC) -- 2.47.2 From aa46e7ad4ae0519ec714fd15cc9e05a19268fa1a Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Sun, 18 Oct 2026 14:14:12 +0400 Subject: [PATCH 23/28] =?UTF-8?q?fix:=20=D0=BA=D0=B2=D0=B0=D0=BD=D1=82?= =?UTF-8?q?=D0=BE=D0=B2=D0=B0=D0=BD=D0=B8=D0=B5=20=D1=81=D1=83=D0=BC=D0=BC?= =?UTF-8?q?=D1=8B=20=C2=AB=D0=BB=D1=83=D1=87=D1=88=D0=B5=D0=B9=20=D0=B4?= =?UTF-8?q?=D0=BE=D0=B3=D0=B0=D0=B4=D0=BA=D0=B8=C2=BB=20=D0=B4=D0=BE=200,0?= =?UTF-8?q?1=20(T20.1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W --- arm_statements/amounts.py | 2 +- arm_statements/tests/golden/aep3_income.expected.json | 2 +- arm_statements/tests/test_amounts.py | 9 +++++++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/arm_statements/amounts.py b/arm_statements/amounts.py index 208eab6..c89c711 100644 --- a/arm_statements/amounts.py +++ b/arm_statements/amounts.py @@ -73,7 +73,7 @@ def _guess(digits: str, text: str, sign_form: SignForm, *, negative: bool) -> Am """«Лучшая догадка»: цифры подряд, не длиннее 16, иначе значение не число.""" if not digits or len(digits) > MAX_INTEGER_DIGITS: return AmountParse(None, text, False, sign_form) - amount = Decimal(digits) + amount = _quantize(Decimal(digits)) return AmountParse(-amount if negative else amount, None, False, sign_form) diff --git a/arm_statements/tests/golden/aep3_income.expected.json b/arm_statements/tests/golden/aep3_income.expected.json index 4c8973e..781405e 100644 --- a/arm_statements/tests/golden/aep3_income.expected.json +++ b/arm_statements/tests/golden/aep3_income.expected.json @@ -241,7 +241,7 @@ "values": [ { "column_index": 2, - "amount": "428311", + "amount": "428311.00", "string_value": null, "text": "42831 1", "confidence": 90.0, diff --git a/arm_statements/tests/test_amounts.py b/arm_statements/tests/test_amounts.py index 859880f..c27a407 100644 --- a/arm_statements/tests/test_amounts.py +++ b/arm_statements/tests/test_amounts.py @@ -117,3 +117,12 @@ def test_sign_inside_parentheses_is_invalid(rules: Rules) -> None: def test_leading_comma_is_edge_trash(rules: Rules) -> None: parsed = parse_amount(",5", OCR, rules) assert (parsed.amount, parsed.is_recognized) == (D("5"), False) + + +def test_guess_amount_is_quantized_to_cents(rules: Rules) -> None: + """T20.1: «лучшая догадка» квантуется как прочие пути; в контракте — две цифры после точки.""" + parsed = parse_amount("42831 1", Source.OCR, rules) + assert parsed.amount == D("428311.00") and parsed.amount.as_tuple().exponent == -2 # noqa: PLR2004 — знаков после запятой + assert format(parsed.amount, "f") == "428311.00" + negative = parse_amount("(42831 1)", Source.OCR, rules) + assert negative.amount is not None and format(negative.amount, "f") == "-428311.00" -- 2.47.2 From be3b9c89052b78d00cefe1d7e620052d907c2322 Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Sun, 18 Oct 2026 14:27:42 +0400 Subject: [PATCH 24/28] =?UTF-8?q?=D0=A1=D0=BA=D0=BB=D0=B5=D0=B9=D0=BA?= =?UTF-8?q?=D0=B0=20=D0=B4=D0=B2=D1=83=D1=85=D1=81=D1=82=D1=80=D0=BE=D1=87?= =?UTF-8?q?=D0=BD=D0=BE=D0=B9=20=D1=88=D0=B0=D0=BF=D0=BA=D0=B8,=20=D0=BA?= =?UTF-8?q?=D0=BE=D0=B3=D0=B4=D0=B0=20=C2=AB=D0=9A=D0=BE=D0=B4=C2=BB=20?= =?UTF-8?q?=D0=B2=20=D0=B2=D0=B5=D1=80=D1=85=D0=BD=D0=B5=D0=B9=20=D1=81?= =?UTF-8?q?=D1=82=D1=80=D0=BE=D0=BA=D0=B5=20(I-1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit После выбора строки по подписи кода проверяется и нижняя строка (годы): если подпись шапки отдельно не период, а вместе с нижней — период, строки склеиваются. Тесты: Excel, OCR, повторная шапка в два ряда. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W --- arm_statements/layout.py | 10 ++++++ arm_statements/tests/test_layout.py | 34 +++++++++++++++++++ arm_statements/tests/test_segments.py | 47 ++++++++++++++++++++++++++- 3 files changed, 90 insertions(+), 1 deletion(-) diff --git a/arm_statements/layout.py b/arm_statements/layout.py index f79798f..3f86a5b 100644 --- a/arm_statements/layout.py +++ b/arm_statements/layout.py @@ -201,6 +201,13 @@ def _upper_candidate(rows: Grid, row: int, first: int, rules: Rules) -> int | No return above if above >= first else None +def _lower_candidate(rows: Grid, row: int, first_code: int, rules: Rules) -> int | None: + below = row + 1 + while below < first_code and is_numbering_row(rows[below], rules): + below += 1 + return below if below < first_code else None + + def find_header_rows(rows: Grid, first: int, first_code: int, code_col: int, rules: Rules) -> tuple[int, ...]: """Строка заголовка (одна или две, склеенные); не найдена — `()`.""" candidates = [r for r in range(first, first_code) if not is_numbering_row(rows[r], rules)] @@ -215,6 +222,9 @@ def find_header_rows(rows: Grid, first: int, first_code: int, code_col: int, rul upper = _upper_candidate(rows, header, first, rules) if upper is not None and _glued_with_upper(rows, upper, header, code_col, rules): return (upper, header) + lower = _lower_candidate(rows, header, first_code, rules) + if lower is not None and _glued_with_upper(rows, header, lower, code_col, rules): + return (header, lower) return (header,) diff --git a/arm_statements/tests/test_layout.py b/arm_statements/tests/test_layout.py index 74c15f3..1b029e3 100644 --- a/arm_statements/tests/test_layout.py +++ b/arm_statements/tests/test_layout.py @@ -137,3 +137,37 @@ def test_name_cell_falls_back_to_left_text(rules: Rules) -> None: cell = name_cell(row, 2, 3, 0) assert cell is not None and cell.text == "Запасы" assert name_cell(ocr([["5.1", "", "", "1210"]]).rows[0], 2, 3, 0) is None + + +def test_two_row_header_with_caption_in_upper_row_is_glued_with_lower(rules: Rules) -> None: + """I-1: «Код» в верхней строке, годы — в нижней (шаблон банка).""" + cand = ocr( + [ + ["Наименование", "Код", "На 30 сентября", "На 31 декабря", "На 31 декабря"], + ["", "", "2025 г.", "2024 г.", "2023 г."], + ["Запасы", "1210", "5", "4", "3"], + ] + ) + header = find_header_rows(cand.rows, 0, 2, 1, rules) + assert header == (0, 1) + assert segment_roles(cand.rows, 0, 2, header, rules) == [RowRole.HEADER, RowRole.HEADER, RowRole.DATA] + + +def test_two_row_header_lower_skips_numbering_and_ignores_unrelated_row(rules: Rules) -> None: + glued = ocr( + [ + ["Наименование", "Код", "На 30 сентября", "На 31 декабря"], + ["", "", "2025 г.", "2024 г."], + ["1", "2", "3", "4"], + ["Запасы", "1210", "5", "4"], + ] + ) + assert find_header_rows(glued.rows, 0, 3, 1, rules) == (0, 1) + plain = ocr( + [ + ["Наименование", "Код", "На 30.09.2025", "На 31.12.2024"], + ["Справочно", "", "", ""], + ["Запасы", "1210", "5", "4"], + ] + ) + assert find_header_rows(plain.rows, 0, 2, 1, rules) == (0,) diff --git a/arm_statements/tests/test_segments.py b/arm_statements/tests/test_segments.py index fe86101..83c536d 100644 --- a/arm_statements/tests/test_segments.py +++ b/arm_statements/tests/test_segments.py @@ -3,7 +3,7 @@ from decimal import Decimal as D from arm_statements.header import parse_header from arm_statements.loader import Rules from arm_statements.model import RowRole -from arm_statements.segments import find_layout +from arm_statements.segments import Layout, find_layout from arm_statements.tests.builders import excel, hdr, ocr, place DATES3 = ["На 30 сентября 2025 г.", "На 31 декабря 2024 г.", "На 31 декабря 2023 г."] @@ -113,3 +113,48 @@ def test_no_codes_anywhere_has_no_segment_warnings(rules: Rules) -> None: layout = find_layout(ocr([["Просто", "текст"], ["ещё", "строка"]]), parse_header(hdr(), rules), rules) assert not layout.code_found and layout.columns == () and layout.warnings == () assert layout.roles == (RowRole.DATA, RowRole.DATA) + + +def _no_date_noise(layout: Layout) -> None: + assert not [w for w in layout.warnings if w.code == "COLUMN_WITHOUT_DATE"] + + +def test_two_row_header_with_caption_on_top_ocr(rules: Rules) -> None: + """I-1: годы во второй строке шапки — не строка данных, колонки с абсолютными датами.""" + rows = [ + ["Наименование", "Код", "На 30 сентября", "На 31 декабря", "На 31 декабря"], + ["", "", "2025 г.", "2024 г.", "2023 г."], + ["Запасы", "1210", "5", "4", "3"], + ] + layout = find_layout(ocr(rows), parse_header(hdr(), rules), rules) + assert layout.roles[:2] == (RowRole.HEADER, RowRole.HEADER) and layout.roles[2] is RowRole.DATA + assert [c.period_kind.value != "none" for c in layout.columns] == [True, True, True] + _no_date_noise(layout) + + +def test_two_row_header_with_caption_on_top_excel(rules: Rules) -> None: + sheet = [ + place({0: "Наименование", 1: "Код", 2: "На 30 сентября", 3: "На 31 декабря", 4: "На 31 декабря"}), + place({2: "2025 г.", 3: "2024 г.", 4: "2023 г."}), + place({0: "Запасы", 1: "1210", 2: D("5"), 3: D("4"), 4: D("3")}), + ] + layout = find_layout(excel([("Баланс", sheet)]), parse_header(hdr(), rules), rules) + assert layout.roles[1] is RowRole.HEADER and layout.roles[2] is RowRole.DATA + assert [c.period_kind.value != "none" for c in layout.columns] == [True, True, True] + _no_date_noise(layout) + + +def test_repeat_header_in_two_rows(rules: Rules) -> None: + """T10.5: повторная шапка в два ряда использует ту же склейку.""" + rows = [ + ["Наименование", "Код", "На 30 сентября", "На 31 декабря"], + ["", "", "2025 г.", "2024 г."], + ["Запасы", "1210", "5", "4"], + ["Наименование", "Код", "На 30 сентября", "На 31 декабря"], + ["", "", "2025 г.", "2024 г."], + ["Итого", "1200", "5", "4"], + ] + layout = find_layout(ocr(rows), parse_header(hdr(), rules), rules) + assert [s.first_row for s in layout.segments] == [0, 3] + assert layout.roles[4] is RowRole.HEADER and layout.roles[5] is RowRole.DATA + _no_date_noise(layout) -- 2.47.2 From 8a63ce519645376b05eaa072b719f7427dae817e Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Sun, 18 Oct 2026 14:28:03 +0400 Subject: [PATCH 25/28] =?UTF-8?q?VALUE=5FOUT=5FOF=5FRANGE:=20line=5Frefs?= =?UTF-8?q?=20=E2=80=94=20ref=20=D1=81=D1=82=D1=80=D0=BE=D0=BA=D0=B8,=20?= =?UTF-8?q?=D0=BD=D0=B5=20=D1=8F=D1=87=D0=B5=D0=B9=D0=BA=D0=B8=20=D0=B7?= =?UTF-8?q?=D0=BD=D0=B0=D1=87=D0=B5=D0=BD=D0=B8=D1=8F=20(I-2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Адрес ячейки остаётся в message. Добавлен инвариант-тест по golden: line_refs предупреждений документа входят в ref строк, locator нет, column_index есть среди колонок. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W --- arm_statements/lines.py | 2 +- arm_statements/tests/test_golden.py | 12 ++++++++++++ arm_statements/tests/test_lines.py | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/arm_statements/lines.py b/arm_statements/lines.py index 0702806..fc06045 100644 --- a/arm_statements/lines.py +++ b/arm_statements/lines.py @@ -42,7 +42,7 @@ def build_lines( message = f"значение ячейки {cell.ref} вне диапазона numeric(18,2)" warnings.append( make_warning( - WarningCode.VALUE_OUT_OF_RANGE, message, rules, line_refs=(cell.ref,), column_index=column_index + WarningCode.VALUE_OUT_OF_RANGE, message, rules, line_refs=(item.ref,), column_index=column_index ) ) lines.append( diff --git a/arm_statements/tests/test_golden.py b/arm_statements/tests/test_golden.py index d00c536..3058676 100644 --- a/arm_statements/tests/test_golden.py +++ b/arm_statements/tests/test_golden.py @@ -157,3 +157,15 @@ def test_npz_continuation_single_code(rules: Rules) -> None: # Review Focus 5 assert [v.amount for v in line_2400.values] == [D("264"), D("238")] assert "," in doc.locator and not [w for w in doc.warnings if w.code.startswith("SEGMENT_")] assert doc.status is DocStatus.DONE + + +@pytest.mark.parametrize("name", NAMES) +def test_document_warnings_reference_document_parts(name: str, rules: Rules) -> None: + """I-2: line_refs предупреждения — ref строк документа; locator нет; column_index — из колонок.""" + doc = run(name, rules) + line_refs = {ln.ref for ln in doc.lines} + column_indexes = {c.column_index for c in doc.columns} + for warning in doc.warnings: + assert set(warning.line_refs) <= line_refs, warning + assert warning.locator is None, warning + assert warning.column_index is None or warning.column_index in column_indexes, warning diff --git a/arm_statements/tests/test_lines.py b/arm_statements/tests/test_lines.py index 1c868e8..12484b0 100644 --- a/arm_statements/tests/test_lines.py +++ b/arm_statements/tests/test_lines.py @@ -37,7 +37,7 @@ def test_value_out_of_range_warning(rules: Rules) -> None: lines, warnings = statement_lines(cand, rules) assert lines[0].values[0].amount is None assert [(w.code, w.severity, w.line_refs, w.column_index) for w in warnings] == [ - ("VALUE_OUT_OF_RANGE", Severity.INFO, ("Баланс!C2",), 2) + ("VALUE_OUT_OF_RANGE", Severity.INFO, ("Баланс!B2",), 2) ] -- 2.47.2 From 598ac83855e877b2019976b7e005dae27dd4921a Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Sun, 18 Oct 2026 14:28:20 +0400 Subject: [PATCH 26/28] =?UTF-8?q?recompute=20=D0=B4=D0=BB=D1=8F=20DISCLOSU?= =?UTF-8?q?RE:=20=D0=BF=D0=BE=D0=BB=D0=BD=D1=8B=D0=B5=20=D1=81=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=D0=B0=D1=80=D0=B8=20needs=5Freview=20=D0=B8=20=D1=80?= =?UTF-8?q?=D0=BE=D0=B4=D0=B8=D1=82=D0=B5=D0=BB=D0=B8=20detail=20(I-3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W --- arm_statements/recompute.py | 7 +++--- arm_statements/tests/test_recompute.py | 31 ++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/arm_statements/recompute.py b/arm_statements/recompute.py index 8e2b335..887f276 100644 --- a/arm_statements/recompute.py +++ b/arm_statements/recompute.py @@ -127,13 +127,14 @@ def recompute(form: StoredForm, rules: Rules | None = None) -> RecomputeResult: """Шаги 6–8 по сохранённым данным: вариант, редакция, §7, `needs_review`, родители `detail`, статус.""" rules = rules if rules is not None else default_rules() if form.document_type is DocumentType.DISCLOSURE: + ordered = sorted(form.lines, key=_sort_key) return RecomputeResult( form_variant=None, edition=edition_by_date(form.effective_date), checks=StatementChecks((), ()), - line_needs_review={}, - value_needs_review={}, - detail_parents={}, + line_needs_review={line.key: False for line in ordered}, + value_needs_review={(line.key, v.column_key): False for line in ordered for v in line.values}, + detail_parents=_detail_parents(ordered), status=DocStatus.DONE, ) lines = sorted(form.lines, key=_sort_key) diff --git a/arm_statements/tests/test_recompute.py b/arm_statements/tests/test_recompute.py index 9564b8a..7e9d3c0 100644 --- a/arm_statements/tests/test_recompute.py +++ b/arm_statements/tests/test_recompute.py @@ -5,6 +5,7 @@ from decimal import Decimal as D from arm_statements import StoredColumn, StoredForm, StoredLine, StoredValue, choose_edition, recompute from arm_statements.loader import Rules from arm_statements.model import DocStatus, DocumentType, Edition, FormVariant, LineStatus, PeriodKind +from arm_statements.structure import edition_by_date OK, DETAIL = LineStatus.OK, LineStatus.DETAIL COL = StoredColumn("c1", 2, date(2025, 9, 30), PeriodKind.ABSOLUTE) @@ -85,3 +86,33 @@ def test_accepted_and_empty_edit_line(rules: Rules) -> None: broken = form(sl("a", 0, "1600", "100"), sl("b", 1, "1700", "90"), sl("e", None, "1220", None)) assert recompute(broken, rules).status is DocStatus.NEEDS_CHECK assert recompute(replace(broken, accepted=True), rules).status is DocStatus.DONE + + +def test_disclosure_has_complete_maps(rules: Rules) -> None: + """I-3: для расшифровки в словарях ключи всех строк и значений, а не пустые словари.""" + stored = form( + sl("a", 0, "1110", "10"), + sl("d", 1, None, "7", status=DETAIL), + sl("e", None, "1120", None), + document_type=DocumentType.DISCLOSURE, + ) + result = recompute(stored, rules) + assert result.line_needs_review == {"a": False, "d": False, "e": False} + assert result.value_needs_review == {("a", "c1"): False, ("d", "c1"): False, ("e", "c1"): False} + assert result.detail_parents == {"d": "a"} + assert (result.form_variant, result.checks.relations_failed, result.status) == (None, (), DocStatus.DONE) + assert result.edition is edition_by_date(date(2025, 9, 30)) + + +def test_edit_line_without_amount_is_excluded_from_checks_and_variant(rules: Rules) -> None: + """T18.6: пустая строка правки не участвует в §7 и не влияет на вариант.""" + base = [sl("a", 0, "1150", "50"), sl("b", 1, "1600", "50"), sl("c", 2, "1300", "50"), sl("d", 3, "1700", "50")] + with_edit = recompute(form(*base, sl("e", None, "1100", None)), rules) + without = recompute(form(*base), rules) + assert with_edit.form_variant is without.form_variant is FormVariant.SIMPLIFIED + assert with_edit.checks == without.checks + + +def test_detail_without_row_index_has_no_parent(rules: Rules) -> None: + stored = form(sl("a", 0, "1210", "10"), sl("d", None, None, "3", status=DETAIL)) + assert recompute(stored, rules).detail_parents == {"d": None} -- 2.47.2 From 5cc4522e4af791b49db3a7a0b75f7b7d0ff2a43d Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Sun, 18 Oct 2026 14:28:38 +0400 Subject: [PATCH 27/28] =?UTF-8?q?status=5Finput:=20=D0=BF=D1=83=D1=81?= =?UTF-8?q?=D1=82=D0=B0=D1=8F=20=D1=81=D1=82=D1=80=D0=BE=D0=BA=D0=B0=20?= =?UTF-8?q?=D0=BE=D1=80=D0=B3=D0=B0=D0=BD=D0=B8=D0=B7=D0=B0=D1=86=D0=B8?= =?UTF-8?q?=D0=B8/=D0=98=D0=9D=D0=9D=20=E2=80=94=20=C2=AB=D0=BD=D0=B5?= =?UTF-8?q?=D1=82=C2=BB=20(M-1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W --- arm_statements/status.py | 2 +- arm_statements/tests/test_status.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/arm_statements/status.py b/arm_statements/status.py index 02be8f8..25a2ccf 100644 --- a/arm_statements/status.py +++ b/arm_statements/status.py @@ -86,7 +86,7 @@ def status_input(document: StatementDocument) -> StatusInput: return StatusInput( document_type=document.document_type, layout_found=document.lines != (), - has_organization_or_inn=header.organization_name.value is not None or header.inn.value is not None, + has_organization_or_inn=bool(header.organization_name.value or header.inn.value), period_known=period_known, needs_review_fields=header.needs_review_fields, lines=tuple(StatusLine(line.line_status, line.needs_review) for line in document.lines), diff --git a/arm_statements/tests/test_status.py b/arm_statements/tests/test_status.py index ec5c9b2..0952f37 100644 --- a/arm_statements/tests/test_status.py +++ b/arm_statements/tests/test_status.py @@ -2,6 +2,7 @@ from dataclasses import replace from datetime import date from decimal import Decimal as D +from arm_statements import status_input_from_contract, to_contract from arm_statements.header import parse_header from arm_statements.issues import make_warning from arm_statements.loader import Rules @@ -10,6 +11,7 @@ from arm_statements.model import ( DocumentType, EnvelopeStatus, FormVariant, + HeaderField, LineStatus, PeriodKind, Severity, @@ -115,3 +117,12 @@ def test_status_input_from_document(rules: Rules) -> None: assert document_status(data) is DONE empty = replace(doc, columns=(), lines=(), checks=None) assert (status_input(empty).layout_found, status_input(empty).period_known) == (False, False) + + +def test_empty_string_organization_and_inn_mean_absent(rules: Rules) -> None: + """M-1: пустая строка в шапке — «нет»; совпадает с расчётом по контракту.""" + empty = HeaderField("", 100.0, "") + head = replace(parse_header(hdr(), rules), organization_name=empty, inn=empty) + doc = StatementDocument(DocumentType.BALANCE, None, "l", head, (), (), None, DONE, ()) + assert status_input(doc).has_organization_or_inn is False + assert status_input_from_contract(to_contract(doc)).has_organization_or_inn is False -- 2.47.2 From 2973cc7966fae6896b091d292081e5c2dbdb9504 Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Sun, 18 Oct 2026 14:29:10 +0400 Subject: [PATCH 28/28] =?UTF-8?q?=D0=94=D0=BE=D0=BA=D1=83=D0=BC=D0=B5?= =?UTF-8?q?=D0=BD=D1=82=D0=B8=D1=80=D0=BE=D0=B2=D0=B0=D0=BD=D0=B8=D0=B5=20?= =?UTF-8?q?warnings=20=D0=B0=D0=B4=D0=B0=D0=BF=D1=82=D0=B5=D1=80=D0=B0,=20?= =?UTF-8?q?=D1=82=D0=B5=D1=81=D1=82=20=D0=BB=D0=B8=D1=81=D1=82=D0=B0-?= =?UTF-8?q?=D0=BF=D1=80=D0=BE=D0=B4=D0=BE=D0=BB=D0=B6=D0=B5=D0=BD=D0=B8?= =?UTF-8?q?=D1=8F,=20=D0=BE=D0=B1=D0=B5=D0=B7=D0=BB=D0=B8=D1=87=D0=B5?= =?UTF-8?q?=D0=BD=D0=BD=D0=B0=D1=8F=20=D1=84=D0=B0=D0=BC=D0=B8=D0=BB=D0=B8?= =?UTF-8?q?=D1=8F=20(M-2,=20M-3,=20M-4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M-2: README и docstring normalize_candidate — при отсутствии документа candidate.warnings хост обрабатывает сам. M-3: тесты сопоставления по дате (переставленные даты) и по порядку (дубль даты). M-4: «Косарев» заменён на «Иванов» в тестах и docstring. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W --- README.md | 6 +++-- arm_statements/pipeline.py | 5 ++++- arm_statements/tests/test_rows.py | 2 +- arm_statements/tests/test_segments.py | 29 +++++++++++++++++++++++++ arm_statements/tests/test_text_codes.py | 2 +- arm_statements/text.py | 2 +- 6 files changed, 40 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 76101d9..25a5692 100644 --- a/README.md +++ b/README.md @@ -96,9 +96,11 @@ NormalizationResult(documents: tuple[StatementDocument, ...], file_warnings: tup ```python normalize_candidate(candidate: FormCandidate, rules: Rules | None = None) -> CandidateResult - # шаги 2–8 одного кандидата; документа нет — file_warnings (TABLE_LOST | TABLE_SKIPPED | SHEET_OTHER) + # шаги 2–8 одного кандидата; документа нет — file_warnings (TABLE_LOST | TABLE_SKIPPED | SHEET_OTHER); + # при этом candidate.warnings адаптера пакет не переносит — хост обрабатывает их сам normalize(candidates: Sequence[FormCandidate], rules: Rules | None = None) -> NormalizationResult - # по порядку; исключение кандидата не перехватывается — хост, которому нужна изоляция кандидатов + # по порядку; у кандидата без документа candidate.warnings не попадают в результат (см. выше); + # исключение кандидата не перехватывается — хост, которому нужна изоляция кандидатов # (TABLE_PARSE_ERROR), зовёт normalize_candidate в цикле recompute(form: StoredForm, rules: Rules | None = None) -> RecomputeResult parse_header(cells: HeaderCells, rules: Rules) -> StatementHeader # form_name = None (ставит конвейер) diff --git a/arm_statements/pipeline.py b/arm_statements/pipeline.py index aadcd0a..8e34d1a 100644 --- a/arm_statements/pipeline.py +++ b/arm_statements/pipeline.py @@ -160,7 +160,10 @@ def _lines_document( # noqa: PLR0913 -- сборка документа со с def normalize_candidate(candidate: FormCandidate, rules: Rules | None = None) -> CandidateResult: - """Шаги 2–8 кандидата; документа нет — предупреждение файла (`TABLE_LOST`, `TABLE_SKIPPED`, `SHEET_OTHER`).""" + """Шаги 2–8 кандидата; документа нет — предупреждение файла (`TABLE_LOST`, `TABLE_SKIPPED`, `SHEET_OTHER`). + + Когда документа нет, `candidate.warnings` (предупреждения адаптера) никуда не попадают: хост обрабатывает их сам. + """ rules = rules if rules is not None else default_rules() header = replace(parse_header(candidate.header, rules), form_name=_form_name(candidate)) layout = find_layout(candidate, header, rules) diff --git a/arm_statements/tests/test_rows.py b/arm_statements/tests/test_rows.py index 2c33bff..87ae28d 100644 --- a/arm_statements/tests/test_rows.py +++ b/arm_statements/tests/test_rows.py @@ -108,7 +108,7 @@ def test_excel_signature_text_is_not_content(rules: Rules) -> None: rows = [ ["Наименование", "", "Код", "На 30.09.2025"], ["Запасы", "", "1150", "5"], - ["Руководитель", "", "", "Косарев"], + ["Руководитель", "", "", "Иванов"], ["", "", "", "(расшифровка подписи)"], ] lines, _ = lines_of(rows, rules, source="excel") diff --git a/arm_statements/tests/test_segments.py b/arm_statements/tests/test_segments.py index 83c536d..d636409 100644 --- a/arm_statements/tests/test_segments.py +++ b/arm_statements/tests/test_segments.py @@ -158,3 +158,32 @@ def test_repeat_header_in_two_rows(rules: Rules) -> None: assert [s.first_row for s in layout.segments] == [0, 3] assert layout.roles[4] is RowRole.HEADER and layout.roles[5] is RowRole.DATA _no_date_noise(layout) + + +def _npz_sheets(second_dates: list[str]) -> list[tuple[str, list[list[str | D | None]]]]: + first = [ + place({1: "Наименование показателя", 12: "Код", 16: DATES3[0], 21: DATES3[1], 24: DATES3[2]}), + place({1: "Нематериальные активы", 12: "1110", 16: D("10"), 21: D("9"), 24: D("8")}), + ] + second = [ + place({0: "Наименование показателя", 6: "Код", 8: second_dates[0], 9: second_dates[1], 10: second_dates[2]}), + place({0: "Заемные средства", 6: "1510", 8: D("3"), 9: D("2"), 10: D("1")}), + ] + return [("Бухгалтерский баланс", first), ("Бухгалтерский баланс. Лист 4", second)] + + +def test_continuation_sheet_with_permuted_dates_maps_by_date_not_order(rules: Rules) -> None: + """M-3: даты листа-продолжения переставлены — колонки ищутся по дате, а не по порядку.""" + permuted = [DATES3[2], DATES3[0], DATES3[1]] # 31.12.2023, 30.09.2025, 31.12.2024 + cand = excel(_npz_sheets(permuted)) + layout = find_layout(cand, parse_header(hdr(report_date="30.09.2025"), rules), rules) + assert layout.segments[1].value_cols == ((8, 24), (9, 16), (10, 21)) + assert not [w for w in layout.warnings if w.code.startswith("SEGMENT_")] + + +def test_continuation_sheet_with_duplicate_date_maps_by_order(rules: Rules) -> None: + """M-3: две колонки на одну дату — сопоставление по порядку.""" + duplicate = [DATES3[0], DATES3[0], DATES3[2]] + cand = excel(_npz_sheets(duplicate)) + layout = find_layout(cand, parse_header(hdr(report_date="30.09.2025"), rules), rules) + assert layout.segments[1].value_cols == ((8, 16), (9, 21), (10, 24)) diff --git a/arm_statements/tests/test_text_codes.py b/arm_statements/tests/test_text_codes.py index 17567c7..77c94d3 100644 --- a/arm_statements/tests/test_text_codes.py +++ b/arm_statements/tests/test_text_codes.py @@ -44,7 +44,7 @@ def test_lookalikes_only_near_digit(rules: Rules) -> None: def test_disguised_digit(rules: Rules) -> None: assert is_disguised_digit("I", rules.lookalikes) and is_disguised_digit("| О", rules.lookalikes) - assert not is_disguised_digit("Косарев", rules.lookalikes) + assert not is_disguised_digit("Иванов", rules.lookalikes) assert not is_disguised_digit("(расшифровка подписи)", rules.lookalikes) diff --git a/arm_statements/text.py b/arm_statements/text.py index 70ac37d..e4dd601 100644 --- a/arm_statements/text.py +++ b/arm_statements/text.py @@ -39,7 +39,7 @@ def has_digit(text: str | None) -> bool: def is_disguised_digit(text: str, lookalikes: Mapping[str, str]) -> bool: - """После замены двойников везде есть цифра и нет других букв («I», «| О» — да; «Косарев» — нет).""" + """После замены двойников везде есть цифра и нет других букв («I», «| О» — да; «Иванов» — нет).""" fixed = fix_lookalikes(text, lookalikes, anywhere=True) return has_digit(fixed) and not any(char.isalpha() for char in fixed) -- 2.47.2