From f29800e645bee7c4dc79752c0de5389de830b941 Mon Sep 17 00:00:00 2001 From: tsygankoviva Date: Sat, 24 Oct 2026 11:44:25 +0300 Subject: [PATCH 1/6] =?UTF-8?q?AURORA-1677:=20arm=5Fstatements=20=D1=82?= =?UTF-8?q?=D0=B5=D0=BF=D0=B5=D1=80=D1=8C=20=D0=B1=D0=B8=D0=B1=D0=BB=D0=B8?= =?UTF-8?q?=D0=BE=D1=82=D0=B5=D0=BA=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CLAUDE.md | 13 +- Dockerfile | 9 +- Makefile | 28 +- README.md | 19 +- api/arm_statements/VERSION | 1 - api/arm_statements/__init__.py | 139 --- api/arm_statements/amounts.py | 194 ---- api/arm_statements/checks.py | 184 ---- api/arm_statements/codes.py | 118 --- api/arm_statements/columns.py | 238 ----- api/arm_statements/contract.py | 151 --- api/arm_statements/dates.py | 276 ------ api/arm_statements/evaluation.py | 36 - api/arm_statements/header.py | 125 --- api/arm_statements/issues.py | 73 -- api/arm_statements/layout.py | 334 ------- api/arm_statements/lines.py | 88 -- api/arm_statements/loader.py | 516 ----------- api/arm_statements/model.py | 246 ----- api/arm_statements/pipeline.py | 218 ----- api/arm_statements/recompute.py | 178 ---- api/arm_statements/review.py | 171 ---- api/arm_statements/rows.py | 253 ----- api/arm_statements/rules/ocr_lookalikes.yaml | 16 - .../rules/statement_checks.yaml | 29 - .../rules/statement_code_orders.yaml | 194 ---- .../rules/statement_code_ranges.yaml | 13 - .../rules/statement_headers.yaml | 78 -- .../rules/statement_periods.yaml | 44 - api/arm_statements/rules/statement_units.yaml | 16 - .../rules/statement_warnings.yaml | 37 - api/arm_statements/segments.py | 294 ------ api/arm_statements/serialize.py | 108 --- api/arm_statements/status.py | 97 -- api/arm_statements/structure.py | 177 ---- api/arm_statements/tests/__init__.py | 0 api/arm_statements/tests/builders.py | 155 ---- api/arm_statements/tests/conftest.py | 10 - .../tests/golden/aep3_balance.candidate.json | 750 --------------- .../tests/golden/aep3_balance.expected.json | 868 ------------------ .../tests/golden/aep3_income.candidate.json | 290 ------ .../tests/golden/aep3_income.expected.json | 356 ------- .../tests/golden/glued_balance.candidate.json | 330 ------- .../tests/golden/glued_balance.expected.json | 411 --------- .../golden/kody_title_balance.candidate.json | 538 ----------- .../golden/kody_title_balance.expected.json | 371 -------- .../tests/golden/npz_income.candidate.json | 433 --------- .../tests/golden/npz_income.expected.json | 488 ---------- api/arm_statements/tests/test_amounts.py | 128 --- api/arm_statements/tests/test_checks.py | 107 --- api/arm_statements/tests/test_columns.py | 213 ----- api/arm_statements/tests/test_contract.py | 99 -- api/arm_statements/tests/test_dates.py | 94 -- api/arm_statements/tests/test_golden.py | 171 ---- api/arm_statements/tests/test_header.py | 56 -- api/arm_statements/tests/test_layout.py | 173 ---- api/arm_statements/tests/test_lines.py | 63 -- api/arm_statements/tests/test_loader.py | 168 ---- api/arm_statements/tests/test_model.py | 118 --- api/arm_statements/tests/test_pipeline.py | 212 ----- api/arm_statements/tests/test_recompute.py | 118 --- api/arm_statements/tests/test_review.py | 139 --- api/arm_statements/tests/test_rows.py | 115 --- api/arm_statements/tests/test_segments.py | 259 ------ api/arm_statements/tests/test_status.py | 128 --- api/arm_statements/tests/test_structure.py | 106 --- api/arm_statements/tests/test_text_codes.py | 74 -- api/arm_statements/tests/test_version.py | 21 - api/arm_statements/text.py | 60 -- api/arm_statements/version.py | 14 - api/pyproject.toml | 17 +- api/pytest.ini | 1 - api/requirements-dev.txt | 2 +- api/requirements.txt | 1 + api/rules/README.md | 2 +- api/src/contract/common.py | 2 +- api/src/parser/__init__.py | 9 +- api/src/parser/book.py | 4 +- api/src/parser/detect.py | 1 + api/src/parser/statements/adapter.py | 1 + api/src/parser/statements/cells.py | 1 + api/src/parser/statements/forms.py | 1 + api/src/parser/statements/header_cells.py | 1 + api/src/parser/statements/pipeline.py | 1 + api/tests/fixtures/test_rules_coverage.py | 2 +- .../fixtures/test_statements_adapter_real.py | 2 +- .../unit/parser/statements/test_adapter.py | 2 +- .../unit/parser/statements/test_cells.py | 2 +- .../unit/parser/statements/test_forms.py | 2 +- .../parser/statements/test_header_cells.py | 2 +- .../unit/parser/statements/test_pipeline.py | 4 +- api/tests/unit/parser/test_book_parser.py | 4 +- api/tests/unit/parser/test_read_limits.py | 2 +- api/tests/unit/test_worker.py | 4 +- .../use-arm-statements-library/.openspec.yaml | 2 + .../use-arm-statements-library/design.md | 61 ++ .../use-arm-statements-library/proposal.md | 39 + .../specs/statement-parsing/spec.md | 50 + .../specs/tabular-recognition-service/spec.md | 105 +++ .../use-arm-statements-library/tasks.md | 36 + 100 files changed, 362 insertions(+), 12348 deletions(-) delete mode 100644 api/arm_statements/VERSION delete mode 100644 api/arm_statements/__init__.py delete mode 100644 api/arm_statements/amounts.py delete mode 100644 api/arm_statements/checks.py delete mode 100644 api/arm_statements/codes.py delete mode 100644 api/arm_statements/columns.py delete mode 100644 api/arm_statements/contract.py delete mode 100644 api/arm_statements/dates.py delete mode 100644 api/arm_statements/evaluation.py delete mode 100644 api/arm_statements/header.py delete mode 100644 api/arm_statements/issues.py delete mode 100644 api/arm_statements/layout.py delete mode 100644 api/arm_statements/lines.py delete mode 100644 api/arm_statements/loader.py delete mode 100644 api/arm_statements/model.py delete mode 100644 api/arm_statements/pipeline.py delete mode 100644 api/arm_statements/recompute.py delete mode 100644 api/arm_statements/review.py delete mode 100644 api/arm_statements/rows.py delete mode 100644 api/arm_statements/rules/ocr_lookalikes.yaml delete mode 100644 api/arm_statements/rules/statement_checks.yaml delete mode 100644 api/arm_statements/rules/statement_code_orders.yaml delete mode 100644 api/arm_statements/rules/statement_code_ranges.yaml delete mode 100644 api/arm_statements/rules/statement_headers.yaml delete mode 100644 api/arm_statements/rules/statement_periods.yaml delete mode 100644 api/arm_statements/rules/statement_units.yaml delete mode 100644 api/arm_statements/rules/statement_warnings.yaml delete mode 100644 api/arm_statements/segments.py delete mode 100644 api/arm_statements/serialize.py delete mode 100644 api/arm_statements/status.py delete mode 100644 api/arm_statements/structure.py delete mode 100644 api/arm_statements/tests/__init__.py delete mode 100644 api/arm_statements/tests/builders.py delete mode 100644 api/arm_statements/tests/conftest.py delete mode 100644 api/arm_statements/tests/golden/aep3_balance.candidate.json delete mode 100644 api/arm_statements/tests/golden/aep3_balance.expected.json delete mode 100644 api/arm_statements/tests/golden/aep3_income.candidate.json delete mode 100644 api/arm_statements/tests/golden/aep3_income.expected.json delete mode 100644 api/arm_statements/tests/golden/glued_balance.candidate.json delete mode 100644 api/arm_statements/tests/golden/glued_balance.expected.json delete mode 100644 api/arm_statements/tests/golden/kody_title_balance.candidate.json delete mode 100644 api/arm_statements/tests/golden/kody_title_balance.expected.json delete mode 100644 api/arm_statements/tests/golden/npz_income.candidate.json delete mode 100644 api/arm_statements/tests/golden/npz_income.expected.json delete mode 100644 api/arm_statements/tests/test_amounts.py delete mode 100644 api/arm_statements/tests/test_checks.py delete mode 100644 api/arm_statements/tests/test_columns.py delete mode 100644 api/arm_statements/tests/test_contract.py delete mode 100644 api/arm_statements/tests/test_dates.py delete mode 100644 api/arm_statements/tests/test_golden.py delete mode 100644 api/arm_statements/tests/test_header.py delete mode 100644 api/arm_statements/tests/test_layout.py delete mode 100644 api/arm_statements/tests/test_lines.py delete mode 100644 api/arm_statements/tests/test_loader.py delete mode 100644 api/arm_statements/tests/test_model.py delete mode 100644 api/arm_statements/tests/test_pipeline.py delete mode 100644 api/arm_statements/tests/test_recompute.py delete mode 100644 api/arm_statements/tests/test_review.py delete mode 100644 api/arm_statements/tests/test_rows.py delete mode 100644 api/arm_statements/tests/test_segments.py delete mode 100644 api/arm_statements/tests/test_status.py delete mode 100644 api/arm_statements/tests/test_structure.py delete mode 100644 api/arm_statements/tests/test_text_codes.py delete mode 100644 api/arm_statements/tests/test_version.py delete mode 100644 api/arm_statements/text.py delete mode 100644 api/arm_statements/version.py create mode 100644 openspec/changes/use-arm-statements-library/.openspec.yaml create mode 100644 openspec/changes/use-arm-statements-library/design.md create mode 100644 openspec/changes/use-arm-statements-library/proposal.md create mode 100644 openspec/changes/use-arm-statements-library/specs/statement-parsing/spec.md create mode 100644 openspec/changes/use-arm-statements-library/specs/tabular-recognition-service/spec.md create mode 100644 openspec/changes/use-arm-statements-library/tasks.md diff --git a/CLAUDE.md b/CLAUDE.md index d9a2abc..ccd9f85 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -14,7 +14,7 @@ Stateless-сервис разбора XLS/XLSX/XLSB: ОСВ по счетам ( новый формат = новая фикстура + синоним, не новый `if`. ## Правила репозитория -- `api/arm_statements/` — копия пакета `arm-statements`, напрямую не править: обновление — `../docs/scripts/sync-statements.sh` +- `arm_statements` — библиотека `pylib_ai_arm_statements==0.1.1`; установка и обновление — `make install` (`--in-place` в PR фичи), свежесть — `make statements-check` (в `check-full`); код возврата 20 при расхождении. Словари форм — в пакете; `api/rules/statement_*.yaml` удалены планом 4 (#54). @@ -104,8 +104,8 @@ property, покрытие словарей); без каталога `../fixtur ## Формы через arm_statements (план 4, #54) -Конвейер форм ББ/ОФР (шаги 2–8, словари форм, перевод в контракт) — в пакете `api/arm_statements/` (копия -`arm-statements`, версия — `arm_statements/VERSION`). Собственного конвейера форм и словарей форм в сервисе нет: удалены +Конвейер форм ББ/ОФР (шаги 2–8, словари форм, перевод в контракт) — в библиотеке `arm_statements` (дистрибутив +`pylib_ai_arm_statements`, версия — из VERSION или метаданных пакета). Собственного конвейера форм и словарей форм в сервисе нет: удалены `api/src/parser/statements/{checks,columns,detect,header,lines,merge,statement}.py` и `api/rules/statement_*.yaml`. @@ -116,18 +116,17 @@ property, покрытие словарей); без каталога `../fixtur `book.py` (`BookParser`) склеивает ОСВ и формы; `get_parser(settings)` создаёт его с `default_rules()` пакета. **Контракт 2.0:** `CONTRACT_VERSION = "2.0"`; у документов `status` и `warnings`; `checks` обязателен у BALANCE/INCOME со -строками; `line_code` может быть пуст (`line_status`). `service.statements_version` — версия копии пакета (без `VERSION` +строками; `line_code` может быть пуст (`line_status`). `service.statements_version` — версия библиотеки (без версии сервис не стартует). Копия `api/contracts/` идентична `../docs/contracts/` (проверяется в `make check`). **Коды `OSV_*`:** предупреждения листа ОСВ — уровень `info`, модуль `osv_warnings.py` (контракт 2.0); предупреждение файла `XLS_NO_CODEPAGE` — из пакета. **Правила расширения форм:** -- Словари форм меняются только в `arm-statements`, затем `sync-statements` (`--in-place` в PR фичи). Новое написание или +- Словари форм меняются только в библиотеке `arm_statements`, затем `make install`. Новое написание или соотношение — там же, плюс фикстура или синтетика, на которой оно срабатывает (`tests/synthetic/generate_statements.py`, `../fixtures/statements-xlsx/`). Не `if` под файл, организацию или лист. -- **Копию `api/arm_statements/` не правим напрямую** (код возврата 20 у `statements-check` при расхождении). Правка в - копии теряется при следующем `sync-statements`. +- Библиотека устанавливается из `../pylib_ai_arm_statements` и не публикуется в PyPI. - Координат в `src/parser/**` нет (кроме `grid.py`, `readers/`): тот же AST-линтер `test_no_coordinates.py`; слов формы в коде сервиса нет. Деньги — `Decimal`; единицы не нормализуются, первичка не мутируется. - Модули ОСВ (`header.py`, `classify.py`, `checks.py`, `hierarchy.py`) под формы не правятся. diff --git a/Dockerfile b/Dockerfile index cc4e97a..19ed694 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,7 @@ +# syntax=docker/dockerfile:1 # Образ для dev и `make check-full`. Контекст сборки — корень репозитория tabular-parser. -# Платформенные пакеты raisa_* (Nexus) в образ не ставятся — та же логика, что в `make venv`. +# Библиотека передаётся именованным контекстом arm_statements (см. make docker-build). +# Платформенные пакеты raisa_* (Nexus) в образ не ставятся. FROM python:3.11-slim ENV PYTHONUNBUFFERED=1 \ @@ -8,8 +10,9 @@ ENV PYTHONUNBUFFERED=1 \ WORKDIR /app/api COPY api/requirements.txt ./requirements.txt -RUN grep -v '^raisa_' requirements.txt > /tmp/req.txt \ - && pip install --no-cache-dir -r /tmp/req.txt \ +RUN --mount=type=bind,from=arm_statements,target=/tmp/arm_statements,rw \ + grep -Ev '^(raisa_|pylib_ai)' requirements.txt > /tmp/req.txt \ + && pip install --no-cache-dir -r /tmp/req.txt /tmp/arm_statements \ && rm /tmp/req.txt COPY api/ ./ diff --git a/Makefile b/Makefile index ae46c50..0311571 100644 --- a/Makefile +++ b/Makefile @@ -1,20 +1,27 @@ # Команды разработки tabular-parser. Backend — api/.venv; запуск из корня репозитория. VENV := api/.venv BIN := $(CURDIR)/$(VENV)/bin +STATEMENTS_DIR ?= ../pylib_ai_arm_statements COMPOSE_FILE ?= ../docs/dev/docker-compose.yml -.PHONY: venv lint test test-integration test-fixtures golden-update golden-check check docker-build check-full up statements-check +.PHONY: venv install lint test test-integration test-fixtures golden-update golden-check check docker-build check-full up -# Python 3.11 (версия платформы); внутренние пакеты raisa_* вне банка недоступны — пропускаем их +# Python 3.11; пропускаем внутренние raisa_* и устанавливаемые локально pylib_ai* venv: cd api && uv venv --python 3.11 --allow-existing .venv - cd api && grep -v '^raisa_' requirements.txt | uv pip install --python .venv/bin/python -r - + cd api && grep -Ev '^(raisa_|pylib_ai)' requirements.txt | uv pip install --python .venv/bin/python -r - cd api && grep -v '^-r' requirements-dev.txt | uv pip install --python .venv/bin/python -r - + uv pip install --python "$(BIN)/python" --reinstall-package pylib-ai-arm-statements "$(STATEMENTS_DIR)" + +# Установить или переустановить библиотеку в окружение backend. +install: + uv venv --python 3.11 --allow-existing "$(VENV)" + uv pip install --python "$(BIN)/python" --reinstall-package pylib-ai-arm-statements "$(STATEMENTS_DIR)" lint: cd api && $(BIN)/ruff check . cd api && $(BIN)/ruff format --check . - cd api && $(BIN)/mypy src arm_statements + cd api && $(BIN)/mypy src test: cd api && $(BIN)/pytest -m "not integration" @@ -39,20 +46,11 @@ check: lint test # Сборка образа из корня репозитория docker-build: - docker build -t tabular-parser . + docker build --build-context arm_statements="$(STATEMENTS_DIR)" -t tabular-parser . # Полная проверка перед PR: быстрые проверки, затем NATS/MinIO из compose (make up), затем образ -check-full: check statements-check test-integration docker-build +check-full: check test-integration docker-build # Только инфраструктура, нужная сервису (без arm-core): NATS и MinIO; postgres — для общей среды compose up: docker compose -f $(COMPOSE_FILE) up -d --wait postgres nats minio - -# Свежесть копии api/arm_statements/ (../docs/dev/checks.md): 0 — синхронна; 10 — отстала или перечень кодов в docs -# новее (цель зелёная, предупреждение вписать в описание PR); 20 и 2 — красная. Читает ../docs и ../arm-statements, -# поэтому только в check-full, не в check -ARM_KA_DOCS ?= ../docs -statements-check: - @rc=0; $(ARM_KA_DOCS)/scripts/sync-statements.sh --check --host-dir . || rc=$$?; \ - if [ $$rc -eq 10 ]; then echo "statements-check: ПРЕДУПРЕЖДЕНИЕ — впишите в описание PR"; \ - elif [ $$rc -ne 0 ]; then exit $$rc; fi diff --git a/README.md b/README.md index 66d6a47..7e35763 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,8 @@ openspec/ требования и дельта текущего ср `arm-core`, этому сервису БД не нужна). - `make`. -Пакеты `raisa_*` из Nexus банка вне банка недоступны: `make venv` пропускает строки `raisa_*` в `requirements.txt`. +Пакеты `raisa_*` из Nexus банка вне банка недоступны: `make venv` пропускает строки `raisa_*` и `pylib_ai*` в `requirements.txt`; +библиотека `pylib_ai_arm_statements==0.1.1` устанавливается локально. ## Команды `make` @@ -87,6 +88,7 @@ openspec/ требования и дельта текущего ср | Команда | Что делает | |---|---| | `make venv` | создаёт `api/.venv` (Python 3.11) и ставит `requirements.txt` и `requirements-dev.txt` через `uv` | +| `make install` | устанавливает или переустанавливает библиотеку из `../pylib_ai_arm_statements` (путь: `STATEMENTS_DIR`) | | `make lint` | `ruff check .`, `ruff format --check .`, `mypy src` | | `make test` | `pytest -m "not integration"` — без Docker; включает тесты маркера `fixtures` (golden, property, покрытие словарей), без каталога `../fixtures` они пропускаются | | `make test-integration` | `pytest -m integration` — против NATS и MinIO из compose | @@ -95,6 +97,7 @@ openspec/ требования и дельта текущего ср | `make golden-check` | то же без записи; код возврата 1, если снимки расходятся с разбором | | `make check` | `make lint` + `make test` (включая идентичность копии контракта с `../docs/contracts`); с каталогом fixtures ≈ 95 с — из-за тестов на 82 файлах | | `make check-full` | `make check` + `make test-integration` + `docker build`; запускать перед PR (нужен поднятый compose, `make up`) | +| `make docker-build` | собирает образ, передавая локальную библиотеку через `--build-context arm_statements="$(STATEMENTS_DIR)"` (нужен BuildKit) | | `make up` | `docker compose -f ../docs/dev/docker-compose.yml up -d --wait postgres nats minio` | Интеграционные тесты (`make test-integration`, `make check-full`) поднимают сервис в процессе теста и вступают в queue group `tabular-parser` на реальной шине: на шине compose не должно работать другого экземпляра tabular-parser (и наоборот: на время теста in-process воркер ответит ошибкой разбора (`UNSUPPORTED_FORMAT` / `UNREADABLE`) на реальные запросы других производителей). @@ -299,12 +302,12 @@ openpyxl 0,58 с; синтетический лист в 2 млн ячеек (63 Книга разбирается `book.py` (`BookParser`): листы ОСВ идут через `osv.py`, листы форм — через адаптер `statements/` к пакету `arm_statements`. Документы результата: сначала ОСВ в порядке листов, затем формы в порядке листов. -**Словари форм и шаги 2–8 живут в пакете `arm_statements`** (копия `api/arm_statements/`, версия — `arm_statements/VERSION`, -`service.statements_version`). Собственного конвейера форм и словарей форм в сервисе нет: синонимы заголовков, периодов, -соотношения §7 — данные пакета (`rules/statement_*.yaml`), код сервиса их не содержит. +**Словари форм и шаги 2–8 живут в библиотеке `arm_statements`** (дистрибутив `pylib_ai_arm_statements`, +версия передаётся в `service.statements_version`). Собственного конвейера форм и словарей форм в сервисе нет: +синонимы заголовков, периодов, соотношения §7 — данные библиотеки (`rules/statement_*.yaml`). -**Копию `api/arm_statements/` не правим напрямую.** Обновление — `docs/scripts/sync-statements.sh` (`--in-place` в PR -фичи); свежесть проверяет `make statements-check` (код возврата 20 при расхождении, входит в `make check-full`). +Библиотека не публикуется в PyPI. `make venv` устанавливает её из `../pylib_ai_arm_statements`; +`make install` переустанавливает её после изменений. Путь можно задать через `STATEMENTS_DIR`. | Модуль (`api/src/parser/statements/`) | Что делает | |---|---| @@ -322,8 +325,8 @@ openpyxl 0,58 с; синтетический лист в 2 млн ячеек (63 `checks` обязателен у BALANCE/INCOME со строками, `line_code` может быть пуст (`line_status`: `ok | unreadable | missing | detail`). Статус книги и критерий `ok` документа вычисляет пакет (шаги 2–8); сервис статус не пересчитывает. -Словарь форм не правится в сервисе. Новое написание формы или соотношение — правка в `arm-statements` с последующим -`sync-statements`, плюс фикстура или синтетика, на которой оно срабатывает. +Словарь форм не правится в сервисе. Новое написание формы или соотношение — правка в библиотеке `arm_statements` с последующим +`make install`, плюс фикстура или синтетика, на которой оно срабатывает. ### Ограничения diff --git a/api/arm_statements/VERSION b/api/arm_statements/VERSION deleted file mode 100644 index 58f755c..0000000 --- a/api/arm_statements/VERSION +++ /dev/null @@ -1 +0,0 @@ -25b2bb81195bb1e99f73b90e336cd175ae8b344c diff --git a/api/arm_statements/__init__.py b/api/arm_statements/__init__.py deleted file mode 100644 index 34ad4e5..0000000 --- a/api/arm_statements/__init__.py +++ /dev/null @@ -1,139 +0,0 @@ -"""Общий код нормализации форм ББ/ОФР. - -Публичный API — только имена `__all__` (раздел «Публичный API пакета» плана 1). Контрактные имена — `contract.py`. -""" - -from arm_statements.amounts import parse_amount -from arm_statements.codes import parse_code, sheet_joins -from arm_statements.columns import amount_columns -from arm_statements.contract import status_input_from_contract, to_contract, warning_to_contract -from arm_statements.dates import parse_date, resolve_relative -from arm_statements.header import continuation_type, is_continuation, parse_header, type_hint -from arm_statements.issues import WarningCode, make_warning -from arm_statements.layout import code_column -from arm_statements.loader import Rules, RulesError, default_rules, load_rules -from arm_statements.model import ( - HEADER_FIELDS, - CandidateResult, - Cell, - CellValue, - DateField, - DocStatus, - DocumentType, - Edition, - EnvelopeStatus, - Evidence, - FailedRelation, - FormCandidate, - FormVariant, - HeaderCells, - HeaderField, - LineStatus, - NormalizationResult, - PeriodKind, - Severity, - SignForm, - Source, - StatementChecks, - StatementColumn, - StatementDocument, - StatementHeader, - StatementLine, - StatementValue, - StatementWarning, -) -from arm_statements.pipeline import normalize, normalize_candidate -from arm_statements.recompute import ( - RecomputeResult, - StoredColumn, - StoredForm, - StoredLine, - StoredValue, - StoredWarning, - recompute, -) -from arm_statements.serialize import candidate_from_dict, to_jsonable -from arm_statements.status import ( - StatusInput, - StatusLine, - StatusValue, - document_status, - envelope_status, - status_input, -) -from arm_statements.structure import choose_edition -from arm_statements.text import fix_lookalikes, normalize_caption, normalize_label -from arm_statements.version import statements_version - -__all__ = [ - "HEADER_FIELDS", - "CandidateResult", - "Cell", - "CellValue", - "DateField", - "DocStatus", - "DocumentType", - "Edition", - "EnvelopeStatus", - "Evidence", - "FailedRelation", - "FormCandidate", - "FormVariant", - "HeaderCells", - "HeaderField", - "LineStatus", - "NormalizationResult", - "PeriodKind", - "RecomputeResult", - "Rules", - "RulesError", - "Severity", - "SignForm", - "Source", - "StatementChecks", - "StatementColumn", - "StatementDocument", - "StatementHeader", - "StatementLine", - "StatementValue", - "StatementWarning", - "StoredColumn", - "StoredForm", - "StoredLine", - "StoredValue", - "StoredWarning", - "StatusInput", - "StatusLine", - "StatusValue", - "WarningCode", - "amount_columns", - "candidate_from_dict", - "choose_edition", - "code_column", - "continuation_type", - "default_rules", - "document_status", - "envelope_status", - "fix_lookalikes", - "is_continuation", - "load_rules", - "make_warning", - "normalize", - "normalize_candidate", - "normalize_caption", - "normalize_label", - "parse_amount", - "parse_code", - "parse_date", - "parse_header", - "recompute", - "resolve_relative", - "sheet_joins", - "statements_version", - "status_input", - "status_input_from_contract", - "to_contract", - "to_jsonable", - "type_hint", - "warning_to_contract", -] diff --git a/api/arm_statements/amounts.py b/api/arm_statements/amounts.py deleted file mode 100644 index c89c711..0000000 --- a/api/arm_statements/amounts.py +++ /dev/null @@ -1,194 +0,0 @@ -"""Сумма и значение ячейки (шаг 4): `parse_amount` — единственное место, где разбор зависит от источника.""" - -import re -from dataclasses import dataclass -from datetime import date -from decimal import ROUND_HALF_UP, Decimal - -from arm_statements.codes import code_parts -from arm_statements.loader import Rules -from arm_statements.model import Cell, SignForm, Source, StatementValue -from arm_statements.text import replace_lookalikes - -MAX_INTEGER_DIGITS = 16 -_CENT = Decimal("0.01") -_DIGITS = frozenset("0123456789") -_CORE_START = _DIGITS | frozenset("(-−+") -_CORE_END = _DIGITS | frozenset(")}]") -_DASHES = frozenset("-–—−") -_MINUSES = frozenset("-−") -_BODY = re.compile(r"[\d\s,.]*") -_DIGIT_SPACE_RUN = re.compile(r"\d+(?:\s+\d+)+") -_SPACE_GROUPS = re.compile(r"\d{1,3}(?:\s\d{3})+") -_COMMA_THOUSANDS = re.compile(r"[1-9]\d{0,2}(?:,\d{3})+") -_DOT_THOUSANDS = re.compile(r"[1-9]\d*(?:\.\d{3})+") -_SEPARATORS = ",." - - -@dataclass(frozen=True, slots=True) -class AmountParse: - amount: Decimal | None - string_value: str | None - is_recognized: bool - sign_form: SignForm | None - - -@dataclass(frozen=True, slots=True) -class ValueBuild: - value: StatementValue - out_of_range: bool - - -def _quantize(amount: Decimal) -> Decimal: - return amount.quantize(_CENT, rounding=ROUND_HALF_UP) - - -def _too_long(amount: Decimal) -> bool: - return len(str(abs(int(amount)))) > MAX_INTEGER_DIGITS - - -def _is_blank(text: str) -> bool: - return all(char.isspace() or char in _DASHES for char in text) - - -def _strip_edge_bars(text: str) -> tuple[str, bool]: - """Крайние `|` (след границы таблицы) — до замены двойников; второй элемент — было ли удаление.""" - stripped = text.strip() - while stripped.startswith("|"): - stripped = stripped[1:].strip() - while stripped.endswith("|"): - stripped = stripped[:-1].strip() - return stripped, stripped != text.strip() - - -def _core_bounds(text: str) -> tuple[int, int] | None: - starts = [index for index, char in enumerate(text) if char in _CORE_START] - ends = [index for index, char in enumerate(text) if char in _CORE_END] - if not starts or not ends or starts[0] > ends[-1]: - return None - return starts[0], ends[-1] - - -def _guess(digits: str, text: str, sign_form: SignForm, *, negative: bool) -> AmountParse: - """«Лучшая догадка»: цифры подряд, не длиннее 16, иначе значение не число.""" - if not digits or len(digits) > MAX_INTEGER_DIGITS: - return AmountParse(None, text, False, sign_form) - amount = _quantize(Decimal(digits)) - return AmountParse(-amount if negative else amount, None, False, sign_form) - - -def _split_core(core: str) -> tuple[str, SignForm, bool, bool] | None: - """Ядро → (тело без знака и скобок, форма знака, отрицательное ли, исправлена ли скобка) или None.""" - fixed = False - sign_form = SignForm.UNSIGNED - negative = False - if core[0] == "(": - if len(core) < 2 or core[-1] not in ")}]": # noqa: PLR2004 — скобки в ядре минимум пара - return None - fixed = core[-1] != ")" - core = core[1:-1].strip() - sign_form, negative = SignForm.PARENTHESES, True - elif core[0] in "-−+": - if core[0] in _MINUSES: - sign_form, negative = SignForm.MINUS, True - core = core[1:].strip() - if not core or not _BODY.fullmatch(core) or not core[0].isdigit(): - return None - return core, sign_form, negative, fixed - - -def _number(body: str, source: Source, text: str, sign_form: SignForm, *, negative: bool) -> AmountParse: - """Стадия 6: разделители. Возвращает догадку с `is_recognized = false` или точный разбор.""" - runs = _DIGIT_SPACE_RUN.findall(body) - spaces_ok = all(_SPACE_GROUPS.fullmatch(run) for run in runs) - compact = "".join(body.split()) - digits = compact.replace(",", "").replace(".", "") - if not spaces_ok: - return _guess(digits, text, sign_form, negative=negative) - if source is Source.OCR and _COMMA_THOUSANDS.fullmatch(compact): - compact = compact.replace(",", "") - elif source is Source.OCR and _DOT_THOUSANDS.fullmatch(compact): - compact = compact.replace(".", "") - separators = [char for char in compact if char in _SEPARATORS] - if not separators: - integer, fraction = compact, "" - elif len(separators) > 1: # смешанные или повторяющиеся разделители - return _guess(digits, text, sign_form, negative=negative) - else: - integer, fraction = compact.replace(",", ".").split(".") - if source is Source.OCR and len(fraction) >= 3: # noqa: PLR2004 — три знака: не копейки - return AmountParse(None, text, False, sign_form) - if len(integer) > MAX_INTEGER_DIGITS: - return AmountParse(None, text, False, sign_form) - amount = _quantize(Decimal(f"{integer}.{fraction}" if fraction else integer)) - return AmountParse(-amount if negative else amount, None, True, sign_form) - - -def parse_amount(text: str | None, source: Source, rules: Rules) -> AmountParse: - """Стадии 1–6 шага «значение ячейки»; от источника зависят только десятичный разделитель и тысячи.""" - if text is None or _is_blank(text): - return AmountParse(None, None, True, None) - work, fixed = _strip_edge_bars(text) - replaced = replace_lookalikes(work, rules.lookalikes) - fixed = fixed or replaced != work - if not any(char in _DIGITS for char in replaced): # стадия 5 - return AmountParse(None, text, False, None) - bounds = _core_bounds(replaced) - if bounds is None: - return AmountParse(None, text, False, None) - start, end = bounds - core = replaced[start : end + 1] - trash = bool((replaced[:start] + replaced[end + 1 :]).strip()) - split = _split_core(core) - if split is None: - return AmountParse(None, text, False, None) - body, sign_form, negative, bracket_fixed = split - parsed = _number(body, source, text, sign_form, negative=negative) - clean = parsed.is_recognized and not (fixed or trash or bracket_fixed) - return AmountParse(parsed.amount, parsed.string_value, clean, parsed.sign_form) - - -def sign_form_of(text: str | None, rules: Rules) -> SignForm | None: - return parse_amount(text, Source.OCR, rules).sign_form - - -def _plain(cell: Cell, column_index: int, parsed: AmountParse, *, recognized: bool | None = None) -> StatementValue: - return StatementValue( - column_index=column_index, - amount=parsed.amount, - string_value=parsed.string_value, - text=cell.text, - confidence=cell.confidence, - is_recognized=parsed.is_recognized if recognized is None else recognized, - ) - - -def build_value( - cell: Cell | None, - column_index: int, - source: Source, - rules: Rules, - *, - split_part: bool = False, -) -> ValueBuild: - if cell is None: - return ValueBuild(StatementValue(column_index, None, None, None, 100.0, True), out_of_range=False) - text = cell.text - string_text = text if text.strip() else None - value = cell.value - if isinstance(value, Decimal): - # длину целой части проверяем до квантования: для 1E+30 и нефинитных значений quantize падает - amount = _quantize(value) if value.is_finite() and value.adjusted() < MAX_INTEGER_DIGITS else None - if amount is None or _too_long(amount): - overflow = AmountParse(None, string_text, False, None) - return ValueBuild(_plain(cell, column_index, overflow), out_of_range=True) - return ValueBuild( - _plain(cell, column_index, AmountParse(amount, None, True, None), recognized=not split_part), - out_of_range=False, - ) - if isinstance(value, date | bool): - return ValueBuild(_plain(cell, column_index, AmountParse(None, string_text, False, None)), out_of_range=False) - if len(code_parts(text)) > 1 and not split_part: - return ValueBuild(_plain(cell, column_index, AmountParse(None, text, False, None)), out_of_range=False) - parsed = parse_amount(text, source, rules) - return ValueBuild(_plain(cell, column_index, parsed, recognized=False if split_part else None), out_of_range=False) diff --git a/api/arm_statements/checks.py b/api/arm_statements/checks.py deleted file mode 100644 index 16a3cf2..0000000 --- a/api/arm_statements/checks.py +++ /dev/null @@ -1,184 +0,0 @@ -"""Шаг 7: самопроверки §7 и вариант формы.""" - -from collections.abc import Iterable, Mapping, Sequence -from dataclasses import dataclass -from decimal import Decimal - -from arm_statements.codes import document_type_of, section_of -from arm_statements.evaluation import EvalLine, is_check_line -from arm_statements.loader import RelationRule, Rules -from arm_statements.model import DocumentType, FailedRelation, FormVariant, StatementChecks - -_ZERO = Decimal("0") -_HUNDRED = Decimal("100") -_PARENT_LEN = 4 -_SUBLINE_LEN = 5 -_GROUP_LEN = 3 # первые цифры, общие у строки и её подстрок (1230 → 12301…12314) - - -def form_variant(document_type: DocumentType, ok_codes: Iterable[str]) -> FormVariant | None: - """Вариант формы по структуре кодов (спека «Шаг 6»); ОКУД признаком не служит.""" - codes = set(ok_codes) - if document_type is DocumentType.BALANCE: - simplified = "1600" in codes and not {"1100", "1200"} & codes - elif document_type is DocumentType.INCOME: - simplified = bool({"2400", "2110"} & codes) and not {"2100", "2200", "2300"} & codes - else: - return None - return FormVariant.SIMPLIFIED if simplified else FormVariant.FULL - - -def subline_parents(codes: Sequence[str]) -> list[str | None]: - """Родитель каждой позиции: у пятизначного кода — ближайший четырёхзначный выше с теми же тремя цифрами, - иначе первые четыре цифры; у прочих — `None`.""" - result: list[str | None] = [] - last_parent: str | None = None - for code in codes: - if len(code) == _PARENT_LEN: - last_parent = code - result.append(None) - elif len(code) == _SUBLINE_LEN: - if last_parent is not None and last_parent[:_GROUP_LEN] == code[:_GROUP_LEN]: - result.append(last_parent) - else: - result.append(code[:_PARENT_LEN]) - else: - result.append(None) - return result - - -@dataclass(frozen=True, slots=True) -class RelationOutcome: - column: str - rule: RelationRule - codes: tuple[str, ...] - computed: bool - expected: Decimal | None # None — цель пуста - actual: Decimal | None # None — пусты все члены - within_tolerance: bool - within_rounding: bool - - @property - def hard_failed(self) -> bool: - return self.rule.strict and self.computed and not self.within_tolerance and not self.within_rounding - - -def _column_sums(lines: Sequence[EvalLine], column: str) -> tuple[dict[str, Decimal], set[str]]: - """Код → сумма `amount` по строкам `is_check_line` в колонке; второй результат — коды с непустой суммой.""" - sums: dict[str, Decimal] = {} - for item in lines: - if not is_check_line(item) or item.line_code is None: - continue - for value in item.values: - if value.column == column and value.amount is not None: - sums[item.line_code] = sums.get(item.line_code, _ZERO) + value.amount - return sums, set(sums) - - -def _rhs(sums: Mapping[str, Decimal], rule: RelationRule, *, absolute: bool) -> Decimal: - total = _ZERO - for sign, code in rule.terms: - value = sums.get(code, _ZERO) - if absolute and code in rule.abs_terms: - value = abs(value) - total += sign * value - return total - - -def _actual(sums: Mapping[str, Decimal], rule: RelationRule, expected: Decimal, tolerance: Decimal) -> Decimal: - """Правая часть: сошедшийся вариант (как записано, затем по модулю), иначе ближайший к цели.""" - plain = _rhs(sums, rule, absolute=False) - if abs(expected - plain) <= tolerance or not rule.abs_terms: - return plain - with_abs = _rhs(sums, rule, absolute=True) - if abs(expected - with_abs) <= tolerance or abs(expected - with_abs) < abs(expected - plain): - return with_abs - return plain - - -def _outcome( - column: str, rule: RelationRule, sums: Mapping[str, Decimal], filled: set[str], rules: Rules -) -> RelationOutcome: - codes = (rule.target, *(code for _, code in rule.terms)) - target_filled = rule.target in filled - members_filled = any(code in filled for _, code in rule.terms) - if not target_filled and not members_filled: - return RelationOutcome(column, rule, codes, False, None, None, True, True) - expected = sums[rule.target] if target_filled else None - if expected is None or not members_filled: - actual = _rhs(sums, rule, absolute=False) if members_filled else None - return RelationOutcome(column, rule, codes, True, expected, actual, False, False) - actual = _actual(sums, rule, expected, rules.checks.tolerance) - diff = abs(expected - actual) - scale = max(abs(expected), *(abs(sums.get(code, _ZERO)) for _, code in rule.terms)) - within_rounding = diff <= scale * rules.checks.rounding_tolerance_percent / _HUNDRED - return RelationOutcome(column, rule, codes, True, expected, actual, diff <= rules.checks.tolerance, within_rounding) - - -def _subline_rules( - lines: Sequence[EvalLine], document_type: DocumentType, variant: FormVariant, rules: Rules -) -> list[RelationRule]: - """«родитель = Σ подстрок» для родителей из отрезка формы (мягкие), в порядке строк документа.""" - codes = [item.line_code for item in lines if is_check_line(item) and item.line_code is not None] - children: dict[str, list[str]] = {} - for code, parent in zip(codes, subline_parents(codes), strict=True): - if parent is None or code in children.get(parent, []): - continue - section = section_of(parent, rules) - if section is None or document_type_of(section) is not document_type: - continue - children.setdefault(parent, []).append(code) - return [ - RelationRule( - form=document_type, - variant=variant, - relation=f"{parent} = {' + '.join(kids)}", - target=parent, - terms=tuple((1, kid) for kid in kids), - strict=False, - abs_terms=frozenset(), - ) - for parent, kids in children.items() - ] - - -def evaluate_relations( - lines: Sequence[EvalLine], - columns: Sequence[str], - document_type: DocumentType, - variant: FormVariant | None, - rules: Rules, -) -> list[RelationOutcome]: - """Все соотношения набора формы и подстрок по каждой колонке. `lines` — в порядке документа.""" - effective = variant or FormVariant.FULL - static = list(rules.checks.relations_for(document_type, effective)) - generated = _subline_rules(lines, document_type, effective, rules) if rules.checks.subline_check else [] - outcomes: list[RelationOutcome] = [] - for column in columns: - sums, filled = _column_sums(lines, column) - outcomes.extend(_outcome(column, rule, sums, filled, rules) for rule in static) - # подстроки — только если хоть одна подстрока в колонке непуста (AURORA-1527) - outcomes.extend( - _outcome(column, rule, sums, filled, rules) - for rule in generated - if any(code in filled for _, code in rule.terms) - ) - return outcomes - - -def checks_from(outcomes: Sequence[RelationOutcome], column_index_of: Mapping[str, int | None]) -> StatementChecks: - hard: list[FailedRelation] = [] - soft: list[FailedRelation] = [] - for item in outcomes: - if not item.computed or item.within_tolerance: - continue - failure = FailedRelation( - column_index=column_index_of.get(item.column), - column_key=item.column, - relation=item.rule.relation, - codes=item.codes, - expected=item.expected, - actual=item.actual, - ) - (hard if item.hard_failed else soft).append(failure) - return StatementChecks(relations_failed=tuple(hard), soft_relations_failed=tuple(soft)) diff --git a/api/arm_statements/codes.py b/api/arm_statements/codes.py deleted file mode 100644 index fb8859c..0000000 --- a/api/arm_statements/codes.py +++ /dev/null @@ -1,118 +0,0 @@ -"""Разбор кода строки и отрезки кодов форм.""" - -import re -from collections import Counter -from collections.abc import Iterable, Mapping, Sequence -from typing import NamedTuple - -from arm_statements.loader import CodeSection, Rules -from arm_statements.model import DocumentType -from arm_statements.text import replace_lookalikes - -_CODE = re.compile(r"\d{4,5}(?:\.0)?") -_BAR = "|" - - -class CodeParse(NamedTuple): - code: str | None - fixed: bool # при разборе заменён двойник - - -def code_parts(text: str | None) -> tuple[str, ...]: - """Непустые строки ячейки, обрезанные.""" - if text is None: - return () - return tuple(part.strip() for part in text.splitlines() if part.strip()) - - -def section_of(code: str, rules: Rules) -> CodeSection | None: - head = int(code[:4]) - for item in rules.code_ranges.ranges: - if item.low <= head <= item.high: - return item.section - return None - - -def _strip_edges(text: str) -> str: - start, end = 0, len(text) - while start < end and not text[start].isalnum(): - start += 1 - while end > start and not text[end - 1].isalnum(): - end -= 1 - return text[start:end] - - -def _as_code(text: str, rules: Rules) -> str | None: - core = _strip_edges(text) - if _CODE.fullmatch(core) is None: - return None - code = core.split(".", 1)[0] - return code if section_of(code, rules) is not None else None - - -def _parse_line(line: str, rules: Rules) -> CodeParse: - compact = "".join(line.split()) - # сначала `|` как двойник единицы; если результата-кода нет — `|` как краевая пунктуация - for candidate in (compact, compact.replace(_BAR, "")): - replaced = replace_lookalikes(candidate, rules.lookalikes) - code = _as_code(replaced, rules) - if code is not None: - return CodeParse(code, replaced != candidate) - return CodeParse(None, False) - - -def parse_code(text: str | None, rules: Rules) -> CodeParse: - """Однострочный текст → код; многострочный → `CodeParse(None, False)`.""" - parts = code_parts(text) - if len(parts) != 1: - return CodeParse(None, False) - return _parse_line(parts[0], rules) - - -def is_code_cell(text: str | None, rules: Rules) -> bool: - parts = code_parts(text) - return bool(parts) and all(_parse_line(part, rules).code is not None for part in parts) - - -def majority_section(codes: Iterable[str], rules: Rules) -> CodeSection | None: - """Отрезок большинства; ничья — первый по порядку отрезков словаря.""" - counts: Counter[CodeSection] = Counter() - for code in codes: - section = section_of(code, rules) - if section is not None: - counts[section] += 1 - best: CodeSection | None = None - for item in rules.code_ranges.ranges: - if counts[item.section] and (best is None or counts[item.section] > counts[best]): - best = item.section - return best - - -def document_type_of(section: CodeSection) -> DocumentType: - mapping: Mapping[CodeSection, DocumentType] = { - CodeSection.BALANCE: DocumentType.BALANCE, - CodeSection.INCOME: DocumentType.INCOME, - } - return mapping.get(section, DocumentType.DISCLOSURE) - - -def sheet_joins( - form_type: DocumentType, - sheet_codes: Sequence[str], - sheet_amount_columns: int, - form_amount_columns: int, - rules: Rules, -) -> bool: - """Лист без маркера и шапки присоединяется к форме: коды в её отрезке; при ≥3 кодах колонки сумм совпадают.""" - if len(sheet_codes) < rules.code_ranges.min_codes_continuation: - return False - form_section = {DocumentType.BALANCE: CodeSection.BALANCE, DocumentType.INCOME: CodeSection.INCOME}.get(form_type) - for code in sheet_codes: - section = section_of(code, rules) - if section is None or document_type_of(section) is not form_type: - return False - if form_section is not None and section is not form_section: - return False - if len(sheet_codes) >= rules.code_ranges.min_codes: - return sheet_amount_columns == form_amount_columns - return True diff --git a/api/arm_statements/columns.py b/api/arm_statements/columns.py deleted file mode 100644 index e3ab9d6..0000000 --- a/api/arm_statements/columns.py +++ /dev/null @@ -1,238 +0,0 @@ -"""Шаг 2: колонки значений, подписи и даты колонок.""" - -from collections.abc import Sequence -from dataclasses import dataclass -from datetime import date -from decimal import Decimal - -from arm_statements.amounts import parse_amount -from arm_statements.codes import is_code_cell -from arm_statements.dates import expected_dates, parse_period_label, resolve_relative -from arm_statements.issues import WarningCode, make_warning -from arm_statements.layout import Grid, cell_at -from arm_statements.loader import Rules -from arm_statements.model import Cell, PeriodKind, Source, StatementColumn, StatementHeader, StatementWarning - -_FULL_CONFIDENCE = 100.0 -_MAX_REFS = 10 - - -@dataclass(frozen=True, slots=True) -class ColumnDraft: - """Колонка сегмента до сопоставления; `date` — только у абсолютной подписи.""" - - grid_col: int - label: str - label_confidence: float | None - kind: PeriodKind - date: date | None - - -@dataclass(frozen=True, slots=True) -class DocumentDates: - effective: date | None - confident_effective: date | None - latest_absolute: date | None - confident_latest_absolute: date | None - - -def _text(cell: Cell | None) -> str: - return "" if cell is None else cell.text.strip() - - -def _code_rows(rows: Grid, data_rows: Sequence[int], code_col: int, rules: Rules) -> list[int]: - return [r for r in data_rows if is_code_cell(_text(cell_at(rows, r, code_col)), rules)] - - -def _is_number(cell: Cell | None, source: Source, rules: Rules) -> bool: - if cell is None: - return False - if isinstance(cell.value, Decimal): - return True - return parse_amount(cell.text, source, rules).amount is not None - - -def has_numbers( # noqa: PLR0913, PLR0917 - сигнатура задана планом (Task 9) - rows: Grid, data_rows: Sequence[int], col: int, code_col: int, source: Source, rules: Rules -) -> bool: - """В строках с кодом есть ячейка столбца с числом (значение-число или разобранная сумма).""" - return any(_is_number(cell_at(rows, r, col), source, rules) for r in _code_rows(rows, data_rows, code_col, rules)) - - -def _width(rows: Grid, row_indexes: Sequence[int]) -> int: - return max((len(rows[r]) for r in row_indexes if 0 <= r < len(rows)), default=0) - - -def amount_columns(rows: Grid, code_col: int, rules: Rules, *, source: Source = Source.EXCEL) -> tuple[int, ...]: - """Столбцы правее кодов, где в строках с кодом есть числа, по возрастанию.""" - every = range(len(rows)) - return tuple( - col - for col in range(code_col + 1, _width(rows, every)) - if has_numbers(rows, every, col, code_col, source, rules) - ) - - -def _label(rows: Grid, header_rows: tuple[int, ...], col: int) -> Cell | None: - """Подпись колонки: строки заголовка через пробел, уверенность — минимум; одна ячейка берётся как есть.""" - cells = [c for r in header_rows if (c := cell_at(rows, r, col)) is not None and c.text.strip()] - if not cells: - return None - if len(cells) == 1: - return cells[0] - text = " ".join(c.text.strip() for c in cells) - return Cell(text, None, min(c.confidence for c in cells), cells[0].ref) - - -def segment_columns( # noqa: PLR0913, PLR0917 - сигнатура задана планом (Task 9) - rows: Grid, - header_rows: tuple[int, ...], - data_rows: Sequence[int], - code_col: int, - source: Source, - rules: Rules, -) -> tuple[list[ColumnDraft], list[StatementWarning]]: - """Колонки значений сегмента и предупреждения о пропущенных/недатированных столбцах.""" - drafts: list[ColumnDraft] = [] - warnings: list[StatementWarning] = [] - columns = range(code_col + 1, _width(rows, [*header_rows, *data_rows])) - last_dated = _last_dated_column(rows, header_rows, data_rows, columns, code_col, source, rules) - outside: list[int] = [] - for col in columns: - numbers = has_numbers(rows, data_rows, col, code_col, source, rules) - if not header_rows: - if numbers: - drafts.append(ColumnDraft(col, "", None, PeriodKind.NONE, None)) - continue - cell = _label(rows, header_rows, col) - parsed = parse_period_label(cell, rules) - if cell is not None and parsed.excluded: - message = f"столбец {col} («{cell.text}») пропущен: не период" - warnings.append(make_warning(WarningCode.COLUMN_SKIPPED, message, rules, column_index=col)) - elif cell is not None and parsed.kind is not PeriodKind.NONE: - drafts.append(ColumnDraft(col, cell.text, cell.confidence, parsed.kind, parsed.date)) - elif numbers and cell is None and last_dated is not None and col > last_dated: - outside.append(col) # числа правее последней колонки с подписью периода — не колонка сумм - elif numbers: - label, conf = (cell.text, cell.confidence) if cell is not None else ("", None) - message = f"у столбца {col} с числами нет разобранной подписи периода" - warnings.append(make_warning(WarningCode.COLUMN_WITHOUT_DATE, message, rules, column_index=col)) - drafts.append(ColumnDraft(col, label, conf, PeriodKind.NONE, None)) - if outside: - warnings.append(_outside_table_warning(rows, data_rows, outside, code_col, source, rules)) - return drafts, warnings - - -def _last_dated_column( # noqa: PLR0913, PLR0917 - внутренняя сборка границы таблицы - rows: Grid, - header_rows: tuple[int, ...], - data_rows: Sequence[int], - columns: range, - code_col: int, - source: Source, - rules: Rules, -) -> int | None: - """Правый столбец сегмента с подписью периода и числами в строках с кодами; без шапки — `None`. - - Подписанный столбец без чисел (потерянная подпись объединённой ячейки) границей таблицы не служит. - """ - dated = [ - col - for col in columns - if header_rows - and (cell := _label(rows, header_rows, col)) is not None - and not (parsed := parse_period_label(cell, rules)).excluded - and parsed.kind is not PeriodKind.NONE - and has_numbers(rows, data_rows, col, code_col, source, rules) - ] - return max(dated, default=None) - - -def _outside_table_warning( # noqa: PLR0913, PLR0917 - внутренняя сборка предупреждения - rows: Grid, data_rows: Sequence[int], outside: Sequence[int], code_col: int, source: Source, rules: Rules -) -> StatementWarning: - refs = [ - cell.ref - for r in _code_rows(rows, data_rows, code_col, rules) - for col in outside - if (cell := cell_at(rows, r, col)) is not None and _is_number(cell, source, rules) - ] - return outside_table_warning(refs, rules) - - -def outside_table_warning(refs: Sequence[str], rules: Rules) -> StatementWarning: - """`NUMBERS_OUTSIDE_TABLE` (info): адреса ячеек правее таблицы (`locator` — все, через перевод строки).""" - shown = ", ".join(refs[:_MAX_REFS]) + (f" и ещё {len(refs) - _MAX_REFS}" if len(refs) > _MAX_REFS else "") - message = f"числа правее таблицы формы не учитываются: {shown}" - return make_warning(WarningCode.NUMBERS_OUTSIDE_TABLE, message, rules, locator="\n".join(refs)) - - -def merge_outside_table(warnings: Sequence[StatementWarning], rules: Rules) -> list[StatementWarning]: - """Предупреждения `NUMBERS_OUTSIDE_TABLE` сегментов — в одно (адреса без повторов).""" - refs: dict[str, None] = {} - rest: list[StatementWarning] = [] - for warning in warnings: - if warning.code == WarningCode.NUMBERS_OUTSIDE_TABLE: - refs.update(dict.fromkeys((warning.locator or "").split("\n"))) - else: - rest.append(warning) - return [*rest, outside_table_warning(list(refs), rules)] if refs else rest - - -def trailing_outside_refs( # noqa: PLR0913, PLR0917 - внутренняя сборка - rows: Grid, data_rows: Sequence[int], cols: Sequence[int], code_col: int, source: Source, rules: Rules -) -> list[str]: - """Адреса чисел в строках с кодами по столбцам `cols` (хвост сегмента правее колонок документа).""" - return [ - cell.ref - for r in _code_rows(rows, data_rows, code_col, rules) - for col in cols - if (cell := cell_at(rows, r, col)) is not None and _is_number(cell, source, rules) - ] - - -def _latest(drafts: Sequence[ColumnDraft], *, confident: bool) -> date | None: - days = [ - d.date - for d in drafts - if d.kind is PeriodKind.ABSOLUTE - and d.date is not None - and (not confident or d.label_confidence == _FULL_CONFIDENCE) - ] - return max(days, default=None) - - -def document_dates(drafts: Sequence[ColumnDraft], header: StatementHeader) -> DocumentDates: - """Эффективная дата и её «уверенный» двойник: неуверенное само себя не подтверждает.""" - latest = _latest(drafts, confident=False) - confident_latest = _latest(drafts, confident=True) - report = header.report_date - sure_report = report.value if report.confidence == _FULL_CONFIDENCE else None - return DocumentDates( - effective=report.value or latest, - confident_effective=sure_report or confident_latest, - latest_absolute=latest, - confident_latest_absolute=confident_latest, - ) - - -def finalize_columns( - drafts: Sequence[ColumnDraft], dates: DocumentDates, rules: Rules -) -> tuple[tuple[StatementColumn, ...], list[StatementWarning]]: - """Колонки документа: даты относительных подписей и проверка неуверенных абсолютных.""" - expected = None if dates.confident_effective is None else expected_dates(dates.confident_effective) - columns: list[StatementColumn] = [] - warnings: list[StatementWarning] = [] - for draft in drafts: - if draft.kind is PeriodKind.ABSOLUTE: - period = draft.date - unsure = draft.label_confidence is not None and draft.label_confidence < _FULL_CONFIDENCE - if unsure and (expected is None or period not in expected): - message = f"дата подписи столбца {draft.grid_col} («{draft.label}») не подтверждена" - warnings.append( - make_warning(WarningCode.COLUMN_DATE_UNCERTAIN, message, rules, column_index=draft.grid_col) - ) - else: - period = resolve_relative(draft.kind, dates.effective) - columns.append(StatementColumn(draft.grid_col, draft.label, draft.label_confidence, period, draft.kind)) - return tuple(columns), warnings diff --git a/api/arm_statements/contract.py b/api/arm_statements/contract.py deleted file mode 100644 index 10e7d80..0000000 --- a/api/arm_statements/contract.py +++ /dev/null @@ -1,151 +0,0 @@ -"""Перевод выхода пакета в форму контракта v2.0 и вход статуса из документа контракта. - -Один перевод на оба сервиса (P1-26): хосты свой `contract_map` не пишут. Схему пакет не читает — -форма держится тестами и разовой сверкой со `statementDocument` схемы v2.0. -""" - -from collections.abc import Mapping -from decimal import Decimal -from typing import Any - -from arm_statements.model import ( - HEADER_FIELDS, - DocumentType, - FailedRelation, - LineStatus, - PeriodKind, - Severity, - StatementDocument, - StatementHeader, - StatementWarning, -) -from arm_statements.status import StatusInput, StatusLine, StatusValue - - -def _money(value: Decimal | None) -> str | None: - return None if value is None else format(value, "f") - - -def warning_to_contract(warning: StatementWarning, *, file_level: bool = False) -> dict[str, Any]: - """`warning` схемы; при `file_level` — `fileWarning` (с `locator`). Ключи без значения опускаются.""" - data: dict[str, Any] = { - "code": warning.code, - "severity": str(warning.severity), - "message": warning.message, - } - if warning.line_refs: - data["line_refs"] = list(warning.line_refs) - if warning.column_index is not None: - data["column_index"] = warning.column_index - if warning.field is not None: - data["field"] = warning.field - if file_level and warning.locator is not None: - data["locator"] = warning.locator - return data - - -def _header(header: StatementHeader) -> dict[str, Any]: - data: dict[str, Any] = {"form_name": header.form_name} - for name in HEADER_FIELDS: - field = getattr(header, name) - value = field.value - data[name] = value.isoformat() if name == "report_date" and value is not None else value - data[f"{name}_text"] = field.text - data[f"{name}_confidence"] = field.confidence - data["needs_review_fields"] = list(header.needs_review_fields) - return data - - -def _relation(relation: FailedRelation) -> dict[str, Any]: - if relation.column_index is None: - raise ValueError(f"у соотношения {relation.relation!r} нет column_index: у сервиса номер колонки есть всегда") - return { - "column_index": relation.column_index, - "relation": relation.relation, - "codes": list(relation.codes), - "expected": _money(relation.expected), - "actual": _money(relation.actual), - } - - -def to_contract(document: StatementDocument) -> dict[str, Any]: - """`statementDocument` схемы v2.0 как json-совместимый `dict`.""" - data: dict[str, Any] = { - "document_type": str(document.document_type), - "form_variant": None if document.form_variant is None else str(document.form_variant), - "locator": document.locator, - "status": str(document.status), - "warnings": [warning_to_contract(w) for w in document.warnings], - "header": _header(document.header), - "columns": [ - { - "column_index": c.column_index, - "label": c.label, - "label_confidence": c.label_confidence, - "period_date": None if c.period_date is None else c.period_date.isoformat(), - "period_kind": str(c.period_kind), - } - for c in document.columns - ], - "lines": [ - { - "row_index": ln.row_index, - "ref": ln.ref, - "line_status": str(ln.line_status), - "line_code": ln.line_code, - "line_code_text": ln.line_code_text, - "line_code_confidence": ln.line_code_confidence, - "line_name": ln.line_name, - "line_name_confidence": ln.line_name_confidence, - "parent_ref": ln.parent_ref, - "needs_review": ln.needs_review, - "values": [ - { - "column_index": v.column_index, - "amount": _money(v.amount), - "string_value": v.string_value, - "text": v.text, - "confidence": v.confidence, - "is_recognized": v.is_recognized, - "needs_review": v.needs_review, - } - for v in ln.values - ], - } - for ln in document.lines - ], - } - if document.checks is not None: - data["checks"] = { - "relations_failed": [_relation(r) for r in document.checks.relations_failed], - "soft_relations_failed": [_relation(r) for r in document.checks.soft_relations_failed], - } - return data - - -def status_input_from_contract(document: Mapping[str, Any]) -> StatusInput: - """Вход функции статуса из `statementDocument` контракта v2.0 (то же правило, что `status_input`). - - Схему не проверяет: битый документ — `KeyError` / `ValueError`. Отсутствующий необязательный ключ и `null` равны. - """ - header = document["header"] - lines = document["lines"] - columns = document["columns"] - period_known = header.get("report_date") is not None or any( - c["period_kind"] == PeriodKind.ABSOLUTE and c.get("period_date") for c in columns - ) - checks = document.get("checks") - return StatusInput( - document_type=DocumentType(document["document_type"]), - layout_found=len(lines) > 0, - has_organization_or_inn=bool(header.get("organization_name") or header.get("inn")), - period_known=period_known, - needs_review_fields=tuple(header.get("needs_review_fields") or ()), - lines=tuple(StatusLine(LineStatus(ln["line_status"]), bool(ln["needs_review"])) for ln in lines), - values=tuple( - StatusValue(LineStatus(ln["line_status"]), bool(v["needs_review"])) for ln in lines for v in ln["values"] - ), - hard_failures=0 if checks is None else len(checks["relations_failed"]), - accepted=False, - warning_severities=tuple(Severity(w["severity"]) for w in document["warnings"]), - ) diff --git a/api/arm_statements/dates.py b/api/arm_statements/dates.py deleted file mode 100644 index b76a216..0000000 --- a/api/arm_statements/dates.py +++ /dev/null @@ -1,276 +0,0 @@ -"""Даты и подписи колонок: разбор текстовых и числовых форм, относительные подписи бланка 66н (§11.3, §12 п. 4). - -Вход `parse_date` — текст после `fix_lookalikes`. Регулярки и месяцы — данные (`rules/statement_periods.yaml`), -здесь только обработчики на каждый вид абсолютной формы.""" - -import calendar -import re -from collections.abc import Callable -from dataclasses import dataclass -from datetime import date - -from arm_statements.loader import PeriodRules, Rules -from arm_statements.model import Cell, PeriodKind -from arm_statements.text import fix_lookalikes, normalize_caption - -__all__ = ["PeriodLabel", "expected_dates", "parse_date", "parse_period_label", "resolve_relative"] - -_BARE_YEAR = re.compile(r"\d{4}") -_DIGITS = re.compile(r"\d+") -_ROMAN = {"I": 1, "II": 2, "III": 3, "IV": 4, "1V": 4, "11": 2, "111": 3} -_ORDINAL = {"перв": 1, "втор": 2, "трет": 3, "четв": 4} -_QUARTER_MONTHS = 3 -_HALF_MONTHS = 6 -_DECEMBER = 12 -_SHORT_YEAR_LIMIT = 100 # меньше — двузначный год → 20YY -_CENTURY = 2000 -_DIGITS_FULL = 8 -_DIGITS_YEAR = 4 -_DIGITS_SHORT = 6 -_PARTS = 3 -_SHORT_PART = 2 - - -@dataclass(frozen=True, slots=True) -class PeriodLabel: - kind: PeriodKind - date: date | None - excluded: bool - bare_year: bool - - -def _year_full(year: str) -> int: - value = int(year) - return value if value >= _SHORT_YEAR_LIMIT else _CENTURY + value - - -def _month_num(word: str, rules: PeriodRules) -> int | None: - lowered = word.lower() - for stem, number in rules.months: - if lowered.startswith(stem): - return number - return None - - -def _month_end(year: int, month: int) -> date | None: - if not 1 <= month <= _DECEMBER: - return None - try: - return date(year, month, calendar.monthrange(year, month)[1]) - except ValueError: - return None - - -def _period_end(year: str, month: int | None) -> date | None: - return _month_end(_year_full(year), month) if month else None - - -def _safe(year: int, month: int | None, day: int) -> date | None: - try: - return date(year, month, day) if month else None - except ValueError: - return None - - -def _ordinal(word: str) -> int | None: - """Номер квартала/полугодия из «1», «I», «IV», «первый», «1-й».""" - token = word.strip().upper().rstrip("-Й") - if token in _ROMAN: - return _ROMAN[token] - if token.isdigit(): - return int(token) - lowered = token.lower() - for stem, number in _ORDINAL.items(): - if lowered.startswith(stem): - return number - return None - - -def _group(match: re.Match[str], name: str) -> str | None: - return match.group(name) if name in match.re.groupindex else None - - -def _handle_range_end(match: re.Match[str], rules: PeriodRules) -> date | None: - return _handle_day_month_year(match, rules) - - -def _handle_day_month_year(match: re.Match[str], rules: PeriodRules) -> date | None: - year, day = _group(match, "y"), _group(match, "d") - if year is None or day is None: - return None - month_word = _group(match, "mw") - month = _month_num(month_word, rules) if month_word is not None else int(_group(match, "m") or 0) - return _safe(_year_full(year), month, int(day)) - - -def _handle_quarter(match: re.Match[str], rules: PeriodRules) -> date | None: - del rules - year, number = _group(match, "y"), _ordinal(_group(match, "q") or "") - return _period_end(year, (number or 0) * _QUARTER_MONTHS) if year else None - - -def _handle_half(match: re.Match[str], rules: PeriodRules) -> date | None: - del rules - year, number = _group(match, "y"), _ordinal(_group(match, "q") or "") - return _period_end(year, (number or 0) * _HALF_MONTHS) if year else None - - -def _handle_months(match: re.Match[str], rules: PeriodRules) -> date | None: - del rules - year, count = _group(match, "y"), _group(match, "n") - return _period_end(year, int(count)) if year and count else None - - -def _handle_month_range(match: re.Match[str], rules: PeriodRules) -> date | None: - year, word = _group(match, "y"), _group(match, "mw") - return _period_end(year, _month_num(word, rules)) if year and word else None - - -def _handle_year(match: re.Match[str], rules: PeriodRules) -> date | None: - year = _group(match, "y") - return _period_end(year, _DECEMBER) if year and _year_ok(int(year), rules) else None - - -_HANDLERS: dict[str, Callable[[re.Match[str], PeriodRules], date | None]] = { - "range_end": _handle_range_end, - "day_month_year": _handle_day_month_year, - "quarter": _handle_quarter, - "half": _handle_half, - "months": _handle_months, - "month_range": _handle_month_range, - "year": _handle_year, -} - - -def _year_ok(year: int, rules: PeriodRules) -> bool: - return rules.min_year <= year <= rules.max_year - - -def _text_date(text: str, rules: PeriodRules) -> date | None: - for item in rules.absolute: - match = item.pattern.search(text) - if match: - found = _HANDLERS[item.kind](match, rules) - if found: - return found - return None - - -def _from_parts(first: str, second: str, third: str) -> date | None: - """ДД ММ ГГ[ГГ] и ГГ[ГГ] ММ ДД (год двух- или четырёхзначный только в позиции года).""" - for day, month, year in ((first, second, third), (third, second, first)): - if len(year) in (_SHORT_PART, _DIGITS_YEAR) and len(day) <= _SHORT_PART and len(month) <= _SHORT_PART: - try: - return date(_year_full(year), int(month), int(day)) - except ValueError: - continue - return None - - -def _from_digits(digits: str) -> date | None: - """8 слитных цифр: ДДММГГГГ, ГГГГММДД, ГГГГДДММ — первый валидный вариант.""" - return ( - _from_parts(digits[:2], digits[2:4], digits[4:]) - or _from_parts(digits[6:], digits[4:6], digits[:4]) - or _from_parts(digits[4:6], digits[6:], digits[:4]) - ) - - -def _from_digits_run(digits: str, groups: list[str], rules: PeriodRules) -> date | None: - """Склеенные цифры «2 0 2 4», «30 09 2 0 2 5»: решает их число.""" - if len(digits) == _DIGITS_FULL: - return _from_digits(digits) - if len(digits) == _DIGITS_YEAR: - return date(int(digits), 12, 31) if _year_ok(int(digits), rules) else None - if len(digits) == _DIGITS_SHORT: - return _from_parts(digits[:2], digits[2:4], digits[4:]) - if len(groups) > _PARTS: # «30.09.2025 0:00:00» - return _from_parts(*groups[:_PARTS]) - return None - - -def _numeric_date(text: str, rules: PeriodRules) -> date | None: - groups = _DIGITS.findall(text) - if not groups: - return None - if len(groups) == _PARTS: - return _from_parts(*groups) - return _from_digits_run("".join(groups), groups, rules) - - -def parse_date(text: str | None, rules: Rules) -> date | None: - """Дата из текста после очистки или None. Длиннее `max_text_length` не разбирается (проверка до регулярок).""" - if text is None: - return None - stripped = text.strip() - if not stripped or len(stripped) > rules.periods.max_text_length: - return None - periods = rules.periods - return ( - _text_date(stripped, periods) - or _numeric_date(stripped.split("\n", 1)[0], periods) - or _numeric_date(stripped.replace("\n", " "), periods) - ) - - -def _is_excluded(caption: str, rules: Rules) -> bool: - return any(rx.search(caption) for rx in rules.headers.column_exclusions) - - -def _classify(text: str, rules: Rules) -> PeriodLabel: - caption = normalize_caption(text) - bare_year = _BARE_YEAR.fullmatch(text.strip()) is not None - if _is_excluded(caption, rules): - return PeriodLabel(PeriodKind.NONE, None, True, False) - for kind, pattern in rules.periods.relative: - if pattern.search(caption): - return PeriodLabel(kind, None, False, bare_year) - found = parse_date(text, rules) - return PeriodLabel(PeriodKind.ABSOLUTE if found else PeriodKind.NONE, found, False, bare_year) - - -def parse_period_label(cell: Cell | None, rules: Rules) -> PeriodLabel: - """Подпись колонки → вид периода и дата. Дата-значение ячейки Excel — абсолютная.""" - if cell is not None and isinstance(cell.value, date): - return PeriodLabel(PeriodKind.ABSOLUTE, cell.value, False, False) - raw = cell.text.strip() if cell is not None else "" - if not raw: - return PeriodLabel(PeriodKind.NONE, None, False, False) - # длинный текст не чистим от двойников (backtracking); parse_date всё равно вернёт None - text = raw if len(raw) > rules.periods.max_text_length else fix_lookalikes(raw, rules.lookalikes) - return _classify(text, rules) - - -def _shift_years(day: date, years: int) -> date: - try: - return day.replace(year=day.year - years) - except ValueError: # 29.02 → 28.02 - return day.replace(year=day.year - years, day=28) - - -def resolve_relative(kind: PeriodKind, effective: date | None) -> date | None: - """Дата относительной подписи; для ABSOLUTE и NONE — None (дата из подписи или от аналитика, P1-24).""" - if effective is None: - return None - if kind is PeriodKind.CURRENT: - return effective - if kind is PeriodKind.PREV_YEAR_END: - return date(effective.year - 1, 12, 31) - if kind is PeriodKind.PREV_PREV_YEAR_END: - return date(effective.year - 2, 12, 31) - if kind is PeriodKind.PREV_PERIOD: - return _shift_years(effective, 1) - return None - - -def expected_dates(effective: date) -> frozenset[date]: - """Даты колонок формы при отчётной дате: она сама, она − 1 и − 2 года, 31.12 года − 1 и года − 2.""" - return frozenset( - { - effective, - _shift_years(effective, 1), - _shift_years(effective, 2), - date(effective.year - 1, 12, 31), - date(effective.year - 2, 12, 31), - } - ) diff --git a/api/arm_statements/evaluation.py b/api/arm_statements/evaluation.py deleted file mode 100644 index c75e4f7..0000000 --- a/api/arm_statements/evaluation.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Общий вид строк шагов 6–8 у сервиса и ядра.""" - -from dataclasses import dataclass -from decimal import Decimal - -from arm_statements.model import LineStatus - - -@dataclass(frozen=True, slots=True) -class EvalValue: - column: str # ключ колонки: str(column_index) у сервиса, ключ хоста у ядра - amount: Decimal | None - text: str | None - confidence: float | None - is_recognized: bool - fixed: bool = False # правлено или подтверждено аналитиком - - -@dataclass(frozen=True, slots=True) -class EvalLine: - key: str # ref у сервиса, ключ хоста у ядра - order: int | None # row_index; у ядра None — строка правки - line_status: LineStatus - line_code: str | None - code_confidence: float | None - code_fixed: bool - values: tuple[EvalValue, ...] - - -def is_check_line(line: EvalLine) -> bool: - """Строка участвует в §7: `ok` с кодом; строка правки (`order is None`) — только с суммой (P1-8).""" - if line.line_status is not LineStatus.OK or line.line_code is None: - return False - if line.order is None: - return any(value.amount is not None for value in line.values) - return True diff --git a/api/arm_statements/header.py b/api/arm_statements/header.py deleted file mode 100644 index a27c8e9..0000000 --- a/api/arm_statements/header.py +++ /dev/null @@ -1,125 +0,0 @@ -"""Шапка кандидата формы и тип-подсказка по ОКУД и заголовку (§12).""" - -import re -from datetime import date - -from arm_statements.dates import parse_date -from arm_statements.loader import Rules -from arm_statements.model import Cell, DateField, DocumentType, HeaderCells, HeaderField, StatementHeader -from arm_statements.text import fix_lookalikes, normalize_caption, normalize_label - -_INN_LENGTHS = (10, 12) -_NOT_DIGIT = re.compile(r"\D") - - -def _field(cell: Cell | None) -> HeaderField: - if cell is None: - return HeaderField(None, None, None) - return HeaderField(cell.text, cell.confidence, None) - - -def _stripped(cell: Cell | None) -> HeaderField: - if cell is None: - return HeaderField(None, None, None) - return HeaderField(cell.text, cell.confidence, cell.text.strip() or None) - - -def _digits(cell: Cell | None) -> HeaderField: - """Только цифры текста (пусто — None); двойники не заменяются.""" - if cell is None: - return HeaderField(None, None, None) - return HeaderField(cell.text, cell.confidence, _NOT_DIGIT.sub("", cell.text) or None) - - -def _inn(cell: Cell | None) -> HeaderField: - base = _digits(cell) - if base.value is not None and len(base.value) not in _INN_LENGTHS: - return HeaderField(base.text, base.confidence, None) - return base - - -def _report_date(cell: Cell | None, rules: Rules) -> DateField: - if cell is None: - return DateField(None, None, None) - value: date | None - if isinstance(cell.value, date): - value = cell.value - else: - value = parse_date(fix_lookalikes(cell.text, rules.lookalikes), rules) - return DateField(cell.text, cell.confidence, value) - - -def _okud(cell: Cell | None, rules: Rules) -> HeaderField: - if cell is None: - return HeaderField(None, None, None) - match = rules.headers.okud_code.search(normalize_label(cell.text)) - return HeaderField(cell.text, cell.confidence, match.group(0) if match else None) - - -def _units(cell: Cell | None, okei: HeaderField, rules: Rules) -> HeaderField: - base = _field(cell) - if cell is not None: - caption = normalize_caption(cell.text) - for pattern, canonical in rules.units.spellings: - if pattern.search(caption): - return HeaderField(base.text, base.confidence, canonical) - if okei.value is not None: - return HeaderField(base.text, base.confidence, rules.units.okei.get(okei.value)) - return base - - -def parse_header(cells: HeaderCells, rules: Rules) -> StatementHeader: - """Шапка кандидата: текст и уверенность — из ячейки, значение — по правилам поля.""" - okei = _digits(cells.okei) - return StatementHeader( - organization_name=_stripped(cells.organization_name), - inn=_inn(cells.inn), - report_date=_report_date(cells.report_date, rules), - units=_units(cells.units, okei, rules), - okud=_okud(cells.okud, rules), - okpo=_digits(cells.okpo), - okopf=_digits(cells.okopf), - okfs=_digits(cells.okfs), - okei=okei, - address=_stripped(cells.address), - needs_review_fields=(), - ) - - -def _type_by_okud(text: str | None, rules: Rules) -> DocumentType | None: - if not text: - return None - label = normalize_label(text) - for doc_type, patterns in rules.headers.okud.items(): - if any(pattern.search(label) for pattern in patterns): - return doc_type - return None - - -def type_hint(okud_text: str | None, title_text: str | None, rules: Rules) -> DocumentType | None: - """Тип по ОКУД, иначе по заголовку формы; заголовки колонок не участвуют.""" - by_okud = _type_by_okud(okud_text, rules) - if by_okud is not None: - return by_okud - if not title_text: - return None - label = normalize_label(title_text) - for doc_type, patterns in rules.headers.form_titles.items(): - if any(pattern.search(label) for pattern in patterns): - return doc_type - return None - - -def is_continuation(text: str | None, rules: Rules) -> bool: - """Маркер страницы-продолжения «Форма 0710001 с.2» (ОКУД из словаря не обязателен).""" - if not text: - return False - label = normalize_label(text) - return any(pattern.search(label) for pattern in rules.headers.continuation) - - -def continuation_type(text: str | None, rules: Rules) -> DocumentType | None: - """Тип по ОКУД из маркера продолжения; не маркер или неизвестный ОКУД — None.""" - if not is_continuation(text, rules): - return None - return _type_by_okud(text, rules) diff --git a/api/arm_statements/issues.py b/api/arm_statements/issues.py deleted file mode 100644 index 0efc9c4..0000000 --- a/api/arm_statements/issues.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Предупреждения нормализации: перечень кодов и фабрика с уровнем из словаря.""" - -from collections.abc import Sequence -from enum import StrEnum -from typing import TYPE_CHECKING - -from arm_statements.model import StatementWarning - -if TYPE_CHECKING: - from arm_statements.loader import Rules - - -class WarningCode(StrEnum): - """Коды предупреждений: по члену на код `rules/statement_warnings.yaml` (имя = значение).""" - - JSON_FAILED = "JSON_FAILED" - TABLE_PARSE_ERROR = "TABLE_PARSE_ERROR" - TABLE_LOST = "TABLE_LOST" - TABLE_SKIPPED = "TABLE_SKIPPED" - CODE_COLUMN_NOT_FOUND = "CODE_COLUMN_NOT_FOUND" - SEGMENT_WITHOUT_CODES = "SEGMENT_WITHOUT_CODES" - SHEET_NOT_JOINED = "SHEET_NOT_JOINED" - CONTINUATION_TYPE_CONFLICT = "CONTINUATION_TYPE_CONFLICT" - SEGMENT_COLUMNS_MISMATCH = "SEGMENT_COLUMNS_MISMATCH" - COLUMN_WITHOUT_DATE = "COLUMN_WITHOUT_DATE" - COLUMN_DATE_UNCERTAIN = "COLUMN_DATE_UNCERTAIN" - HEADER_ROW_NOT_FOUND = "HEADER_ROW_NOT_FOUND" - TYPE_CONFLICT = "TYPE_CONFLICT" - NO_ORGANIZATION_NO_INN = "NO_ORGANIZATION_NO_INN" - PERIOD_UNKNOWN = "PERIOD_UNKNOWN" - REPORT_DATE_UNCERTAIN = "REPORT_DATE_UNCERTAIN" - XLSB_SCALE_SUSPECT = "XLSB_SCALE_SUSPECT" - INN_NOT_FOUND = "INN_NOT_FOUND" - INN_UNCERTAIN = "INN_UNCERTAIN" - ORGANIZATION_NOT_FOUND = "ORGANIZATION_NOT_FOUND" - ORGANIZATION_UNCERTAIN = "ORGANIZATION_UNCERTAIN" - REPORT_DATE_UNPARSED = "REPORT_DATE_UNPARSED" - UNITS_NOT_FOUND = "UNITS_NOT_FOUND" - FILE_TYPE_MISMATCH = "FILE_TYPE_MISMATCH" - XLS_NO_CODEPAGE = "XLS_NO_CODEPAGE" - COLUMN_SKIPPED = "COLUMN_SKIPPED" - FRACTIONAL_IN_THOUSANDS = "FRACTIONAL_IN_THOUSANDS" - XLSB_NO_FORMATS = "XLSB_NO_FORMATS" - NUMBER_FORMAT_UNPARSED = "NUMBER_FORMAT_UNPARSED" - VALUE_OUT_OF_RANGE = "VALUE_OUT_OF_RANGE" - SHEET_OTHER = "SHEET_OTHER" - ROW_SPLIT = "ROW_SPLIT" - NUMBERS_OUTSIDE_TABLE = "NUMBERS_OUTSIDE_TABLE" - - -def make_warning( # noqa: PLR0913 -- сигнатура задана публичным API пакета - code: str, - message: str, - rules: "Rules", - *, - line_refs: Sequence[str] = (), - column_index: int | None = None, - field: str | None = None, - locator: str | None = None, -) -> StatementWarning: - """Предупреждение с уровнем из словаря; неизвестный код — `KeyError` (выдуманный код не создаётся).""" - rule = rules.warnings.get(code) - if rule is None: - raise KeyError(f"код предупреждения {code} не задан в statement_warnings.yaml") - return StatementWarning( - code=str(code), - severity=rule.severity, - message=message, - line_refs=tuple(line_refs), - column_index=column_index, - field=field, - locator=locator, - ) diff --git a/api/arm_statements/layout.py b/api/arm_statements/layout.py deleted file mode 100644 index 9e9dc86..0000000 --- a/api/arm_statements/layout.py +++ /dev/null @@ -1,334 +0,0 @@ -"""Шаг 2: столбец кодов, нумерация, строка заголовка, роли строк, столбец наименования.""" - -import re -from collections import Counter -from collections.abc import Sequence -from decimal import Decimal - -from arm_statements.codes import code_parts, is_code_cell, parse_code, section_of -from arm_statements.dates import parse_period_label -from arm_statements.loader import CodeSection, Rules -from arm_statements.model import Cell, DocumentType, PeriodKind, RowRole -from arm_statements.text import fix_lookalikes, normalize_caption - -Grid = Sequence[Sequence[Cell | None]] - -_MIN_NUMBERING_CELLS = 3 -_SHORT_NOTE = 3 # «Пояснения» без подписи: значения не длиннее 3 символов -_SKIP_STEP = 2 # скачок через одну пропущенную цифру нумерации -_NUMBER = re.compile(r"[0-9]+") -_NUMBERING_TAIL = re.compile(r"\s+[0-9]{1,2}$") -_FULL_CONFIDENCE = 100.0 - - -def _text(cell: Cell | None) -> str: - return "" if cell is None else cell.text.strip() - - -def caption_text(text: str) -> str: - """Подпись шапки для сравнения: `normalize_caption` без номера столбца, склеенного снизу («Коды\\n2» → «коды»).""" - return _NUMBERING_TAIL.sub("", normalize_caption(text)) - - -def cell_at(rows: Grid, row: int, col: int) -> Cell | None: - """Ячейка сетки; вне строки или сетки — `None`.""" - if row < 0 or row >= len(rows) or col < 0: - return None - line = rows[row] - return line[col] if col < len(line) else None - - -def _width(rows: Grid, first: int, last: int) -> int: - return max((len(rows[r]) for r in range(max(first, 0), min(last, len(rows) - 1) + 1)), default=0) - - -def _relaxed_count(rows: Grid, span: range, col: int, section: CodeSection, rules: Rules) -> int: - count = 0 - for row in span: - text = _text(cell_at(rows, row, col)) - if not text or not is_code_cell(text, rules): - continue - codes = [parse_code(part, rules).code for part in code_parts(text)] - if all(code is not None and section_of(code, rules) is section for code in codes): - count += 1 - return count - - -def _share_ok(codes: int, nonempty: int, min_share: Decimal) -> bool: - return nonempty > 0 and Decimal(codes) >= min_share * Decimal(nonempty) - - -def _tally(rows: Grid, start: int, last: int, col: int, rules: Rules) -> tuple[int, int]: - """(кодов, непустых ячеек) столбца в строках `start..last`.""" - codes = nonempty = 0 - for row in range(start, last + 1): - text = _text(cell_at(rows, row, col)) - if text: - nonempty += 1 - if is_code_cell(text, rules): - codes += 1 - return codes, nonempty - - -def _best(candidates: dict[int, int]) -> int | None: - """Столбец с наибольшим числом кодов; ничья — левый.""" - best: int | None = None - for col in sorted(candidates): - if best is None or candidates[col] > candidates[best]: - best = col - return best - - -def _caption_candidates(rows: Grid, first: int, last: int, width: int, rules: Rules) -> dict[int, int]: - """Правило (а): столбец с подписью кода; под подписью — ячейки ниже неё до `last`.""" - ranges = rules.code_ranges - found: dict[int, int] = {} - for col in range(width): - first_code = next( - (r for r in range(first, last + 1) if is_code_cell(_text(cell_at(rows, r, col)), rules)), last + 1 - ) - captions = [ - r - for r in range(first, min(first_code, last + 1)) - if rules.headers.code_caption.search(normalize_caption(_text(cell_at(rows, r, col)))) - ] - if not captions: - continue - codes, nonempty = _tally(rows, captions[-1] + 1, last, col, rules) - if codes > 0 and (codes >= ranges.min_codes or _share_ok(codes, nonempty, ranges.min_share)): - found[col] = codes - return found - - -def find_code_column( - rows: Grid, first: int, last: int, rules: Rules, *, relaxed_section: CodeSection | None = None -) -> int | None: - """Столбец кодов по правилу (а)/(б) шага 2; `relaxed_section` — ослабленный поиск продолжения.""" - width = _width(rows, first, last) - if relaxed_section is not None: - counts = { - col: _relaxed_count(rows, range(first, last + 1), col, relaxed_section, rules) for col in range(width) - } - counts = {col: n for col, n in counts.items() if n >= rules.code_ranges.min_codes_continuation} - return _best(counts) - by_caption = _best(_caption_candidates(rows, first, last, width, rules)) - if by_caption is not None: - return by_caption - ranges = rules.code_ranges - scored: dict[int, int] = {} - for col in range(width): - codes, nonempty = _tally(rows, first, last, col, rules) - if codes >= ranges.min_codes and _share_ok(codes, nonempty, ranges.min_share): - scored[col] = codes - return _best(scored) - - -def code_column(rows: Grid, rules: Rules, *, relaxed: DocumentType | None = None) -> int | None: - """Столбец кодов всей сетки — то же правило, что у шага 2 (для адаптеров).""" - section: CodeSection | None = None - if relaxed is DocumentType.DISCLOSURE: - raise ValueError("relaxed=DISCLOSURE не поддерживается: у пояснений несколько отрезков кодов") - if relaxed is DocumentType.BALANCE: - section = CodeSection.BALANCE - elif relaxed is DocumentType.INCOME: - section = CodeSection.INCOME - if not rows: - return None - return find_code_column(rows, 0, len(rows) - 1, rules, relaxed_section=section) - - -def first_code_row(rows: Grid, first: int, last: int, code_col: int, rules: Rules) -> int | None: - for row in range(first, last + 1): - if is_code_cell(_text(cell_at(rows, row, code_col)), rules): - return row - return None - - -def is_numbering_row(cells: Sequence[Cell | None], rules: Rules, *, noisy: bool = False) -> bool: - """≥ 3 непустых ячеек-целых, слева направо +1, начало 1 или 2, не больше одного пропуска. - - `noisy` — строка сразу под шапкой (AURORA-1517): ячейка со сбоем OCR даёт своё единственное целое («• ■ 1» → 1, - «3гола» → 3), ячейка без цифр («/», «V») — пропуск, два целых в ячейке — не нумерация. - """ - numbers: list[int] = [] - for cell in cells: - text = _text(cell) - if not text: - continue - fixed = fix_lookalikes(text, rules.lookalikes, anywhere=True).strip() - if _NUMBER.fullmatch(fixed) is not None: - numbers.append(int(fixed)) - continue - if not noisy: - return False - found = _NUMBER.findall(text) - if len(found) > 1: - return False - numbers.extend(int(number) for number in found) - if len(numbers) < _MIN_NUMBERING_CELLS or numbers[0] not in (1, 2): - return False - skips = 0 - for prev, current in zip(numbers, numbers[1:], strict=False): - step = current - prev - if step == 1: - continue - if step == _SKIP_STEP and skips == 0: - skips += 1 - continue - return False - return True - - -def _is_period(text: str, ref: str, rules: Rules) -> bool: - cell = Cell(text, None, _FULL_CONFIDENCE, ref) - return parse_period_label(cell, rules).kind is not PeriodKind.NONE - - -def _has_header_periods(rows: Grid, row: int, code_col: int, rules: Rules) -> bool: - """Две и более подписи-периода правее кода, либо одна — не голый год.""" - labels = [ - parse_period_label(cell, rules) - for col, cell in enumerate(rows[row]) - if col > code_col and cell is not None and cell.text.strip() - ] - periods = [label for label in labels if label.kind is not PeriodKind.NONE] - strong = [label for label in periods if not label.bare_year] - return bool(strong) or len(periods) >= 2 # noqa: PLR2004 - две подписи подряд — заголовок, одна «2025» — титул - - -def _glued_with_upper(rows: Grid, upper: int, lower: int, code_col: int, rules: Rules) -> bool: - """P1-9: подпись верхней строки отдельно — не период, а «верхняя + нижняя» — период.""" - for col, cell in enumerate(rows[upper]): - if col <= code_col: - continue - top = _text(cell) - bottom = _text(cell_at(rows, lower, col)) - if not top or not bottom: - continue - if not _is_period(top, "", rules) and _is_period(f"{top} {bottom}", "", rules): - return True - return False - - -def _upper_candidate(rows: Grid, row: int, first: int, rules: Rules) -> int | None: - above = row - 1 - while above >= first and is_numbering_row(rows[above], rules): - above -= 1 - return above if above >= first else None - - -def _lower_candidate(rows: Grid, row: int, first_code: int, rules: Rules) -> int | None: - below = row + 1 - while below < first_code and is_numbering_row(rows[below], rules): - below += 1 - return below if below < first_code else None - - -def find_header_rows(rows: Grid, first: int, first_code: int, code_col: int, rules: Rules) -> tuple[int, ...]: - """Строка заголовка (одна или две, склеенные); не найдена — `()`.""" - candidates = [r for r in range(first, first_code) if not is_numbering_row(rows[r], rules)] - captioned = [ - r for r in candidates if rules.headers.code_caption.search(caption_text(_text(cell_at(rows, r, code_col)))) - ] - header: int | None = captioned[-1] if captioned else None - if header is None: - header = next((r for r in reversed(candidates) if _has_header_periods(rows, r, code_col, rules)), None) - if header is None: - return () - upper = _upper_candidate(rows, header, first, rules) - if upper is not None and _glued_with_upper(rows, upper, header, code_col, rules): - return (upper, header) - lower = _lower_candidate(rows, header, first_code, rules) - if lower is not None and _glued_with_upper(rows, header, lower, code_col, rules): - return (header, lower) - return (header,) - - -def segment_roles(rows: Grid, first: int, last: int, header_rows: tuple[int, ...], rules: Rules) -> list[RowRole]: - """Роли строк `first..last`: выше заголовка — TITLE (только при найденном заголовке); строка сразу под заголовком — - нумерация и при сбое OCR (`is_numbering_row(noisy=True)`).""" - roles: list[RowRole] = [] - top = min(header_rows) if header_rows else None - under = max(header_rows) + 1 if header_rows else None - for row in range(first, last + 1): - if row in header_rows: - roles.append(RowRole.HEADER) - elif top is not None and row < top: - roles.append(RowRole.TITLE) - elif is_numbering_row(rows[row], rules, noisy=row == under): - roles.append(RowRole.NUMBERING) - else: - roles.append(RowRole.DATA) - return roles - - -def _caption_column( - rows: Grid, header_rows: tuple[int, ...], code_col: int, pattern: re.Pattern[str], *, left_of_code: bool -) -> int | None: - for row in header_rows: - for col, cell in enumerate(rows[row]): - if col == code_col or (left_of_code and col > code_col): - continue - if pattern.search(caption_text(_text(cell))): - return col - return None - - -def _code_rows(rows: Grid, span: range, skip: tuple[int, ...], code_col: int, rules: Rules) -> list[int]: - return [r for r in span if r not in skip and is_code_cell(_text(cell_at(rows, r, code_col)), rules)] - - -def _short_columns(rows: Grid, data: Sequence[int], code_col: int, skip: int | None) -> list[int]: - result: list[int] = [] - for col in range(code_col): - if col == skip: - continue - values = [_text(cell_at(rows, r, col)) for r in data] - filled = [v for v in values if v] - if filled and all(len(v) <= _SHORT_NOTE for v in filled): - result.append(col) - return result - - -def _text_column(rows: Grid, data: Sequence[int], code_col: int, skip: set[int]) -> int | None: - counts: Counter[int] = Counter() - for col in range(code_col): - if col in skip: - continue - counts[col] = sum(1 for r in data if any(ch.isalpha() for ch in _text(cell_at(rows, r, col)))) - best: int | None = None - for col in sorted(counts): # ничья — ближайший к коду - if counts[col] > 0 and (best is None or counts[col] >= counts[best]): - best = col - return best - - -def find_name_column( # noqa: PLR0913, PLR0917 - сигнатура задана планом (Task 8) - rows: Grid, header_rows: tuple[int, ...], first: int, last: int, code_col: int, rules: Rules -) -> tuple[int | None, int | None]: - """(столбец наименования, столбец «Пояснения»).""" - name_col = _caption_column(rows, header_rows, code_col, rules.headers.name_caption, left_of_code=False) - notes_col = _caption_column(rows, header_rows, code_col, rules.headers.explanations_caption, left_of_code=True) - data = _code_rows(rows, range(first, last + 1), header_rows, code_col, rules) - if name_col is None: - skip = {notes_col} if notes_col is not None else set(_short_columns(rows, data, code_col, None)) - name_col = _text_column(rows, data, code_col, skip) - if notes_col is None: - short = _short_columns(rows, data, code_col, name_col) - notes_col = short[-1] if short else None - return name_col, notes_col - - -def name_cell( - cells: Sequence[Cell | None], name_col: int | None, code_col: int, explanations_col: int | None -) -> Cell | None: - """Ячейка наименования; пустая — ближайшая левее кода ячейка с буквами, кроме «Пояснений».""" - if name_col is not None and name_col < len(cells): - cell = cells[name_col] - if cell is not None and cell.text.strip(): - return cell - for col in range(min(code_col, len(cells)) - 1, -1, -1): - cell = cells[col] - if col != explanations_col and cell is not None and any(ch.isalpha() for ch in cell.text): - return cell - return None diff --git a/api/arm_statements/lines.py b/api/arm_statements/lines.py deleted file mode 100644 index fc06045..0000000 --- a/api/arm_statements/lines.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Шаги 4–5: значения ячеек в строки формы и предупреждения единиц измерения.""" - -from collections.abc import Sequence -from decimal import Decimal - -from arm_statements.amounts import build_value -from arm_statements.issues import WarningCode, make_warning -from arm_statements.loader import Rules -from arm_statements.model import ( - LineStatus, - Source, - StatementHeader, - StatementLine, - StatementValue, - StatementWarning, -) -from arm_statements.rows import RawLine - -CODE_CONFIDENCE_CAP = 99.0 -_PERCENT = 100 - - -def _code_confidence(raw: RawLine) -> float | None: - if raw.code_cell is None: - return None - confidence = raw.code_cell.confidence - return min(confidence, CODE_CONFIDENCE_CAP) if raw.code_fixed else confidence - - -def build_lines( - raw: Sequence[RawLine], source: Source, rules: Rules -) -> tuple[list[StatementLine], list[StatementWarning]]: - """Строки формы со значениями; `VALUE_OUT_OF_RANGE` (`info`) на каждое значение вне диапазона.""" - lines: list[StatementLine] = [] - warnings: list[StatementWarning] = [] - for item in raw: - values: list[StatementValue] = [] - for column_index, cell in item.cells: - built = build_value(cell, column_index, source, rules, split_part=item.split) - values.append(built.value) - if built.out_of_range and cell is not None: - message = f"значение ячейки {cell.ref} вне диапазона numeric(18,2)" - warnings.append( - make_warning( - WarningCode.VALUE_OUT_OF_RANGE, message, rules, line_refs=(item.ref,), column_index=column_index - ) - ) - lines.append( - StatementLine( - row_index=item.row_index, - ref=item.ref, - line_status=item.line_status, - line_code=item.code if item.line_status is LineStatus.OK else None, - line_code_text=item.code_cell.text if item.code_cell is not None else None, - line_code_confidence=_code_confidence(item), - line_name=item.name.text if item.name is not None else None, - line_name_confidence=item.name.confidence if item.name is not None else None, - parent_ref=item.parent_ref, - needs_review=False, - values=tuple(values), - ) - ) - return lines, warnings - - -def _has_fraction(amount: Decimal) -> bool: - return amount != amount.to_integral_value() - - -def units_warnings( - lines: Sequence[StatementLine], - header: StatementHeader, - adapter_warnings: Sequence[StatementWarning], - rules: Rules, -) -> list[StatementWarning]: - """Единицы «тыс. руб.» и доля дробных не ниже порога: `FRACTIONAL_IN_THOUSANDS` / `XLSB_SCALE_SUSPECT`.""" - if header.units.value != rules.units.thousands: - return [] - amounts = [v.amount for line in lines for v in line.values if v.amount is not None] - if not amounts: - return [] - fractional = sum(1 for amount in amounts if _has_fraction(amount)) - if fractional * _PERCENT < rules.units.fractional_share_percent * len(amounts): - return [] - no_formats = any(w.code == WarningCode.XLSB_NO_FORMATS for w in adapter_warnings) - code = WarningCode.XLSB_SCALE_SUSPECT if no_formats else WarningCode.FRACTIONAL_IN_THOUSANDS - message = f"единицы «{rules.units.thousands}», но {fractional} из {len(amounts)} значений с дробной частью" - return [make_warning(code, message, rules)] diff --git a/api/arm_statements/loader.py b/api/arm_statements/loader.py deleted file mode 100644 index 2ffc8f2..0000000 --- a/api/arm_statements/loader.py +++ /dev/null @@ -1,516 +0,0 @@ -"""Загрузчик словарей `rules/*.yaml` в типизированный `Rules` с проверкой при загрузке.""" - -import re -from collections.abc import Mapping -from dataclasses import dataclass -from decimal import Decimal -from enum import StrEnum -from functools import cache -from pathlib import Path -from typing import Literal, TypeVar - -import yaml - -from arm_statements.issues import WarningCode -from arm_statements.model import DocumentType, Edition, FormVariant, PeriodKind, Severity - -LABEL_KEYS: tuple[str, ...] = ( - "organization", - "inn", - "okpo", - "okopf", - "okfs", - "okopf_okfs", - "okei", - "units", - "address", - "date", -) - -_CHECKS = "statement_checks.yaml" -_PERIODS = "statement_periods.yaml" -_HEADERS = "statement_headers.yaml" -_UNITS = "statement_units.yaml" -_RANGES = "statement_code_ranges.yaml" -_ORDERS = "statement_code_orders.yaml" -_WARNINGS = "statement_warnings.yaml" -_LOOKALIKES = "ocr_lookalikes.yaml" - -_RELATION = re.compile(r"^(\d{4}) = (\d{4})((?: [+-] \d{4})*)$") -_RELATION_TERM = re.compile(r"([+-]) (\d{4})") -_PERIOD_GROUPS = frozenset({"d", "m", "mw", "y", "q", "n"}) -_ABSOLUTE_KINDS = frozenset({"range_end", "day_month_year", "quarter", "half", "months", "month_range", "year"}) -_MONTH_MAX = 12 -_PERCENT_MAX = 100 - -_E = TypeVar("_E", bound=StrEnum) - - -class RulesError(ValueError): - """Словарь не прошёл проверку (в сообщении — файл и ключ).""" - - -class CodeSection(StrEnum): - BALANCE = "BALANCE" - INCOME = "INCOME" - CAPITAL = "CAPITAL" - CASH_FLOW = "CASH_FLOW" - NOTES = "NOTES" - TARGET_USE = "TARGET_USE" - - -@dataclass(frozen=True, slots=True) -class CodeRange: - section: CodeSection - low: int - high: int - - -@dataclass(frozen=True, slots=True) -class CodeRanges: - ranges: tuple[CodeRange, ...] - min_share: Decimal - min_codes: int - min_codes_continuation: int - - -@dataclass(frozen=True, slots=True) -class RelationRule: - form: DocumentType - variant: FormVariant - relation: str - target: str - terms: tuple[tuple[int, str], ...] - strict: bool - abs_terms: frozenset[str] - - -@dataclass(frozen=True, slots=True) -class CheckRules: - tolerance: Decimal - rounding_tolerance_percent: Decimal - subline_check: bool - relations: tuple[RelationRule, ...] - - def relations_for(self, form: DocumentType, variant: FormVariant) -> tuple[RelationRule, ...]: - return tuple(r for r in self.relations if r.form == form and r.variant == variant) - - def abs_codes(self, form: DocumentType, variant: FormVariant) -> frozenset[str]: - found: set[str] = set() - for rule in self.relations_for(form, variant): - found |= rule.abs_terms - return frozenset(found) - - -@dataclass(frozen=True, slots=True) -class PeriodPattern: - kind: str - pattern: re.Pattern[str] - - -@dataclass(frozen=True, slots=True) -class PeriodRules: - relative: tuple[tuple[PeriodKind, re.Pattern[str]], ...] - absolute: tuple[PeriodPattern, ...] - months: tuple[tuple[str, int], ...] - min_year: int - max_year: int - max_text_length: int - - -@dataclass(frozen=True, slots=True) -class HeaderRules: - form_titles: Mapping[DocumentType, tuple[re.Pattern[str], ...]] - okud: Mapping[DocumentType, tuple[re.Pattern[str], ...]] - okud_code: re.Pattern[str] - continuation: tuple[re.Pattern[str], ...] - code_caption: re.Pattern[str] - name_caption: re.Pattern[str] - explanations_caption: re.Pattern[str] - column_exclusions: tuple[re.Pattern[str], ...] - labels: Mapping[str, tuple[re.Pattern[str], ...]] - auditor: tuple[re.Pattern[str], ...] - legal_form: tuple[re.Pattern[str], ...] - - -@dataclass(frozen=True, slots=True) -class UnitRules: - spellings: tuple[tuple[re.Pattern[str], str], ...] - okei: Mapping[str, str] - thousands: str - fractional_share_percent: int - - -@dataclass(frozen=True, slots=True) -class CodeOrders: - editions: tuple[Edition, ...] - sequences: Mapping[tuple[Edition, DocumentType, FormVariant], tuple[str, ...]] - - def sequence(self, edition: Edition, form: DocumentType, variant: FormVariant) -> tuple[str, ...]: - return self.sequences.get((edition, form, variant), ()) - - -@dataclass(frozen=True, slots=True) -class WarningRule: - code: str - severity: Severity - scope: Literal["document", "file"] - - -@dataclass(frozen=True, slots=True) -class Rules: - checks: CheckRules - periods: PeriodRules - headers: HeaderRules - units: UnitRules - code_ranges: CodeRanges - code_orders: CodeOrders - lookalikes: Mapping[str, str] - warnings: Mapping[str, WarningRule] - - -# --- проверяющие помощники: Any из YAML дальше них не уходит --- - - -def _fail(file: str, key: str, message: str) -> RulesError: - return RulesError(f"{file}: {key}: {message}") - - -def _read(directory: Path, name: str) -> dict[str, object]: - path = directory / name - try: - data = yaml.safe_load(path.read_text(encoding="utf-8")) - except (OSError, yaml.YAMLError) as exc: - raise _fail(name, "<файл>", f"не удалось прочитать: {exc}") from exc - if not isinstance(data, dict): - raise _fail(name, "<корень>", "ожидается словарь") - return {str(k): v for k, v in data.items()} - - -def _map(file: str, key: str, node: object) -> dict[str, object]: - if not isinstance(node, dict): - raise _fail(file, key, "ожидается словарь") - return {str(k): v for k, v in node.items()} - - -def _seq(file: str, key: str, node: object) -> list[object]: - if not isinstance(node, list): - raise _fail(file, key, "ожидается список") - return list(node) - - -def _get(file: str, data: Mapping[str, object], key: str) -> object: - if key not in data: - raise _fail(file, key, "ключ отсутствует") - return data[key] - - -def _str(file: str, key: str, node: object) -> str: - if not isinstance(node, str): - raise _fail(file, key, "ожидается строка") - return node - - -def _int(file: str, key: str, node: object) -> int: - if isinstance(node, bool) or not isinstance(node, int): - raise _fail(file, key, "ожидается целое число") - return node - - -def _bool(file: str, key: str, node: object) -> bool: - if not isinstance(node, bool): - raise _fail(file, key, "ожидается true/false") - return node - - -def _number(file: str, key: str, node: object) -> Decimal: - if isinstance(node, bool) or not isinstance(node, int | float): - raise _fail(file, key, "ожидается число") - return Decimal(str(node)) - - -def _rx(file: str, key: str, node: object) -> re.Pattern[str]: - text = _str(file, key, node) - try: - return re.compile(text, re.IGNORECASE) - except re.error as exc: - raise _fail(file, key, f"регулярка не компилируется: {exc}") from exc - - -def _rx_list(file: str, key: str, node: object) -> tuple[re.Pattern[str], ...]: - return tuple(_rx(file, f"{key}[{i}]", item) for i, item in enumerate(_seq(file, key, node))) - - -def _enum(file: str, key: str, enum_type: type[_E], node: object) -> _E: - try: - return enum_type(_str(file, key, node)) - except ValueError as exc: - raise _fail(file, key, f"недопустимое значение {node!r}") from exc - - -def _doc_type_patterns(file: str, key: str, node: object) -> dict[DocumentType, tuple[re.Pattern[str], ...]]: - return { - _enum(file, f"{key}.{name}", DocumentType, name): _rx_list(file, f"{key}.{name}", value) - for name, value in _map(file, key, node).items() - } - - -# --- разделы --- - - -def _parse_relation(file: str, key: str, text: str) -> tuple[str, tuple[tuple[int, str], ...]]: - match = _RELATION.match(text) - if match is None: - raise _fail(file, key, f"соотношение {text!r} не вида «<код> = <код> [+|-] <код> …»") - terms = [(1, match.group(2))] - terms += [(1 if sign == "+" else -1, code) for sign, code in _RELATION_TERM.findall(match.group(3))] - return match.group(1), tuple(terms) - - -def _load_checks(directory: Path) -> CheckRules: - file = _CHECKS - data = _read(directory, file) - relations: list[RelationRule] = [] - for i, item in enumerate(_seq(file, "relations", _get(file, data, "relations"))): - key = f"relations[{i}]" - entry = _map(file, key, item) - form = _enum(file, f"{key}.form", DocumentType, _get(file, entry, "form")) - variant = _enum(file, f"{key}.variant", FormVariant, _get(file, entry, "variant")) - text = _str(file, f"{key}.relation", _get(file, entry, "relation")) - target, terms = _parse_relation(file, f"{key}.relation", text) - raw_abs = _seq(file, f"{key}.abs_terms", entry.get("abs_terms", [])) - abs_terms = frozenset(_str(file, f"{key}.abs_terms", code) for code in raw_abs) - if not abs_terms <= {code for _, code in terms}: - raise _fail(file, f"{key}.abs_terms", "коды должны быть членами правой части соотношения") - relations.append( - RelationRule( - form=form, - variant=variant, - relation=text, - target=target, - terms=terms, - strict=_bool(file, f"{key}.strict", _get(file, entry, "strict")), - abs_terms=abs_terms, - ) - ) - return CheckRules( - tolerance=_number(file, "tolerance", _get(file, data, "tolerance")), - rounding_tolerance_percent=_number( - file, "rounding_tolerance_percent", _get(file, data, "rounding_tolerance_percent") - ), - subline_check=_bool(file, "subline_check", _get(file, data, "subline_check")), - relations=tuple(relations), - ) - - -def _load_periods(directory: Path) -> PeriodRules: - file = _PERIODS - data = _read(directory, file) - relative: list[tuple[PeriodKind, re.Pattern[str]]] = [] - for i, item in enumerate(_seq(file, "relative", _get(file, data, "relative"))): - key = f"relative[{i}]" - entry = _map(file, key, item) - kind = _enum(file, f"{key}.kind", PeriodKind, _get(file, entry, "kind")) - relative.append((kind, _rx(file, f"{key}.pattern", _get(file, entry, "pattern")))) - absolute: list[PeriodPattern] = [] - for i, item in enumerate(_seq(file, "absolute", _get(file, data, "absolute"))): - key = f"absolute[{i}]" - entry = _map(file, key, item) - abs_kind = _str(file, f"{key}.kind", _get(file, entry, "kind")) - if abs_kind not in _ABSOLUTE_KINDS: - raise _fail(file, f"{key}.kind", f"недопустимый вид {abs_kind!r}") - pattern = _rx(file, f"{key}.pattern", _get(file, entry, "pattern")) - if not set(pattern.groupindex) <= _PERIOD_GROUPS: - raise _fail(file, f"{key}.pattern", f"допустимые имена групп: {sorted(_PERIOD_GROUPS)}") - absolute.append(PeriodPattern(abs_kind, pattern)) - months: list[tuple[str, int]] = [] - for stem, number in _map(file, "months", _get(file, data, "months")).items(): - month = _int(file, f"months.{stem}", number) - if not 1 <= month <= _MONTH_MAX: - raise _fail(file, f"months.{stem}", "номер месяца вне 1..12") - months.append((stem, month)) - years = _map(file, "years", _get(file, data, "years")) - min_year = _int(file, "years.min", _get(file, years, "min")) - max_year = _int(file, "years.max", _get(file, years, "max")) - if min_year > max_year: - raise _fail(file, "years", "min больше max") - return PeriodRules( - relative=tuple(relative), - absolute=tuple(absolute), - months=tuple(months), - min_year=min_year, - max_year=max_year, - max_text_length=_int(file, "max_text_length", _get(file, data, "max_text_length")), - ) - - -def _load_headers(directory: Path) -> HeaderRules: - file = _HEADERS - data = _read(directory, file) - captions = _map(file, "captions", _get(file, data, "captions")) - labels_raw = _map(file, "labels", _get(file, data, "labels")) - if tuple(labels_raw) != LABEL_KEYS: - raise _fail(file, "labels", f"ключи должны быть ровно {LABEL_KEYS}, получены {tuple(labels_raw)}") - return HeaderRules( - form_titles=_doc_type_patterns(file, "form_titles", _get(file, data, "form_titles")), - okud=_doc_type_patterns(file, "okud", _get(file, data, "okud")), - okud_code=_rx(file, "okud_code", _get(file, data, "okud_code")), - continuation=_rx_list(file, "continuation", _get(file, data, "continuation")), - code_caption=_rx(file, "captions.code", _get(file, captions, "code")), - name_caption=_rx(file, "captions.name", _get(file, captions, "name")), - explanations_caption=_rx(file, "captions.explanations", _get(file, captions, "explanations")), - column_exclusions=_rx_list(file, "captions.exclusions", _get(file, captions, "exclusions")), - labels={name: _rx_list(file, f"labels.{name}", value) for name, value in labels_raw.items()}, - auditor=_rx_list(file, "auditor", _get(file, data, "auditor")), - legal_form=_rx_list(file, "legal_form", _get(file, data, "legal_form")), - ) - - -def _load_units(directory: Path) -> UnitRules: - file = _UNITS - data = _read(directory, file) - spellings: list[tuple[re.Pattern[str], str]] = [] - canonical_all: set[str] = set() - for i, item in enumerate(_seq(file, "units", _get(file, data, "units"))): - key = f"units[{i}]" - entry = _map(file, key, item) - canonical = _str(file, f"{key}.canonical", _get(file, entry, "canonical")) - canonical_all.add(canonical) - spellings.extend((rx, canonical) for rx in _rx_list(file, f"{key}.patterns", _get(file, entry, "patterns"))) - okei = { - code: _str(file, f"okei.{code}", value) for code, value in _map(file, "okei", _get(file, data, "okei")).items() - } - for code, canonical in okei.items(): - if canonical not in canonical_all: - raise _fail(file, f"okei.{code}", f"единица {canonical!r} не среди canonical") - thousands = _str(file, "thousands", _get(file, data, "thousands")) - if thousands not in canonical_all: - raise _fail(file, "thousands", f"единица {thousands!r} не среди canonical") - percent = _int(file, "fractional_share_percent", _get(file, data, "fractional_share_percent")) - if not 1 <= percent <= _PERCENT_MAX: - raise _fail(file, "fractional_share_percent", "ожидается 1..100") - return UnitRules(tuple(spellings), okei, thousands, percent) - - -def _load_ranges(directory: Path) -> CodeRanges: - file = _RANGES - data = _read(directory, file) - ranges: list[CodeRange] = [] - for name, bounds in _map(file, "sections", _get(file, data, "sections")).items(): - section = _enum(file, f"sections.{name}", CodeSection, name) - pair = _seq(file, f"sections.{name}", bounds) - if len(pair) != len(("low", "high")): - raise _fail(file, f"sections.{name}", "ожидается [low, high]") - low, high = (_int(file, f"sections.{name}", v) for v in pair) - if low > high: - raise _fail(file, f"sections.{name}", "low больше high") - ranges.append(CodeRange(section, low, high)) - min_codes = _int(file, "min_codes", _get(file, data, "min_codes")) - continuation = _int(file, "min_codes_continuation", _get(file, data, "min_codes_continuation")) - if not 1 <= continuation <= min_codes: - raise _fail(file, "min_codes_continuation", "должно быть 1 <= min_codes_continuation <= min_codes") - return CodeRanges( - ranges=tuple(ranges), - min_share=_number(file, "min_share", _get(file, data, "min_share")), - min_codes=min_codes, - min_codes_continuation=continuation, - ) - - -def _load_orders(directory: Path) -> CodeOrders: - file = _ORDERS - data = _read(directory, file) - if _get(file, data, "version") != 1: - raise _fail(file, "version", "поддерживается только версия 1") - repeats = _map(file, "repeats", _get(file, data, "repeats")) - editions_raw = _map(file, "editions", _get(file, data, "editions")) - for ed_name, forms_node in repeats.items(): - if ed_name not in editions_raw: - raise _fail(file, f"repeats.{ed_name}", "редакции нет в editions") - known_forms = _map(file, f"editions.{ed_name}", editions_raw[ed_name]) - for form_name in _map(file, f"repeats.{ed_name}", forms_node): - if form_name not in known_forms: - raise _fail(file, f"repeats.{ed_name}.{form_name}", "формы нет в editions этой редакции") - editions: list[Edition] = [] - sequences: dict[tuple[Edition, DocumentType, FormVariant], tuple[str, ...]] = {} - for ed_name, forms_node in editions_raw.items(): - edition = _enum(file, f"editions.{ed_name}", Edition, ed_name) - editions.append(edition) - forms = _map(file, f"editions.{ed_name}", forms_node) - edition_repeats = _map(file, f"repeats.{ed_name}", repeats.get(ed_name, {})) - for form_name, variants_node in forms.items(): - form = _enum(file, f"editions.{ed_name}.{form_name}", DocumentType, form_name) - repeat_codes = [ - _str(file, f"repeats.{ed_name}.{form_name}", c) - for c in _seq(file, f"repeats.{ed_name}.{form_name}", edition_repeats.get(form_name, [])) - ] - variants = _map(file, f"editions.{ed_name}.{form_name}", variants_node) - for var_name, codes_node in variants.items(): - key = f"editions.{ed_name}.{form_name}.{var_name}" - variant = _enum(file, key, FormVariant, var_name) - codes = tuple(_str(file, key, c) for c in _seq(file, key, codes_node)) - for code in codes: - if not re.fullmatch(r"\d{4}", code): - raise _fail(file, key, f"код {code!r} не из четырёх цифр") - _check_repeat_counts(file, key, variant, codes, repeat_codes) - sequences[(edition, form, variant)] = codes - return CodeOrders(tuple(editions), sequences) - - -def _check_repeat_counts(file: str, key: str, variant: FormVariant, codes: tuple[str, ...], repeats: list[str]) -> None: - expected = {code: 1 + repeats.count(code) for code in set(codes) | set(repeats)} - for code, limit in sorted(expected.items()): - actual = codes.count(code) - if variant is FormVariant.FULL and actual != limit: - raise _fail(file, key, f"код {code} встречается {actual} раз, ожидается {limit} (1 + repeats)") - if variant is FormVariant.SIMPLIFIED and actual > limit: - raise _fail(file, key, f"код {code} встречается {actual} раз, допустимо не более {limit}") - - -def _load_warnings(directory: Path) -> dict[str, WarningRule]: - file = _WARNINGS - data = _read(directory, file) - warnings: dict[str, WarningRule] = {} - for code, node in _map(file, "warnings", _get(file, data, "warnings")).items(): - entry = _map(file, f"warnings.{code}", node) - severity = _enum(file, f"warnings.{code}.severity", Severity, _get(file, entry, "severity")) - scope = _str(file, f"warnings.{code}.scope", _get(file, entry, "scope")) - if scope == "document": - warnings[code] = WarningRule(code, severity, "document") - elif scope == "file": - warnings[code] = WarningRule(code, severity, "file") - else: - raise _fail(file, f"warnings.{code}.scope", f"недопустимая область {scope!r}") - if set(warnings) != {c.value for c in WarningCode}: - diff = sorted(set(warnings) ^ {c.value for c in WarningCode}) - raise _fail(file, "warnings", f"коды словаря и WarningCode расходятся: {diff}") - return warnings - - -def _load_lookalikes(directory: Path) -> dict[str, str]: - file = _LOOKALIKES - data = _read(directory, file) - return { - char: _str(file, f"lookalikes.{char}", value) - for char, value in _map(file, "lookalikes", _get(file, data, "lookalikes")).items() - } - - -def load_rules(directory: Path | None = None) -> Rules: - """Прочитать и проверить словари каталога (по умолчанию — `rules/` пакета).""" - base = directory if directory is not None else Path(__file__).parent / "rules" - return Rules( - checks=_load_checks(base), - periods=_load_periods(base), - headers=_load_headers(base), - units=_load_units(base), - code_ranges=_load_ranges(base), - code_orders=_load_orders(base), - lookalikes=_load_lookalikes(base), - warnings=_load_warnings(base), - ) - - -@cache -def default_rules() -> Rules: - return load_rules() diff --git a/api/arm_statements/model.py b/api/arm_statements/model.py deleted file mode 100644 index 8b6e451..0000000 --- a/api/arm_statements/model.py +++ /dev/null @@ -1,246 +0,0 @@ -"""Типы входа (кандидат формы) и выхода (нормализованный документ) пакета.""" - -from dataclasses import dataclass -from datetime import date -from decimal import Decimal -from enum import StrEnum - - -class Source(StrEnum): - OCR = "ocr" - EXCEL = "excel" - - -class DocumentType(StrEnum): - BALANCE = "BALANCE" - INCOME = "INCOME" - DISCLOSURE = "DISCLOSURE" - - -class FormVariant(StrEnum): - FULL = "full" - SIMPLIFIED = "simplified" - - -class Edition(StrEnum): - RAS_66N = "66n" - FSBU4 = "fsbu4" - - -class PeriodKind(StrEnum): - ABSOLUTE = "absolute" - CURRENT = "relative:current" - PREV_YEAR_END = "relative:prev_year_end" - PREV_PREV_YEAR_END = "relative:prev_prev_year_end" - PREV_PERIOD = "relative:prev_period" - NONE = "none" - - -class LineStatus(StrEnum): - OK = "ok" - UNREADABLE = "unreadable" - MISSING = "missing" - DETAIL = "detail" - - -class Severity(StrEnum): - ERROR = "error" - INFO = "info" - - -class DocStatus(StrEnum): - DONE = "done" - NEEDS_CHECK = "needs_check" - - -class EnvelopeStatus(StrEnum): - DONE = "done" - NEEDS_CHECK = "needs_check" - ERROR = "error" - - -class SignForm(StrEnum): - PARENTHESES = "parentheses" - MINUS = "minus" - UNSIGNED = "unsigned" - - -class RowRole(StrEnum): - TITLE = "title" - HEADER = "header" - NUMBERING = "numbering" - DATA = "data" - - -CellValue = Decimal | date | bool | None - - -@dataclass(frozen=True, slots=True) -class Cell: - text: str - value: CellValue - confidence: float - ref: str - - -@dataclass(frozen=True, slots=True) -class Evidence: - okud: Cell | None = None - title: Cell | None = None - - -@dataclass(frozen=True, slots=True) -class HeaderCells: - organization_name: Cell | None = None - inn: Cell | None = None - report_date: Cell | None = None - units: Cell | None = None - okud: Cell | None = None - okpo: Cell | None = None - okopf: Cell | None = None - okfs: Cell | None = None - okei: Cell | None = None - address: Cell | None = None - - -HEADER_FIELDS: tuple[str, ...] = ( - "organization_name", - "inn", - "report_date", - "units", - "okud", - "okpo", - "okopf", - "okfs", - "okei", - "address", -) - - -@dataclass(frozen=True, slots=True) -class StatementWarning: - code: str - severity: Severity - message: str - line_refs: tuple[str, ...] = () - column_index: int | None = None - field: str | None = None - locator: str | None = None - - -@dataclass(frozen=True, slots=True) -class FormCandidate: - source: Source - locator: str - type_hint: DocumentType | None - evidence: Evidence - header: HeaderCells - rows: tuple[tuple[Cell | None, ...], ...] - segment_breaks: tuple[int, ...] = () - warnings: tuple[StatementWarning, ...] = () - - -@dataclass(frozen=True, slots=True) -class HeaderField: - text: str | None - confidence: float | None - value: str | None - - -@dataclass(frozen=True, slots=True) -class DateField: - text: str | None - confidence: float | None - value: date | None - - -@dataclass(frozen=True, slots=True) -class StatementHeader: - organization_name: HeaderField - inn: HeaderField - report_date: DateField - units: HeaderField - okud: HeaderField - okpo: HeaderField - okopf: HeaderField - okfs: HeaderField - okei: HeaderField - address: HeaderField - form_name: str | None = None - needs_review_fields: tuple[str, ...] = () - - -@dataclass(frozen=True, slots=True) -class StatementColumn: - column_index: int - label: str - label_confidence: float | None - period_date: date | None - period_kind: PeriodKind - - -@dataclass(frozen=True, slots=True) -class StatementValue: - column_index: int - amount: Decimal | None - string_value: str | None - text: str | None - confidence: float - is_recognized: bool - needs_review: bool = False - - -@dataclass(frozen=True, slots=True) -class StatementLine: - row_index: int - ref: str - line_status: LineStatus - line_code: str | None - line_code_text: str | None - line_code_confidence: float | None - line_name: str | None - line_name_confidence: float | None - parent_ref: str | None - needs_review: bool - values: tuple[StatementValue, ...] - - -@dataclass(frozen=True, slots=True) -class FailedRelation: - column_index: int | None - column_key: str - relation: str - codes: tuple[str, ...] - expected: Decimal | None - actual: Decimal | None - - -@dataclass(frozen=True, slots=True) -class StatementChecks: - relations_failed: tuple[FailedRelation, ...] - soft_relations_failed: tuple[FailedRelation, ...] - - -@dataclass(frozen=True, slots=True) -class StatementDocument: - document_type: DocumentType - form_variant: FormVariant | None - locator: str - header: StatementHeader - columns: tuple[StatementColumn, ...] - lines: tuple[StatementLine, ...] - checks: StatementChecks | None - status: DocStatus - warnings: tuple[StatementWarning, ...] - - -@dataclass(frozen=True, slots=True) -class CandidateResult: - document: StatementDocument | None - file_warnings: tuple[StatementWarning, ...] - - -@dataclass(frozen=True, slots=True) -class NormalizationResult: - documents: tuple[StatementDocument, ...] - file_warnings: tuple[StatementWarning, ...] diff --git a/api/arm_statements/pipeline.py b/api/arm_statements/pipeline.py deleted file mode 100644 index 8e34d1a..0000000 --- a/api/arm_statements/pipeline.py +++ /dev/null @@ -1,218 +0,0 @@ -"""Шаг 6 и конвейер кандидата: шаги 2–8 одного кандидата и нормализация набора кандидатов.""" - -from collections.abc import Sequence -from dataclasses import replace - -from arm_statements.checks import checks_from -from arm_statements.codes import document_type_of, majority_section -from arm_statements.columns import DocumentDates -from arm_statements.evaluation import EvalLine, EvalValue -from arm_statements.header import parse_header -from arm_statements.issues import WarningCode, make_warning -from arm_statements.lines import build_lines, units_warnings -from arm_statements.loader import Rules, default_rules -from arm_statements.model import ( - CandidateResult, - DocStatus, - DocumentType, - FormCandidate, - FormVariant, - LineStatus, - NormalizationResult, - Source, - StatementDocument, - StatementHeader, - StatementLine, - StatementWarning, -) -from arm_statements.review import evaluate, review_header -from arm_statements.rows import RawLine, collect_raw_lines -from arm_statements.segments import Layout, find_layout -from arm_statements.status import document_status, status_input -from arm_statements.text import has_digit, is_disguised_digit - -_SHEET_PREFIX = "sheet:" - - -def _eval_lines(lines: Sequence[StatementLine]) -> list[EvalLine]: - """Строки сервиса в общий вид шагов 6–8: ключ — `ref`, колонка — `str(column_index)`.""" - return [ - EvalLine( - key=line.ref, - order=line.row_index, - line_status=line.line_status, - line_code=line.line_code, - code_confidence=line.line_code_confidence, - code_fixed=False, - values=tuple( - EvalValue(str(v.column_index), v.amount, v.text, v.confidence, v.is_recognized, fixed=False) - for v in line.values - ), - ) - for line in lines - ] - - -def _form_name(candidate: FormCandidate) -> str | None: - title = candidate.evidence.title - if title is None or not title.text.strip(): - return None - return title.text - - -def _has_content(candidate: FormCandidate, rules: Rules) -> bool: - """Содержимое таблицы: ячейка с цифрой или «цифрой под видом буквы».""" - return any( - cell is not None and (has_digit(cell.text) or is_disguised_digit(cell.text, rules.lookalikes)) - for row in candidate.rows - for cell in row - ) - - -def _lost_table(candidate: FormCandidate, rules: Rules) -> StatementWarning: - """Документа нет (нет кодов и `type_hint`): предупреждение файла по источнику и содержимому.""" - if candidate.source is Source.EXCEL: - sheet = candidate.locator.removeprefix(_SHEET_PREFIX).split(",")[0] - message = f"лист «{sheet}» не содержит формы ББ/ОФР" - return make_warning(WarningCode.SHEET_OTHER, message, rules, locator=f"{_SHEET_PREFIX}{sheet}") - if _has_content(candidate, rules): - message = "таблица с данными не распознана как форма: нет столбца кодов и типа формы" - return make_warning(WarningCode.TABLE_LOST, message, rules, locator=candidate.locator) - return make_warning(WarningCode.TABLE_SKIPPED, "таблица без данных пропущена", rules, locator=candidate.locator) - - -def _without_lines( # noqa: PLR0913 -- сборка документа без строк: поля заданы схемой - candidate: FormCandidate, - document_type: DocumentType, - header: StatementHeader, - warnings: Sequence[StatementWarning], - *, - dates: DocumentDates | None, - rules: Rules, -) -> StatementDocument: - """Документ без строк: `DISCLOSURE` (`dates=None` — шапку не проверяем, P1-6) или без столбца кодов (P1-19).""" - collected = list(warnings) - if dates is not None: - header, header_warnings = review_header(header, dates, rules) - collected.extend(header_warnings) - document = StatementDocument( - document_type=document_type, - form_variant=None if document_type is DocumentType.DISCLOSURE else FormVariant.FULL, - locator=candidate.locator, - header=header, - columns=(), - lines=(), - checks=None, - status=DocStatus.DONE, - warnings=tuple(collected), - ) - return replace(document, status=document_status(status_input(document))) - - -def _lines_document( # noqa: PLR0913 -- сборка документа со строками: поля заданы схемой - candidate: FormCandidate, - document_type: DocumentType, - header: StatementHeader, - layout: Layout, - *, - raw: Sequence[RawLine], - row_warnings: Sequence[StatementWarning], - conflict: Sequence[StatementWarning], - rules: Rules, -) -> StatementDocument: - lines, range_warnings = build_lines(raw, candidate.source, rules) - units = units_warnings(lines, header, candidate.warnings, rules) - column_keys = [str(column.column_index) for column in layout.columns] - evaluation = evaluate(_eval_lines(lines), column_keys, document_type, layout.dates.effective, rules, accepted=False) - checks = checks_from(evaluation.outcomes, {key: int(key) for key in column_keys}) - header, header_warnings = review_header(header, dates=layout.dates, rules=rules) - reviewed = tuple( - replace( - line, - needs_review=evaluation.line_needs_review.get(line.ref, False), - values=tuple( - replace(v, needs_review=evaluation.value_needs_review.get((line.ref, str(v.column_index)), False)) - for v in line.values - ), - ) - for line in lines - ) - document = StatementDocument( - document_type=document_type, - form_variant=evaluation.variant, - locator=candidate.locator, - header=header, - columns=layout.columns, - lines=reviewed, - checks=checks, - status=DocStatus.DONE, - warnings=( - *candidate.warnings, - *layout.warnings, - *row_warnings, - *range_warnings, - *units, - *conflict, - *header_warnings, - ), - ) - return replace(document, status=document_status(status_input(document))) - - -def normalize_candidate(candidate: FormCandidate, rules: Rules | None = None) -> CandidateResult: - """Шаги 2–8 кандидата; документа нет — предупреждение файла (`TABLE_LOST`, `TABLE_SKIPPED`, `SHEET_OTHER`). - - Когда документа нет, `candidate.warnings` (предупреждения адаптера) никуда не попадают: хост обрабатывает их сам. - """ - rules = rules if rules is not None else default_rules() - header = replace(parse_header(candidate.header, rules), form_name=_form_name(candidate)) - layout = find_layout(candidate, header, rules) - hint = candidate.type_hint - - if not layout.code_found: - if hint is None: - return CandidateResult(None, (_lost_table(candidate, rules),)) - base = [*candidate.warnings, *layout.warnings] - if hint is DocumentType.DISCLOSURE: - return CandidateResult( - _without_lines(candidate, hint, header, candidate.warnings, dates=None, rules=rules), () - ) - message = "столбец кодов строк не найден" - base.append(make_warning(WarningCode.CODE_COLUMN_NOT_FOUND, message, rules)) - return CandidateResult(_without_lines(candidate, hint, header, base, dates=layout.dates, rules=rules), ()) - - raw, row_warnings = collect_raw_lines(candidate, layout, rules) - ok_codes = [item.code for item in raw if item.line_status is LineStatus.OK and item.code is not None] - section = majority_section(ok_codes, rules) - if section is not None: - document_type = document_type_of(section) - elif hint is not None: - document_type = hint - else: - return CandidateResult(None, (_lost_table(candidate, rules),)) - - conflict: list[StatementWarning] = [] - if hint is not None and hint is not document_type: - message = f"тип по кодам строк ({document_type}) противоречит типу по заголовку ({hint})" - conflict.append(make_warning(WarningCode.TYPE_CONFLICT, message, rules)) - - if document_type is DocumentType.DISCLOSURE: - warnings = [*candidate.warnings, *conflict] - return CandidateResult(_without_lines(candidate, document_type, header, warnings, dates=None, rules=rules), ()) - document = _lines_document( - candidate, document_type, header, layout, raw=raw, row_warnings=row_warnings, conflict=conflict, rules=rules - ) - return CandidateResult(document, ()) - - -def normalize(candidates: Sequence[FormCandidate], rules: Rules | None = None) -> NormalizationResult: - """Кандидаты по порядку; исключения не перехватываются (изоляцию кандидатов делает хост).""" - rules = rules if rules is not None else default_rules() - documents: list[StatementDocument] = [] - file_warnings: list[StatementWarning] = [] - for candidate in candidates: - result = normalize_candidate(candidate, rules) - if result.document is not None: - documents.append(result.document) - file_warnings.extend(result.file_warnings) - return NormalizationResult(tuple(documents), tuple(file_warnings)) diff --git a/api/arm_statements/recompute.py b/api/arm_statements/recompute.py deleted file mode 100644 index 887f276..0000000 --- a/api/arm_statements/recompute.py +++ /dev/null @@ -1,178 +0,0 @@ -"""Точка входа пересчёта формы по сохранённым данным (спека «Точка входа пакета для пересчёта…»). - -Ядро собирает `StoredForm` из своих строк БД; сервисы функцией не пользуются. Пересчёт не меняет `line_status`, -суммы и коды — только выводит признаки (Р-34: подтверждение кода — `code_fixed`, уверенность кода не меняется). -""" - -from collections.abc import Mapping, Sequence -from dataclasses import dataclass -from datetime import date -from decimal import Decimal - -from arm_statements.checks import checks_from -from arm_statements.evaluation import EvalLine, EvalValue -from arm_statements.loader import Rules, default_rules -from arm_statements.model import ( - DocStatus, - DocumentType, - Edition, - FormVariant, - LineStatus, - PeriodKind, - Severity, - StatementChecks, -) -from arm_statements.review import evaluate -from arm_statements.status import StatusInput, StatusLine, StatusValue, document_status -from arm_statements.structure import edition_by_date - - -@dataclass(frozen=True, slots=True) -class StoredColumn: - key: str - column_index: int | None - period_date: date | None - period_kind: PeriodKind - - -@dataclass(frozen=True, slots=True) -class StoredValue: - column_key: str - amount: Decimal | None - text: str | None # source_text; у значения правки None - confidence: float | None - is_recognized: bool - value_fixed: bool # is_modified или is_confirmed - - -@dataclass(frozen=True, slots=True) -class StoredLine: - key: str - row_index: int | None # None — строка правки - line_status: LineStatus - line_code: str | None - line_code_confidence: float | None - line_name: str | None - code_fixed: bool # line_code_modified или is_confirmed строки - parent_key: str | None - values: tuple[StoredValue, ...] - - -@dataclass(frozen=True, slots=True) -class StoredWarning: - code: str - severity: Severity - - -@dataclass(frozen=True, slots=True) -class StoredForm: - document_type: DocumentType - effective_date: date | None - has_organization_or_inn: bool - period_known: bool - needs_review_fields: tuple[str, ...] - accepted: bool - warnings: tuple[StoredWarning, ...] - columns: tuple[StoredColumn, ...] - lines: tuple[StoredLine, ...] - - -@dataclass(frozen=True, slots=True) -class RecomputeResult: - form_variant: FormVariant | None - edition: Edition | None - checks: StatementChecks - line_needs_review: Mapping[str, bool] - value_needs_review: Mapping[tuple[str, str], bool] - detail_parents: Mapping[str, str | None] - status: DocStatus - - -def _sort_key(line: StoredLine) -> tuple[bool, int]: - return (line.row_index is None, line.row_index or 0) - - -def _eval_line(line: StoredLine) -> EvalLine: - return EvalLine( - key=line.key, - order=line.row_index, - line_status=line.line_status, - line_code=line.line_code, - code_confidence=line.line_code_confidence, - code_fixed=line.code_fixed, - values=tuple( - EvalValue(v.column_key, v.amount, v.text, v.confidence, v.is_recognized, fixed=v.value_fixed) - for v in line.values - ), - ) - - -def _detail_parents(lines: Sequence[StoredLine]) -> dict[str, str | None]: - """Ближайшая выше по `row_index` строка `ok` с `row_index`; у `detail` без `row_index` и без такой — `None`.""" - parents: dict[str, str | None] = {} - last_ok: str | None = None - for line in lines: # уже отсортированы: строки правки в конце - if line.row_index is None: - if line.line_status is LineStatus.DETAIL: - parents[line.key] = None - continue - if line.line_status is LineStatus.DETAIL: - parents[line.key] = last_ok - elif line.line_status is LineStatus.OK: - last_ok = line.key - return parents - - -def recompute(form: StoredForm, rules: Rules | None = None) -> RecomputeResult: - """Шаги 6–8 по сохранённым данным: вариант, редакция, §7, `needs_review`, родители `detail`, статус.""" - rules = rules if rules is not None else default_rules() - if form.document_type is DocumentType.DISCLOSURE: - ordered = sorted(form.lines, key=_sort_key) - return RecomputeResult( - form_variant=None, - edition=edition_by_date(form.effective_date), - checks=StatementChecks((), ()), - line_needs_review={line.key: False for line in ordered}, - value_needs_review={(line.key, v.column_key): False for line in ordered for v in line.values}, - detail_parents=_detail_parents(ordered), - status=DocStatus.DONE, - ) - lines = sorted(form.lines, key=_sort_key) - evaluation = evaluate( - [_eval_line(line) for line in lines], - [column.key for column in form.columns], - form.document_type, - form.effective_date, - rules, - accepted=form.accepted, - ) - checks = checks_from(evaluation.outcomes, {column.key: column.column_index for column in form.columns}) - status = document_status( - StatusInput( - document_type=form.document_type, - layout_found=any(line.row_index is not None for line in lines), - has_organization_or_inn=form.has_organization_or_inn, - period_known=form.period_known, - needs_review_fields=form.needs_review_fields, - lines=tuple( - StatusLine(line.line_status, evaluation.line_needs_review.get(line.key, False)) for line in lines - ), - values=tuple( - StatusValue(line.line_status, evaluation.value_needs_review.get((line.key, v.column_key), False)) - for line in lines - for v in line.values - ), - hard_failures=evaluation.hard_failures, - accepted=form.accepted, - warning_severities=tuple(w.severity for w in form.warnings), - ) - ) - return RecomputeResult( - form_variant=evaluation.variant, - edition=evaluation.edition, - checks=checks, - line_needs_review=evaluation.line_needs_review, - value_needs_review=evaluation.value_needs_review, - detail_parents=_detail_parents(lines), - status=status, - ) diff --git a/api/arm_statements/review.py b/api/arm_statements/review.py deleted file mode 100644 index 3b3ad7d..0000000 --- a/api/arm_statements/review.py +++ /dev/null @@ -1,171 +0,0 @@ -"""Шаг 8: `needs_review` значений и строк, проверки полей шапки, сводная `evaluate`.""" - -from collections import Counter -from collections.abc import Mapping, Sequence -from dataclasses import dataclass, replace -from datetime import date - -from arm_statements.amounts import sign_form_of -from arm_statements.checks import RelationOutcome, evaluate_relations, form_variant -from arm_statements.columns import DocumentDates -from arm_statements.evaluation import EvalLine, EvalValue, is_check_line -from arm_statements.issues import WarningCode, make_warning -from arm_statements.loader import Rules -from arm_statements.model import ( - DocumentType, - Edition, - FormVariant, - HeaderField, - LineStatus, - SignForm, - StatementHeader, - StatementWarning, -) -from arm_statements.structure import StructureResult, confirm_codes - -_CONFIDENT = 100 - - -@dataclass(frozen=True, slots=True) -class Evaluation: - variant: FormVariant | None - edition: Edition | None - outcomes: tuple[RelationOutcome, ...] - line_needs_review: Mapping[str, bool] - value_needs_review: Mapping[tuple[str, str], bool] - hard_failures: int - - -def _needs_coverage(value: EvalValue) -> bool: - return (value.confidence is not None and value.confidence < _CONFIDENT) or not value.is_recognized - - -def _is_covered(line: EvalLine, value: EvalValue, outcomes: Sequence[RelationOutcome]) -> bool: - if value.amount is None or line.line_code is None: - return False - for item in outcomes: - if item.column != value.column or line.line_code not in item.codes or not item.computed: - continue - if item.within_tolerance or (item.rule.strict and item.within_rounding): - return True - return False - - -def _sign_confirmed( - line: EvalLine, value: EvalValue, lines: Sequence[EvalLine], abs_codes: frozenset[str], rules: Rules -) -> bool: - """Форма знака ячейки совпадает с формой строгого большинства прочих членов `abs_terms` колонки.""" - forms: Counter[SignForm] = Counter() - for other in lines: - if other is line or not is_check_line(other) or other.line_code not in abs_codes: - continue - for item in other.values: - if item.column != value.column or item.amount is None or not item.is_recognized or item.fixed: - continue - form = sign_form_of(item.text, rules) - if form is not None: - forms[form] += 1 - total = sum(forms.values()) - own = sign_form_of(value.text, rules) - return own is not None and forms[own] * 2 > total - - -def value_needs_review( - lines: Sequence[EvalLine], - outcomes: Sequence[RelationOutcome], - abs_codes: frozenset[str], - rules: Rules, - *, - accepted: bool = False, -) -> dict[tuple[str, str], bool]: - hard_codes: set[tuple[str, str]] = set() - if not accepted: - for item in outcomes: - if item.hard_failed: - hard_codes |= {(item.column, code) for code in item.codes} - result: dict[tuple[str, str], bool] = {} - for line in lines: - for value in line.values: - key = (line.key, value.column) - if line.line_status is LineStatus.DETAIL or value.fixed: - result[key] = False - elif is_check_line(line) and (value.column, line.line_code or "") in hard_codes: - result[key] = True - elif not _needs_coverage(value): - result[key] = False - else: - covered = _is_covered(line, value, outcomes) - signed = line.line_code not in abs_codes or _sign_confirmed(line, value, lines, abs_codes, rules) - result[key] = not (covered and signed) - return result - - -def line_needs_review(lines: Sequence[EvalLine], structure: StructureResult) -> dict[str, bool]: - return { - line.key: line.key in structure.unconfirmed and not line.code_fixed and line.line_status is LineStatus.OK - for line in lines - } - - -def evaluate( # noqa: PLR0913 -- сигнатура задана публичным API пакета - lines: Sequence[EvalLine], - columns: Sequence[str], - document_type: DocumentType, - effective: date | None, - rules: Rules, - *, - accepted: bool = False, -) -> Evaluation: - codes = [item.line_code for item in lines if is_check_line(item) and item.line_code is not None] - variant = form_variant(document_type, codes) - outcomes = evaluate_relations(lines, columns, document_type, variant, rules) - structure = confirm_codes(lines, document_type, variant, effective, rules) - abs_codes = rules.checks.abs_codes(document_type, variant or FormVariant.FULL) - return Evaluation( - variant=variant, - edition=structure.edition, - outcomes=tuple(outcomes), - line_needs_review=line_needs_review(lines, structure), - value_needs_review=value_needs_review(lines, outcomes, abs_codes, rules, accepted=accepted), - hard_failures=sum(1 for item in outcomes if item.hard_failed), - ) - - -def _uncertain(field: HeaderField) -> bool: - return field.confidence is not None and field.confidence < _CONFIDENT - - -def review_header( - header: StatementHeader, dates: DocumentDates, rules: Rules -) -> tuple[StatementHeader, list[StatementWarning]]: - """Поля шапки (спека «Шаг 8 — поля шапки»). Для `DISCLOSURE` не вызывается: предупреждений шапки там нет (P1-6).""" - warnings: list[StatementWarning] = [] - review_fields: list[str] = [] - org, inn, report = header.organization_name, header.inn, header.report_date - - def warn(code: WarningCode, message: str, field: str | None = None) -> None: - warnings.append(make_warning(code, message, rules, field=field)) - - if org.value is None and inn.value is None: - warn(WarningCode.NO_ORGANIZATION_NO_INN, "В шапке нет ни организации, ни ИНН") - elif inn.value is None: - warn(WarningCode.INN_NOT_FOUND, "ИНН не найден", "inn") - elif org.value is None: - warn(WarningCode.ORGANIZATION_NOT_FOUND, "Организация не найдена", "organization_name") - if org.value is not None and _uncertain(org): - warn(WarningCode.ORGANIZATION_UNCERTAIN, "Название организации распознано неуверенно", "organization_name") - if inn.value is not None and _uncertain(inn): - warn(WarningCode.INN_UNCERTAIN, "ИНН распознан неуверенно", "inn") - - if report.value is not None: - confidence = report.confidence - if confidence is not None and confidence < _CONFIDENT and report.value != dates.confident_latest_absolute: - review_fields.append("report_date") - warn(WarningCode.REPORT_DATE_UNCERTAIN, "Дата отчёта неуверенна и не подтверждена колонками", "report_date") - elif report.text is not None and report.text.strip() and dates.latest_absolute is not None: - warn(WarningCode.REPORT_DATE_UNPARSED, "Дата отчёта не разобрана", "report_date") - if report.value is None and dates.latest_absolute is None: - warn(WarningCode.PERIOD_UNKNOWN, "Отчётный период не определён: нет ни даты отчёта, ни дат колонок") - if header.units.value is None: - warn(WarningCode.UNITS_NOT_FOUND, "Единицы измерения не найдены", "units") - return replace(header, needs_review_fields=tuple(review_fields)), warnings diff --git a/api/arm_statements/rows.py b/api/arm_statements/rows.py deleted file mode 100644 index cf05c63..0000000 --- a/api/arm_statements/rows.py +++ /dev/null @@ -1,253 +0,0 @@ -"""Шаг 3: строки данных, их статусы, `parent_ref` и разлепление склеенных строк.""" - -from collections.abc import Sequence -from dataclasses import dataclass - -from arm_statements.amounts import parse_amount -from arm_statements.codes import code_parts, parse_code, section_of -from arm_statements.issues import WarningCode, make_warning -from arm_statements.layout import Grid, cell_at, name_cell -from arm_statements.loader import Rules -from arm_statements.model import ( - Cell, - FormCandidate, - LineStatus, - RowRole, - Source, - StatementWarning, -) -from arm_statements.segments import Layout, SegmentLayout -from arm_statements.text import has_digit, is_disguised_digit - -_FULL_CONFIDENCE = 100.0 -_MIN_SPLIT_PARTS = 2 - -CellsByColumn = tuple[tuple[int, Cell | None], ...] - - -@dataclass(frozen=True, slots=True) -class RawLine: - """Строка данных до разбора значений: ячейки и статус; `cells` — по всем колонкам документа по порядку.""" - - row_index: int - ref: str - grid_row: int - code_cell: Cell | None - code: str | None - code_fixed: bool - line_status: LineStatus - name: Cell | None - parent_ref: str | None - cells: CellsByColumn - split: bool - - -@dataclass(slots=True) -class _State: - """Сквозное состояние документа: счётчик строк и ближайшая строка `ok`.""" - - next_index: int = 0 - last_ok_ref: str | None = None - - -def _blank(cell: Cell | None) -> bool: - return cell is None or not cell.text.strip() - - -def _is_content(cell: Cell | None, source: Source, rules: Rules) -> bool: - """Содержимое ячейки значения: сумма, неуверенный непустой текст или «цифра под видом буквы».""" - if cell is None: - return False - if cell.value is not None: - return True - text = cell.text - if not text.strip(): - return False - if parse_amount(text, source, rules).amount is not None: - return True - return cell.confidence < _FULL_CONFIDENCE or is_disguised_digit(text, rules.lookalikes) - - -def _first_ref(row: Sequence[Cell | None], code_cell: Cell | None) -> str: - if code_cell is not None: - return code_cell.ref - for cell in row: - if not _blank(cell): - return cell.ref if cell is not None else "" - return next((c.ref for c in row if c is not None), "") - - -def _grid_cols(seg: SegmentLayout, layout: Layout) -> dict[int, int]: - """`column_index` колонки документа → столбец сетки сегмента.""" - by_column = {column_index: grid_col for grid_col, column_index in seg.value_cols} - return {c.column_index: by_column[c.column_index] for c in layout.columns if c.column_index in by_column} - - -def _segment_cells(rows: Grid, row: int, mapping: dict[int, int], layout: Layout) -> CellsByColumn: - return tuple( - (c.column_index, cell_at(rows, row, mapping[c.column_index]) if c.column_index in mapping else None) - for c in layout.columns - ) - - -def _plain_name(cells: Sequence[Cell | None]) -> tuple[Cell | None, int]: - """Сегмент без столбца кодов: наименование — левая ячейка с буквами; второй элемент — её столбец.""" - for col, cell in enumerate(cells): - if cell is not None and any(ch.isalpha() for ch in cell.text): - return cell, col - return None, -1 - - -def _has_value_content(cells: CellsByColumn, source: Source, rules: Rules) -> bool: - return any(_is_content(cell, source, rules) for _, cell in cells) - - -def _part(cell: Cell, k: int, text: str) -> Cell: - return Cell(text, None, cell.confidence, f"{cell.ref}#{k}") - - -def _split_parts(code_cell: Cell | None, cells: CellsByColumn, layout: Layout, rules: Rules) -> tuple[str, ...] | None: - """Части кода склеенной строки или `None`, если разлеплять нельзя.""" - if code_cell is None or layout.column_section is None: - return None - parts = code_parts(code_cell.text) - count = len(parts) - if count < _MIN_SPLIT_PARTS: - return None - if any(not _blank(cell) and len(code_parts(cell.text if cell else None)) != count for _, cell in cells): - return None - in_range = ( - (code := parse_code(part, rules).code) is not None and section_of(code, rules) is layout.column_section - for part in parts - ) - return parts if any(in_range) else None - - -def _status(code_cell: Cell | None, code: str | None, *, has_content: bool, state: _State) -> LineStatus | None: - """Статус строки без разлепления; `None` — строка данных без содержимого.""" - if code_cell is not None and has_digit(code_cell.text): - return LineStatus.OK if code is not None else LineStatus.UNREADABLE - if not has_content: - return None - return LineStatus.DETAIL if state.last_ok_ref is not None else LineStatus.MISSING - - -def _emit( # noqa: PLR0913 - сборка одной строки - state: _State, - *, - ref: str, - grid_row: int, - code_cell: Cell | None, - code: str | None, - fixed: bool, - status: LineStatus, - name: Cell | None, - cells: CellsByColumn, - split: bool, -) -> RawLine: - parent = state.last_ok_ref if status is LineStatus.DETAIL else None - line = RawLine(state.next_index, ref, grid_row, code_cell, code, fixed, status, name, parent, cells, split) - state.next_index += 1 - if status is LineStatus.OK: - state.last_ok_ref = ref - return line - - -def _split_row( # noqa: PLR0913, PLR0917 - разлепление одной строки сетки - state: _State, - grid_row: int, - base_ref: str, - parts: tuple[str, ...], - code_cell: Cell, - name: Cell | None, - cells: CellsByColumn, - rules: Rules, -) -> list[RawLine]: - count = len(parts) - name_parts = code_parts(name.text) if name is not None else () - lines: list[RawLine] = [] - for k, part in enumerate(parts, start=1): - parsed = parse_code(part, rules) - derived_cells: CellsByColumn = tuple( - (index, cell if _blank(cell) or cell is None else _part(cell, k, code_parts(cell.text)[k - 1])) - for index, cell in cells - ) - derived_name = name - if name is not None and len(name_parts) == count: - derived_name = _part(name, k, name_parts[k - 1]) - lines.append( - _emit( - state, - ref=f"{base_ref}#{k}", - grid_row=grid_row, - code_cell=_part(code_cell, k, part), - code=parsed.code, - fixed=parsed.fixed, - status=LineStatus.OK if parsed.code is not None else LineStatus.UNREADABLE, - name=derived_name, - cells=derived_cells, - split=True, - ) - ) - return lines - - -def _row_lines( # noqa: PLR0913, PLR0917 - разбор одной строки сетки - candidate: FormCandidate, layout: Layout, seg: SegmentLayout, grid_row: int, state: _State, rules: Rules -) -> list[RawLine]: - rows = candidate.rows - row = rows[grid_row] - cells = _segment_cells(rows, grid_row, _grid_cols(seg, layout), layout) - code_cell = cell_at(rows, grid_row, seg.code_col) if seg.code_col is not None else None - if seg.code_col is not None: - name = name_cell(row, seg.name_col, seg.code_col, seg.explanations_col) - content_cells = cells - else: - name, name_col = _plain_name(row) - content_cells = ( - cells if seg.value_cols else tuple((col, cell) for col, cell in enumerate(row) if col > name_col) - ) - base_ref = _first_ref(row, code_cell) - parts = _split_parts(code_cell, cells, layout, rules) - if parts is not None and code_cell is not None: - return _split_row(state, grid_row, base_ref, parts, code_cell, name, cells, rules) - parsed = parse_code(code_cell.text if code_cell is not None else None, rules) - has_content = _has_value_content(content_cells, candidate.source, rules) - status = _status(code_cell, parsed.code, has_content=has_content, state=state) - if status is None: - return [] - ref = base_ref or _first_ref(row, None) - return [ - _emit( - state, - ref=ref, - grid_row=grid_row, - code_cell=code_cell, - code=parsed.code, - fixed=parsed.fixed, - status=status, - name=name, - cells=cells, - split=False, - ) - ] - - -def collect_raw_lines( - candidate: FormCandidate, layout: Layout, rules: Rules -) -> tuple[list[RawLine], list[StatementWarning]]: - """Строки данных по сегментам и предупреждения `ROW_SPLIT` (`info`) на каждую разлепленную строку сетки.""" - state = _State() - lines: list[RawLine] = [] - warnings: list[StatementWarning] = [] - for seg in layout.segments: - for grid_row in range(seg.first_row, seg.last_row + 1): - if grid_row >= len(candidate.rows) or layout.roles[grid_row] is not RowRole.DATA: - continue - produced = _row_lines(candidate, layout, seg, grid_row, state, rules) - lines.extend(produced) - if produced and produced[0].split: - message = f"склеенная строка сетки {grid_row} разделена на {len(produced)} строк" - refs = tuple(line.ref for line in produced) - warnings.append(make_warning(WarningCode.ROW_SPLIT, message, rules, line_refs=refs)) - return lines, warnings diff --git a/api/arm_statements/rules/ocr_lookalikes.yaml b/api/arm_statements/rules/ocr_lookalikes.yaml deleted file mode 100644 index 486a8a9..0000000 --- a/api/arm_statements/rules/ocr_lookalikes.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Буквы и знаки, которые OCR ставит вместо цифр (§11.1 п. 1). Замена — только вплотную к цифре, в несколько проходов. -lookalikes: - "О": "0" - "о": "0" - "O": "0" - "o": "0" - "З": "3" - "з": "3" - "б": "6" - "S": "5" - "s": "5" - "I": "1" - "l": "1" - "|": "1" - "і": "1" - "Ч": "4" diff --git a/api/arm_statements/rules/statement_checks.yaml b/api/arm_statements/rules/statement_checks.yaml deleted file mode 100644 index 8466325..0000000 --- a/api/arm_statements/rules/statement_checks.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Соотношения §7 (05-domain-rules.md), ББ и ОФР. Данные, не код: новое соотношение — строка здесь. -# variant: full — полный набор; simplified — упрощённый набор (спека «Шаг 7», §7 «Упрощённый набор»). -# relation: «<код> = <код> [+|-] <код> …», только четырёхзначные коды; strict: нарушение переводит документ в needs_check -# (false — только мягкое предупреждение, ASSUMPTION(OQ-12)); abs_terms: коды, которые 1С выгружает со знаком минус, а 66н — -# в скобках: правая часть считается и как записано, и с этими членами по модулю. -tolerance: 1 # допуск в единицах формы (тыс. руб. округляются построчно) -# OQ-138: единицы шапки верны, расхождение до 0,001 % итоговой строки — округление (мягкое нарушение, статус не портит). -rounding_tolerance_percent: 0.001 -subline_check: true # «сумма пятизначных подстрок = родитель» — мягкая проверка, генерируется по коду -relations: - - {form: BALANCE, variant: full, relation: "1100 = 1105 + 1110 + 1120 + 1130 + 1140 + 1150 + 1160 + 1170 + 1180 + 1190", strict: true} - - {form: BALANCE, variant: full, relation: "1200 = 1210 + 1215 + 1220 + 1230 + 1240 + 1250 + 1260", strict: true} - - {form: BALANCE, variant: full, relation: "1300 = 1310 + 1320 + 1340 + 1350 + 1360 + 1370", strict: true} - - {form: BALANCE, variant: full, relation: "1400 = 1410 + 1420 + 1430 + 1450", strict: true} - - {form: BALANCE, variant: full, relation: "1500 = 1510 + 1520 + 1530 + 1540 + 1550", strict: true} - - {form: BALANCE, variant: full, relation: "1600 = 1100 + 1200", strict: true} - - {form: BALANCE, variant: full, relation: "1700 = 1300 + 1400 + 1500", strict: true} - - {form: BALANCE, variant: full, relation: "1600 = 1700", strict: true} - - {form: INCOME, variant: full, relation: "2100 = 2110 - 2120", strict: true, abs_terms: ["2120"]} - - {form: INCOME, variant: full, relation: "2200 = 2100 - 2210 - 2220", strict: true, abs_terms: ["2210", "2220"]} - - {form: INCOME, variant: full, relation: "2300 = 2200 + 2310 + 2320 - 2330 + 2340 - 2350", strict: true, abs_terms: ["2330", "2350"]} - - {form: INCOME, variant: full, relation: "2400 = 2300 - 2410 + 2420", strict: false, abs_terms: ["2410"]} - - {form: INCOME, variant: full, relation: "2400 = 2300 + 2410 + 2460", strict: false} - - {form: INCOME, variant: full, relation: "2410 = 2411 + 2412", strict: false} - - {form: INCOME, variant: full, relation: "2500 = 2400 + 2510 + 2520 + 2530", strict: false} - - {form: BALANCE, variant: simplified, relation: "1600 = 1150 + 1170 + 1210 + 1230 + 1250", strict: true} - - {form: BALANCE, variant: simplified, relation: "1700 = 1300 + 1350 + 1360 + 1410 + 1450 + 1510 + 1520 + 1550", strict: true} - - {form: BALANCE, variant: simplified, relation: "1600 = 1700", strict: true} - - {form: INCOME, variant: simplified, relation: "2400 = 2110 - 2120 - 2330 + 2340 - 2350 - 2410", strict: false, abs_terms: ["2120", "2330", "2350", "2410"]} diff --git a/api/arm_statements/rules/statement_code_orders.yaml b/api/arm_statements/rules/statement_code_orders.yaml deleted file mode 100644 index de9bce1..0000000 --- a/api/arm_statements/rules/statement_code_orders.yaml +++ /dev/null @@ -1,194 +0,0 @@ -# Упорядоченные перечни кодов строк форм ББ/ОФР по редакциям и вариантам — данные для структурного подтверждения -# неуверенного кода (05-domain-rules.md §11.9 «Код строки»; change unified-statement-normalization, K-5, Р-20). -# -# Что это: порядок кодов бланка (итоги после слагаемых: 1110 … 1190, 1100), без наименований (Р-3: «код → наименование» -# знает только ядро, arm-core/api/rules/statement_lines.yaml). Только четырёхзначные коды: пятизначная подстрока -# подтверждается через родителя (§11.9). Перечень — объединение кодов шаблонов банка (словарь имён ядра, -# fixtures/statements-xlsx/templates/) и кодов печатных форм 1С и сканов, найденных в fixtures (источник — комментарий -# у кода). Код без образца в fixtures сюда не добавляется: кандидаты — OQ-168. -# -# Где живёт: этот файл — источник; в пакете arm_statements лежит его байтовая копия (rules/statement_code_orders.yaml), -# идентичность проверяет sync-statements --check (как копии схем контракта в сервисах). Ядро берёт порядок и состав -# редакций из копии пакета; sync-statements --check сверяет, что все коды словаря имён ядра есть в перечне своей -# редакции. Правка перечня — PR сюда, затем в arm-statements, затем sync-statements в потребителях («сначала ядро»). -# -# Как читается (§11.9): четырёхзначные коды строк ok документа — последовательность; для каждой редакции берётся -# наибольшая подпоследовательность, согласованная с порядком перечня варианта формы документа (form_variant); выбирается -# редакция с более длинной (при равенстве — по эффективной дате: fsbu4 после 2024-12-31). Повтор кода допустим только -# там, где он есть в перечне (repeats): тогда структура подтверждает обе копии; повтор вне перечня с неуверенной копией — -# needs_review у неуверенной копии (Р-16, Р-20). -# -# Проверка формы файла — scripts/validate_contracts.py (make validate-contracts). -version: 1 - -# Законные повторы кода в одном перечне (Р-20). Ключ — редакция и форма; значение — коды, которые встречаются в -# перечне больше одного раза: в варианте full код стоит ровно 1 + (число его вхождений в этот список) раз, в simplified — -# не чаще (сейчас повторов в simplified нет; validate_contracts.py проверяет оба правила). -repeats: - 66n: - # раздел III для некоммерческих организаций («Целевое финансирование») печатается после раздела III коммерческих - # (fixtures/ocr/aep_3.json, 31.12.2024: 1310 «Паевой фонд», 1320 «Целевой капитал», 1350 «Целевые средства», - # 1360 «Фонд недвижимого и особо ценного движимого имущества», 1370 «Резервный и иные целевые фонды») - BALANCE: ["1310", "1320", "1350", "1360", "1370", "1300"] - INCOME: [] - fsbu4: - # печатная форма 1С по ФСБУ 4/2023 (fixtures/ocr/aep_6.json, «БО 1 кв 2025.json», «Бухгалтерская отчетность - # 1 кв.25.json»): 1220 «НДС, начисленный по отгрузке», 1520 «Обязательство по аренде» (AURORA-1431) - BALANCE: ["1220", "1520"] - # 2410 «Налог при упрощенной системе налогообложения» после 2412 (fixtures/ocr/aep_6.json, OQ-152) - INCOME: ["2410"] - -editions: - 66n: - BALANCE: - full: - - "1110" - - "1120" - - "1130" - - "1140" - - "1150" - - "1160" - - "1170" - - "1180" - - "1190" - - "1100" - - "1210" - - "1220" - - "1230" - - "1231" # код клиента: unagrande-group/2024-FY/ББ. Лист 1 .xls - - "1240" - - "1250" - - "1260" - - "1200" - - "1600" - - "1310" - - "1320" - - "1340" - - "1350" - - "1360" - - "1370" - - "1300" - - "1310" # раздел III НКО (aep_3), см. repeats - - "1320" - - "1350" - - "1360" - - "1370" - - "1300" - - "1410" - - "1420" - - "1430" - - "1450" - - "1400" - - "1510" - - "1520" - - "1530" - - "1540" - - "1550" - - "1500" - - "1700" - # упрощённая форма, приказ 66н прил. 5 — по бланку, образца в fixtures нет (OQ-166) - simplified: ["1150", "1170", "1210", "1250", "1230", "1600", "1300", "1350", "1360", "1410", "1450", "1510", - "1520", "1550", "1700"] - INCOME: - full: - - "2110" - - "2120" - - "2100" - - "2210" - - "2220" - - "2200" - - "2310" - - "2320" - - "2330" - - "2340" - - "2350" - - "2300" - - "2410" - - "2411" - - "2412" - - "2460" - - "2400" - - "2510" - - "2520" - - "2530" # печатная форма 1С: unagrande-co/2024-FY/ОФР. Лист 2 .xls, mezheninovskaya/2025-Q3 - - "2500" - - "2900" - - "2910" - - "2930" # печатная форма 1С: fixtures/ocr/aep_3.json (после 2900) - # упрощённая форма, приказ 66н прил. 5 — по бланку, образца нет (OQ-166) - simplified: ["2110", "2120", "2330", "2340", "2350", "2410", "2400"] - fsbu4: - BALANCE: - full: - - "1105" # печатная форма 1С: npz/2025-Q3, fixtures/ocr/aep_6.json, «БО 1 кв 2025.json» - - "1110" - - "1130" # печатная форма 1С: npz/2025-Q3, «БО 1 кв 2025.json», «Бухгалтерская отчетность 1 кв.25.json» - - "1140" # то же и aep_6.json - - "1150" - - "1160" - - "1170" - - "1180" - - "1190" - - "1100" - - "1210" - - "1215" - - "1220" - - "1220" # «НДС, начисленный по отгрузке», см. repeats - - "1230" - - "1240" - - "1250" - - "1260" - - "1200" - - "1600" - - "1310" - - "1320" - - "1340" - - "1350" - - "1360" - - "1370" - - "1300" - - "1410" - - "1420" - - "1430" - - "1450" - - "1400" - - "1510" - - "1520" - - "1520" # «Обязательство по аренде», см. repeats - - "1530" # печатная форма 1С: npz/2025-Q3, aep_6.json, оба json 1 кв. 2025 - - "1540" - - "1550" - - "1500" - - "1700" - # упрощённая форма по ФСБУ 4/2023 — состав принят как у 66н до образца (OQ-166, OQ-168) - simplified: ["1150", "1170", "1210", "1250", "1230", "1600", "1300", "1350", "1360", "1410", "1450", "1510", - "1520", "1550", "1700"] - INCOME: - full: - - "2110" - - "2120" - - "2100" - - "2210" - - "2220" - - "2200" - - "2310" - - "2320" - - "2330" - - "2340" - - "2350" - - "2300" - - "2410" - - "2411" - - "2412" - - "2410" # «Налог при упрощенной системе налогообложения», см. repeats - - "2420" - - "2460" # печатная форма 1С: npz/2025-Q3, aep_6.json, оба json 1 кв. 2025 - - "2400" - - "2510" - - "2520" - - "2530" - - "2500" - - "2900" - - "2910" - # упрощённая форма по ФСБУ 4/2023 — как у 66н до образца (OQ-166, OQ-168) - simplified: ["2110", "2120", "2330", "2340", "2350", "2410", "2400"] diff --git a/api/arm_statements/rules/statement_code_ranges.yaml b/api/arm_statements/rules/statement_code_ranges.yaml deleted file mode 100644 index bf994db..0000000 --- a/api/arm_statements/rules/statement_code_ranges.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# Коды строк форм: диапазоны четырёхзначных кодов (пятизначные — по первым четырём цифрам), доля кодов в столбце, -# минимум кодов, чтобы столбец считался столбцом кодов. -sections: - BALANCE: [1100, 1799] - INCOME: [2100, 2999] - CAPITAL: [3100, 3699] - CASH_FLOW: [4100, 4499] - NOTES: [5000, 5999] - TARGET_USE: [6100, 6499] -min_share: 0.6 -min_codes: 3 -# слабое продолжение (отрезок формы без шапки, лист-продолжение без маркера): хватает одного кода -min_codes_continuation: 1 diff --git a/api/arm_statements/rules/statement_headers.yaml b/api/arm_statements/rules/statement_headers.yaml deleted file mode 100644 index 2bc897b..0000000 --- a/api/arm_statements/rules/statement_headers.yaml +++ /dev/null @@ -1,78 +0,0 @@ -# Словарь форм ББ/ОФР (05-domain-rules.md §12): заголовки форм и ОКУД, маркер продолжения, подписи колонок, метки шапки. -# Шаблоны ищутся (re.search) без учёта регистра. Текст перед поиском: normalize_label (пунктуация сохраняется) для -# form_titles, okud, okud_code, continuation, labels, auditor, legal_form; normalize_caption (пунктуация -> пробел) -# для captions. Новый синоним = строка в YAML. - -# Заголовок формы -> тип документа. Титул «Бухгалтерская отчетность» (НПЗ) типом не является и здесь не перечислен. -form_titles: - BALANCE: - - 'бухгалтерский\s+баланс' - INCOME: - - 'отчет\s+о\s+финансовых\s+результатах' - - 'отчет\s+о\s+прибылях\s+и\s+убытках' - DISCLOSURE: - - 'пояснени\w*\s+к\s+бухгалтерскому\s+балансу' - - '^пояснени' - - '^расшифровк' - - 'отчет\s+об\s+изменениях\s+капитала' - - 'отчет\s+о\s+движении\s+денежных\s+средств' - - 'отчет\s+о\s+целевом\s+использовании' - -# Код ОКУД -> тип (ячейка «Форма по ОКУД 0710001» или сам код). Прочие формы 0710xxx — расшифровки -okud: - BALANCE: - - '\b0710001\b' - INCOME: - - '\b0710002\b' - DISCLOSURE: - - '\b07100(0(3|4|5|6|7|8|9)|(1|2|3|4|5|6|7|8|9)\d)\b' - -# Сам код ОКУД формы внутри ячейки («Форма по ОКУД 0710001» -> «0710001») -okud_code: '0710\d{3}' - -# Маркер страницы-продолжения в 1С-выгрузке: «Форма 0710001 с.2» -continuation: - - '\bформа\s+0710\d{3}\s+с\.?\s*\d+' - -# Подписи колонок заголовка таблицы (ячейка целиком; исключения — подстроки) -captions: - code: '^код(ы|\s+строки|\s+показателя)?$' - name: '^наименование' - explanations: '^пояснени' - exclusions: ['изменени', 'отклонени', '%', 'темп', 'доля', 'примечани'] - -# Метки шапки документа (значение — правее в той же строке либо в следующей строке) -labels: - organization: - - '^организация\b' - inn: - - '^инн\b' - - 'идентификационный\s+номер\s+налогоплательщика' - okpo: - - 'по\s+окпо' - okopf: - - 'по\s+окопф\b(?!\s*/)' - okfs: - - 'по\s+окфс' - okopf_okfs: - - 'по\s+окопф\s*/\s*окфс' - okei: - - 'по\s+океи' - units: - - 'единица\s+измерения' - address: - - 'местонахождение\s*\(адрес\)' - - 'адрес\s+в\s+пределах\s+места\s+нахождения' - - '^адрес\b' - date: - - '^(отчетная\s+)?дата\b' - -# Строки блока аудитора: ИНН ниже этой метки — ИНН аудитора, не организации -auditor: - - 'аудитор' - -# Признак, что строка содержит название организации (для имени над меткой «Организация», НПЗ) -legal_form: - - '\b(ооо|ао|пао|зао|оао|нао|ип|гуп|муп|фгуп)\b' - - '"' - - '«' diff --git a/api/arm_statements/rules/statement_periods.yaml b/api/arm_statements/rules/statement_periods.yaml deleted file mode 100644 index 4adfad2..0000000 --- a/api/arm_statements/rules/statement_periods.yaml +++ /dev/null @@ -1,44 +0,0 @@ -# Периоды форм ББ/ОФР: подпись колонки или подзаголовок формы -> дата (конец периода). §11.3, §12 п. 4. -# Регулярки ищутся без учёта регистра. Относительные подписи применяются к подписи после normalize_caption. -max_text_length: 200 # длиннее — не дата (защита от backtracking) -years: {min: 1990, max: 2100} # одинокий год вне диапазона датой не считается - -# Основа названия месяца -> номер (сопоставляется по началу слова) -months: - январ: 1 - феврал: 2 - март: 3 - апрел: 4 - ма: 5 - июн: 6 - июл: 7 - август: 8 - сентябр: 9 - октябр: 10 - ноябр: 11 - декабр: 12 - -# Относительные подписи колонок (по порядку проверки) -relative: - - {kind: "relative:prev_prev_year_end", pattern: 'предшествующ\w*\s+предыдущ'} - - {kind: "relative:prev_year_end", pattern: 'декабр\w*\s+предыдущ'} - - {kind: "relative:prev_period", pattern: 'аналогичн\w*\s+период\w*\s+предыдущ'} - - {kind: "relative:current", pattern: 'отч[её]тн\w*\s+(дат|период)'} - -# Абсолютные формы по приоритету (§11.3 п. 3). Именованные группы: d (день), m (месяц числом), mw (месяц словом), -# y (год, 2 или 4 цифры), q (номер квартала/полугодия словом, римской или арабской цифрой), n (число месяцев). -absolute: - - kind: range_end # «с 01.01.2025 по 30.09.2025»: конец диапазона - pattern: 'по\s*(?P\d{1,2})[.\-/ ](?P\d{1,2})[.\-/ ](?P\d{4}|\d{2})' - - kind: day_month_year # «31 декабря 2025 г.», «на 30 сентября 2025 года» - pattern: '(?P\d{1,2})\s+(?P[а-яё]{3,8})\s+(?P\d{4}|\d{2})\s*(?:г\.?|r\.?|год\w*)?' - - kind: quarter # «I квартал 2026», «1-й кв. 2026», «первый квартал» - pattern: '(?PI{1,3}V?|IV|\d|перв\w*|втор\w*|трет\w*|четв\w*)\s*-?\s*(?:й|ый|ого|го)?\s*кв\w*\.?\s*(?P\d{4}|\d{2})\s*(?:г\.?|r\.?|год\w*)?' - - kind: half # «1 полугодие 2026», «первое полугодие», «1 п/г 2026» - pattern: '(?PI{1,3}V?|IV|\d|перв\w*|втор\w*|трет\w*|четв\w*)\s*-?\s*(?:е|ое|го)?\s*(?:полугоди\w*|п/г)\.?\s*(?P\d{4}|\d{2})\s*(?:г\.?|r\.?|год\w*)?' - - kind: months # «9 месяцев 2025», «9 мес. 2025» - pattern: '(?P\d{1,2})\s*мес\w*\.?\s*(?P\d{4}|\d{2})\s*(?:г\.?|r\.?|год\w*)?' - - kind: month_range # «январь - сентябрь 2025»: последний месяц - pattern: '[а-яё]{3,8}\s*[-–—]\s*(?P[а-яё]{3,8})\s*(?P\d{4}|\d{2})\s*(?:г\.?|r\.?|год\w*)?' - - kind: year # «за 2025 г.», «2025 год», «2025г» - pattern: '(?:за\s*)?(?:отч[её]тный\s*)?(?P\d{4})\s*(?:г\b|г\.|r\b|r\.|год)' diff --git a/api/arm_statements/rules/statement_units.yaml b/api/arm_statements/rules/statement_units.yaml deleted file mode 100644 index dd2c217..0000000 --- a/api/arm_statements/rules/statement_units.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Единицы измерения форм ББ/ОФР. Подпись проверяется по порядку: первая совпавшая запись определяет единицу. -units: - - canonical: "тыс. руб." - patterns: ['тыс\w*\.?\s*руб', 'тысяч\w*\s+рубл'] - - canonical: "млн руб." - patterns: ['млн\w*\.?\s*руб', 'миллион\w*\s+рубл'] - - canonical: "руб." - patterns: ['руб'] -# Код ОКЕИ -> каноническая единица -okei: - "384": "тыс. руб." - "383": "руб." - "385": "млн руб." -thousands: "тыс. руб." # единица, для которой проверяется доля дробных значений -# Порог для предупреждения о дробных значениях при единицах в тыс. руб. (процент; 1..100) -fractional_share_percent: 10 diff --git a/api/arm_statements/rules/statement_warnings.yaml b/api/arm_statements/rules/statement_warnings.yaml deleted file mode 100644 index 03699c9..0000000 --- a/api/arm_statements/rules/statement_warnings.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# Предупреждения нормализации: код -> уровень и область (спека «Шаг 8 — предупреждения»). -# scope: document — предупреждение документа; file — предупреждение файла (metrics.warnings). -# Множество кодов равно перечислению WarningCode (arm_statements/issues.py); код добавляется в обоих местах. -warnings: - JSON_FAILED: {severity: error, scope: file} - TABLE_PARSE_ERROR: {severity: error, scope: file} - TABLE_LOST: {severity: error, scope: file} - TABLE_SKIPPED: {severity: info, scope: file} - CODE_COLUMN_NOT_FOUND: {severity: error, scope: document} - SEGMENT_WITHOUT_CODES: {severity: error, scope: document} - SHEET_NOT_JOINED: {severity: error, scope: document} - CONTINUATION_TYPE_CONFLICT: {severity: error, scope: document} - SEGMENT_COLUMNS_MISMATCH: {severity: error, scope: document} - COLUMN_WITHOUT_DATE: {severity: error, scope: document} - COLUMN_DATE_UNCERTAIN: {severity: error, scope: document} - HEADER_ROW_NOT_FOUND: {severity: error, scope: document} - TYPE_CONFLICT: {severity: error, scope: document} - NO_ORGANIZATION_NO_INN: {severity: error, scope: document} - PERIOD_UNKNOWN: {severity: error, scope: document} - REPORT_DATE_UNCERTAIN: {severity: error, scope: document} - XLSB_SCALE_SUSPECT: {severity: error, scope: document} - INN_NOT_FOUND: {severity: info, scope: document} - INN_UNCERTAIN: {severity: info, scope: document} - ORGANIZATION_NOT_FOUND: {severity: info, scope: document} - ORGANIZATION_UNCERTAIN: {severity: info, scope: document} - REPORT_DATE_UNPARSED: {severity: info, scope: document} - UNITS_NOT_FOUND: {severity: info, scope: document} - FILE_TYPE_MISMATCH: {severity: info, scope: file} - XLS_NO_CODEPAGE: {severity: info, scope: file} - COLUMN_SKIPPED: {severity: info, scope: document} - FRACTIONAL_IN_THOUSANDS: {severity: info, scope: document} - XLSB_NO_FORMATS: {severity: info, scope: document} - NUMBER_FORMAT_UNPARSED: {severity: info, scope: document} - VALUE_OUT_OF_RANGE: {severity: info, scope: document} - SHEET_OTHER: {severity: info, scope: file} - ROW_SPLIT: {severity: info, scope: document} - NUMBERS_OUTSIDE_TABLE: {severity: info, scope: document} diff --git a/api/arm_statements/segments.py b/api/arm_statements/segments.py deleted file mode 100644 index 821925e..0000000 --- a/api/arm_statements/segments.py +++ /dev/null @@ -1,294 +0,0 @@ -"""Шаг 2: сегменты сетки, повторные шапки, границы листов и сопоставление колонок с первым сегментом.""" - -from collections.abc import Sequence -from dataclasses import dataclass, replace -from typing import NamedTuple - -from arm_statements.codes import code_parts, is_code_cell, majority_section, parse_code -from arm_statements.columns import ( - ColumnDraft, - DocumentDates, - document_dates, - finalize_columns, - merge_outside_table, - outside_table_warning, - segment_columns, - trailing_outside_refs, -) -from arm_statements.issues import WarningCode, make_warning -from arm_statements.layout import ( - Grid, - caption_text, - cell_at, - find_code_column, - find_header_rows, - find_name_column, - first_code_row, - is_numbering_row, - segment_roles, -) -from arm_statements.loader import CodeSection, Rules -from arm_statements.model import ( - FormCandidate, - PeriodKind, - RowRole, - StatementColumn, - StatementHeader, - StatementWarning, -) - - -@dataclass(frozen=True, slots=True) -class SegmentLayout: - """Раскладка сегмента; `value_cols` — (столбец сетки сегмента, `column_index` колонки документа).""" - - first_row: int - last_row: int - header_rows: tuple[int, ...] - code_col: int | None - name_col: int | None - explanations_col: int | None - value_cols: tuple[tuple[int, int], ...] - extra_cols: tuple[int, ...] - - -@dataclass(frozen=True, slots=True) -class Layout: - """Раскладка кандидата; `roles` — по всем строкам сетки.""" - - code_found: bool - column_section: CodeSection | None - columns: tuple[StatementColumn, ...] - segments: tuple[SegmentLayout, ...] - roles: tuple[RowRole, ...] - dates: DocumentDates - warnings: tuple[StatementWarning, ...] - - -@dataclass(frozen=True, slots=True) -class _Analysed: - """Сегмент до сопоставления колонок.""" - - first: int - last: int - header_rows: tuple[int, ...] - code_col: int | None - name_col: int | None - explanations_col: int | None - roles: tuple[RowRole, ...] - drafts: tuple[ColumnDraft, ...] - draft_warnings: tuple[StatementWarning, ...] - has_content: bool - - -class _Bounds(NamedTuple): - last: int - reference: frozenset[str] - - -def _text(rows: Grid, row: int, col: int) -> str: - cell = cell_at(rows, row, col) - return "" if cell is None else cell.text.strip() - - -def _chunks(rows: Grid, breaks: Sequence[int]) -> list[tuple[int, int]]: - """Куски сетки по границам листов; границы вне сетки и повторы отбрасываются.""" - cuts = sorted({b for b in breaks if 0 < b < len(rows)}) - starts = [0, *cuts] - ends = [*(c - 1 for c in cuts), len(rows) - 1] - return list(zip(starts, ends, strict=True)) - - -def _caption_cells(rows: Grid, header_rows: Sequence[int]) -> frozenset[str]: - cells = (caption_text(_text(rows, r, c)) for r in header_rows for c in range(len(rows[r]))) - return frozenset(c for c in cells if c) - - -def _is_repeat_header(rows: Grid, row: int, code_col: int | None, reference: frozenset[str], rules: Rules) -> bool: - """Строка-шапка: ячейка подписи кода или не менее двух ячеек, равных ячейкам шапки первого сегмента.""" - if is_numbering_row(rows[row], rules): - return False - if code_col is not None and is_code_cell(_text(rows, row, code_col), rules): - return False - normalized = [caption_text(c.text) for c in rows[row] if c is not None and c.text.strip()] - if any(rules.headers.code_caption.search(text) for text in normalized): - return True - return sum(1 for text in normalized if text and text in reference) >= _MIN_SAME_CELLS - - -_MIN_SAME_CELLS = 2 - - -def _segment_bounds(rows: Grid, start: int, chunk_last: int, reference: frozenset[str], rules: Rules) -> _Bounds: - """Последняя строка сегмента: до следующей шапки, стоящей после первой строки с кодом.""" - code_col = find_code_column(rows, start, chunk_last, rules) - first_code = None if code_col is None else first_code_row(rows, start, chunk_last, code_col, rules) - if code_col is None or first_code is None: - return _Bounds(chunk_last, reference) - ref = reference or _caption_cells(rows, find_header_rows(rows, start, first_code, code_col, rules)) - for row in range(first_code + 1, chunk_last + 1): - if _is_repeat_header(rows, row, code_col, ref, rules): - return _Bounds(row - 1, ref) - return _Bounds(chunk_last, ref) - - -def _segment_codes(rows: Grid, first: int, last: int, code_col: int, rules: Rules) -> list[str]: - codes: list[str] = [] - for row in range(first, last + 1): - text = _text(rows, row, code_col) - if is_code_cell(text, rules): - codes.extend(c for part in code_parts(text) if (c := parse_code(part, rules).code) is not None) - return codes - - -def _analyse( # noqa: PLR0913, PLR0917 - внутренняя сборка сегмента - candidate: FormCandidate, first: int, last: int, section: CodeSection | None, rules: Rules -) -> _Analysed: - rows = candidate.rows - code_col = find_code_column(rows, first, last, rules) - if code_col is None and section is not None: - code_col = find_code_column(rows, first, last, rules, relaxed_section=section) - has_content = any(c is not None and c.text.strip() for r in range(first, last + 1) for c in rows[r]) - first_code = None if code_col is None else first_code_row(rows, first, last, code_col, rules) - if code_col is None or first_code is None: - return _Analysed(first, last, (), None, None, None, (RowRole.DATA,) * (last - first + 1), (), (), has_content) - header_rows = find_header_rows(rows, first, first_code, code_col, rules) - roles = tuple(segment_roles(rows, first, last, header_rows, rules)) - data_rows = [r for r, role in zip(range(first, last + 1), roles, strict=True) if role is RowRole.DATA] - name_col, notes_col = find_name_column(rows, header_rows, first, last, code_col, rules) - drafts, warnings = segment_columns(rows, header_rows, data_rows, code_col, candidate.source, rules) - return _Analysed( - first, last, header_rows, code_col, name_col, notes_col, roles, tuple(drafts), tuple(warnings), has_content - ) - - -def _split_and_analyse(candidate: FormCandidate, rules: Rules) -> tuple[list[_Analysed], CodeSection | None]: - rows = candidate.rows - result: list[_Analysed] = [] - reference: frozenset[str] = frozenset() - section: CodeSection | None = None - for chunk_first, chunk_last in _chunks(rows, candidate.segment_breaks): - start = chunk_first - while start <= chunk_last: - bounds = _segment_bounds(rows, start, chunk_last, reference, rules) - reference = reference or bounds.reference - seg = _analyse(candidate, start, bounds.last, section, rules) - if section is None and seg.code_col is not None: - section = majority_section(_segment_codes(rows, seg.first, seg.last, seg.code_col, rules), rules) - result.append(seg) - start = bounds.last + 1 - return result, section - - -def _map_by_date( - drafts: Sequence[ColumnDraft], columns: Sequence[StatementColumn] -) -> tuple[tuple[int, int], ...] | None: - """Сопоставление по `period_date`: у каждой колонки дата есть и совпадает ровно с одной колонкой документа.""" - if not drafts or any(d.date is None for d in drafts): - return None - pairs: list[tuple[int, int]] = [] - for draft in drafts: - found = [c.column_index for c in columns if c.period_date == draft.date] - if len(found) != 1: - return None - pairs.append((draft.grid_col, found[0])) - if len({target for _, target in pairs}) != len(pairs): - return None - return tuple(pairs) - - -def _map_segment( - seg: _Analysed, columns: tuple[StatementColumn, ...], rules: Rules -) -> tuple[SegmentLayout, StatementWarning | None]: - """Колонки сегмента → колонки документа; по порядку и разное число — `SEGMENT_COLUMNS_MISMATCH`.""" - drafts = seg.drafts - mapped = _map_by_date(drafts, columns) if seg.header_rows else None - warning: StatementWarning | None = None - extra: tuple[int, ...] = () - if mapped is None: - size = min(len(drafts), len(columns)) - mapped = tuple((d.grid_col, c.column_index) for d, c in zip(drafts[:size], columns[:size], strict=False)) - extra = tuple(d.grid_col for d in drafts[size:]) - if len(drafts) != len(columns): - tail = f"; лишние столбцы сегмента: {', '.join(str(c) for c in extra)}" if extra else "" - message = ( - f"в сегменте со строки {seg.first} колонок значений {len(drafts)}, " - f"в первом сегменте {len(columns)}{tail}" - ) - warning = make_warning(WarningCode.SEGMENT_COLUMNS_MISMATCH, message, rules, locator=None) - layout = SegmentLayout( - seg.first, seg.last, seg.header_rows, seg.code_col, seg.name_col, seg.explanations_col, mapped, extra - ) - return layout, warning - - -def _trim_outside(seg: _Analysed, width: int, candidate: FormCandidate, rules: Rules) -> _Analysed: - """Сегмент без подписанных колонок: колонки сверх `width` документа — числа правее таблицы, не колонки.""" - if seg.code_col is None or len(seg.drafts) <= width or any(d.kind is not PeriodKind.NONE for d in seg.drafts): - return seg - tail = seg.drafts[width:] - if any(d.label for d in tail): # столбец с текстом подписи — не формула аналитика, остаётся расхождением - return seg - data = [r for r, role in zip(range(seg.first, seg.last + 1), seg.roles, strict=True) if role is RowRole.DATA] - refs = trailing_outside_refs( - candidate.rows, data, [d.grid_col for d in tail], seg.code_col, candidate.source, rules - ) - return replace( - seg, drafts=seg.drafts[:width], draft_warnings=(*seg.draft_warnings, outside_table_warning(refs, rules)) - ) - - -def find_layout(candidate: FormCandidate, header: StatementHeader, rules: Rules) -> Layout: - """Раскладка кандидата: сегменты, колонки документа, роли всех строк и предупреждения шага 2.""" - segments, section = _split_and_analyse(candidate, rules) - coded = [s for s in segments if s.code_col is not None] - defining = next((s for s in coded if s.header_rows), coded[0] if coded else None) - warnings: list[StatementWarning] = [] - columns: tuple[StatementColumn, ...] = () - dates = document_dates([], header) - if defining is not None: - dates = document_dates(defining.drafts, header) - columns, finalize_warnings = finalize_columns(defining.drafts, dates, rules) - # числа правее таблицы — по всем сегментам (формула аналитика может стоять на любой странице формы); - # сегмент без подписанных колонок (лист-продолжение) сверх колонок документа тоже «правее таблицы» - if any(d.kind is not PeriodKind.NONE for d in defining.drafts): - segments = [ - seg if seg is defining else _trim_outside(seg, len(columns), candidate, rules) for seg in segments - ] - warnings.extend( - merge_outside_table( - [ - w - for seg in segments - for w in seg.draft_warnings - if seg is defining or w.code == WarningCode.NUMBERS_OUTSIDE_TABLE - ], - rules, - ) - ) - warnings.extend(finalize_warnings) - if not any(s.header_rows for s in coded): - warnings.append( - make_warning(WarningCode.HEADER_ROW_NOT_FOUND, "строка заголовка таблицы не найдена", rules) - ) - laid: list[SegmentLayout] = [] - for seg in segments: - if seg.code_col is None: - laid.append(SegmentLayout(seg.first, seg.last, (), None, None, None, (), ())) - if coded and seg.has_content: - message = f"в сегменте со строки {seg.first} не найден столбец кодов" - warnings.append(make_warning(WarningCode.SEGMENT_WITHOUT_CODES, message, rules)) - elif seg is defining: - own = tuple((c.column_index, c.column_index) for c in columns) - laid.append( - SegmentLayout( - seg.first, seg.last, seg.header_rows, seg.code_col, seg.name_col, seg.explanations_col, own, () - ) - ) - else: - segment_layout, mismatch = _map_segment(seg, columns, rules) - laid.append(segment_layout) - if mismatch is not None: - warnings.append(mismatch) - roles = tuple(role for seg in segments for role in seg.roles) - return Layout(bool(coded), section, columns, tuple(laid), roles, dates, tuple(warnings)) diff --git a/api/arm_statements/serialize.py b/api/arm_statements/serialize.py deleted file mode 100644 index 03ebf8a..0000000 --- a/api/arm_statements/serialize.py +++ /dev/null @@ -1,108 +0,0 @@ -"""Сериализация в json-совместимые значения и обратно (для golden-тестов и обмена).""" - -from collections.abc import Mapping -from dataclasses import fields, is_dataclass -from datetime import date -from decimal import Decimal -from enum import Enum -from typing import Any - -from arm_statements.model import ( - Cell, - CellValue, - DocumentType, - Evidence, - FormCandidate, - HeaderCells, - Severity, - Source, - StatementWarning, -) - - -def _value_to_jsonable(value: CellValue) -> Any: - """Тег типа значения ячейки; bool проверяется раньше прочего (не подкласс Decimal/date, но явно).""" - if value is None: - return None - if isinstance(value, bool): - return {"bool": value} - if isinstance(value, Decimal): - return {"decimal": str(value)} - return {"date": value.isoformat()} - - -def _scalar_to_jsonable(obj: object) -> Any: - if isinstance(obj, Enum): - return obj.value - if isinstance(obj, Decimal): - return str(obj) - if isinstance(obj, date): - return obj.isoformat() - return obj - - -def to_jsonable(obj: object) -> Any: - """Dataclass пакета → json-совместимое значение.""" - if isinstance(obj, Cell): - return { - "text": obj.text, - "value": _value_to_jsonable(obj.value), - "confidence": obj.confidence, - "ref": obj.ref, - } - if is_dataclass(obj) and not isinstance(obj, type): - return {f.name: to_jsonable(getattr(obj, f.name)) for f in fields(obj)} - if isinstance(obj, tuple | list): - return [to_jsonable(item) for item in obj] - return _scalar_to_jsonable(obj) - - -def _value_from_dict(raw: Mapping[str, Any] | None) -> CellValue: - if raw is None: - return None - if "decimal" in raw: - return Decimal(raw["decimal"]) - if "date" in raw: - return date.fromisoformat(raw["date"]) - if "bool" in raw: - return bool(raw["bool"]) - raise ValueError(f"неизвестный тег значения ячейки: {sorted(raw)}") - - -def _cell_from_dict(raw: Mapping[str, Any] | None) -> Cell | None: - if raw is None: - return None - return Cell( - text=raw["text"], - value=_value_from_dict(raw["value"]), - confidence=float(raw["confidence"]), - ref=raw["ref"], - ) - - -def _warning_from_dict(raw: Mapping[str, Any]) -> StatementWarning: - return StatementWarning( - code=raw["code"], - severity=Severity(raw["severity"]), - message=raw["message"], - line_refs=tuple(raw.get("line_refs", ())), - column_index=raw.get("column_index"), - field=raw.get("field"), - locator=raw.get("locator"), - ) - - -def candidate_from_dict(data: Mapping[str, Any]) -> FormCandidate: - """Обратная к `to_jsonable` для кандидата.""" - type_hint = data["type_hint"] - evidence = data["evidence"] - return FormCandidate( - source=Source(data["source"]), - locator=data["locator"], - type_hint=None if type_hint is None else DocumentType(type_hint), - evidence=Evidence(okud=_cell_from_dict(evidence["okud"]), title=_cell_from_dict(evidence["title"])), - header=HeaderCells(**{name: _cell_from_dict(raw) for name, raw in data["header"].items()}), - rows=tuple(tuple(_cell_from_dict(raw) for raw in row) for row in data["rows"]), - segment_breaks=tuple(data.get("segment_breaks", ())), - warnings=tuple(_warning_from_dict(raw) for raw in data.get("warnings", ())), - ) diff --git a/api/arm_statements/status.py b/api/arm_statements/status.py deleted file mode 100644 index 25a2ccf..0000000 --- a/api/arm_statements/status.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Статус документа и конверта (спека «Шаг 8 — статус документа и конверта»). - -Одна функция на сервис и ядро: вход не зависит от источника строк. -""" - -from collections.abc import Sequence -from dataclasses import dataclass - -from arm_statements.model import ( - DocStatus, - DocumentType, - EnvelopeStatus, - LineStatus, - PeriodKind, - Severity, - StatementDocument, - StatementWarning, -) - -_ACCEPTABLE_LINES = frozenset({LineStatus.OK, LineStatus.DETAIL}) - - -@dataclass(frozen=True, slots=True) -class StatusLine: - line_status: LineStatus - needs_review: bool - - -@dataclass(frozen=True, slots=True) -class StatusValue: - line_status: LineStatus - needs_review: bool - - -@dataclass(frozen=True, slots=True) -class StatusInput: - document_type: DocumentType - layout_found: bool - has_organization_or_inn: bool - period_known: bool - needs_review_fields: tuple[str, ...] - lines: tuple[StatusLine, ...] - values: tuple[StatusValue, ...] - hard_failures: int - accepted: bool - warning_severities: tuple[Severity, ...] - - -def document_status(data: StatusInput) -> DocStatus: - """`DISCLOSURE` — всегда `done`; иначе `done` только при выполнении всех условий спеки.""" - if data.document_type is DocumentType.DISCLOSURE: - return DocStatus.DONE - done = ( - data.layout_found - and data.has_organization_or_inn - and data.period_known - and all(line.line_status in _ACCEPTABLE_LINES for line in data.lines) - and not any(line.needs_review for line in data.lines) - # значения строк detail в статус не входят (Р-23) - and not any(v.needs_review for v in data.values if v.line_status is not LineStatus.DETAIL) - and not data.needs_review_fields - and (data.hard_failures == 0 or data.accepted) - and Severity.ERROR not in data.warning_severities - ) - return DocStatus.DONE if done else DocStatus.NEEDS_CHECK - - -def envelope_status( - document_statuses: Sequence[DocStatus], file_warnings: Sequence[StatementWarning] -) -> EnvelopeStatus: - if not document_statuses: - return EnvelopeStatus.ERROR - if all(s is DocStatus.DONE for s in document_statuses) and all( - w.severity is not Severity.ERROR for w in file_warnings - ): - return EnvelopeStatus.DONE - return EnvelopeStatus.NEEDS_CHECK - - -def status_input(document: StatementDocument) -> StatusInput: - """Вход функции статуса из документа шагов 2–8.""" - header = document.header - period_known = header.report_date.value is not None or any( - c.period_kind is PeriodKind.ABSOLUTE and c.period_date is not None for c in document.columns - ) - return StatusInput( - document_type=document.document_type, - layout_found=document.lines != (), - has_organization_or_inn=bool(header.organization_name.value or header.inn.value), - period_known=period_known, - needs_review_fields=header.needs_review_fields, - lines=tuple(StatusLine(line.line_status, line.needs_review) for line in document.lines), - values=tuple(StatusValue(line.line_status, v.needs_review) for line in document.lines for v in line.values), - hard_failures=0 if document.checks is None else len(document.checks.relations_failed), - accepted=False, - warning_severities=tuple(w.severity for w in document.warnings), - ) diff --git a/api/arm_statements/structure.py b/api/arm_statements/structure.py deleted file mode 100644 index c24ccb6..0000000 --- a/api/arm_statements/structure.py +++ /dev/null @@ -1,177 +0,0 @@ -"""Шаг 8 — подтверждение неуверенного кода структурой бланка и выбор редакции (спека «Шаг 8»).""" - -from collections.abc import Sequence -from dataclasses import dataclass -from datetime import date - -from arm_statements.checks import form_variant, subline_parents -from arm_statements.evaluation import EvalLine, is_check_line -from arm_statements.loader import Rules -from arm_statements.model import DocumentType, Edition, FormVariant - -FSBU4_FROM = date(2025, 1, 1) # эффективная дата после 2024-12-31 → fsbu4 - -_CODE_LEN = 4 -_SUBLINE_LEN = 5 -_CONFIDENT = 100 - - -def edition_by_date(effective: date | None) -> Edition | None: - if effective is None: - return None - return Edition.FSBU4 if effective >= FSBU4_FROM else Edition.RAS_66N - - -@dataclass(frozen=True, slots=True) -class StructureResult: - edition: Edition | None - unconfirmed: frozenset[str] # ключи строк с неуверенным кодом, не подтверждённым структурой - - -# Элемент последовательности: (код, уверенный). -_Item = tuple[str, bool] - - -def _optimum(items: Sequence[_Item], order: Sequence[str], scale: int, skip: int | None = None) -> int: - """Взвешенная LCS `items` × `order` по равенству кода; вес позиции `scale * уверенный + 1`. - - `skip` — индекс позиции `items`, которой запрещено входить. Сложность O(|items|·|order|).""" - if not items or not order: - return 0 - previous = [0] * (len(order) + 1) - for index, (code, confident) in enumerate(items): - weight = None if index == skip else scale * int(confident) + 1 - current = [0] * (len(order) + 1) - for j, order_code in enumerate(order, start=1): - best = max(previous[j], current[j - 1]) - if weight is not None and code == order_code: - best = max(best, previous[j - 1] + weight) - current[j] = best - previous = current - return previous[-1] - - -def _editions(rules: Rules) -> tuple[Edition, ...]: - return rules.code_orders.editions or tuple(Edition) - - -def _pick(optima: dict[Edition, int], effective: date | None) -> tuple[Edition | None, tuple[Edition, ...]]: - """Редакция с большим оптимумом; равенство — по дате. Второе значение — редакции, чьи оптимумы лучшие.""" - top = max(optima.values()) - tied = tuple(edition for edition, value in optima.items() if value == top) - if len(tied) == 1: - return tied[0], tied - by_date = edition_by_date(effective) - return (by_date if by_date in tied else None), tied - - -def _filtered(items: Sequence[_Item], order: Sequence[str]) -> tuple[list[_Item], list[int]]: - allowed = set(order) - kept = [(item, index) for index, item in enumerate(items) if item[0] in allowed] - return [item for item, _ in kept], [index for _, index in kept] - - -def _select( - document_type: DocumentType, - items: Sequence[_Item], - variant: FormVariant, - effective: date | None, - rules: Rules, -) -> tuple[Edition | None, tuple[Edition, ...], dict[Edition, int]]: - scale = len(items) + 1 - optima = { - edition: _optimum( - _filtered(items, order := rules.code_orders.sequence(edition, document_type, variant))[0], - order, - scale, - ) - for edition in _editions(rules) - } - edition, tied = _pick(optima, effective) - return edition, tied, optima - - -def choose_edition( - document_type: DocumentType, - codes: Sequence[tuple[str, bool]], - effective: date | None, - rules: Rules, - *, - variant: FormVariant | None = None, -) -> Edition | None: - """Шаги 2–3 и 6 без подтверждения кодов; `codes` — пары `(код, уверенный)` строк `ok` по порядку.""" - if document_type is DocumentType.DISCLOSURE: - return edition_by_date(effective) - if variant is None: - variant = form_variant(document_type, [code for code, _ in codes]) or FormVariant.FULL - items = [(code, confident) for code, confident in codes if len(code) == _CODE_LEN] - return _select(document_type, items, variant, effective, rules)[0] - - -def _is_uncertain(line: EvalLine) -> bool: - return (line.code_confidence is None or line.code_confidence < _CONFIDENT) and not line.code_fixed - - -def _confirm_sublines(checked: Sequence[EvalLine], codes: Sequence[str], confirmed_keys: set[str]) -> set[str]: - """Пятизначные неуверенные коды: подтверждены родителем, стоящим раньше (§11.9); возвращает неподтверждённые.""" - unconfirmed: set[str] = set() - parents = subline_parents(codes) - for position, line in enumerate(checked): - code = line.line_code or "" - if len(code) == _CODE_LEN or not _is_uncertain(line): - continue - parent = parents[position] - found = None - if len(code) == _SUBLINE_LEN and parent is not None: - found = next((e for e in reversed(checked[:position]) if e.line_code == parent), None) - if found is not None and (not _is_uncertain(found) or found.key in confirmed_keys): - confirmed_keys.add(line.key) - else: - unconfirmed.add(line.key) - return unconfirmed - - -def confirm_codes( - lines: Sequence[EvalLine], - document_type: DocumentType, - variant: FormVariant | None, - effective: date | None, - rules: Rules, -) -> StructureResult: - if document_type is DocumentType.DISCLOSURE: - return StructureResult(edition_by_date(effective), frozenset()) - checked = sorted( - (line for line in lines if is_check_line(line) and line.order is not None), - key=lambda line: line.order or 0, - ) - codes = [line.line_code or "" for line in checked] - if variant is None: - variant = form_variant(document_type, codes) or FormVariant.FULL - four = [line for line in checked if len(line.line_code or "") == _CODE_LEN] - items: list[_Item] = [(line.line_code or "", not _is_uncertain(line)) for line in four] - edition, tied, _ = _select(document_type, items, variant, effective, rules) - - # Четырёхзначные неуверенные коды: входят во все лучшие подпоследовательности выбранной редакции - # (при равенстве без даты — во все лучшие обеих редакций, P1-4). - unconfirmed: set[str] = set() - confirmed_keys: set[str] = set() - scale = len(items) + 1 - for index, line in enumerate(four): - if not _is_uncertain(line): - confirmed_keys.add(line.key) - continue - confirmed = True - for candidate in tied if edition is None else (edition,): - order = rules.code_orders.sequence(candidate, document_type, variant) - if items[index][0] not in order: - confirmed = False - break - kept, indices = _filtered(items, order) - total = _optimum(kept, order, scale) - if _optimum(kept, order, scale, skip=indices.index(index)) >= total: - confirmed = False - break - (confirmed_keys if confirmed else unconfirmed).add(line.key) - - unconfirmed |= _confirm_sublines(checked, codes, confirmed_keys) - return StructureResult(edition, frozenset(unconfirmed)) diff --git a/api/arm_statements/tests/__init__.py b/api/arm_statements/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/api/arm_statements/tests/builders.py b/api/arm_statements/tests/builders.py deleted file mode 100644 index 7b54b00..0000000 --- a/api/arm_statements/tests/builders.py +++ /dev/null @@ -1,155 +0,0 @@ -"""Сборщики тестовых кандидатов (только для тестов пакета).""" - -from collections.abc import Mapping, Sequence -from decimal import Decimal -from typing import overload - -from arm_statements.model import ( - HEADER_FIELDS, - Cell, - DocumentType, - Evidence, - FormCandidate, - HeaderCells, - Source, - StatementWarning, -) - -OcrInput = str | tuple[str, float] | None -ExcelInput = str | Decimal | None -PlaceInput = str | Decimal | tuple[str, float] - -_FULL_CONFIDENCE = 100.0 -_LATIN_LETTERS = 26 - - -def _text_conf(item: str | tuple[str, float]) -> tuple[str, float]: - if isinstance(item, tuple): - return item - return item, _FULL_CONFIDENCE - - -def hdr(**fields: str | tuple[str, float]) -> HeaderCells: - """Шапка из текстов; ref — `hdr:<поле>`.""" - unknown = set(fields) - set(HEADER_FIELDS) - if unknown: - raise TypeError(f"неизвестные поля шапки: {sorted(unknown)}") - cells: dict[str, Cell] = {} - for name, item in fields.items(): - text, conf = _text_conf(item) - cells[name] = Cell(text, None, conf, f"hdr:{name}") - return HeaderCells(**cells) - - -def _evidence(okud: str | None, title: str | None) -> Evidence: - return Evidence( - okud=None if okud is None else Cell(okud, None, _FULL_CONFIDENCE, "evidence:okud"), - title=None if title is None else Cell(title, None, _FULL_CONFIDENCE, "evidence:title"), - ) - - -def ocr( # noqa: PLR0913 - сигнатура задана планом (Task 2) - rows: Sequence[Sequence[OcrInput]], - *, - type_hint: DocumentType | None = None, - header: HeaderCells | None = None, - title: str | None = None, - okud: str | None = None, - locator: str = "file:t.json;table:0", -) -> FormCandidate: - """Кандидат OCR: строка r → ячейки с ref `row:`.""" - grid: list[tuple[Cell | None, ...]] = [] - for r, row in enumerate(rows): - cells: list[Cell | None] = [] - for item in row: - if item is None: - cells.append(None) - else: - text, conf = _text_conf(item) - cells.append(Cell(text, None, conf, f"row:{r}")) - grid.append(tuple(cells)) - return FormCandidate( - source=Source.OCR, - locator=locator, - type_hint=type_hint, - evidence=_evidence(okud, title), - header=header if header is not None else HeaderCells(), - rows=tuple(grid), - ) - - -def _column_letters(index: int) -> str: - """0 → A, 25 → Z, 26 → AA.""" - letters = "" - n = index + 1 - while n > 0: - n, rem = divmod(n - 1, _LATIN_LETTERS) - letters = chr(ord("A") + rem) + letters - return letters - - -def _sheet_prefix(name: str) -> str: - """Имя листа в ссылке по правилу Excel: кавычки, если не только буквы/цифры/_ или начинается с цифры.""" - plain = all(ch.isalpha() or ch.isdigit() or ch == "_" for ch in name) and not name[:1].isdigit() - if plain: - return name - return "'" + name.replace("'", "''") + "'" - - -def _thousands(value: Decimal) -> str: - return f"{value:,}".replace(",", " ") - - -def excel( # noqa: PLR0913 - сигнатура задана планом (Task 2) - sheets: Sequence[tuple[str, Sequence[Sequence[ExcelInput]]]], - *, - type_hint: DocumentType | None = None, - header: HeaderCells | None = None, - title: str | None = None, - okud: str | None = None, - warnings: Sequence[StatementWarning] = (), -) -> FormCandidate: - """Кандидат Excel: листы подряд в одной сетке, ref `<лист>!`.""" - grid: list[tuple[Cell | None, ...]] = [] - breaks: list[int] = [] - for sheet_no, (name, sheet_rows) in enumerate(sheets): - if sheet_no > 0: - breaks.append(len(grid)) - prefix = _sheet_prefix(name) - for r, row in enumerate(sheet_rows): - cells: list[Cell | None] = [] - for c, item in enumerate(row): - ref = f"{prefix}!{_column_letters(c)}{r + 1}" - if item is None: - cells.append(None) - elif isinstance(item, Decimal): - cells.append(Cell(_thousands(item), item, _FULL_CONFIDENCE, ref)) - else: - cells.append(Cell(item, None, _FULL_CONFIDENCE, ref)) - grid.append(tuple(cells)) - return FormCandidate( - source=Source.EXCEL, - locator="sheet:" + ",".join(name for name, _ in sheets), - type_hint=type_hint, - evidence=_evidence(okud, title), - header=header if header is not None else HeaderCells(), - rows=tuple(grid), - segment_breaks=tuple(breaks), - warnings=tuple(warnings), - ) - - -@overload -def place(cells: Mapping[int, str | Decimal], width: int | None = None) -> list[str | Decimal | None]: ... - - -@overload -def place(cells: Mapping[int, PlaceInput], width: int | None = None) -> list[PlaceInput | None]: ... - - -def place( - cells: Mapping[int, PlaceInput] | Mapping[int, str | Decimal], width: int | None = None -) -> list[PlaceInput | None] | list[str | Decimal | None]: - """Строка с ячейками в заданных столбцах, остальные None.""" - size = width if width is not None else (max(cells) + 1 if cells else 0) - return [cells.get(i) for i in range(size)] diff --git a/api/arm_statements/tests/conftest.py b/api/arm_statements/tests/conftest.py deleted file mode 100644 index 6e1508e..0000000 --- a/api/arm_statements/tests/conftest.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Общие фикстуры тестов пакета.""" - -import pytest - -from arm_statements.loader import Rules, default_rules - - -@pytest.fixture -def rules() -> Rules: - return default_rules() diff --git a/api/arm_statements/tests/golden/aep3_balance.candidate.json b/api/arm_statements/tests/golden/aep3_balance.candidate.json deleted file mode 100644 index a3c898e..0000000 --- a/api/arm_statements/tests/golden/aep3_balance.candidate.json +++ /dev/null @@ -1,750 +0,0 @@ -{ - "source": "ocr", - "locator": "file:aep3.json;table:0", - "type_hint": "BALANCE", - "evidence": { - "okud": null, - "title": { - "text": "Бухгалтерский баланс", - "value": null, - "confidence": 100.0, - "ref": "evidence:title" - } - }, - "header": { - "organization_name": { - "text": "НКО «Пример»", - "value": null, - "confidence": 100.0, - "ref": "hdr:organization_name" - }, - "inn": { - "text": "3|1 21 2|", - "value": null, - "confidence": 80.0, - "ref": "hdr:inn" - }, - "report_date": { - "text": "31.12.2024", - "value": null, - "confidence": 75.0, - "ref": "hdr:report_date" - }, - "units": { - "text": "(тыс. руб. - год по ОКЕИ)", - "value": null, - "confidence": 69.0, - "ref": "hdr:units" - }, - "okud": null, - "okpo": null, - "okopf": null, - "okfs": null, - "okei": null, - "address": null - }, - "rows": [ - [ - { - "text": "Наименование показателя", - "value": null, - "confidence": 100.0, - "ref": "row:0" - }, - { - "text": "Код", - "value": null, - "confidence": 100.0, - "ref": "row:0" - }, - { - "text": "На 31 декабря 2024 г.", - "value": null, - "confidence": 100.0, - "ref": "row:0" - } - ], - [ - { - "text": "Нематериальные активы", - "value": null, - "confidence": 100.0, - "ref": "row:1" - }, - { - "text": "1110", - "value": null, - "confidence": 100.0, - "ref": "row:1" - }, - { - "text": "30127", - "value": null, - "confidence": 60.0, - "ref": "row:1" - } - ], - [ - { - "text": "в том числе лицензии", - "value": null, - "confidence": 100.0, - "ref": "row:2" - }, - { - "text": "НИ", - "value": null, - "confidence": 0.0, - "ref": "row:2" - }, - { - "text": "100", - "value": null, - "confidence": 60.0, - "ref": "row:2" - } - ], - [ - { - "text": "Материальные поисковые\nактивы", - "value": null, - "confidence": 100.0, - "ref": "row:3" - }, - { - "text": "| 140", - "value": null, - "confidence": 40.0, - "ref": "row:3" - }, - { - "text": "500", - "value": null, - "confidence": 100.0, - "ref": "row:3" - } - ], - [ - { - "text": "Нематериальные поисковые\nактивы", - "value": null, - "confidence": 100.0, - "ref": "row:4" - }, - { - "text": "изо :", - "value": null, - "confidence": 80.0, - "ref": "row:4" - }, - { - "text": "3", - "value": null, - "confidence": 0.0, - "ref": "row:4" - } - ], - [ - { - "text": "Основные средства", - "value": null, - "confidence": 100.0, - "ref": "row:5" - }, - { - "text": "1150", - "value": null, - "confidence": 100.0, - "ref": "row:5" - }, - { - "text": "2875532", - "value": null, - "confidence": 100.0, - "ref": "row:5" - } - ], - [ - { - "text": "Итого по разделу I", - "value": null, - "confidence": 100.0, - "ref": "row:6" - }, - { - "text": "1100", - "value": null, - "confidence": 80.0, - "ref": "row:6" - }, - { - "text": "2906159", - "value": null, - "confidence": 100.0, - "ref": "row:6" - } - ], - [ - { - "text": "Запасы", - "value": null, - "confidence": 100.0, - "ref": "row:7" - }, - { - "text": "1210", - "value": null, - "confidence": 100.0, - "ref": "row:7" - }, - { - "text": "1108817", - "value": null, - "confidence": 100.0, - "ref": "row:7" - } - ], - [ - { - "text": "Налог на добавленную стоимость\nпо приобретённым ценностям", - "value": null, - "confidence": 100.0, - "ref": "row:8" - }, - { - "text": "1220", - "value": null, - "confidence": 100.0, - "ref": "row:8" - }, - { - "text": "157", - "value": null, - "confidence": 100.0, - "ref": "row:8" - } - ], - [ - { - "text": "Дебиторская задолженность", - "value": null, - "confidence": 100.0, - "ref": "row:9" - }, - { - "text": "1230", - "value": null, - "confidence": 100.0, - "ref": "row:9" - }, - { - "text": "1285022", - "value": null, - "confidence": 100.0, - "ref": "row:9" - } - ], - [ - { - "text": "Финансовые вложения", - "value": null, - "confidence": 100.0, - "ref": "row:10" - }, - { - "text": "1240", - "value": null, - "confidence": 100.0, - "ref": "row:10" - }, - { - "text": "183600", - "value": null, - "confidence": 100.0, - "ref": "row:10" - } - ], - [ - { - "text": "Денежные средства", - "value": null, - "confidence": 100.0, - "ref": "row:11" - }, - { - "text": "1250", - "value": null, - "confidence": 100.0, - "ref": "row:11" - }, - { - "text": "4447", - "value": null, - "confidence": 100.0, - "ref": "row:11" - } - ], - [ - { - "text": "Прочие оборотные активы", - "value": null, - "confidence": 100.0, - "ref": "row:12" - }, - { - "text": "1260", - "value": null, - "confidence": 100.0, - "ref": "row:12" - }, - { - "text": "30524", - "value": null, - "confidence": 100.0, - "ref": "row:12" - } - ], - [ - { - "text": "Итого по разделу II", - "value": null, - "confidence": 100.0, - "ref": "row:13" - }, - { - "text": "1200", - "value": null, - "confidence": 100.0, - "ref": "row:13" - }, - { - "text": "2612567", - "value": null, - "confidence": 100.0, - "ref": "row:13" - } - ], - [ - { - "text": "БАЛАНС", - "value": null, - "confidence": 100.0, - "ref": "row:14" - }, - { - "text": "1600", - "value": null, - "confidence": 100.0, - "ref": "row:14" - }, - { - "text": "5518726", - "value": null, - "confidence": 100.0, - "ref": "row:14" - } - ], - [ - { - "text": "Переоценка внеоборотных активов", - "value": null, - "confidence": 100.0, - "ref": "row:15" - }, - { - "text": "1340", - "value": null, - "confidence": 100.0, - "ref": "row:15" - }, - { - "text": "146927", - "value": null, - "confidence": 100.0, - "ref": "row:15" - } - ], - [ - { - "text": "Добавочный капитал", - "value": null, - "confidence": 100.0, - "ref": "row:16" - }, - { - "text": "1350", - "value": null, - "confidence": 100.0, - "ref": "row:16" - }, - { - "text": "24", - "value": null, - "confidence": 100.0, - "ref": "row:16" - } - ], - [ - { - "text": "Резервный капитал", - "value": null, - "confidence": 100.0, - "ref": "row:17" - }, - { - "text": "1360", - "value": null, - "confidence": 100.0, - "ref": "row:17" - }, - { - "text": "2416", - "value": null, - "confidence": 100.0, - "ref": "row:17" - } - ], - [ - { - "text": "Нераспределенная прибыль", - "value": null, - "confidence": 100.0, - "ref": "row:18" - }, - { - "text": "1370", - "value": null, - "confidence": 100.0, - "ref": "row:18" - }, - { - "text": "3262545", - "value": null, - "confidence": 100.0, - "ref": "row:18" - } - ], - [ - { - "text": "Итого по разделу III", - "value": null, - "confidence": 100.0, - "ref": "row:19" - }, - { - "text": "1300", - "value": null, - "confidence": 100.0, - "ref": "row:19" - }, - { - "text": "3411912", - "value": null, - "confidence": 100.0, - "ref": "row:19" - } - ], - [ - { - "text": "Паевой фонд", - "value": null, - "confidence": 100.0, - "ref": "row:20" - }, - { - "text": "1310", - "value": null, - "confidence": 100.0, - "ref": "row:20" - }, - { - "text": "", - "value": null, - "confidence": 100.0, - "ref": "row:20" - } - ], - [ - { - "text": "Целевой капитал", - "value": null, - "confidence": 100.0, - "ref": "row:21" - }, - { - "text": "1320", - "value": null, - "confidence": 100.0, - "ref": "row:21" - }, - { - "text": "", - "value": null, - "confidence": 100.0, - "ref": "row:21" - } - ], - [ - { - "text": "Целевые средства", - "value": null, - "confidence": 100.0, - "ref": "row:22" - }, - { - "text": "1350", - "value": null, - "confidence": 100.0, - "ref": "row:22" - }, - { - "text": "", - "value": null, - "confidence": 100.0, - "ref": "row:22" - } - ], - [ - { - "text": "Фонд недвижимого и особо\nценного движимого имущества", - "value": null, - "confidence": 100.0, - "ref": "row:23" - }, - { - "text": "1360", - "value": null, - "confidence": 100.0, - "ref": "row:23" - }, - { - "text": "", - "value": null, - "confidence": 100.0, - "ref": "row:23" - } - ], - [ - { - "text": "Резервный и иные целевые\nфонды", - "value": null, - "confidence": 100.0, - "ref": "row:24" - }, - { - "text": "1370", - "value": null, - "confidence": 100.0, - "ref": "row:24" - }, - { - "text": "", - "value": null, - "confidence": 100.0, - "ref": "row:24" - } - ], - [ - { - "text": "Итого по разделу III", - "value": null, - "confidence": 100.0, - "ref": "row:25" - }, - { - "text": "1300", - "value": null, - "confidence": 100.0, - "ref": "row:25" - }, - { - "text": "", - "value": null, - "confidence": 100.0, - "ref": "row:25" - } - ], - [ - { - "text": "Заемные средства", - "value": null, - "confidence": 100.0, - "ref": "row:26" - }, - { - "text": "1410", - "value": null, - "confidence": 100.0, - "ref": "row:26" - }, - { - "text": "628256", - "value": null, - "confidence": 100.0, - "ref": "row:26" - } - ], - [ - { - "text": "Отложенные налоговые\nобязательства", - "value": null, - "confidence": 100.0, - "ref": "row:27" - }, - { - "text": "1420", - "value": null, - "confidence": 100.0, - "ref": "row:27" - }, - { - "text": "43094", - "value": null, - "confidence": 100.0, - "ref": "row:27" - } - ], - [ - { - "text": "Итого по разделу IV", - "value": null, - "confidence": 100.0, - "ref": "row:28" - }, - { - "text": "1400", - "value": null, - "confidence": 100.0, - "ref": "row:28" - }, - { - "text": "671350", - "value": null, - "confidence": 100.0, - "ref": "row:28" - } - ], - [ - { - "text": "Заемные средства", - "value": null, - "confidence": 100.0, - "ref": "row:29" - }, - { - "text": "1510", - "value": null, - "confidence": 100.0, - "ref": "row:29" - }, - { - "text": "570663", - "value": null, - "confidence": 100.0, - "ref": "row:29" - } - ], - [ - { - "text": "Кредиторская задолженность", - "value": null, - "confidence": 100.0, - "ref": "row:30" - }, - { - "text": "1520", - "value": null, - "confidence": 100.0, - "ref": "row:30" - }, - { - "text": "817769", - "value": null, - "confidence": 100.0, - "ref": "row:30" - } - ], - [ - { - "text": "Оценочные обязательства", - "value": null, - "confidence": 100.0, - "ref": "row:31" - }, - { - "text": "1540", - "value": null, - "confidence": 100.0, - "ref": "row:31" - }, - { - "text": "47001", - "value": null, - "confidence": 100.0, - "ref": "row:31" - } - ], - [ - { - "text": "Прочие обязательства", - "value": null, - "confidence": 100.0, - "ref": "row:32" - }, - { - "text": "1550", - "value": null, - "confidence": 100.0, - "ref": "row:32" - }, - { - "text": "31", - "value": null, - "confidence": 100.0, - "ref": "row:32" - } - ], - [ - { - "text": "Итого по разделу V", - "value": null, - "confidence": 100.0, - "ref": "row:33" - }, - { - "text": "1500", - "value": null, - "confidence": 100.0, - "ref": "row:33" - }, - { - "text": "1435464", - "value": null, - "confidence": 100.0, - "ref": "row:33" - } - ], - [ - { - "text": "БАЛАНС", - "value": null, - "confidence": 100.0, - "ref": "row:34" - }, - { - "text": "1700", - "value": null, - "confidence": 100.0, - "ref": "row:34" - }, - { - "text": "5518726", - "value": null, - "confidence": 100.0, - "ref": "row:34" - } - ] - ], - "segment_breaks": [], - "warnings": [] -} diff --git a/api/arm_statements/tests/golden/aep3_balance.expected.json b/api/arm_statements/tests/golden/aep3_balance.expected.json deleted file mode 100644 index a8366b6..0000000 --- a/api/arm_statements/tests/golden/aep3_balance.expected.json +++ /dev/null @@ -1,868 +0,0 @@ -{ - "document_type": "BALANCE", - "form_variant": "full", - "locator": "file:aep3.json;table:0", - "header": { - "organization_name": { - "text": "НКО «Пример»", - "confidence": 100.0, - "value": "НКО «Пример»" - }, - "inn": { - "text": "3|1 21 2|", - "confidence": 80.0, - "value": null - }, - "report_date": { - "text": "31.12.2024", - "confidence": 75.0, - "value": "2024-12-31" - }, - "units": { - "text": "(тыс. руб. - год по ОКЕИ)", - "confidence": 69.0, - "value": "тыс. руб." - }, - "okud": { - "text": null, - "confidence": null, - "value": null - }, - "okpo": { - "text": null, - "confidence": null, - "value": null - }, - "okopf": { - "text": null, - "confidence": null, - "value": null - }, - "okfs": { - "text": null, - "confidence": null, - "value": null - }, - "okei": { - "text": null, - "confidence": null, - "value": null - }, - "address": { - "text": null, - "confidence": null, - "value": null - }, - "form_name": "Бухгалтерский баланс", - "needs_review_fields": [] - }, - "columns": [ - { - "column_index": 2, - "label": "На 31 декабря 2024 г.", - "label_confidence": 100.0, - "period_date": "2024-12-31", - "period_kind": "absolute" - } - ], - "lines": [ - { - "row_index": 0, - "ref": "row:1", - "line_status": "ok", - "line_code": "1110", - "line_code_text": "1110", - "line_code_confidence": 100.0, - "line_name": "Нематериальные активы", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "30127.00", - "string_value": null, - "text": "30127", - "confidence": 60.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 1, - "ref": "row:2", - "line_status": "detail", - "line_code": null, - "line_code_text": "НИ", - "line_code_confidence": 0.0, - "line_name": "в том числе лицензии", - "line_name_confidence": 100.0, - "parent_ref": "row:1", - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "100.00", - "string_value": null, - "text": "100", - "confidence": 60.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 2, - "ref": "row:3", - "line_status": "ok", - "line_code": "1140", - "line_code_text": "| 140", - "line_code_confidence": 40.0, - "line_name": "Материальные поисковые\nактивы", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "500.00", - "string_value": null, - "text": "500", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 3, - "ref": "row:4", - "line_status": "detail", - "line_code": null, - "line_code_text": "изо :", - "line_code_confidence": 80.0, - "line_name": "Нематериальные поисковые\nактивы", - "line_name_confidence": 100.0, - "parent_ref": "row:3", - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "3.00", - "string_value": null, - "text": "3", - "confidence": 0.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 4, - "ref": "row:5", - "line_status": "ok", - "line_code": "1150", - "line_code_text": "1150", - "line_code_confidence": 100.0, - "line_name": "Основные средства", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "2875532.00", - "string_value": null, - "text": "2875532", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 5, - "ref": "row:6", - "line_status": "ok", - "line_code": "1100", - "line_code_text": "1100", - "line_code_confidence": 80.0, - "line_name": "Итого по разделу I", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "2906159.00", - "string_value": null, - "text": "2906159", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 6, - "ref": "row:7", - "line_status": "ok", - "line_code": "1210", - "line_code_text": "1210", - "line_code_confidence": 100.0, - "line_name": "Запасы", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "1108817.00", - "string_value": null, - "text": "1108817", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 7, - "ref": "row:8", - "line_status": "ok", - "line_code": "1220", - "line_code_text": "1220", - "line_code_confidence": 100.0, - "line_name": "Налог на добавленную стоимость\nпо приобретённым ценностям", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "157.00", - "string_value": null, - "text": "157", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 8, - "ref": "row:9", - "line_status": "ok", - "line_code": "1230", - "line_code_text": "1230", - "line_code_confidence": 100.0, - "line_name": "Дебиторская задолженность", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "1285022.00", - "string_value": null, - "text": "1285022", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 9, - "ref": "row:10", - "line_status": "ok", - "line_code": "1240", - "line_code_text": "1240", - "line_code_confidence": 100.0, - "line_name": "Финансовые вложения", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "183600.00", - "string_value": null, - "text": "183600", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 10, - "ref": "row:11", - "line_status": "ok", - "line_code": "1250", - "line_code_text": "1250", - "line_code_confidence": 100.0, - "line_name": "Денежные средства", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "4447.00", - "string_value": null, - "text": "4447", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 11, - "ref": "row:12", - "line_status": "ok", - "line_code": "1260", - "line_code_text": "1260", - "line_code_confidence": 100.0, - "line_name": "Прочие оборотные активы", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "30524.00", - "string_value": null, - "text": "30524", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 12, - "ref": "row:13", - "line_status": "ok", - "line_code": "1200", - "line_code_text": "1200", - "line_code_confidence": 100.0, - "line_name": "Итого по разделу II", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "2612567.00", - "string_value": null, - "text": "2612567", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 13, - "ref": "row:14", - "line_status": "ok", - "line_code": "1600", - "line_code_text": "1600", - "line_code_confidence": 100.0, - "line_name": "БАЛАНС", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "5518726.00", - "string_value": null, - "text": "5518726", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 14, - "ref": "row:15", - "line_status": "ok", - "line_code": "1340", - "line_code_text": "1340", - "line_code_confidence": 100.0, - "line_name": "Переоценка внеоборотных активов", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "146927.00", - "string_value": null, - "text": "146927", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 15, - "ref": "row:16", - "line_status": "ok", - "line_code": "1350", - "line_code_text": "1350", - "line_code_confidence": 100.0, - "line_name": "Добавочный капитал", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "24.00", - "string_value": null, - "text": "24", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 16, - "ref": "row:17", - "line_status": "ok", - "line_code": "1360", - "line_code_text": "1360", - "line_code_confidence": 100.0, - "line_name": "Резервный капитал", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "2416.00", - "string_value": null, - "text": "2416", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 17, - "ref": "row:18", - "line_status": "ok", - "line_code": "1370", - "line_code_text": "1370", - "line_code_confidence": 100.0, - "line_name": "Нераспределенная прибыль", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "3262545.00", - "string_value": null, - "text": "3262545", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 18, - "ref": "row:19", - "line_status": "ok", - "line_code": "1300", - "line_code_text": "1300", - "line_code_confidence": 100.0, - "line_name": "Итого по разделу III", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "3411912.00", - "string_value": null, - "text": "3411912", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 19, - "ref": "row:20", - "line_status": "ok", - "line_code": "1310", - "line_code_text": "1310", - "line_code_confidence": 100.0, - "line_name": "Паевой фонд", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": null, - "string_value": null, - "text": "", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 20, - "ref": "row:21", - "line_status": "ok", - "line_code": "1320", - "line_code_text": "1320", - "line_code_confidence": 100.0, - "line_name": "Целевой капитал", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": null, - "string_value": null, - "text": "", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 21, - "ref": "row:22", - "line_status": "ok", - "line_code": "1350", - "line_code_text": "1350", - "line_code_confidence": 100.0, - "line_name": "Целевые средства", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": null, - "string_value": null, - "text": "", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 22, - "ref": "row:23", - "line_status": "ok", - "line_code": "1360", - "line_code_text": "1360", - "line_code_confidence": 100.0, - "line_name": "Фонд недвижимого и особо\nценного движимого имущества", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": null, - "string_value": null, - "text": "", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 23, - "ref": "row:24", - "line_status": "ok", - "line_code": "1370", - "line_code_text": "1370", - "line_code_confidence": 100.0, - "line_name": "Резервный и иные целевые\nфонды", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": null, - "string_value": null, - "text": "", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 24, - "ref": "row:25", - "line_status": "ok", - "line_code": "1300", - "line_code_text": "1300", - "line_code_confidence": 100.0, - "line_name": "Итого по разделу III", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": null, - "string_value": null, - "text": "", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 25, - "ref": "row:26", - "line_status": "ok", - "line_code": "1410", - "line_code_text": "1410", - "line_code_confidence": 100.0, - "line_name": "Заемные средства", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "628256.00", - "string_value": null, - "text": "628256", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 26, - "ref": "row:27", - "line_status": "ok", - "line_code": "1420", - "line_code_text": "1420", - "line_code_confidence": 100.0, - "line_name": "Отложенные налоговые\nобязательства", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "43094.00", - "string_value": null, - "text": "43094", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 27, - "ref": "row:28", - "line_status": "ok", - "line_code": "1400", - "line_code_text": "1400", - "line_code_confidence": 100.0, - "line_name": "Итого по разделу IV", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "671350.00", - "string_value": null, - "text": "671350", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 28, - "ref": "row:29", - "line_status": "ok", - "line_code": "1510", - "line_code_text": "1510", - "line_code_confidence": 100.0, - "line_name": "Заемные средства", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "570663.00", - "string_value": null, - "text": "570663", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 29, - "ref": "row:30", - "line_status": "ok", - "line_code": "1520", - "line_code_text": "1520", - "line_code_confidence": 100.0, - "line_name": "Кредиторская задолженность", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "817769.00", - "string_value": null, - "text": "817769", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 30, - "ref": "row:31", - "line_status": "ok", - "line_code": "1540", - "line_code_text": "1540", - "line_code_confidence": 100.0, - "line_name": "Оценочные обязательства", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "47001.00", - "string_value": null, - "text": "47001", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 31, - "ref": "row:32", - "line_status": "ok", - "line_code": "1550", - "line_code_text": "1550", - "line_code_confidence": 100.0, - "line_name": "Прочие обязательства", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "31.00", - "string_value": null, - "text": "31", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 32, - "ref": "row:33", - "line_status": "ok", - "line_code": "1500", - "line_code_text": "1500", - "line_code_confidence": 100.0, - "line_name": "Итого по разделу V", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "1435464.00", - "string_value": null, - "text": "1435464", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 33, - "ref": "row:34", - "line_status": "ok", - "line_code": "1700", - "line_code_text": "1700", - "line_code_confidence": 100.0, - "line_name": "БАЛАНС", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "5518726.00", - "string_value": null, - "text": "5518726", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - } - ], - "checks": { - "relations_failed": [], - "soft_relations_failed": [] - }, - "status": "done", - "warnings": [ - { - "code": "INN_NOT_FOUND", - "severity": "info", - "message": "ИНН не найден", - "line_refs": [], - "column_index": null, - "field": "inn", - "locator": null - } - ] -} diff --git a/api/arm_statements/tests/golden/aep3_income.candidate.json b/api/arm_statements/tests/golden/aep3_income.candidate.json deleted file mode 100644 index 265443e..0000000 --- a/api/arm_statements/tests/golden/aep3_income.candidate.json +++ /dev/null @@ -1,290 +0,0 @@ -{ - "source": "ocr", - "locator": "file:aep3_ofr.json;table:0", - "type_hint": "INCOME", - "evidence": { - "okud": null, - "title": { - "text": "Отчет о финансовых результатах", - "value": null, - "confidence": 100.0, - "ref": "evidence:title" - } - }, - "header": { - "organization_name": { - "text": "НКО «Пример»", - "value": null, - "confidence": 100.0, - "ref": "hdr:organization_name" - }, - "inn": { - "text": "7700000000", - "value": null, - "confidence": 100.0, - "ref": "hdr:inn" - }, - "report_date": { - "text": "31.12.2024", - "value": null, - "confidence": 75.0, - "ref": "hdr:report_date" - }, - "units": { - "text": "тыс. руб.", - "value": null, - "confidence": 100.0, - "ref": "hdr:units" - }, - "okud": null, - "okpo": null, - "okopf": null, - "okfs": null, - "okei": null, - "address": null - }, - "rows": [ - [ - { - "text": "Наименование показателя", - "value": null, - "confidence": 100.0, - "ref": "row:0" - }, - { - "text": "Код", - "value": null, - "confidence": 100.0, - "ref": "row:0" - }, - { - "text": "За 2024 г.", - "value": null, - "confidence": 100.0, - "ref": "row:0" - } - ], - [ - { - "text": "Выручка", - "value": null, - "confidence": 100.0, - "ref": "row:1" - }, - { - "text": "2110", - "value": null, - "confidence": 100.0, - "ref": "row:1" - }, - { - "text": "8294952", - "value": null, - "confidence": 100.0, - "ref": "row:1" - } - ], - [ - { - "text": "Себестоимость продаж", - "value": null, - "confidence": 100.0, - "ref": "row:2" - }, - { - "text": "2120", - "value": null, - "confidence": 100.0, - "ref": "row:2" - }, - { - "text": "(7744536}", - "value": null, - "confidence": 88.9, - "ref": "row:2" - } - ], - [ - { - "text": "Валовая прибыль (убыток)", - "value": null, - "confidence": 100.0, - "ref": "row:3" - }, - { - "text": "2100", - "value": null, - "confidence": 100.0, - "ref": "row:3" - }, - { - "text": "550416", - "value": null, - "confidence": 100.0, - "ref": "row:3" - } - ], - [ - { - "text": "Коммерческие расходы", - "value": null, - "confidence": 100.0, - "ref": "row:4" - }, - { - "text": "2210", - "value": null, - "confidence": 100.0, - "ref": "row:4" - }, - { - "text": "(100)", - "value": null, - "confidence": 100.0, - "ref": "row:4" - } - ], - [ - { - "text": "Управленческие расходы", - "value": null, - "confidence": 100.0, - "ref": "row:5" - }, - { - "text": "2220", - "value": null, - "confidence": 100.0, - "ref": "row:5" - }, - { - "text": "(200)", - "value": null, - "confidence": 100.0, - "ref": "row:5" - } - ], - [ - { - "text": "Прибыль (убыток) от продаж", - "value": null, - "confidence": 100.0, - "ref": "row:6" - }, - { - "text": "2200", - "value": null, - "confidence": 100.0, - "ref": "row:6" - }, - { - "text": "550116", - "value": null, - "confidence": 100.0, - "ref": "row:6" - } - ], - [ - { - "text": "Доходы от участия в других\nорганизациях", - "value": null, - "confidence": 100.0, - "ref": "row:7" - }, - { - "text": "2310", - "value": null, - "confidence": 100.0, - "ref": "row:7" - }, - { - "text": "211(1 |", - "value": null, - "confidence": 85.7, - "ref": "row:7" - } - ], - [ - { - "text": "Проценты к получению", - "value": null, - "confidence": 100.0, - "ref": "row:8" - }, - { - "text": "2320", - "value": null, - "confidence": 100.0, - "ref": "row:8" - }, - { - "text": "42831 1", - "value": null, - "confidence": 90.0, - "ref": "row:8" - } - ], - [ - { - "text": "Прибыль (убыток) до налогообложения", - "value": null, - "confidence": 100.0, - "ref": "row:9" - }, - { - "text": "2300", - "value": null, - "confidence": 100.0, - "ref": "row:9" - }, - { - "text": "978427", - "value": null, - "confidence": 100.0, - "ref": "row:9" - } - ], - [ - { - "text": "Налог на прибыль", - "value": null, - "confidence": 100.0, - "ref": "row:10" - }, - { - "text": "2410", - "value": null, - "confidence": 100.0, - "ref": "row:10" - }, - { - "text": "(108450)", - "value": null, - "confidence": 100.0, - "ref": "row:10" - } - ], - [ - { - "text": "Чистая прибыль (убыток)", - "value": null, - "confidence": 100.0, - "ref": "row:11" - }, - { - "text": "2400", - "value": null, - "confidence": 100.0, - "ref": "row:11" - }, - { - "text": "869977", - "value": null, - "confidence": 100.0, - "ref": "row:11" - } - ] - ], - "segment_breaks": [], - "warnings": [] -} diff --git a/api/arm_statements/tests/golden/aep3_income.expected.json b/api/arm_statements/tests/golden/aep3_income.expected.json deleted file mode 100644 index 781405e..0000000 --- a/api/arm_statements/tests/golden/aep3_income.expected.json +++ /dev/null @@ -1,356 +0,0 @@ -{ - "document_type": "INCOME", - "form_variant": "full", - "locator": "file:aep3_ofr.json;table:0", - "header": { - "organization_name": { - "text": "НКО «Пример»", - "confidence": 100.0, - "value": "НКО «Пример»" - }, - "inn": { - "text": "7700000000", - "confidence": 100.0, - "value": "7700000000" - }, - "report_date": { - "text": "31.12.2024", - "confidence": 75.0, - "value": "2024-12-31" - }, - "units": { - "text": "тыс. руб.", - "confidence": 100.0, - "value": "тыс. руб." - }, - "okud": { - "text": null, - "confidence": null, - "value": null - }, - "okpo": { - "text": null, - "confidence": null, - "value": null - }, - "okopf": { - "text": null, - "confidence": null, - "value": null - }, - "okfs": { - "text": null, - "confidence": null, - "value": null - }, - "okei": { - "text": null, - "confidence": null, - "value": null - }, - "address": { - "text": null, - "confidence": null, - "value": null - }, - "form_name": "Отчет о финансовых результатах", - "needs_review_fields": [] - }, - "columns": [ - { - "column_index": 2, - "label": "За 2024 г.", - "label_confidence": 100.0, - "period_date": "2024-12-31", - "period_kind": "absolute" - } - ], - "lines": [ - { - "row_index": 0, - "ref": "row:1", - "line_status": "ok", - "line_code": "2110", - "line_code_text": "2110", - "line_code_confidence": 100.0, - "line_name": "Выручка", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "8294952.00", - "string_value": null, - "text": "8294952", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 1, - "ref": "row:2", - "line_status": "ok", - "line_code": "2120", - "line_code_text": "2120", - "line_code_confidence": 100.0, - "line_name": "Себестоимость продаж", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "-7744536.00", - "string_value": null, - "text": "(7744536}", - "confidence": 88.9, - "is_recognized": false, - "needs_review": false - } - ] - }, - { - "row_index": 2, - "ref": "row:3", - "line_status": "ok", - "line_code": "2100", - "line_code_text": "2100", - "line_code_confidence": 100.0, - "line_name": "Валовая прибыль (убыток)", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "550416.00", - "string_value": null, - "text": "550416", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 3, - "ref": "row:4", - "line_status": "ok", - "line_code": "2210", - "line_code_text": "2210", - "line_code_confidence": 100.0, - "line_name": "Коммерческие расходы", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "-100.00", - "string_value": null, - "text": "(100)", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 4, - "ref": "row:5", - "line_status": "ok", - "line_code": "2220", - "line_code_text": "2220", - "line_code_confidence": 100.0, - "line_name": "Управленческие расходы", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "-200.00", - "string_value": null, - "text": "(200)", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 5, - "ref": "row:6", - "line_status": "ok", - "line_code": "2200", - "line_code_text": "2200", - "line_code_confidence": 100.0, - "line_name": "Прибыль (убыток) от продаж", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "550116.00", - "string_value": null, - "text": "550116", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 6, - "ref": "row:7", - "line_status": "ok", - "line_code": "2310", - "line_code_text": "2310", - "line_code_confidence": 100.0, - "line_name": "Доходы от участия в других\nорганизациях", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": null, - "string_value": "211(1 |", - "text": "211(1 |", - "confidence": 85.7, - "is_recognized": false, - "needs_review": true - } - ] - }, - { - "row_index": 7, - "ref": "row:8", - "line_status": "ok", - "line_code": "2320", - "line_code_text": "2320", - "line_code_confidence": 100.0, - "line_name": "Проценты к получению", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "428311.00", - "string_value": null, - "text": "42831 1", - "confidence": 90.0, - "is_recognized": false, - "needs_review": false - } - ] - }, - { - "row_index": 8, - "ref": "row:9", - "line_status": "ok", - "line_code": "2300", - "line_code_text": "2300", - "line_code_confidence": 100.0, - "line_name": "Прибыль (убыток) до налогообложения", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "978427.00", - "string_value": null, - "text": "978427", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 9, - "ref": "row:10", - "line_status": "ok", - "line_code": "2410", - "line_code_text": "2410", - "line_code_confidence": 100.0, - "line_name": "Налог на прибыль", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "-108450.00", - "string_value": null, - "text": "(108450)", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 10, - "ref": "row:11", - "line_status": "ok", - "line_code": "2400", - "line_code_text": "2400", - "line_code_confidence": 100.0, - "line_name": "Чистая прибыль (убыток)", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "869977.00", - "string_value": null, - "text": "869977", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - } - ], - "checks": { - "relations_failed": [], - "soft_relations_failed": [ - { - "column_index": 2, - "column_key": "2", - "relation": "2410 = 2411 + 2412", - "codes": [ - "2410", - "2411", - "2412" - ], - "expected": "-108450.00", - "actual": null - }, - { - "column_index": 2, - "column_key": "2", - "relation": "2500 = 2400 + 2510 + 2520 + 2530", - "codes": [ - "2500", - "2400", - "2510", - "2520", - "2530" - ], - "expected": null, - "actual": "869977.00" - } - ] - }, - "status": "needs_check", - "warnings": [] -} diff --git a/api/arm_statements/tests/golden/glued_balance.candidate.json b/api/arm_statements/tests/golden/glued_balance.candidate.json deleted file mode 100644 index b88ba09..0000000 --- a/api/arm_statements/tests/golden/glued_balance.candidate.json +++ /dev/null @@ -1,330 +0,0 @@ -{ - "source": "ocr", - "locator": "file:glued.json;table:0", - "type_hint": "BALANCE", - "evidence": { - "okud": null, - "title": { - "text": "Бухгалтерский баланс", - "value": null, - "confidence": 100.0, - "ref": "evidence:title" - } - }, - "header": { - "organization_name": { - "text": "ООО «Пример»", - "value": null, - "confidence": 100.0, - "ref": "hdr:organization_name" - }, - "inn": { - "text": "7700000000", - "value": null, - "confidence": 100.0, - "ref": "hdr:inn" - }, - "report_date": { - "text": "31.12.2024", - "value": null, - "confidence": 100.0, - "ref": "hdr:report_date" - }, - "units": { - "text": "тыс. руб.", - "value": null, - "confidence": 100.0, - "ref": "hdr:units" - }, - "okud": null, - "okpo": null, - "okopf": null, - "okfs": null, - "okei": null, - "address": null - }, - "rows": [ - [ - { - "text": "Наименование показателя", - "value": null, - "confidence": 100.0, - "ref": "row:0" - }, - { - "text": "Код", - "value": null, - "confidence": 100.0, - "ref": "row:0" - }, - { - "text": "На 31 декабря 2024 г.", - "value": null, - "confidence": 100.0, - "ref": "row:0" - } - ], - [ - { - "text": "Основные средства", - "value": null, - "confidence": 100.0, - "ref": "row:1" - }, - { - "text": "1150", - "value": null, - "confidence": 100.0, - "ref": "row:1" - }, - { - "text": "600000", - "value": null, - "confidence": 100.0, - "ref": "row:1" - } - ], - [ - { - "text": "Итого по разделу I", - "value": null, - "confidence": 100.0, - "ref": "row:2" - }, - { - "text": "1100", - "value": null, - "confidence": 100.0, - "ref": "row:2" - }, - { - "text": "600000", - "value": null, - "confidence": 100.0, - "ref": "row:2" - } - ], - [ - { - "text": "Запасы", - "value": null, - "confidence": 100.0, - "ref": "row:3" - }, - { - "text": "1210", - "value": null, - "confidence": 100.0, - "ref": "row:3" - }, - { - "text": "1771333", - "value": null, - "confidence": 100.0, - "ref": "row:3" - } - ], - [ - { - "text": "Итого по разделу II", - "value": null, - "confidence": 100.0, - "ref": "row:4" - }, - { - "text": "1200", - "value": null, - "confidence": 100.0, - "ref": "row:4" - }, - { - "text": "1771333", - "value": null, - "confidence": 100.0, - "ref": "row:4" - } - ], - [ - { - "text": "БАЛАНС", - "value": null, - "confidence": 100.0, - "ref": "row:5" - }, - { - "text": "1600", - "value": null, - "confidence": 100.0, - "ref": "row:5" - }, - { - "text": "2371333", - "value": null, - "confidence": 100.0, - "ref": "row:5" - } - ], - [ - { - "text": "Уставный капитал", - "value": null, - "confidence": 100.0, - "ref": "row:6" - }, - { - "text": "1310", - "value": null, - "confidence": 100.0, - "ref": "row:6" - }, - { - "text": "1000000", - "value": null, - "confidence": 100.0, - "ref": "row:6" - } - ], - [ - { - "text": "Нераспределенная прибыль", - "value": null, - "confidence": 100.0, - "ref": "row:7" - }, - { - "text": "1370", - "value": null, - "confidence": 100.0, - "ref": "row:7" - }, - { - "text": "253558", - "value": null, - "confidence": 100.0, - "ref": "row:7" - } - ], - [ - { - "text": "Итого по разделу III", - "value": null, - "confidence": 100.0, - "ref": "row:8" - }, - { - "text": "1300", - "value": null, - "confidence": 100.0, - "ref": "row:8" - }, - { - "text": "1253558", - "value": null, - "confidence": 100.0, - "ref": "row:8" - } - ], - [ - { - "text": "Заемные средства", - "value": null, - "confidence": 100.0, - "ref": "row:9" - }, - { - "text": "1410", - "value": null, - "confidence": 100.0, - "ref": "row:9" - }, - { - "text": "200000", - "value": null, - "confidence": 100.0, - "ref": "row:9" - } - ], - [ - { - "text": "Итого по разделу IV", - "value": null, - "confidence": 100.0, - "ref": "row:10" - }, - { - "text": "1400", - "value": null, - "confidence": 100.0, - "ref": "row:10" - }, - { - "text": "200000", - "value": null, - "confidence": 100.0, - "ref": "row:10" - } - ], - [ - { - "text": "Заемные средства", - "value": null, - "confidence": 100.0, - "ref": "row:11" - }, - { - "text": "1510", - "value": null, - "confidence": 100.0, - "ref": "row:11" - }, - { - "text": "400000", - "value": null, - "confidence": 100.0, - "ref": "row:11" - } - ], - [ - { - "text": "Кредиторская задолженность", - "value": null, - "confidence": 100.0, - "ref": "row:12" - }, - { - "text": "1520", - "value": null, - "confidence": 100.0, - "ref": "row:12" - }, - { - "text": "517775", - "value": null, - "confidence": 100.0, - "ref": "row:12" - } - ], - [ - { - "text": "Итого по разделу V\nБАЛАНС", - "value": null, - "confidence": 100.0, - "ref": "row:13" - }, - { - "text": "1500\n1700", - "value": null, - "confidence": 90.0, - "ref": "row:13" - }, - { - "text": "917 775\n2 371 333", - "value": null, - "confidence": 90.0, - "ref": "row:13" - } - ] - ], - "segment_breaks": [], - "warnings": [] -} diff --git a/api/arm_statements/tests/golden/glued_balance.expected.json b/api/arm_statements/tests/golden/glued_balance.expected.json deleted file mode 100644 index ce2716c..0000000 --- a/api/arm_statements/tests/golden/glued_balance.expected.json +++ /dev/null @@ -1,411 +0,0 @@ -{ - "document_type": "BALANCE", - "form_variant": "full", - "locator": "file:glued.json;table:0", - "header": { - "organization_name": { - "text": "ООО «Пример»", - "confidence": 100.0, - "value": "ООО «Пример»" - }, - "inn": { - "text": "7700000000", - "confidence": 100.0, - "value": "7700000000" - }, - "report_date": { - "text": "31.12.2024", - "confidence": 100.0, - "value": "2024-12-31" - }, - "units": { - "text": "тыс. руб.", - "confidence": 100.0, - "value": "тыс. руб." - }, - "okud": { - "text": null, - "confidence": null, - "value": null - }, - "okpo": { - "text": null, - "confidence": null, - "value": null - }, - "okopf": { - "text": null, - "confidence": null, - "value": null - }, - "okfs": { - "text": null, - "confidence": null, - "value": null - }, - "okei": { - "text": null, - "confidence": null, - "value": null - }, - "address": { - "text": null, - "confidence": null, - "value": null - }, - "form_name": "Бухгалтерский баланс", - "needs_review_fields": [] - }, - "columns": [ - { - "column_index": 2, - "label": "На 31 декабря 2024 г.", - "label_confidence": 100.0, - "period_date": "2024-12-31", - "period_kind": "absolute" - } - ], - "lines": [ - { - "row_index": 0, - "ref": "row:1", - "line_status": "ok", - "line_code": "1150", - "line_code_text": "1150", - "line_code_confidence": 100.0, - "line_name": "Основные средства", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "600000.00", - "string_value": null, - "text": "600000", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 1, - "ref": "row:2", - "line_status": "ok", - "line_code": "1100", - "line_code_text": "1100", - "line_code_confidence": 100.0, - "line_name": "Итого по разделу I", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "600000.00", - "string_value": null, - "text": "600000", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 2, - "ref": "row:3", - "line_status": "ok", - "line_code": "1210", - "line_code_text": "1210", - "line_code_confidence": 100.0, - "line_name": "Запасы", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "1771333.00", - "string_value": null, - "text": "1771333", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 3, - "ref": "row:4", - "line_status": "ok", - "line_code": "1200", - "line_code_text": "1200", - "line_code_confidence": 100.0, - "line_name": "Итого по разделу II", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "1771333.00", - "string_value": null, - "text": "1771333", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 4, - "ref": "row:5", - "line_status": "ok", - "line_code": "1600", - "line_code_text": "1600", - "line_code_confidence": 100.0, - "line_name": "БАЛАНС", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "2371333.00", - "string_value": null, - "text": "2371333", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 5, - "ref": "row:6", - "line_status": "ok", - "line_code": "1310", - "line_code_text": "1310", - "line_code_confidence": 100.0, - "line_name": "Уставный капитал", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "1000000.00", - "string_value": null, - "text": "1000000", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 6, - "ref": "row:7", - "line_status": "ok", - "line_code": "1370", - "line_code_text": "1370", - "line_code_confidence": 100.0, - "line_name": "Нераспределенная прибыль", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "253558.00", - "string_value": null, - "text": "253558", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 7, - "ref": "row:8", - "line_status": "ok", - "line_code": "1300", - "line_code_text": "1300", - "line_code_confidence": 100.0, - "line_name": "Итого по разделу III", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "1253558.00", - "string_value": null, - "text": "1253558", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 8, - "ref": "row:9", - "line_status": "ok", - "line_code": "1410", - "line_code_text": "1410", - "line_code_confidence": 100.0, - "line_name": "Заемные средства", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "200000.00", - "string_value": null, - "text": "200000", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 9, - "ref": "row:10", - "line_status": "ok", - "line_code": "1400", - "line_code_text": "1400", - "line_code_confidence": 100.0, - "line_name": "Итого по разделу IV", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "200000.00", - "string_value": null, - "text": "200000", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 10, - "ref": "row:11", - "line_status": "ok", - "line_code": "1510", - "line_code_text": "1510", - "line_code_confidence": 100.0, - "line_name": "Заемные средства", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "400000.00", - "string_value": null, - "text": "400000", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 11, - "ref": "row:12", - "line_status": "ok", - "line_code": "1520", - "line_code_text": "1520", - "line_code_confidence": 100.0, - "line_name": "Кредиторская задолженность", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "517775.00", - "string_value": null, - "text": "517775", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 12, - "ref": "row:13#1", - "line_status": "ok", - "line_code": "1500", - "line_code_text": "1500", - "line_code_confidence": 90.0, - "line_name": "Итого по разделу V", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "917775.00", - "string_value": null, - "text": "917 775", - "confidence": 90.0, - "is_recognized": false, - "needs_review": false - } - ] - }, - { - "row_index": 13, - "ref": "row:13#2", - "line_status": "ok", - "line_code": "1700", - "line_code_text": "1700", - "line_code_confidence": 90.0, - "line_name": "БАЛАНС", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 2, - "amount": "2371333.00", - "string_value": null, - "text": "2 371 333", - "confidence": 90.0, - "is_recognized": false, - "needs_review": false - } - ] - } - ], - "checks": { - "relations_failed": [], - "soft_relations_failed": [] - }, - "status": "done", - "warnings": [ - { - "code": "ROW_SPLIT", - "severity": "info", - "message": "склеенная строка сетки 13 разделена на 2 строк", - "line_refs": [ - "row:13#1", - "row:13#2" - ], - "column_index": null, - "field": null, - "locator": null - } - ] -} diff --git a/api/arm_statements/tests/golden/kody_title_balance.candidate.json b/api/arm_statements/tests/golden/kody_title_balance.candidate.json deleted file mode 100644 index e9a164b..0000000 --- a/api/arm_statements/tests/golden/kody_title_balance.candidate.json +++ /dev/null @@ -1,538 +0,0 @@ -{ - "source": "excel", - "locator": "sheet:Баланс", - "type_hint": "BALANCE", - "evidence": { - "okud": { - "text": "0710001", - "value": null, - "confidence": 100.0, - "ref": "evidence:okud" - }, - "title": { - "text": "Бухгалтерский баланс", - "value": null, - "confidence": 100.0, - "ref": "evidence:title" - } - }, - "header": { - "organization_name": { - "text": "ООО «Пример»", - "value": null, - "confidence": 100.0, - "ref": "hdr:organization_name" - }, - "inn": { - "text": "7700000000", - "value": null, - "confidence": 100.0, - "ref": "hdr:inn" - }, - "report_date": { - "text": "30.09.2025", - "value": null, - "confidence": 100.0, - "ref": "hdr:report_date" - }, - "units": null, - "okud": { - "text": "0710001", - "value": null, - "confidence": 100.0, - "ref": "hdr:okud" - }, - "okpo": null, - "okopf": { - "text": "12267", - "value": null, - "confidence": 100.0, - "ref": "hdr:okopf" - }, - "okfs": null, - "okei": { - "text": "384", - "value": null, - "confidence": 100.0, - "ref": "hdr:okei" - }, - "address": null - }, - "rows": [ - [ - null, - null, - null, - null, - null, - { - "text": "Коды", - "value": null, - "confidence": 100.0, - "ref": "Баланс!F1" - } - ], - [ - { - "text": "Бухгалтерский баланс", - "value": null, - "confidence": 100.0, - "ref": "Баланс!A2" - }, - null, - null, - null, - { - "text": "Форма по ОКУД", - "value": null, - "confidence": 100.0, - "ref": "Баланс!E2" - }, - { - "text": "0710001", - "value": null, - "confidence": 100.0, - "ref": "Баланс!F2" - } - ], - [ - { - "text": "на 30 сентября 2025 г.", - "value": null, - "confidence": 100.0, - "ref": "Баланс!A3" - }, - null, - null, - null, - { - "text": "Дата (число, месяц, год)", - "value": null, - "confidence": 100.0, - "ref": "Баланс!E3" - }, - { - "text": "30 | 09 | 2025", - "value": null, - "confidence": 100.0, - "ref": "Баланс!F3" - } - ], - [ - { - "text": "Организация ООО «Пример»", - "value": null, - "confidence": 100.0, - "ref": "Баланс!A4" - }, - null, - null, - null, - { - "text": "по ОКПО", - "value": null, - "confidence": 100.0, - "ref": "Баланс!E4" - }, - { - "text": "12 345 678", - "value": { - "decimal": "12345678" - }, - "confidence": 100.0, - "ref": "Баланс!F4" - } - ], - [ - { - "text": "Идентификационный номер налогоплательщика", - "value": null, - "confidence": 100.0, - "ref": "Баланс!A5" - }, - null, - null, - null, - { - "text": "ИНН", - "value": null, - "confidence": 100.0, - "ref": "Баланс!E5" - }, - { - "text": "7700000000", - "value": null, - "confidence": 100.0, - "ref": "Баланс!F5" - } - ], - [ - { - "text": "Организационно-правовая форма / форма собственности", - "value": null, - "confidence": 100.0, - "ref": "Баланс!A6" - }, - null, - null, - null, - { - "text": "по ОКОПФ / ОКФС", - "value": null, - "confidence": 100.0, - "ref": "Баланс!E6" - }, - { - "text": "12 267", - "value": { - "decimal": "12267" - }, - "confidence": 100.0, - "ref": "Баланс!F6" - } - ], - [ - { - "text": "Единица измерения: тыс. руб.", - "value": null, - "confidence": 100.0, - "ref": "Баланс!A7" - }, - null, - null, - null, - { - "text": "по ОКЕИ", - "value": null, - "confidence": 100.0, - "ref": "Баланс!E7" - }, - { - "text": "384", - "value": { - "decimal": "384" - }, - "confidence": 100.0, - "ref": "Баланс!F7" - } - ], - [ - null, - null, - null, - null, - null, - { - "text": "16", - "value": { - "decimal": "16" - }, - "confidence": 100.0, - "ref": "Баланс!F8" - } - ], - [ - { - "text": "Наименование показателя", - "value": null, - "confidence": 100.0, - "ref": "Баланс!A9" - }, - null, - null, - { - "text": "На 30 сентября 2025 г.", - "value": null, - "confidence": 100.0, - "ref": "Баланс!D9" - }, - { - "text": "На 31 декабря 2024 г.", - "value": null, - "confidence": 100.0, - "ref": "Баланс!E9" - } - ], - [ - { - "text": "Основные средства", - "value": null, - "confidence": 100.0, - "ref": "Баланс!A10" - }, - null, - { - "text": "1150", - "value": null, - "confidence": 100.0, - "ref": "Баланс!C10" - }, - { - "text": "50 000", - "value": { - "decimal": "50000" - }, - "confidence": 100.0, - "ref": "Баланс!D10" - }, - { - "text": "40 000", - "value": { - "decimal": "40000" - }, - "confidence": 100.0, - "ref": "Баланс!E10" - } - ], - [ - { - "text": "Итого по разделу I", - "value": null, - "confidence": 100.0, - "ref": "Баланс!A11" - }, - null, - { - "text": "1100", - "value": null, - "confidence": 100.0, - "ref": "Баланс!C11" - }, - { - "text": "50 000", - "value": { - "decimal": "50000" - }, - "confidence": 100.0, - "ref": "Баланс!D11" - }, - { - "text": "40 000", - "value": { - "decimal": "40000" - }, - "confidence": 100.0, - "ref": "Баланс!E11" - } - ], - [ - { - "text": "Запасы", - "value": null, - "confidence": 100.0, - "ref": "Баланс!A12" - }, - null, - { - "text": "1210", - "value": null, - "confidence": 100.0, - "ref": "Баланс!C12" - }, - { - "text": "60 000", - "value": { - "decimal": "60000" - }, - "confidence": 100.0, - "ref": "Баланс!D12" - }, - { - "text": "50 000", - "value": { - "decimal": "50000" - }, - "confidence": 100.0, - "ref": "Баланс!E12" - } - ], - [ - { - "text": "Дебиторская задолженность", - "value": null, - "confidence": 100.0, - "ref": "Баланс!A13" - }, - null, - { - "text": "1230", - "value": null, - "confidence": 100.0, - "ref": "Баланс!C13" - }, - { - "text": "40 000", - "value": { - "decimal": "40000" - }, - "confidence": 100.0, - "ref": "Баланс!D13" - }, - { - "text": "30 000", - "value": { - "decimal": "30000" - }, - "confidence": 100.0, - "ref": "Баланс!E13" - } - ], - [ - { - "text": "Итого по разделу II", - "value": null, - "confidence": 100.0, - "ref": "Баланс!A14" - }, - null, - { - "text": "1200", - "value": null, - "confidence": 100.0, - "ref": "Баланс!C14" - }, - { - "text": "100 000", - "value": { - "decimal": "100000" - }, - "confidence": 100.0, - "ref": "Баланс!D14" - }, - { - "text": "80 000", - "value": { - "decimal": "80000" - }, - "confidence": 100.0, - "ref": "Баланс!E14" - } - ], - [ - { - "text": "БАЛАНС", - "value": null, - "confidence": 100.0, - "ref": "Баланс!A15" - }, - null, - { - "text": "1600", - "value": null, - "confidence": 100.0, - "ref": "Баланс!C15" - }, - { - "text": "150 000", - "value": { - "decimal": "150000" - }, - "confidence": 100.0, - "ref": "Баланс!D15" - }, - { - "text": "120 000", - "value": { - "decimal": "120000" - }, - "confidence": 100.0, - "ref": "Баланс!E15" - } - ], - [ - { - "text": "Нераспределенная прибыль", - "value": null, - "confidence": 100.0, - "ref": "Баланс!A16" - }, - null, - { - "text": "1370", - "value": null, - "confidence": 100.0, - "ref": "Баланс!C16" - }, - { - "text": "150 000", - "value": { - "decimal": "150000" - }, - "confidence": 100.0, - "ref": "Баланс!D16" - }, - { - "text": "120 000", - "value": { - "decimal": "120000" - }, - "confidence": 100.0, - "ref": "Баланс!E16" - } - ], - [ - { - "text": "Итого по разделу III", - "value": null, - "confidence": 100.0, - "ref": "Баланс!A17" - }, - null, - { - "text": "1300", - "value": null, - "confidence": 100.0, - "ref": "Баланс!C17" - }, - { - "text": "150 000", - "value": { - "decimal": "150000" - }, - "confidence": 100.0, - "ref": "Баланс!D17" - }, - { - "text": "120 000", - "value": { - "decimal": "120000" - }, - "confidence": 100.0, - "ref": "Баланс!E17" - } - ], - [ - { - "text": "БАЛАНС", - "value": null, - "confidence": 100.0, - "ref": "Баланс!A18" - }, - null, - { - "text": "1700", - "value": null, - "confidence": 100.0, - "ref": "Баланс!C18" - }, - { - "text": "150 000", - "value": { - "decimal": "150000" - }, - "confidence": 100.0, - "ref": "Баланс!D18" - }, - { - "text": "120 000", - "value": { - "decimal": "120000" - }, - "confidence": 100.0, - "ref": "Баланс!E18" - } - ] - ], - "segment_breaks": [], - "warnings": [] -} diff --git a/api/arm_statements/tests/golden/kody_title_balance.expected.json b/api/arm_statements/tests/golden/kody_title_balance.expected.json deleted file mode 100644 index b24d123..0000000 --- a/api/arm_statements/tests/golden/kody_title_balance.expected.json +++ /dev/null @@ -1,371 +0,0 @@ -{ - "document_type": "BALANCE", - "form_variant": "full", - "locator": "sheet:Баланс", - "header": { - "organization_name": { - "text": "ООО «Пример»", - "confidence": 100.0, - "value": "ООО «Пример»" - }, - "inn": { - "text": "7700000000", - "confidence": 100.0, - "value": "7700000000" - }, - "report_date": { - "text": "30.09.2025", - "confidence": 100.0, - "value": "2025-09-30" - }, - "units": { - "text": null, - "confidence": null, - "value": "тыс. руб." - }, - "okud": { - "text": "0710001", - "confidence": 100.0, - "value": "0710001" - }, - "okpo": { - "text": null, - "confidence": null, - "value": null - }, - "okopf": { - "text": "12267", - "confidence": 100.0, - "value": "12267" - }, - "okfs": { - "text": null, - "confidence": null, - "value": null - }, - "okei": { - "text": "384", - "confidence": 100.0, - "value": "384" - }, - "address": { - "text": null, - "confidence": null, - "value": null - }, - "form_name": "Бухгалтерский баланс", - "needs_review_fields": [] - }, - "columns": [ - { - "column_index": 3, - "label": "На 30 сентября 2025 г.", - "label_confidence": 100.0, - "period_date": "2025-09-30", - "period_kind": "absolute" - }, - { - "column_index": 4, - "label": "На 31 декабря 2024 г.", - "label_confidence": 100.0, - "period_date": "2024-12-31", - "period_kind": "absolute" - } - ], - "lines": [ - { - "row_index": 0, - "ref": "Баланс!C10", - "line_status": "ok", - "line_code": "1150", - "line_code_text": "1150", - "line_code_confidence": 100.0, - "line_name": "Основные средства", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 3, - "amount": "50000.00", - "string_value": null, - "text": "50 000", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - }, - { - "column_index": 4, - "amount": "40000.00", - "string_value": null, - "text": "40 000", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 1, - "ref": "Баланс!C11", - "line_status": "ok", - "line_code": "1100", - "line_code_text": "1100", - "line_code_confidence": 100.0, - "line_name": "Итого по разделу I", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 3, - "amount": "50000.00", - "string_value": null, - "text": "50 000", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - }, - { - "column_index": 4, - "amount": "40000.00", - "string_value": null, - "text": "40 000", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 2, - "ref": "Баланс!C12", - "line_status": "ok", - "line_code": "1210", - "line_code_text": "1210", - "line_code_confidence": 100.0, - "line_name": "Запасы", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 3, - "amount": "60000.00", - "string_value": null, - "text": "60 000", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - }, - { - "column_index": 4, - "amount": "50000.00", - "string_value": null, - "text": "50 000", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 3, - "ref": "Баланс!C13", - "line_status": "ok", - "line_code": "1230", - "line_code_text": "1230", - "line_code_confidence": 100.0, - "line_name": "Дебиторская задолженность", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 3, - "amount": "40000.00", - "string_value": null, - "text": "40 000", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - }, - { - "column_index": 4, - "amount": "30000.00", - "string_value": null, - "text": "30 000", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 4, - "ref": "Баланс!C14", - "line_status": "ok", - "line_code": "1200", - "line_code_text": "1200", - "line_code_confidence": 100.0, - "line_name": "Итого по разделу II", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 3, - "amount": "100000.00", - "string_value": null, - "text": "100 000", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - }, - { - "column_index": 4, - "amount": "80000.00", - "string_value": null, - "text": "80 000", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 5, - "ref": "Баланс!C15", - "line_status": "ok", - "line_code": "1600", - "line_code_text": "1600", - "line_code_confidence": 100.0, - "line_name": "БАЛАНС", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 3, - "amount": "150000.00", - "string_value": null, - "text": "150 000", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - }, - { - "column_index": 4, - "amount": "120000.00", - "string_value": null, - "text": "120 000", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 6, - "ref": "Баланс!C16", - "line_status": "ok", - "line_code": "1370", - "line_code_text": "1370", - "line_code_confidence": 100.0, - "line_name": "Нераспределенная прибыль", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 3, - "amount": "150000.00", - "string_value": null, - "text": "150 000", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - }, - { - "column_index": 4, - "amount": "120000.00", - "string_value": null, - "text": "120 000", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 7, - "ref": "Баланс!C17", - "line_status": "ok", - "line_code": "1300", - "line_code_text": "1300", - "line_code_confidence": 100.0, - "line_name": "Итого по разделу III", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 3, - "amount": "150000.00", - "string_value": null, - "text": "150 000", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - }, - { - "column_index": 4, - "amount": "120000.00", - "string_value": null, - "text": "120 000", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 8, - "ref": "Баланс!C18", - "line_status": "ok", - "line_code": "1700", - "line_code_text": "1700", - "line_code_confidence": 100.0, - "line_name": "БАЛАНС", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 3, - "amount": "150000.00", - "string_value": null, - "text": "150 000", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - }, - { - "column_index": 4, - "amount": "120000.00", - "string_value": null, - "text": "120 000", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - } - ], - "checks": { - "relations_failed": [], - "soft_relations_failed": [] - }, - "status": "done", - "warnings": [] -} diff --git a/api/arm_statements/tests/golden/npz_income.candidate.json b/api/arm_statements/tests/golden/npz_income.candidate.json deleted file mode 100644 index 03c177e..0000000 --- a/api/arm_statements/tests/golden/npz_income.candidate.json +++ /dev/null @@ -1,433 +0,0 @@ -{ - "source": "excel", - "locator": "sheet:ОФР,ОФР_2", - "type_hint": "INCOME", - "evidence": { - "okud": null, - "title": { - "text": "Отчет о финансовых результатах", - "value": null, - "confidence": 100.0, - "ref": "evidence:title" - } - }, - "header": { - "organization_name": { - "text": "ООО «Пример»", - "value": null, - "confidence": 100.0, - "ref": "hdr:organization_name" - }, - "inn": { - "text": "7700000000", - "value": null, - "confidence": 100.0, - "ref": "hdr:inn" - }, - "report_date": { - "text": "30.09.2025", - "value": null, - "confidence": 100.0, - "ref": "hdr:report_date" - }, - "units": { - "text": "тыс. руб.", - "value": null, - "confidence": 100.0, - "ref": "hdr:units" - }, - "okud": null, - "okpo": null, - "okopf": null, - "okfs": null, - "okei": null, - "address": null - }, - "rows": [ - [ - { - "text": "Отчет о финансовых результатах", - "value": null, - "confidence": 100.0, - "ref": "ОФР!A1" - } - ], - [ - { - "text": "Наименование показателя", - "value": null, - "confidence": 100.0, - "ref": "ОФР!A2" - }, - null, - null, - { - "text": "Код", - "value": null, - "confidence": 100.0, - "ref": "ОФР!D2" - }, - { - "text": "За 9 месяцев 2025 г.", - "value": null, - "confidence": 100.0, - "ref": "ОФР!E2" - }, - { - "text": "За 9 месяцев 2024 г.", - "value": null, - "confidence": 100.0, - "ref": "ОФР!F2" - } - ], - [ - { - "text": "Выручка", - "value": null, - "confidence": 100.0, - "ref": "ОФР!A3" - }, - null, - null, - { - "text": "2110", - "value": null, - "confidence": 100.0, - "ref": "ОФР!D3" - }, - { - "text": "1 000", - "value": { - "decimal": "1000" - }, - "confidence": 100.0, - "ref": "ОФР!E3" - }, - { - "text": "900", - "value": { - "decimal": "900" - }, - "confidence": 100.0, - "ref": "ОФР!F3" - } - ], - [ - { - "text": "Себестоимость продаж", - "value": null, - "confidence": 100.0, - "ref": "ОФР!A4" - }, - null, - null, - { - "text": "2120", - "value": null, - "confidence": 100.0, - "ref": "ОФР!D4" - }, - { - "text": "-600", - "value": { - "decimal": "-600" - }, - "confidence": 100.0, - "ref": "ОФР!E4" - }, - { - "text": "-540", - "value": { - "decimal": "-540" - }, - "confidence": 100.0, - "ref": "ОФР!F4" - } - ], - [ - { - "text": "Валовая прибыль (убыток)", - "value": null, - "confidence": 100.0, - "ref": "ОФР!A5" - }, - null, - null, - { - "text": "2100", - "value": null, - "confidence": 100.0, - "ref": "ОФР!D5" - }, - { - "text": "400", - "value": { - "decimal": "400" - }, - "confidence": 100.0, - "ref": "ОФР!E5" - }, - { - "text": "360", - "value": { - "decimal": "360" - }, - "confidence": 100.0, - "ref": "ОФР!F5" - } - ], - [ - { - "text": "Коммерческие расходы", - "value": null, - "confidence": 100.0, - "ref": "ОФР!A6" - }, - null, - null, - { - "text": "2210", - "value": null, - "confidence": 100.0, - "ref": "ОФР!D6" - }, - { - "text": "-50", - "value": { - "decimal": "-50" - }, - "confidence": 100.0, - "ref": "ОФР!E6" - }, - { - "text": "-45", - "value": { - "decimal": "-45" - }, - "confidence": 100.0, - "ref": "ОФР!F6" - } - ], - [ - { - "text": "Управленческие расходы", - "value": null, - "confidence": 100.0, - "ref": "ОФР!A7" - }, - null, - null, - { - "text": "2220", - "value": null, - "confidence": 100.0, - "ref": "ОФР!D7" - }, - { - "text": "-30", - "value": { - "decimal": "-30" - }, - "confidence": 100.0, - "ref": "ОФР!E7" - }, - { - "text": "-27", - "value": { - "decimal": "-27" - }, - "confidence": 100.0, - "ref": "ОФР!F7" - } - ], - [ - { - "text": "Прибыль (убыток) от продаж", - "value": null, - "confidence": 100.0, - "ref": "ОФР!A8" - }, - null, - null, - { - "text": "2200", - "value": null, - "confidence": 100.0, - "ref": "ОФР!D8" - }, - { - "text": "320", - "value": { - "decimal": "320" - }, - "confidence": 100.0, - "ref": "ОФР!E8" - }, - { - "text": "288", - "value": { - "decimal": "288" - }, - "confidence": 100.0, - "ref": "ОФР!F8" - } - ], - [ - { - "text": "Прочие доходы", - "value": null, - "confidence": 100.0, - "ref": "ОФР!A9" - }, - null, - null, - { - "text": "2340", - "value": null, - "confidence": 100.0, - "ref": "ОФР!D9" - }, - { - "text": "10", - "value": { - "decimal": "10" - }, - "confidence": 100.0, - "ref": "ОФР!E9" - }, - { - "text": "9", - "value": { - "decimal": "9" - }, - "confidence": 100.0, - "ref": "ОФР!F9" - } - ], - [ - { - "text": "Прибыль (убыток) до налогообложения", - "value": null, - "confidence": 100.0, - "ref": "ОФР!A10" - }, - null, - null, - { - "text": "2300", - "value": null, - "confidence": 100.0, - "ref": "ОФР!D10" - }, - { - "text": "330", - "value": { - "decimal": "330" - }, - "confidence": 100.0, - "ref": "ОФР!E10" - }, - { - "text": "297", - "value": { - "decimal": "297" - }, - "confidence": 100.0, - "ref": "ОФР!F10" - } - ], - [ - { - "text": "Налог на прибыль", - "value": null, - "confidence": 100.0, - "ref": "ОФР!A11" - }, - null, - null, - { - "text": "2410", - "value": null, - "confidence": 100.0, - "ref": "ОФР!D11" - }, - { - "text": "-66", - "value": { - "decimal": "-66" - }, - "confidence": 100.0, - "ref": "ОФР!E11" - }, - { - "text": "-59", - "value": { - "decimal": "-59" - }, - "confidence": 100.0, - "ref": "ОФР!F11" - } - ], - [ - { - "text": "Прочее", - "value": null, - "confidence": 100.0, - "ref": "ОФР!A12" - }, - null, - null, - { - "text": "2460", - "value": null, - "confidence": 100.0, - "ref": "ОФР!D12" - }, - { - "text": "0", - "value": { - "decimal": "0" - }, - "confidence": 100.0, - "ref": "ОФР!E12" - }, - { - "text": "0", - "value": { - "decimal": "0" - }, - "confidence": 100.0, - "ref": "ОФР!F12" - } - ], - [ - null, - { - "text": "2400", - "value": null, - "confidence": 100.0, - "ref": "ОФР_2!B1" - }, - { - "text": "264", - "value": { - "decimal": "264" - }, - "confidence": 100.0, - "ref": "ОФР_2!C1" - }, - { - "text": "238", - "value": { - "decimal": "238" - }, - "confidence": 100.0, - "ref": "ОФР_2!D1" - } - ] - ], - "segment_breaks": [ - 12 - ], - "warnings": [] -} diff --git a/api/arm_statements/tests/golden/npz_income.expected.json b/api/arm_statements/tests/golden/npz_income.expected.json deleted file mode 100644 index d0e41cc..0000000 --- a/api/arm_statements/tests/golden/npz_income.expected.json +++ /dev/null @@ -1,488 +0,0 @@ -{ - "document_type": "INCOME", - "form_variant": "full", - "locator": "sheet:ОФР,ОФР_2", - "header": { - "organization_name": { - "text": "ООО «Пример»", - "confidence": 100.0, - "value": "ООО «Пример»" - }, - "inn": { - "text": "7700000000", - "confidence": 100.0, - "value": "7700000000" - }, - "report_date": { - "text": "30.09.2025", - "confidence": 100.0, - "value": "2025-09-30" - }, - "units": { - "text": "тыс. руб.", - "confidence": 100.0, - "value": "тыс. руб." - }, - "okud": { - "text": null, - "confidence": null, - "value": null - }, - "okpo": { - "text": null, - "confidence": null, - "value": null - }, - "okopf": { - "text": null, - "confidence": null, - "value": null - }, - "okfs": { - "text": null, - "confidence": null, - "value": null - }, - "okei": { - "text": null, - "confidence": null, - "value": null - }, - "address": { - "text": null, - "confidence": null, - "value": null - }, - "form_name": "Отчет о финансовых результатах", - "needs_review_fields": [] - }, - "columns": [ - { - "column_index": 4, - "label": "За 9 месяцев 2025 г.", - "label_confidence": 100.0, - "period_date": "2025-09-30", - "period_kind": "absolute" - }, - { - "column_index": 5, - "label": "За 9 месяцев 2024 г.", - "label_confidence": 100.0, - "period_date": "2024-09-30", - "period_kind": "absolute" - } - ], - "lines": [ - { - "row_index": 0, - "ref": "ОФР!D3", - "line_status": "ok", - "line_code": "2110", - "line_code_text": "2110", - "line_code_confidence": 100.0, - "line_name": "Выручка", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 4, - "amount": "1000.00", - "string_value": null, - "text": "1 000", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - }, - { - "column_index": 5, - "amount": "900.00", - "string_value": null, - "text": "900", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 1, - "ref": "ОФР!D4", - "line_status": "ok", - "line_code": "2120", - "line_code_text": "2120", - "line_code_confidence": 100.0, - "line_name": "Себестоимость продаж", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 4, - "amount": "-600.00", - "string_value": null, - "text": "-600", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - }, - { - "column_index": 5, - "amount": "-540.00", - "string_value": null, - "text": "-540", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 2, - "ref": "ОФР!D5", - "line_status": "ok", - "line_code": "2100", - "line_code_text": "2100", - "line_code_confidence": 100.0, - "line_name": "Валовая прибыль (убыток)", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 4, - "amount": "400.00", - "string_value": null, - "text": "400", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - }, - { - "column_index": 5, - "amount": "360.00", - "string_value": null, - "text": "360", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 3, - "ref": "ОФР!D6", - "line_status": "ok", - "line_code": "2210", - "line_code_text": "2210", - "line_code_confidence": 100.0, - "line_name": "Коммерческие расходы", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 4, - "amount": "-50.00", - "string_value": null, - "text": "-50", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - }, - { - "column_index": 5, - "amount": "-45.00", - "string_value": null, - "text": "-45", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 4, - "ref": "ОФР!D7", - "line_status": "ok", - "line_code": "2220", - "line_code_text": "2220", - "line_code_confidence": 100.0, - "line_name": "Управленческие расходы", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 4, - "amount": "-30.00", - "string_value": null, - "text": "-30", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - }, - { - "column_index": 5, - "amount": "-27.00", - "string_value": null, - "text": "-27", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 5, - "ref": "ОФР!D8", - "line_status": "ok", - "line_code": "2200", - "line_code_text": "2200", - "line_code_confidence": 100.0, - "line_name": "Прибыль (убыток) от продаж", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 4, - "amount": "320.00", - "string_value": null, - "text": "320", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - }, - { - "column_index": 5, - "amount": "288.00", - "string_value": null, - "text": "288", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 6, - "ref": "ОФР!D9", - "line_status": "ok", - "line_code": "2340", - "line_code_text": "2340", - "line_code_confidence": 100.0, - "line_name": "Прочие доходы", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 4, - "amount": "10.00", - "string_value": null, - "text": "10", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - }, - { - "column_index": 5, - "amount": "9.00", - "string_value": null, - "text": "9", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 7, - "ref": "ОФР!D10", - "line_status": "ok", - "line_code": "2300", - "line_code_text": "2300", - "line_code_confidence": 100.0, - "line_name": "Прибыль (убыток) до налогообложения", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 4, - "amount": "330.00", - "string_value": null, - "text": "330", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - }, - { - "column_index": 5, - "amount": "297.00", - "string_value": null, - "text": "297", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 8, - "ref": "ОФР!D11", - "line_status": "ok", - "line_code": "2410", - "line_code_text": "2410", - "line_code_confidence": 100.0, - "line_name": "Налог на прибыль", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 4, - "amount": "-66.00", - "string_value": null, - "text": "-66", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - }, - { - "column_index": 5, - "amount": "-59.00", - "string_value": null, - "text": "-59", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 9, - "ref": "ОФР!D12", - "line_status": "ok", - "line_code": "2460", - "line_code_text": "2460", - "line_code_confidence": 100.0, - "line_name": "Прочее", - "line_name_confidence": 100.0, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 4, - "amount": "0.00", - "string_value": null, - "text": "0", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - }, - { - "column_index": 5, - "amount": "0.00", - "string_value": null, - "text": "0", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - }, - { - "row_index": 10, - "ref": "ОФР_2!B1", - "line_status": "ok", - "line_code": "2400", - "line_code_text": "2400", - "line_code_confidence": 100.0, - "line_name": null, - "line_name_confidence": null, - "parent_ref": null, - "needs_review": false, - "values": [ - { - "column_index": 4, - "amount": "264.00", - "string_value": null, - "text": "264", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - }, - { - "column_index": 5, - "amount": "238.00", - "string_value": null, - "text": "238", - "confidence": 100.0, - "is_recognized": true, - "needs_review": false - } - ] - } - ], - "checks": { - "relations_failed": [], - "soft_relations_failed": [ - { - "column_index": 4, - "column_key": "4", - "relation": "2410 = 2411 + 2412", - "codes": [ - "2410", - "2411", - "2412" - ], - "expected": "-66.00", - "actual": null - }, - { - "column_index": 4, - "column_key": "4", - "relation": "2500 = 2400 + 2510 + 2520 + 2530", - "codes": [ - "2500", - "2400", - "2510", - "2520", - "2530" - ], - "expected": null, - "actual": "264.00" - }, - { - "column_index": 5, - "column_key": "5", - "relation": "2410 = 2411 + 2412", - "codes": [ - "2410", - "2411", - "2412" - ], - "expected": "-59.00", - "actual": null - }, - { - "column_index": 5, - "column_key": "5", - "relation": "2500 = 2400 + 2510 + 2520 + 2530", - "codes": [ - "2500", - "2400", - "2510", - "2520", - "2530" - ], - "expected": null, - "actual": "238.00" - } - ] - }, - "status": "done", - "warnings": [] -} diff --git a/api/arm_statements/tests/test_amounts.py b/api/arm_statements/tests/test_amounts.py deleted file mode 100644 index c27a407..0000000 --- a/api/arm_statements/tests/test_amounts.py +++ /dev/null @@ -1,128 +0,0 @@ -from datetime import date -from decimal import Decimal as D - -import pytest - -from arm_statements.amounts import build_value, parse_amount, sign_form_of -from arm_statements.loader import Rules -from arm_statements.model import Cell, SignForm, Source - -OCR, EXCEL = Source.OCR, Source.EXCEL - - -@pytest.mark.parametrize( - ("raw", "amount", "string_value", "recognized"), - [ - ("1 2О5", D("1205"), None, False), - ("42831 1", D("428311"), None, False), - ("(7744536}", D("-7744536"), None, False), - ("211(1 |", None, "211(1 |", False), - ("(Ю4)", None, "(Ю4)", False), - (": : 2856983 = ■", D("2856983"), None, False), - ("| : 122477", D("122477"), None, False), - ("|122477", D("122477"), None, False), - ("{38378)", None, "{38378)", False), - ("4,200", D("4200"), None, True), - ("1,234,567", D("1234567"), None, True), - ("837.994", D("837994"), None, True), - ("1 234.567", D("1234567"), None, True), - ("4,2", D("4.2"), None, True), - ("1 234,56", D("1234.56"), None, True), - ("1.234,56", D("123456"), None, False), - ("0,125", None, "0,125", False), - ("1 234 567", D("1234567"), None, True), - ("(1 234)", D("-1234"), None, True), - ("−500", D("-500"), None, True), - ("12:34", None, "12:34", False), - ("1 234 руб.", D("1234"), None, False), - ("—", None, None, True), - ("-", None, None, True), - ("", None, None, True), - ("I", None, "I", False), - ], -) -def test_parse_amount_ocr(rules: Rules, raw: str, amount: D | None, string_value: str | None, recognized: bool) -> None: - parsed = parse_amount(raw, OCR, rules) - assert (parsed.amount, parsed.string_value, parsed.is_recognized) == (amount, string_value, recognized) - - -def test_comma_depends_on_source(rules: Rules) -> None: - assert parse_amount("4,200", EXCEL, rules).amount == D("4.20") - assert parse_amount("1.500", EXCEL, rules).amount == D("1.5") - - -def test_sign_forms(rules: Rules) -> None: - assert sign_form_of("(7744536}", rules) is SignForm.PARENTHESES - assert sign_form_of("-1 234", rules) is SignForm.MINUS - assert sign_form_of("38378", rules) is SignForm.UNSIGNED - assert sign_form_of("{38378)", rules) is None - - -def test_build_value_excel_number_and_out_of_range(rules: Rules) -> None: - ok = build_value(Cell("106 311", D("106311.456"), 100.0, "Баланс!D27"), 3, EXCEL, rules) - assert (ok.value.amount, ok.value.is_recognized, ok.out_of_range) == (D("106311.46"), True, False) - big = build_value(Cell("1E+17", D("123456789012345678"), 100.0, "Баланс!D28"), 3, EXCEL, rules) - assert (big.value.amount, big.value.string_value, big.value.is_recognized, big.out_of_range) == ( - None, - "1E+17", - False, - True, - ) - - -def test_build_value_date_missing_multiline_split(rules: Rules) -> None: - as_date = build_value(Cell("30.09.2025", date(2025, 9, 30), 100.0, "Баланс!D3"), 3, EXCEL, rules).value - assert (as_date.amount, as_date.string_value, as_date.is_recognized) == (None, "30.09.2025", False) - missing = build_value(None, 4, OCR, rules).value - assert (missing.amount, missing.string_value, missing.text, missing.is_recognized) == (None, None, None, True) - assert missing.confidence == pytest.approx(100.0) - multi = build_value(Cell("917 775\n2 371 333", None, 100.0, "row:9"), 2, OCR, rules).value - assert (multi.amount, multi.string_value, multi.is_recognized) == (None, "917 775\n2 371 333", False) - part = build_value(Cell("917 775", None, 100.0, "row:9#1"), 2, OCR, rules, split_part=True).value - assert (part.amount, part.is_recognized) == (D("917775"), False) - dash = build_value(Cell("—", None, 100.0, "row:3"), 2, OCR, rules).value - assert (dash.amount, dash.string_value, dash.is_recognized, dash.text) == (None, None, True, "—") - - -@pytest.mark.parametrize("raw", ["1E+30", "Infinity", "NaN", "-Infinity", "10000000000000000"]) -def test_build_value_decimal_out_of_range(rules: Rules, raw: str) -> None: - built = build_value(Cell(raw, D(raw), 100.0, "Баланс!D1"), 3, EXCEL, rules) - assert (built.value.amount, built.value.string_value, built.value.is_recognized, built.out_of_range) == ( - None, - raw, - False, - True, - ) - - -def test_build_value_decimal_empty_text_and_boundary(rules: Rules) -> None: - empty = build_value(Cell("", D("1E+30"), 100.0, "Баланс!D1"), 3, EXCEL, rules) - assert (empty.value.string_value, empty.out_of_range) == (None, True) - edge = build_value(Cell("x", D("9999999999999999"), 100.0, "Баланс!D1"), 3, EXCEL, rules) - assert (edge.value.amount, edge.out_of_range) == (D("9999999999999999"), False) - carry = build_value(Cell("x", D("9999999999999999.999"), 100.0, "Баланс!D1"), 3, EXCEL, rules) - assert (carry.value.amount, carry.out_of_range) == (None, True) - - -def test_build_value_rounds_half_up(rules: Rules) -> None: - assert build_value(Cell("0,125", D("0.125"), 100.0, "Баланс!D1"), 3, EXCEL, rules).value.amount == D("0.13") - assert build_value(Cell("-0,125", D("-0.125"), 100.0, "Баланс!D1"), 3, EXCEL, rules).value.amount == D("-0.13") - - -def test_sign_inside_parentheses_is_invalid(rules: Rules) -> None: - parsed = parse_amount("(-5)", OCR, rules) - assert (parsed.amount, parsed.string_value, parsed.is_recognized, parsed.sign_form) == (None, "(-5)", False, None) - - -def test_leading_comma_is_edge_trash(rules: Rules) -> None: - parsed = parse_amount(",5", OCR, rules) - assert (parsed.amount, parsed.is_recognized) == (D("5"), False) - - -def test_guess_amount_is_quantized_to_cents(rules: Rules) -> None: - """T20.1: «лучшая догадка» квантуется как прочие пути; в контракте — две цифры после точки.""" - parsed = parse_amount("42831 1", Source.OCR, rules) - assert parsed.amount == D("428311.00") and parsed.amount.as_tuple().exponent == -2 # noqa: PLR2004 — знаков после запятой - assert format(parsed.amount, "f") == "428311.00" - negative = parse_amount("(42831 1)", Source.OCR, rules) - assert negative.amount is not None and format(negative.amount, "f") == "-428311.00" diff --git a/api/arm_statements/tests/test_checks.py b/api/arm_statements/tests/test_checks.py deleted file mode 100644 index d7ab03c..0000000 --- a/api/arm_statements/tests/test_checks.py +++ /dev/null @@ -1,107 +0,0 @@ -from decimal import Decimal as D - -from arm_statements.checks import checks_from, evaluate_relations, form_variant, subline_parents -from arm_statements.evaluation import EvalLine, EvalValue, is_check_line -from arm_statements.loader import Rules -from arm_statements.model import DocumentType, FormVariant, LineStatus, StatementChecks - -BAL, INC = DocumentType.BALANCE, DocumentType.INCOME - - -def line(code: str | None, *amounts: str | None, status: LineStatus = LineStatus.OK, order: int = 0) -> EvalLine: - values = tuple(EvalValue(str(i), None if a is None else D(a), a, 100.0, True) for i, a in enumerate(amounts)) - return EvalLine(f"k{order}{code}", order, status, code, 100.0, False, values) - - -def failed(lines: list[EvalLine], doc: DocumentType, variant: FormVariant | None, rules: Rules) -> StatementChecks: - outcomes = evaluate_relations(lines, ["0"], doc, variant, rules) - return checks_from(outcomes, {"0": 0}) - - -def test_form_variant() -> None: - assert form_variant(BAL, ["1150", "1210", "1600", "1310", "1700"]) is FormVariant.SIMPLIFIED - assert form_variant(BAL, ["1100", "1600"]) is FormVariant.FULL - assert form_variant(INC, ["2110", "2400"]) is FormVariant.SIMPLIFIED - assert form_variant(INC, ["2110", "2100", "2400"]) is FormVariant.FULL - assert form_variant(DocumentType.DISCLOSURE, ["5101"]) is None - - -def test_empty_target_is_violation(rules: Rules) -> None: - lines = [ - line("1110", "30"), - line("1100", "30"), - line("1210", "70"), - line("1200", "70"), - line(None, "100", status=LineStatus.DETAIL), - ] - checks = failed(lines, BAL, FormVariant.FULL, rules) - assert [f.relation for f in checks.relations_failed] == ["1600 = 1100 + 1200"] - assert (checks.relations_failed[0].expected, checks.relations_failed[0].actual) == (None, D("100")) - - -def test_target_without_members_is_violation(rules: Rules) -> None: - lines = [line("1110", "30"), line("1100", "30"), line("1210", "70"), line("1200", "70"), line("1600", "100")] - checks = failed(lines, BAL, FormVariant.FULL, rules) - assert [f.relation for f in checks.relations_failed] == ["1600 = 1700"] - assert checks.relations_failed[0].codes == ("1600", "1700") - assert (checks.relations_failed[0].expected, checks.relations_failed[0].actual) == (D("100"), None) - - -def test_zero_target_with_empty_members_is_violation(rules: Rules) -> None: - lines = [line("1600", "0"), line("1110", "0"), line("1100", "0")] - hard = failed(lines, BAL, FormVariant.FULL, rules).relations_failed - assert [(f.relation, f.expected, f.actual) for f in hard] == [("1600 = 1700", D("0"), None)] - - -def test_duplicate_code_is_summed(rules: Rules) -> None: - lines = [line("2300", "849"), line("2410", "100"), line("2410", "649"), line("2400", "100")] - outcomes = evaluate_relations(lines, ["0"], INC, FormVariant.FULL, rules) - target = next(o for o in outcomes if o.rule.relation == "2400 = 2300 - 2410 + 2420") - assert target.within_tolerance - - -def test_simplified_income_soft(rules: Rules) -> None: - lines = [line("2110", "1000"), line("2120", "-600"), line("2410", "50"), line("2400", "500")] - checks = failed(lines, INC, FormVariant.SIMPLIFIED, rules) - assert checks.relations_failed == () - assert [f.relation for f in checks.soft_relations_failed] == ["2400 = 2110 - 2120 - 2330 + 2340 - 2350 - 2410"] - - -def test_rounding_goes_soft(rules: Rules) -> None: - lines = [ - line("1600", "17000000009.02"), - line("1700", "17000000000"), - line("1100", "17000000009.02"), - line("1300", "17000000000"), - ] - checks = failed(lines, BAL, FormVariant.FULL, rules) - assert "1600 = 1700" in [f.relation for f in checks.soft_relations_failed] - assert "1600 = 1700" not in [f.relation for f in checks.relations_failed] - - -def test_abs_terms_hold_with_minus(rules: Rules) -> None: - lines = [line("2110", "100"), line("2120", "-60"), line("2100", "40")] - hard = failed(lines, INC, FormVariant.FULL, rules).relations_failed - assert "2100 = 2110 - 2120" not in [f.relation for f in hard] - - -def test_sublines(rules: Rules) -> None: - assert subline_parents(["1230", "12301", "12302", "1240", "11101"]) == [None, "1230", "1230", None, "1110"] - # расхождение 2 (>1): в плане было 10 против 4 + 5 = 9, это в пределах допуска ±1 — спека «Шаг 7» (T13.1) - lines = [line("1230", "10"), line("12301", "4"), line("12302", "4")] - soft = failed(lines, BAL, FormVariant.FULL, rules).soft_relations_failed - assert [f.relation for f in soft] == ["1230 = 12301 + 12302"] - within = [line("1230", "10"), line("12301", "4"), line("12302", "5")] - assert failed(within, BAL, FormVariant.FULL, rules).soft_relations_failed == () - empty = [line("1230", "10"), line("12301", None), line("12302", None)] - assert failed(empty, BAL, FormVariant.FULL, rules).soft_relations_failed == () - - -def test_is_check_line() -> None: - assert is_check_line(line("1600", "1")) - assert not is_check_line(line("1600", "1", status=LineStatus.DETAIL)) - assert not is_check_line(line(None, "1")) - correction = EvalLine("c", None, LineStatus.OK, "1600", 100.0, False, line("1600", None).values) - assert not is_check_line(correction) - with_amount = EvalLine("c", None, LineStatus.OK, "1600", 100.0, False, line("1600", "5").values) - assert is_check_line(with_amount) diff --git a/api/arm_statements/tests/test_columns.py b/api/arm_statements/tests/test_columns.py deleted file mode 100644 index c516d4d..0000000 --- a/api/arm_statements/tests/test_columns.py +++ /dev/null @@ -1,213 +0,0 @@ -from collections.abc import Sequence -from datetime import date -from decimal import Decimal as D -from typing import Any - -from arm_statements.columns import amount_columns, document_dates, finalize_columns, segment_columns -from arm_statements.header import parse_header -from arm_statements.loader import Rules -from arm_statements.model import HeaderCells, PeriodKind, Severity, Source, StatementColumn, StatementWarning -from arm_statements.tests.builders import excel, hdr, ocr - -OCR, EXCEL = Source.OCR, Source.EXCEL - - -def run( # noqa: PLR0913, PLR0917 - вспомогательная функция тестов, сигнатура задана планом (Task 9) - rows: Sequence[Sequence[Any]], - source: Source, - rules: Rules, - header: HeaderCells | None = None, - header_rows: tuple[int, ...] = (0,), - data_rows: tuple[int, ...] = (1, 2), -) -> tuple[tuple[StatementColumn, ...], list[StatementWarning]]: - cand = ocr(rows) if source is OCR else excel([("Лист1", rows)]) - drafts, warnings = segment_columns(cand.rows, header_rows, data_rows, 1 if source is OCR else 2, source, rules) - head = parse_header(header or hdr(), rules) - columns, more = finalize_columns(drafts, document_dates(drafts, head), rules) - return columns, [*warnings, *more] - - -def test_blank_66n_relative_labels(rules: Rules) -> None: - rows: list[list[Any]] = [ - ["Наименование", "Код", "На отчетную дату", "На 31 декабря предыдущего года", - "На 31 декабря года, предшествующего предыдущему"], - ["Запасы", "1210", "5", "4", "3"], - ["Итого", "1200", "5", "4", "3"], - ] # fmt: skip - columns, warnings = run(rows, OCR, rules, header=hdr(report_date="30.09.2025")) - assert [c.period_date for c in columns] == [date(2025, 9, 30), date(2024, 12, 31), date(2023, 12, 31)] - assert [c.period_kind for c in columns] == [ - PeriodKind.CURRENT, - PeriodKind.PREV_YEAR_END, - PeriodKind.PREV_PREV_YEAR_END, - ] - assert warnings == [] - - -def test_excel_skipped_change_column(rules: Rules) -> None: - rows: list[list[Any]] = [ - ["Наименование", "", "Код", "На 30.09.2025", "На 31.12.2024", "Изменение, %"], - ["Запасы", "", "1210", D("5"), D("4"), D("25")], - ["Итого", "", "1200", D("5"), D("4"), D("25")], - ] - columns, warnings = run(rows, EXCEL, rules) - assert [c.column_index for c in columns] == [3, 4] - assert [(w.code, w.severity, w.column_index) for w in warnings] == [("COLUMN_SKIPPED", Severity.INFO, 5)] - assert {c.label_confidence for c in columns} == {100.0} - - -def test_junk_label_over_numbers_is_kept(rules: Rules) -> None: - rows: list[list[Any]] = [ - ["Наименование", "Код", "На 30.09.2025", ("| | г", 17.0)], - ["Запасы", "1210", "5", "4"], - ["Итого", "1200", "5", "4"], - ] - columns, warnings = run(rows, OCR, rules, header=hdr(report_date="30.09.2025")) - junk = columns[1] - assert (junk.label, junk.period_date, junk.period_kind) == ("| | г", None, PeriodKind.NONE) - assert [(w.code, w.severity, w.column_index) for w in warnings] == [("COLUMN_WITHOUT_DATE", Severity.ERROR, 3)] - - -def test_empty_label_right_of_dated_columns_is_outside_table(rules: Rules) -> None: - rows: list[list[Any]] = [ - ["Наименование", "Код", "На 30.09.2025", ""], - ["Запасы", "1210", "5", "4"], - ["Итого", "1200", "5", "4"], - ] - columns, warnings = run(rows, OCR, rules) - assert [(c.column_index, c.period_kind) for c in columns] == [(2, PeriodKind.ABSOLUTE)] - assert [(w.code, w.severity, w.column_index) for w in warnings] == [("NUMBERS_OUTSIDE_TABLE", Severity.INFO, None)] - assert warnings[0].locator == "row:1\nrow:2" - - -def test_numbers_right_of_last_dated_column_are_not_amounts(rules: Rules) -> None: - rows: list[list[Any]] = [ - ["Наименование", "", "Код", "На 30.09.2025", "На 31.12.2024", None, None], - ["Запасы", "", "1210", D("5"), D("4"), None, D("9")], - ["Баланс", "", "1600", D("5"), D("4"), D("7"), None], - ["Итого", "", "1700", D("5"), D("4"), None, None], - ] - columns, warnings = run(rows, EXCEL, rules, data_rows=(1, 2, 3)) - assert [c.column_index for c in columns] == [3, 4] - assert [(w.code, w.severity) for w in warnings] == [("NUMBERS_OUTSIDE_TABLE", Severity.INFO)] - assert warnings[0].locator == "Лист1!G2\nЛист1!F3" - assert "G2" in warnings[0].message and "F3" in warnings[0].message - - -def test_numbers_outside_table_do_not_depend_on_cell_count(rules: Rules) -> None: - rows: list[list[Any]] = [["Наименование", "", "Код", "На 30.09.2025", None]] - rows += [["Строка", "", str(1200 + i), D("1"), D("2")] for i in range(12)] - cand = excel([("Лист1", rows)]) - _, warnings = segment_columns(cand.rows, (0,), tuple(range(1, 13)), 2, EXCEL, rules) - assert len(warnings) == 1 - assert warnings[0].message.endswith("и ещё 2") - assert warnings[0].message.count("Лист1!E") == 10 # noqa: PLR2004 - assert len((warnings[0].locator or "").split("\n")) == 12 # noqa: PLR2004 - - -def test_no_dated_columns_keeps_old_behaviour(rules: Rules) -> None: - rows: list[list[Any]] = [ - ["Наименование", "", "Код", "", ""], - ["Запасы", "", "1210", D("5"), D("4")], - ["Итого", "", "1200", D("5"), D("4")], - ] - columns, warnings = run(rows, EXCEL, rules) - assert [c.column_index for c in columns] == [3, 4] - assert [(w.code, w.column_index) for w in warnings] == [("COLUMN_WITHOUT_DATE", 3), ("COLUMN_WITHOUT_DATE", 4)] - - -def test_undated_column_between_dated_keeps_old_behaviour(rules: Rules) -> None: - rows: list[list[Any]] = [ - ["Наименование", "", "Код", "На 30.09.2025", None, "На 31.12.2024"], - ["Запасы", "", "1210", D("5"), D("9"), D("4")], - ["Итого", "", "1200", D("5"), D("9"), D("4")], - ] - columns, warnings = run(rows, EXCEL, rules) - assert [c.column_index for c in columns] == [3, 4, 5] - assert [(w.code, w.severity, w.column_index) for w in warnings] == [("COLUMN_WITHOUT_DATE", Severity.ERROR, 4)] - - -def test_two_row_header_is_joined_with_min_confidence(rules: Rules) -> None: - rows: list[list[Any]] = [ - ["", "", ("На 31 декабря", 90.0)], - ["Наименование", "Код", ("2024 г.", 80.0)], - ["Запасы", "1210", "5"], - ] - cand = ocr(rows) - drafts, _ = segment_columns(cand.rows, (0, 1), (2,), 1, OCR, rules) - assert [(d.grid_col, d.label, d.label_confidence, d.date) for d in drafts] == [ - (2, "На 31 декабря 2024 г.", 80.0, date(2024, 12, 31)) - ] - - -def test_sparse_ocr_columns(rules: Rules) -> None: - rows: list[list[Any]] = [ - ["Наименование", "Код", "На 30.09.2025", None, "На 31.12.2024", "На 31.12.2023"], - ["Запасы", "1210", "5", None, "4", "3"], - ["Итого", "1200", "5", None, "4", "3"], - ] - columns, _ = run(rows, OCR, rules) - assert [c.column_index for c in columns] == [2, 4, 5] - - -def test_uncertain_label_does_not_confirm_itself(rules: Rules) -> None: - rows: list[list[Any]] = [ - ["Наименование", "Код", ("На 30.09.2О25", 60.0), ""], - ["Запасы", "1210", "5", "4"], - ["Итого", "1200", "5", "4"], - ] - cand = ocr(rows) - drafts, _ = segment_columns(cand.rows, (0,), (1, 2), 1, OCR, rules) - dates = document_dates(drafts, parse_header(hdr(), rules)) - assert (dates.effective, dates.confident_effective) == (date(2025, 9, 30), None) - _, warnings = finalize_columns(drafts, dates, rules) - assert [(w.code, w.column_index) for w in warnings] == [("COLUMN_DATE_UNCERTAIN", 2)] - - -def test_uncertain_label_confirmed_by_report_date(rules: Rules) -> None: - rows: list[list[Any]] = [ - ["Наименование", "Код", "На 30.09.2025", ("На 31.12.2024", 60.0), ("На 31.03.2025", 60.0)], - ["Запасы", "1210", "5", "4", "3"], - ["Итого", "1200", "5", "4", "3"], - ] - _, warnings = run(rows, OCR, rules, header=hdr(report_date="30.09.2025")) - assert [(w.code, w.column_index) for w in warnings] == [("COLUMN_DATE_UNCERTAIN", 4)] - - -def test_segment_without_header(rules: Rules) -> None: - cand = ocr([["Запасы", "1210", "5", "", "4"], ["Итого", "1200", "5", "", "4"]]) - drafts, warnings = segment_columns(cand.rows, (), (0, 1), 1, OCR, rules) - assert [(d.grid_col, d.label, d.kind) for d in drafts] == [(2, "", PeriodKind.NONE), (4, "", PeriodKind.NONE)] - assert warnings == [] - - -def test_public_amount_columns(rules: Rules) -> None: - rows: list[list[Any]] = [ - ["Наименование", "", "Код", "На 30.09.2025", "На 31.12.2024", "Изменение"], - ["Запасы", "", "1210", D("5"), D("4"), "рост"], - ["Итого", "", "1200", D("5"), "", "рост"], - ] - assert amount_columns(excel([("Лист1", rows)]).rows, 2, rules) == (3, 4) - assert amount_columns(ocr([["Запасы", "1210", "5", "", "4"]]).rows, 1, rules, source=OCR) == (2, 4) - - -def test_dated_column_without_numbers_is_not_the_table_edge(rules: Rules) -> None: - rows: list[list[Any]] = [ - ["Наименование", "", "Код", "На 30.09.2025", "", "На 31.12.2024", ""], - ["Запасы", "", "1210", None, D("5"), None, D("4")], - ["Итого", "", "1200", None, D("5"), None, D("4")], - ] # подпись объединённой ячейки стоит левее значений: границы таблицы нет, колонки остаются - columns, warnings = run(rows, EXCEL, rules) - assert [c.column_index for c in columns] == [3, 4, 5, 6][:0] or [w.code for w in warnings].count( - "NUMBERS_OUTSIDE_TABLE" - ) == 0 - - -def test_sheet_name_with_comma_keeps_addresses(rules: Rules) -> None: - rows: list[list[Any]] = [ - ["Наименование", "", "Код", "На 30.09.2025", None], - ["Запасы", "", "1210", D("5"), D("9")], - ] - cand = excel([("Баланс, 2025", rows)]) - _, warnings = segment_columns(cand.rows, (0,), (1,), 2, EXCEL, rules) - assert (warnings[0].locator or "").split("\n") == ["'Баланс, 2025'!E2"] diff --git a/api/arm_statements/tests/test_contract.py b/api/arm_statements/tests/test_contract.py deleted file mode 100644 index fe63ba8..0000000 --- a/api/arm_statements/tests/test_contract.py +++ /dev/null @@ -1,99 +0,0 @@ -import json -from dataclasses import replace -from decimal import Decimal as D - -import pytest - -from arm_statements import ( - DocStatus, - DocumentType, - document_status, - make_warning, - normalize_candidate, - status_input, - status_input_from_contract, - to_contract, - warning_to_contract, -) -from arm_statements.loader import Rules -from arm_statements.model import FailedRelation, StatementChecks, StatementDocument -from arm_statements.tests.builders import hdr, ocr - -HEAD = ["Наименование показателя", "Код", "На 30.09.2025"] -FULL_HEADER = hdr( - organization_name="ООО «Пример»", inn=("7700000000", 80.0), report_date="30.09.2025", units="тыс. руб." -) -HEADER_KEYS = {"form_name", "needs_review_fields"} | { - f"{name}{suffix}" - for name in ("organization_name", "inn", "report_date", "units", "okud", "okpo", "okopf", "okfs", "okei", "address") - for suffix in ("", "_text", "_confidence") -} - - -def balance(rules: Rules, *rows: list[str]) -> StatementDocument: - cand = ocr([HEAD, *rows], header=FULL_HEADER, type_hint=DocumentType.BALANCE, title="Бухгалтерский баланс") - doc = normalize_candidate(cand, rules).document - assert doc is not None - return doc - - -def test_document_in_contract_form(rules: Rules) -> None: - doc = balance(rules, ["Запасы", "1210", "60"], ["Итого II", "1200", "60"], ["НИ", "", "7"]) - data = json.loads(json.dumps(to_contract(doc), ensure_ascii=False)) - assert set(data["header"]) == HEADER_KEYS - assert (data["header"]["form_name"], data["header"]["inn"], data["header"]["inn_confidence"]) == ( - "Бухгалтерский баланс", - "7700000000", - 80.0, - ) - assert data["header"]["report_date"] == "2025-09-30" and data["header"]["okei"] is None - assert data["lines"][0]["values"][0]["amount"] == "60.00" - assert {ln["line_status"] for ln in data["lines"]} >= {"ok"} - assert "column_key" not in json.dumps(data) and "row:" not in json.dumps(data["header"]) - assert "locator" not in json.dumps(data["warnings"]) - assert set(data["checks"]) == {"relations_failed", "soft_relations_failed"} - - -def test_empty_target_is_null_in_contract(rules: Rules) -> None: - doc = balance(rules, ["Запасы", "1210", "60"], ["Итого II", "1200", "60"], ["Итого I", "1100", "40"]) - failed = to_contract(doc)["checks"]["relations_failed"] - target = next(f for f in failed if f["relation"] == "1600 = 1100 + 1200") - assert (target["expected"], target["actual"], target["column_index"]) == (None, "100.00", 2) - - -def test_no_lines_no_checks_key(rules: Rules) -> None: - doc = normalize_candidate( - ocr([["Итого", "15 000"]], type_hint=DocumentType.BALANCE, header=FULL_HEADER), rules - ).document - assert doc is not None and "checks" not in to_contract(doc) and to_contract(doc)["lines"] == [] - - -def test_relation_without_column_index_is_error(rules: Rules) -> None: - doc = balance(rules, ["Запасы", "1210", "60"], ["Итого II", "1200", "60"]) - bad = FailedRelation(None, "c1", "1600 = 1700", ("1600", "1700"), D("1"), None) - with pytest.raises(ValueError, match="column_index"): - to_contract(replace(doc, checks=StatementChecks((bad,), ()))) - - -def test_warning_keys(rules: Rules) -> None: - w = make_warning("ROW_SPLIT", "разлеплено", rules, line_refs=("row:3#1", "row:3#2"), locator="file:a.json") - assert warning_to_contract(w) == { - "code": "ROW_SPLIT", - "severity": "info", - "message": "разлеплено", - "line_refs": ["row:3#1", "row:3#2"], - } - lost = make_warning("TABLE_LOST", "таблица", rules, locator="file:a.json;table:1") - assert warning_to_contract(lost, file_level=True) == { - "code": "TABLE_LOST", - "severity": "error", - "message": "таблица", - "locator": "file:a.json;table:1", - } - - -def test_status_input_roundtrip(rules: Rules) -> None: - doc = balance(rules, ["Запасы", "1210", "60"], ["Итого II", "1200", "60"], ["НИ", "", "7"]) - data = status_input_from_contract(json.loads(json.dumps(to_contract(doc), ensure_ascii=False))) - assert data == status_input(doc) - assert document_status(data) is doc.status is DocStatus.NEEDS_CHECK diff --git a/api/arm_statements/tests/test_dates.py b/api/arm_statements/tests/test_dates.py deleted file mode 100644 index 887de26..0000000 --- a/api/arm_statements/tests/test_dates.py +++ /dev/null @@ -1,94 +0,0 @@ -from datetime import date - -import pytest - -from arm_statements.dates import expected_dates, parse_date, parse_period_label, resolve_relative -from arm_statements.loader import Rules -from arm_statements.model import Cell, PeriodKind -from arm_statements.text import fix_lookalikes - -# Обязательные строки scan-gateway test_dates.py::test_parses_real_forms (спека «Шаг 2 — подпись и дата колонки») -REAL_FORMS = [ - ("на 30 сентября 2025 г.", date(2025, 9, 30)), - ("На 30 июня\n2025 г", date(2025, 6, 30)), - ("31 декабря 2024 года", date(2024, 12, 31)), - ("30 сентября 2025 r.", date(2025, 9, 30)), - ("с 01.01.2025 по 30.09.2025", date(2025, 9, 30)), - ("I квартал 2026", date(2026, 3, 31)), - ("1-й кв. 2026", date(2026, 3, 31)), - ("первый квартал 2025 г.", date(2025, 3, 31)), - ("1 полугодие 2026", date(2026, 6, 30)), - ("1 п/г 2026", date(2026, 6, 30)), - ("9 месяцев 2025", date(2025, 9, 30)), - ("9 мес. 2025", date(2025, 9, 30)), - ("За январь - июнь\n2025 г.", date(2025, 6, 30)), - ("январь – сентябрь 2025", date(2025, 9, 30)), - ("за 2025 г.", date(2025, 12, 31)), - ("2024 год", date(2024, 12, 31)), - ("30.09.2025", date(2025, 9, 30)), - ("2025 03 31", date(2025, 3, 31)), - ("30/09/25", date(2025, 9, 30)), - ("2 0 2 4", date(2024, 12, 31)), - ("30 09 2 0 2 5", date(2025, 9, 30)), - ("30092025", date(2025, 9, 30)), - ("20250930", date(2025, 9, 30)), - ("30 09 25", date(2025, 9, 30)), - ("30.09.2025 0:00:00", date(2025, 9, 30)), - ("2|0|2|4|", date(2024, 12, 31)), - ("3l.12.2О24", date(2024, 12, 31)), -] - - -@pytest.mark.parametrize(("raw", "expected"), REAL_FORMS) -def test_parses_real_forms(rules: Rules, raw: str, expected: date) -> None: - assert parse_date(fix_lookalikes(raw, rules.lookalikes), rules) == expected - label = parse_period_label(Cell(raw, None, 100.0, "row:0"), rules) - assert (label.kind, label.date) == (PeriodKind.ABSOLUTE, expected) - - -@pytest.mark.parametrize("raw", ["Общество с ограниченной ответственностью", "", "+", "тыс. руб.", "1999999999999"]) -def test_not_a_date(rules: Rules, raw: str) -> None: - assert parse_date(raw, rules) is None - - -def test_long_text_not_parsed(rules: Rules) -> None: - assert parse_date("на 30 сентября 2025 г." + "x" * 200, rules) is None - - -@pytest.mark.parametrize( - ("raw", "kind"), - [ - ("На отчетную дату", PeriodKind.CURRENT), - ("За отчётный период", PeriodKind.CURRENT), - ("На 31 декабря предыдущего года", PeriodKind.PREV_YEAR_END), - ("На 31 декабря года, предшествующего предыдущему", PeriodKind.PREV_PREV_YEAR_END), - ("За аналогичный период предыдущего года", PeriodKind.PREV_PERIOD), - ], -) -def test_relative_labels(rules: Rules, raw: str, kind: PeriodKind) -> None: - assert parse_period_label(Cell(raw, None, 100.0, "row:0"), rules).kind is kind - - -def test_exclusion_and_excel_date_value(rules: Rules) -> None: - assert parse_period_label(Cell("Изменение, %", None, 100.0, "A!F1"), rules).excluded - label = parse_period_label(Cell("30.09.2025", date(2025, 9, 30), 100.0, "A!D1"), rules) - assert (label.kind, label.date) == (PeriodKind.ABSOLUTE, date(2025, 9, 30)) - assert parse_period_label(Cell("2024", None, 100.0, "A!E1"), rules).bare_year - - -def test_empty_label(rules: Rules) -> None: - assert parse_period_label(None, rules).kind is PeriodKind.NONE - label = parse_period_label(Cell(" ", None, 100.0, "A!G1"), rules) - assert (label.kind, label.date, label.excluded, label.bare_year) == (PeriodKind.NONE, None, False, False) - - -def test_resolve_relative_and_expected() -> None: - eff = date(2025, 9, 30) - assert resolve_relative(PeriodKind.PREV_YEAR_END, eff) == date(2024, 12, 31) - assert resolve_relative(PeriodKind.PREV_PREV_YEAR_END, eff) == date(2023, 12, 31) - assert resolve_relative(PeriodKind.PREV_PERIOD, date(2024, 2, 29)) == date(2023, 2, 28) - assert resolve_relative(PeriodKind.CURRENT, None) is None - assert resolve_relative(PeriodKind.ABSOLUTE, eff) is None and resolve_relative(PeriodKind.NONE, eff) is None - assert expected_dates(eff) == frozenset( - {eff, date(2024, 9, 30), date(2023, 9, 30), date(2024, 12, 31), date(2023, 12, 31)} - ) diff --git a/api/arm_statements/tests/test_golden.py b/api/arm_statements/tests/test_golden.py deleted file mode 100644 index 3058676..0000000 --- a/api/arm_statements/tests/test_golden.py +++ /dev/null @@ -1,171 +0,0 @@ -"""Golden на обезличенных входах Review Focus 1–5 (план 1, задача 20). - -Входы `golden/*.candidate.json` собраны из `builders` один раз и лежат файлами; мусорный текст ячеек — дословно из -реальных распознаваний, названия, ИНН и суммы — обезличенные и синтетические. Эталон `*.expected.json` — -`to_jsonable(normalize_candidate(...).document)`; пересборка — `make golden-update` (`GOLDEN_UPDATE=1`). -""" - -import json -import os -from collections.abc import Callable -from decimal import Decimal as D -from pathlib import Path - -import pytest - -from arm_statements import ( - document_status, - normalize_candidate, - status_input, - status_input_from_contract, - to_contract, -) -from arm_statements.loader import Rules -from arm_statements.model import DocStatus, DocumentType, FormCandidate, LineStatus, StatementDocument -from arm_statements.serialize import candidate_from_dict, to_jsonable -from arm_statements.tests.builders import hdr, ocr - -GOLDEN = Path(__file__).parent / "golden" -FULL_CONFIDENCE = 100.0 -NAMES = ["aep3_balance", "aep3_income", "glued_balance", "kody_title_balance", "npz_income"] -HEAD = ["Наименование показателя", "Код", "На 30.09.2025"] -FULL_HEADER = hdr(organization_name="ООО «Пример»", inn="7700000000", report_date="30.09.2025", units="тыс. руб.") - - -def run(name: str, rules: Rules) -> StatementDocument: - data = json.loads((GOLDEN / f"{name}.candidate.json").read_text(encoding="utf-8")) - result = normalize_candidate(candidate_from_dict(data), rules) - assert result.document is not None - return result.document - - -@pytest.mark.parametrize("name", NAMES) -def test_golden_snapshot(name: str, rules: Rules) -> None: - actual = to_jsonable(run(name, rules)) - target = GOLDEN / f"{name}.expected.json" - if os.environ.get("GOLDEN_UPDATE") == "1": - target.write_text(json.dumps(actual, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - pytest.fail(f"эталон {target.name} пересобран — просмотрите diff и запустите без GOLDEN_UPDATE") - assert actual == json.loads(target.read_text(encoding="utf-8")) - - -def assert_status_is_function_of_contract(doc: StatementDocument) -> None: - contract = json.loads(json.dumps(to_contract(doc), ensure_ascii=False)) - assert status_input_from_contract(contract) == status_input(doc) - assert document_status(status_input(doc)) is doc.status - - -@pytest.mark.parametrize("name", NAMES) -def test_status_is_function_of_contract_document(name: str, rules: Rules) -> None: - assert_status_is_function_of_contract(run(name, rules)) - - -def _disclosure() -> FormCandidate: - return ocr([["Текст"]], type_hint=DocumentType.DISCLOSURE) - - -def _without_lines() -> FormCandidate: - return ocr([["Итого", "15 000"]], type_hint=DocumentType.BALANCE, header=FULL_HEADER) - - -def _absolute_column_without_report_date() -> FormCandidate: - rows = [ - HEAD, - ["Запасы", "1210", "60"], - ["Дебиторы", "1230", "40"], - ["Итого II", "1200", "100"], - ] - header = hdr(organization_name="ООО «Пример»", inn="7700000000", units="тыс. руб.") - return ocr(rows, type_hint=DocumentType.BALANCE, header=header) - - -EXTRA: dict[str, Callable[[], FormCandidate]] = { - "disclosure": _disclosure, - "without_lines": _without_lines, - "absolute_column_without_report_date": _absolute_column_without_report_date, -} - - -@pytest.mark.parametrize("case", sorted(EXTRA)) -def test_status_is_function_of_contract_document_edge_cases(case: str, rules: Rules) -> None: - doc = normalize_candidate(EXTRA[case](), rules).document - assert doc is not None - assert_status_is_function_of_contract(doc) - - -def test_edge_cases_cover_their_branch(rules: Rules) -> None: - disclosure = normalize_candidate(_disclosure(), rules).document - assert disclosure is not None and disclosure.document_type is DocumentType.DISCLOSURE - assert disclosure.status is DocStatus.DONE - empty = normalize_candidate(_without_lines(), rules).document - assert empty is not None and empty.lines == () - assert [w.code for w in empty.warnings if w.code == "CODE_COLUMN_NOT_FOUND"] == ["CODE_COLUMN_NOT_FOUND"] - assert status_input(empty).layout_found is False and empty.status is DocStatus.NEEDS_CHECK - absolute = normalize_candidate(_absolute_column_without_report_date(), rules).document - assert absolute is not None and absolute.header.report_date.value is None - assert status_input(absolute).period_known is True - - -def test_aep3_junk_codes(rules: Rules) -> None: # Review Focus 1 - doc = run("aep3_balance", rules) - by_text = {ln.line_code_text: ln for ln in doc.lines} - assert (by_text["| 140"].line_status, by_text["| 140"].line_code) == (LineStatus.OK, "1140") - confidence = by_text["| 140"].line_code_confidence - assert confidence is not None and confidence < FULL_CONFIDENCE - for junk in ("НИ", "изо :"): - assert by_text[junk].line_status is LineStatus.DETAIL and by_text[junk].parent_ref is not None - refs = [ln.ref for ln in doc.lines] - assert len(refs) == len(set(refs)) and [ln.row_index for ln in doc.lines] == list(range(len(doc.lines))) - assert by_text["НИ"].parent_ref == by_text["1110"].ref and by_text["изо :"].parent_ref == by_text["| 140"].ref - assert len(doc.lines) == len(json.loads((GOLDEN / "aep3_balance.candidate.json").read_text("utf-8"))["rows"]) - 1 - - -def test_aep3_dirty_income_amounts(rules: Rules) -> None: # Review Focus 2 - doc = run("aep3_income", rules) - values = {v.text: v for ln in doc.lines for v in ln.values if v.text} - assert (values["(7744536}"].amount, values["(7744536}"].needs_review) == (D("-7744536"), False) - assert (values["211(1 |"].amount, values["211(1 |"].needs_review) == (None, True) - assert (values["42831 1"].amount, values["42831 1"].needs_review) == (D("428311"), False) - assert doc.checks is not None and doc.checks.relations_failed == () - assert doc.status is DocStatus.NEEDS_CHECK - - -def test_glued_rows(rules: Rules) -> None: # Review Focus 3 - doc = run("glued_balance", rules) - split = [ln for ln in doc.lines if "#" in ln.ref] - assert [(ln.line_code, ln.values[0].amount) for ln in split] == [("1500", D("917775")), ("1700", D("2371333"))] - assert [ln.ref for ln in split] == ["row:13#1", "row:13#2"] - assert len({ln.ref for ln in doc.lines}) == len(doc.lines) - assert [w.code for w in doc.warnings if w.code == "ROW_SPLIT"] == ["ROW_SPLIT"] - assert not [v for ln in doc.lines for v in ln.values if v.needs_review] - assert not [ln for ln in doc.lines if ln.needs_review] - - -def test_kody_title_block(rules: Rules) -> None: # Review Focus 4 - doc = run("kody_title_balance", rules) - assert "12267" not in [ln.line_code for ln in doc.lines] - assert doc.header.okopf.value == "12267" and doc.status is DocStatus.DONE - assert not [w for w in doc.warnings if w.code == "CODE_COLUMN_NOT_FOUND"] - assert all(ln.ref.startswith("Баланс!C") for ln in doc.lines) - - -def test_npz_continuation_single_code(rules: Rules) -> None: # Review Focus 5 - doc = run("npz_income", rules) - (line_2400,) = [ln for ln in doc.lines if ln.line_code == "2400"] - assert [v.column_index for v in line_2400.values] == [c.column_index for c in doc.columns] - assert all(v.amount is not None for v in line_2400.values) - assert [v.amount for v in line_2400.values] == [D("264"), D("238")] - assert "," in doc.locator and not [w for w in doc.warnings if w.code.startswith("SEGMENT_")] - assert doc.status is DocStatus.DONE - - -@pytest.mark.parametrize("name", NAMES) -def test_document_warnings_reference_document_parts(name: str, rules: Rules) -> None: - """I-2: line_refs предупреждения — ref строк документа; locator нет; column_index — из колонок.""" - doc = run(name, rules) - line_refs = {ln.ref for ln in doc.lines} - column_indexes = {c.column_index for c in doc.columns} - for warning in doc.warnings: - assert set(warning.line_refs) <= line_refs, warning - assert warning.locator is None, warning - assert warning.column_index is None or warning.column_index in column_indexes, warning diff --git a/api/arm_statements/tests/test_header.py b/api/arm_statements/tests/test_header.py deleted file mode 100644 index 071216e..0000000 --- a/api/arm_statements/tests/test_header.py +++ /dev/null @@ -1,56 +0,0 @@ -from datetime import date - -from arm_statements.header import continuation_type, is_continuation, parse_header, type_hint -from arm_statements.loader import Rules -from arm_statements.model import DocumentType, HeaderCells -from arm_statements.tests.builders import hdr - - -def test_inn_digits_only_without_lookalikes(rules: Rules) -> None: - head = parse_header(hdr(inn="|7700000000|"), rules) - assert head.inn.value == "7700000000" - dirty = parse_header(hdr(inn=("3|1 21 2|", 80.0)), rules) - assert dirty.inn.value is None - assert (dirty.inn.text, dirty.inn.confidence) == ("3|1 21 2|", 80.0) - - -def test_units_by_text_then_okei(rules: Rules) -> None: - assert parse_header(hdr(units="в тыс. рублей"), rules).units.value == "тыс. руб." - by_okei = parse_header(hdr(okei="384"), rules) - assert (by_okei.units.value, by_okei.okei.value, by_okei.units.text) == ("тыс. руб.", "384", None) - assert parse_header(hdr(units="руб."), rules).units.value == "руб." - - -def test_report_date_and_codes(rules: Rules) -> None: - head = parse_header(hdr(report_date="30.09.2О25", okud="Форма по ОКУД 0710001", okopf="12267"), rules) - assert head.report_date.value == date(2025, 9, 30) - assert (head.okud.value, head.okopf.value) == ("0710001", "12267") - - -def test_type_hint_okud_wins(rules: Rules) -> None: - assert type_hint("0710002", "Бухгалтерский баланс", rules) is DocumentType.INCOME - assert type_hint("Форма по ОКУД 0710001", None, rules) is DocumentType.BALANCE - assert type_hint("0710005", None, rules) is DocumentType.DISCLOSURE - assert type_hint(None, "Пояснения к бухгалтерскому балансу", rules) is DocumentType.DISCLOSURE - assert type_hint(None, "Отчет о прибылях и убытках", rules) is DocumentType.INCOME - assert type_hint(None, "Бухгалтерская отчетность", rules) is None - - -def test_continuation_marker(rules: Rules) -> None: - assert continuation_type("Форма 0710002 с.2", rules) is DocumentType.INCOME - assert continuation_type("Бухгалтерский баланс", rules) is None - assert is_continuation("Форма 0710000 с. 3", rules) and continuation_type("Форма 0710000 с. 3", rules) is None - assert not is_continuation("Бухгалтерский баланс", rules) and not is_continuation(None, rules) - - -def test_type_hint_uses_label_normalization(rules: Rules) -> None: - assert type_hint(None, "ОТЧЁТ О ФИНАНСОВЫХ\nРЕЗУЛЬТАТАХ", rules) is DocumentType.INCOME - - -def test_absent_cells_and_empty_values(rules: Rules) -> None: - head = parse_header(HeaderCells(), rules) - assert (head.inn.text, head.inn.confidence, head.inn.value) == (None, None, None) - assert head.report_date.value is None and head.needs_review_fields == () - blank = parse_header(hdr(okpo=" ", organization_name=" ООО «Пример» "), rules) - assert blank.okpo.value is None - assert blank.organization_name.value == "ООО «Пример»" diff --git a/api/arm_statements/tests/test_layout.py b/api/arm_statements/tests/test_layout.py deleted file mode 100644 index 1b029e3..0000000 --- a/api/arm_statements/tests/test_layout.py +++ /dev/null @@ -1,173 +0,0 @@ -from decimal import Decimal as D - -import pytest - -from arm_statements.layout import ( - code_column, - find_code_column, - find_header_rows, - find_name_column, - first_code_row, - is_numbering_row, - name_cell, - segment_roles, -) -from arm_statements.loader import CodeSection, Rules -from arm_statements.model import Cell, DocumentType, RowRole -from arm_statements.tests.builders import excel, ocr, place - -BALANCE_ROWS = [ - ["Наименование показателя", "Код", "На 30.09.2025", "На 31.12.2024"], - ["1", "2", "3", "4"], - ["Нематериальные активы", "1110", "10", "9"], - ["Основные средства", "1150", "20", "19"], - ["Итого по разделу I", "1100", "30", "28"], -] - - -def test_code_column_by_caption(rules: Rules) -> None: - cand = ocr(BALANCE_ROWS) - assert (find_code_column(cand.rows, 0, 4, rules), first_code_row(cand.rows, 0, 4, 1, rules)) == (1, 2) - - -def test_title_block_kody_is_not_code_column(rules: Rules) -> None: - title = [ - place({5: v}) for v in ["Коды", "0710001", "30 09 2025", "12345678", "7700000000", "12267", "16", "384", "", ""] - ] - table = [ - place({0: n, 2: c, 3: D(a)}) - for n, c, a in [ - ("Запасы", "1210", "5"), - ("НДС", "1220", "1"), - ("Дебиторы", "1230", "4"), - ("Итого", "1200", "10"), - ] - ] - cand = excel([("Баланс", [*title, *table])]) - assert [find_code_column(cand.rows, 0, len(cand.rows) - 1, rules)] == [2] - - -def test_two_line_document(rules: Rules) -> None: - cand = ocr([["Наименование", "Код", "2025"], ["Выручка", "2110", "5"], ["Прибыль", "2400", "1"]]) - assert find_code_column(cand.rows, 0, 2, rules) == 1 - - -def test_relaxed_search_finds_single_code(rules: Rules) -> None: - cand = ocr([["Чистая прибыль", "", "2400", "7", "6"]]) - assert find_code_column(cand.rows, 0, 0, rules) is None - assert [find_code_column(cand.rows, 0, 0, rules, relaxed_section=CodeSection.INCOME)] == [2] - - -def test_public_code_column(rules: Rules) -> None: - assert code_column(ocr(BALANCE_ROWS).rows, rules) == 1 - single = ocr([["Чистая прибыль", "", "2400", "7", "6"]]).rows - assert (code_column(single, rules), code_column(single, rules, relaxed=DocumentType.INCOME)) == (None, 2) - assert code_column(single, rules, relaxed=DocumentType.BALANCE) is None and code_column((), rules) is None - with pytest.raises(ValueError, match="DISCLOSURE"): - code_column(single, rules, relaxed=DocumentType.DISCLOSURE) - - -def test_numbering_rows(rules: Rules) -> None: - def row(*texts: str) -> tuple[Cell | None, ...]: - return ocr([list(texts)]).rows[0] - - assert is_numbering_row(row("1", "2", "3", "4", "5"), rules) - assert is_numbering_row(row("1", "2", "З", "4"), rules) - assert is_numbering_row(row("2", "3", "4", "5"), rules) - assert is_numbering_row(row("1", "2", "4", "5"), rules) - assert not is_numbering_row(row("1", "3", "5"), rules) - assert not is_numbering_row(row("1", "2"), rules) - - -def test_header_row_roles_and_title(rules: Rules) -> None: - rows = [["Бухгалтерский баланс", "", "", ""], *BALANCE_ROWS] - cand = ocr(rows) - header = find_header_rows(cand.rows, 0, 3, 1, rules) - assert header == (1,) - roles = segment_roles(cand.rows, 0, 5, header, rules) - assert roles == [RowRole.TITLE, RowRole.HEADER, RowRole.NUMBERING, RowRole.DATA, RowRole.DATA, RowRole.DATA] - assert RowRole.TITLE not in segment_roles(cand.rows, 3, 5, (), rules) - - -def test_two_row_header_is_glued(rules: Rules) -> None: - cand = ocr( - [ - ["", "", "На 30 сентября", "На 31 декабря"], - ["Наименование", "Код", "2025 г.", "2024 г."], - ["Запасы", "1210", "5", "4"], - ] - ) - assert find_header_rows(cand.rows, 0, 2, 1, rules) == (0, 1) - single = ocr( - [ - ["Раздел", "", "", ""], - ["Наименование", "Код", "На 30.09.2025", "На 31.12.2024"], - ["Запасы", "1210", "5", "4"], - ] - ) - assert find_header_rows(single.rows, 0, 2, 1, rules) == (1,) - - -def test_name_column_without_caption(rules: Rules) -> None: - cand = ocr( - [ - ["5.1", "Запасы", "1210", "5"], - ["5.2", "Дебиторская задолженность", "1230", "4"], - ["", "Итого", "1200", "9"], - ] - ) - assert find_name_column(cand.rows, (), 0, 2, 2, rules) == (1, 0) - - -def test_header_without_code_caption_skips_numbering(rules: Rules) -> None: - cand = ocr( - [ - ["Титул 2025", "", "", ""], - ["Показатель", "", "На 30.09.2025", "На 31.12.2024"], - ["1", "2", "3", "4"], - ["Запасы", "1210", "5", "4"], - ] - ) - assert find_header_rows(cand.rows, 0, 3, 1, rules) == (1,) - assert find_header_rows(cand.rows, 3, 3, 1, rules) == () - - -def test_name_cell_falls_back_to_left_text(rules: Rules) -> None: - row = ocr([["5.1", "Запасы", "", "1210"]]).rows[0] - cell = name_cell(row, 2, 3, 0) - assert cell is not None and cell.text == "Запасы" - assert name_cell(ocr([["5.1", "", "", "1210"]]).rows[0], 2, 3, 0) is None - - -def test_two_row_header_with_caption_in_upper_row_is_glued_with_lower(rules: Rules) -> None: - """I-1: «Код» в верхней строке, годы — в нижней (шаблон банка).""" - cand = ocr( - [ - ["Наименование", "Код", "На 30 сентября", "На 31 декабря", "На 31 декабря"], - ["", "", "2025 г.", "2024 г.", "2023 г."], - ["Запасы", "1210", "5", "4", "3"], - ] - ) - header = find_header_rows(cand.rows, 0, 2, 1, rules) - assert header == (0, 1) - assert segment_roles(cand.rows, 0, 2, header, rules) == [RowRole.HEADER, RowRole.HEADER, RowRole.DATA] - - -def test_two_row_header_lower_skips_numbering_and_ignores_unrelated_row(rules: Rules) -> None: - glued = ocr( - [ - ["Наименование", "Код", "На 30 сентября", "На 31 декабря"], - ["", "", "2025 г.", "2024 г."], - ["1", "2", "3", "4"], - ["Запасы", "1210", "5", "4"], - ] - ) - assert find_header_rows(glued.rows, 0, 3, 1, rules) == (0, 1) - plain = ocr( - [ - ["Наименование", "Код", "На 30.09.2025", "На 31.12.2024"], - ["Справочно", "", "", ""], - ["Запасы", "1210", "5", "4"], - ] - ) - assert find_header_rows(plain.rows, 0, 2, 1, rules) == (0,) diff --git a/api/arm_statements/tests/test_lines.py b/api/arm_statements/tests/test_lines.py deleted file mode 100644 index 12484b0..0000000 --- a/api/arm_statements/tests/test_lines.py +++ /dev/null @@ -1,63 +0,0 @@ -from decimal import Decimal as D - -from arm_statements.header import parse_header -from arm_statements.issues import make_warning -from arm_statements.lines import build_lines, units_warnings -from arm_statements.loader import Rules -from arm_statements.model import FormCandidate, Severity, StatementLine, StatementWarning -from arm_statements.rows import collect_raw_lines -from arm_statements.segments import find_layout -from arm_statements.tests.builders import excel, hdr, ocr - -HEAD = ["Наименование показателя", "Код", "На 30.09.2025"] -CODE_CONFIDENCE = 99.0 -NAME_CONFIDENCE = 71.0 - - -def statement_lines(cand: FormCandidate, rules: Rules) -> tuple[list[StatementLine], list[StatementWarning]]: - layout = find_layout(cand, parse_header(hdr(), rules), rules) - raw, _ = collect_raw_lines(cand, layout, rules) - return build_lines(raw, cand.source, rules) - - -def test_ocr_name_and_code_confidence(rules: Rules) -> None: - cand = ocr([HEAD, [("Дебиторская задолжен-ность", 71.0), "1230", "5"], ["Итого", "I1О0", "5"]]) - lines, _ = statement_lines(cand, rules) - assert (lines[0].line_name, lines[0].line_name_confidence) == ("Дебиторская задолжен-ность", NAME_CONFIDENCE) - assert (lines[1].line_code, lines[1].line_code_text, lines[1].line_code_confidence) == ( - "1100", - "I1О0", - CODE_CONFIDENCE, - ) - assert [v.column_index for v in lines[0].values] == [2] - - -def test_value_out_of_range_warning(rules: Rules) -> None: - cand = excel([("Баланс", [["Наименование", "Код", "На 30.09.2025"], ["Запасы", "1210", D("123456789012345678")]])]) - lines, warnings = statement_lines(cand, rules) - assert lines[0].values[0].amount is None - assert [(w.code, w.severity, w.line_refs, w.column_index) for w in warnings] == [ - ("VALUE_OUT_OF_RANGE", Severity.INFO, ("Баланс!B2",), 2) - ] - - -def test_excel_code_ref_and_clean_confidence(rules: Rules) -> None: - cand = excel([("Баланс", [["Наименование", "Код", "На 30.09.2025"], ["Запасы", "1210", D("5")]])]) - lines, warnings = statement_lines(cand, rules) - assert (lines[0].ref, lines[0].line_code, lines[0].line_code_text) == ("Баланс!B2", "1210", "1210") - assert (lines[0].line_code_confidence, lines[0].needs_review, warnings) == (100.0, False, []) - - -def test_fractional_in_thousands(rules: Rules) -> None: - pairs = [("1210", "1.5"), ("1220", "2.25"), ("1230", "3"), ("1240", "4"), ("1200", "10.75")] - title: list[str | D | None] = ["Наименование", "Код", "На 30.09.2025"] - body: list[list[str | D | None]] = [[f"Строка {i}", c, D(v)] for i, (c, v) in enumerate(pairs)] - rows = [title, *body] - cand = excel([("Баланс", rows)]) - lines, _ = statement_lines(cand, rules) - head = parse_header(hdr(units="тыс. руб."), rules) - assert [w.code for w in units_warnings(lines, head, (), rules)] == ["FRACTIONAL_IN_THOUSANDS"] - xlsb = make_warning("XLSB_NO_FORMATS", "формат чисел xlsb не читается", rules) - (scale,) = units_warnings(lines, head, (xlsb,), rules) - assert (scale.code, scale.severity) == ("XLSB_SCALE_SUSPECT", Severity.ERROR) - assert units_warnings(lines, parse_header(hdr(units="руб."), rules), (), rules) == [] diff --git a/api/arm_statements/tests/test_loader.py b/api/arm_statements/tests/test_loader.py deleted file mode 100644 index fe1e0f9..0000000 --- a/api/arm_statements/tests/test_loader.py +++ /dev/null @@ -1,168 +0,0 @@ -import re -from decimal import Decimal -from pathlib import Path - -import pytest -import yaml - -from arm_statements.issues import WarningCode, make_warning -from arm_statements.loader import LABEL_KEYS, Rules, RulesError, load_rules -from arm_statements.model import DocumentType, Edition, FormVariant, Severity - -RULES_DIR = Path(__file__).parent.parent / "rules" - -SPEC_WARNINGS = { - "JSON_FAILED": ("error", "file"), - "TABLE_PARSE_ERROR": ("error", "file"), - "TABLE_LOST": ("error", "file"), - "TABLE_SKIPPED": ("info", "file"), - "CODE_COLUMN_NOT_FOUND": ("error", "document"), - "SEGMENT_WITHOUT_CODES": ("error", "document"), - "SHEET_NOT_JOINED": ("error", "document"), - "CONTINUATION_TYPE_CONFLICT": ("error", "document"), - "SEGMENT_COLUMNS_MISMATCH": ("error", "document"), - "COLUMN_WITHOUT_DATE": ("error", "document"), - "COLUMN_DATE_UNCERTAIN": ("error", "document"), - "HEADER_ROW_NOT_FOUND": ("error", "document"), - "TYPE_CONFLICT": ("error", "document"), - "NO_ORGANIZATION_NO_INN": ("error", "document"), - "PERIOD_UNKNOWN": ("error", "document"), - "REPORT_DATE_UNCERTAIN": ("error", "document"), - "XLSB_SCALE_SUSPECT": ("error", "document"), - "INN_NOT_FOUND": ("info", "document"), - "INN_UNCERTAIN": ("info", "document"), - "ORGANIZATION_NOT_FOUND": ("info", "document"), - "ORGANIZATION_UNCERTAIN": ("info", "document"), - "REPORT_DATE_UNPARSED": ("info", "document"), - "UNITS_NOT_FOUND": ("info", "document"), - "FILE_TYPE_MISMATCH": ("info", "file"), - "COLUMN_SKIPPED": ("info", "document"), - "FRACTIONAL_IN_THOUSANDS": ("info", "document"), - "XLSB_NO_FORMATS": ("info", "document"), - "NUMBER_FORMAT_UNPARSED": ("info", "document"), - "VALUE_OUT_OF_RANGE": ("info", "document"), - "SHEET_OTHER": ("info", "file"), - "ROW_SPLIT": ("info", "document"), - "NUMBERS_OUTSIDE_TABLE": ("info", "document"), - "XLS_NO_CODEPAGE": ("info", "file"), -} - - -def test_warning_table_matches_spec(rules: Rules) -> None: - assert {c: (w.severity.value, w.scope) for c, w in rules.warnings.items()} == SPEC_WARNINGS - - -def test_simplified_income_relation_is_soft(rules: Rules) -> None: - (rule,) = rules.checks.relations_for(DocumentType.INCOME, FormVariant.SIMPLIFIED) - assert rule.target == "2400" and not rule.strict - assert rule.abs_terms == frozenset({"2120", "2330", "2350", "2410"}) - assert rules.checks.abs_codes(DocumentType.INCOME, FormVariant.FULL) == frozenset( - {"2120", "2210", "2220", "2330", "2350", "2410"} - ) - - -def test_code_orders_with_repeats(rules: Rules) -> None: - fsbu = rules.code_orders.sequence(Edition.FSBU4, DocumentType.BALANCE, FormVariant.FULL) - assert fsbu.count("1220") == len(("1220", "1220")) and "1120" not in fsbu - ras = rules.code_orders.sequence(Edition.RAS_66N, DocumentType.BALANCE, FormVariant.FULL) - assert "1120" in ras and ras.index("1110") < ras.index("1100") - assert rules.code_orders.editions == (Edition.RAS_66N, Edition.FSBU4) - assert rules.code_orders.sequence(Edition.FSBU4, DocumentType.DISCLOSURE, FormVariant.FULL) == () - - -def test_public_rule_types(rules: Rules) -> None: - assert isinstance(rules.checks.tolerance, Decimal) and rules.checks.tolerance == Decimal("1") - assert all(isinstance(e, Edition) for e in rules.code_orders.editions) - fsbu = rules.code_orders.sequence(Edition.FSBU4, DocumentType.INCOME, FormVariant.SIMPLIFIED) - assert isinstance(fsbu, tuple) and all(isinstance(c, str) for c in fsbu) - assert (rules.code_ranges.min_codes_continuation, rules.code_ranges.min_codes) == (1, 3) - assert tuple(rules.headers.labels) == LABEL_KEYS - - -def test_warning_code_enum_matches_dictionary(rules: Rules) -> None: - assert {c.value for c in WarningCode} == set(rules.warnings) == set(SPEC_WARNINGS) - assert WarningCode.XLS_NO_CODEPAGE == "XLS_NO_CODEPAGE" - - -def test_no_code_to_name_mapping_anywhere() -> None: - code = re.compile(r"^\d{4,5}$") - - def walk(node: object) -> None: - if isinstance(node, dict): - for key, value in node.items(): - assert not (code.match(str(key)) and isinstance(value, str) and re.search(r"[А-Яа-яЁё]", value)) - walk(value) - elif isinstance(node, list): - for item in node: - walk(item) - - for path in RULES_DIR.glob("*.yaml"): - walk(yaml.safe_load(path.read_text(encoding="utf-8"))) - - -def test_broken_repeat_count_rejected(tmp_path: Path) -> None: - for path in RULES_DIR.glob("*.yaml"): - (tmp_path / path.name).write_bytes(path.read_bytes()) - orders = tmp_path / "statement_code_orders.yaml" - orders.write_text( - orders.read_text(encoding="utf-8").replace('- "1520" # «Обязательство', '# "1520"'), encoding="utf-8" - ) - with pytest.raises(RulesError, match="statement_code_orders"): - load_rules(tmp_path) - - -def test_unknown_severity_rejected(tmp_path: Path) -> None: - for path in RULES_DIR.glob("*.yaml"): - (tmp_path / path.name).write_bytes(path.read_bytes()) - target = tmp_path / "statement_warnings.yaml" - target.write_text( - target.read_text(encoding="utf-8").replace("severity: info", "severity: warn", 1), encoding="utf-8" - ) - with pytest.raises(RulesError, match="statement_warnings"): - load_rules(tmp_path) - - -def test_make_warning_takes_severity_from_dictionary(rules: Rules) -> None: - warning = make_warning(WarningCode.COLUMN_SKIPPED, "колонка «Изменение, %» пропущена", rules, column_index=5) - assert (warning.code, warning.severity, warning.column_index) == ("COLUMN_SKIPPED", Severity.INFO, 5) - assert make_warning("JSON_FAILED", "x", rules).severity is Severity.ERROR - - -def test_make_warning_unknown_code_raises(rules: Rules) -> None: - with pytest.raises(KeyError, match="NOT_A_CODE"): - make_warning("NOT_A_CODE", "x", rules) - - -def test_label_keys_are_checked(tmp_path: Path) -> None: - for path in RULES_DIR.glob("*.yaml"): - (tmp_path / path.name).write_bytes(path.read_bytes()) - target = tmp_path / "statement_headers.yaml" - target.write_text(target.read_text(encoding="utf-8").replace(" okpo:", " okpo_x:", 1), encoding="utf-8") - with pytest.raises(RulesError, match="statement_headers"): - load_rules(tmp_path) - - -def test_absolute_period_kinds_come_from_dictionary(rules: Rules) -> None: - expected = [ - "range_end", - "day_month_year", - "quarter", - "half", - "months", - "month_range", - "year", - ] - data = yaml.safe_load((RULES_DIR / "statement_periods.yaml").read_text(encoding="utf-8")) - assert [p["kind"] for p in data["absolute"]] == expected - assert [p.kind for p in rules.periods.absolute] == expected - - -def test_repeats_for_unknown_form_rejected(tmp_path: Path) -> None: - for path in RULES_DIR.glob("*.yaml"): - (tmp_path / path.name).write_bytes(path.read_bytes()) - orders = tmp_path / "statement_code_orders.yaml" - orders.write_text( - orders.read_text(encoding="utf-8").replace(" INCOME: []", " DISCLOSURE: []", 1), encoding="utf-8" - ) - with pytest.raises(RulesError, match="repeats"): - load_rules(tmp_path) diff --git a/api/arm_statements/tests/test_model.py b/api/arm_statements/tests/test_model.py deleted file mode 100644 index 6885794..0000000 --- a/api/arm_statements/tests/test_model.py +++ /dev/null @@ -1,118 +0,0 @@ -import json -from datetime import date -from decimal import Decimal as D - -import arm_statements -from arm_statements.model import Cell, DocumentType, Evidence, FormCandidate, HeaderCells, Source -from arm_statements.serialize import candidate_from_dict, to_jsonable -from arm_statements.tests.builders import excel, ocr - - -def test_candidate_roundtrip_through_json() -> None: - cand = FormCandidate( - source=Source.EXCEL, - locator="sheet:Баланс", - type_hint=DocumentType.BALANCE, - evidence=Evidence(okud=Cell("0710001", None, 100.0, "Баланс!F3"), title=None), - header=HeaderCells(report_date=Cell("30.09.2025", date(2025, 9, 30), 100.0, "Баланс!C5")), - rows=((Cell("106 311", D("106311"), 100.0, "Баланс!D27"), None, Cell("да", True, 100.0, "Баланс!F27")),), - ) - data = json.loads(json.dumps(to_jsonable(cand), ensure_ascii=False)) - assert data["rows"][0][0]["value"] == {"decimal": "106311"} - assert data["header"]["report_date"]["value"] == {"date": "2025-09-30"} - assert candidate_from_dict(data) == cand - - -def test_excel_builder_refs_follow_excel_quoting() -> None: - cand = excel([("Баланс", [["", "", "", "1110"]]), ("Бухгалтерский баланс. Лист 1", [["", "", "1150"]])]) - assert cand.rows[0][3] is not None and cand.rows[0][3].ref == "Баланс!D1" - assert cand.rows[1][2] is not None and cand.rows[1][2].ref == "'Бухгалтерский баланс. Лист 1'!C1" - assert cand.segment_breaks == (1,) - assert cand.locator == "sheet:Баланс,Бухгалтерский баланс. Лист 1" - - -def test_ocr_builder_cells() -> None: - cand = ocr([["Выручка", ("[190 :", 25.0), None]]) - cell = cand.rows[0][1] - assert cell == Cell("[190 :", None, 25.0, "row:0") - assert cand.rows[0][2] is None and cand.source is Source.OCR - - -PUBLIC = { - "Source", - "DocumentType", - "FormVariant", - "Edition", - "PeriodKind", - "LineStatus", - "Severity", - "DocStatus", - "EnvelopeStatus", - "SignForm", - "CellValue", - "Cell", - "Evidence", - "HeaderCells", - "HEADER_FIELDS", - "StatementWarning", - "FormCandidate", - "HeaderField", - "DateField", - "StatementHeader", - "StatementColumn", - "StatementValue", - "StatementLine", - "FailedRelation", - "StatementChecks", - "StatementDocument", - "CandidateResult", - "NormalizationResult", - "Rules", - "RulesError", - "load_rules", - "default_rules", - "normalize_candidate", - "normalize", - "parse_header", - "type_hint", - "continuation_type", - "sheet_joins", - "parse_code", - "parse_amount", - "parse_date", - "resolve_relative", - "fix_lookalikes", - "make_warning", - "StatusInput", - "StatusLine", - "StatusValue", - "document_status", - "envelope_status", - "StoredForm", - "StoredColumn", - "StoredLine", - "StoredValue", - "StoredWarning", - "RecomputeResult", - "recompute", - "to_jsonable", - "candidate_from_dict", - "statements_version", - "WarningCode", - "choose_edition", - "status_input", - "status_input_from_contract", - "to_contract", - "warning_to_contract", - "code_column", - "amount_columns", - "normalize_label", - "normalize_caption", - "is_continuation", -} - - -def test_public_api() -> None: - assert set(arm_statements.__all__) == PUBLIC - assert len(arm_statements.__all__) == len(set(arm_statements.__all__)) - assert all(hasattr(arm_statements, name) for name in PUBLIC) diff --git a/api/arm_statements/tests/test_pipeline.py b/api/arm_statements/tests/test_pipeline.py deleted file mode 100644 index 4c5ad8a..0000000 --- a/api/arm_statements/tests/test_pipeline.py +++ /dev/null @@ -1,212 +0,0 @@ -from decimal import Decimal as D - -from arm_statements import document_status, normalize, normalize_candidate, status_input -from arm_statements.issues import make_warning -from arm_statements.loader import Rules -from arm_statements.model import DocStatus, DocumentType, FormVariant, Severity -from arm_statements.tests.builders import excel, hdr, ocr - -HEAD = ["Наименование показателя", "Код", "На 30.09.2025"] -FULL_CONFIDENCE = 100.0 -FULL_HEADER = hdr(organization_name="ООО «Пример»", inn="7700000000", report_date="30.09.2025", units="тыс. руб.") - - -def bal_rows() -> list[list[str]]: - return [ - HEAD, - ["Запасы", "1210", "60"], - ["Дебиторы", "1230", "40"], - ["Итого II", "1200", "100"], - ["НМА", "1110", "50"], - ["Итого I", "1100", "50"], - ["БАЛАНС", "1600", "150"], - ["Капитал", "1310", "150"], - ["Итого III", "1300", "150"], - ["БАЛАНС", "1700", "150"], - ] - - -def test_clean_ocr_balance_done(rules: Rules) -> None: - cand = ocr(bal_rows(), header=FULL_HEADER, type_hint=DocumentType.BALANCE, title="Бухгалтерский баланс") - result = normalize_candidate(cand, rules) - doc = result.document - assert doc is not None and result.file_warnings == () - assert doc.header.form_name == "Бухгалтерский баланс" - assert document_status(status_input(doc)) is doc.status - assert (doc.document_type, doc.form_variant, doc.status) == (DocumentType.BALANCE, FormVariant.FULL, DocStatus.DONE) - assert doc.checks is not None and doc.checks.relations_failed == () - assert doc.header.inn.value == "7700000000" - - -def test_disclosure_hint_with_balance_codes(rules: Rules) -> None: - doc = normalize_candidate(ocr(bal_rows(), header=FULL_HEADER, type_hint=DocumentType.DISCLOSURE), rules).document - assert doc is not None and doc.document_type is DocumentType.BALANCE - assert [(w.code, w.severity) for w in doc.warnings if w.code == "TYPE_CONFLICT"] == [ - ("TYPE_CONFLICT", Severity.ERROR) - ] - - -def test_balance_hint_with_income_codes(rules: Rules) -> None: - rows = [ - HEAD, - ["Выручка", "2110", "100"], - ["Себестоимость", "2120", "(60)"], - ["Валовая", "2100", "40"], - ["Коммерческие", "2210", "(10)"], - ["Прибыль от продаж", "2200", "30"], - ] - doc = normalize_candidate(ocr(rows, header=FULL_HEADER, type_hint=DocumentType.BALANCE), rules).document - assert doc is not None and doc.document_type is DocumentType.INCOME - assert "TYPE_CONFLICT" in [w.code for w in doc.warnings] - - -def test_excel_other_sheet(rules: Rules) -> None: - result = normalize_candidate(excel([("Примечания", [["Пояснительный текст без цифр"]])]), rules) - assert result.document is None - assert [(w.code, w.severity, w.locator) for w in result.file_warnings] == [ - ("SHEET_OTHER", Severity.INFO, "sheet:Примечания") - ] - - -def test_ocr_lost_and_skipped_tables(rules: Rules) -> None: - lost = normalize_candidate(ocr([["Итого", "", "15 000"]], locator="file:a.json;table:1"), rules) - assert [(w.code, w.locator) for w in lost.file_warnings] == [("TABLE_LOST", "file:a.json;table:1")] - skipped = normalize_candidate(ocr([["Руководитель", "Иванов"]]), rules) - assert [w.code for w in skipped.file_warnings] == ["TABLE_SKIPPED"] and skipped.document is None - - -def test_hint_without_code_column(rules: Rules) -> None: - doc = normalize_candidate( - ocr([["Итого", "15 000"]], type_hint=DocumentType.BALANCE, header=FULL_HEADER), rules - ).document - assert doc is not None and (doc.lines, doc.columns, doc.checks) == ((), (), None) - assert doc.form_variant is FormVariant.FULL - assert doc.status is DocStatus.NEEDS_CHECK and "CODE_COLUMN_NOT_FOUND" in [w.code for w in doc.warnings] - disc = normalize_candidate(ocr([["Текст"]], type_hint=DocumentType.DISCLOSURE), rules).document - assert disc is not None and (disc.status, disc.form_variant, disc.lines) == (DocStatus.DONE, None, ()) - assert disc.header.form_name is None - - -def test_disclosure_by_codes(rules: Rules) -> None: - rows = [HEAD, ["Пояснение", "5101", "1"], ["Пояснение", "5102", "2"], ["Пояснение", "5100", "3"]] - doc = normalize_candidate(ocr(rows, header=FULL_HEADER), rules).document - assert doc is not None and (doc.document_type, doc.lines, doc.status) == ( - DocumentType.DISCLOSURE, - (), - DocStatus.DONE, - ) - - -def test_simplified_balance(rules: Rules) -> None: - rows = [ - HEAD, - ["Матактивы", "1150", "50"], - ["Запасы", "1210", "50"], - ["БАЛАНС", "1600", "100"], - ["Капитал", "1300", "100"], - ["БАЛАНС", "1700", "100"], - ] - doc = normalize_candidate(ocr(rows, header=FULL_HEADER), rules).document - assert doc is not None and doc.form_variant is FormVariant.SIMPLIFIED - assert doc.checks is not None and doc.checks.relations_failed == () - - -def test_income_without_inn_still_done(rules: Rules) -> None: - rows: list[list[str | D]] = [ - ["Наименование", "Код", "За 9 месяцев 2025 г."], - ["Выручка", "2110", D("100")], - ["Себестоимость", "2120", D("-60")], - ["Валовая прибыль", "2100", D("40")], - ["Прибыль от продаж", "2200", D("40")], - ["Прибыль до налогообложения", "2300", D("40")], - ["Чистая прибыль", "2400", D("40")], - ] - head = hdr(organization_name="ООО «Пример»", units="тыс. руб.") - doc = normalize_candidate(excel([("ОФР", rows)], header=head), rules).document - assert doc is not None and doc.status is DocStatus.DONE - assert [(w.code, w.severity) for w in doc.warnings] == [("INN_NOT_FOUND", Severity.INFO)] - - -def test_normalize_collects(rules: Rules) -> None: - result = normalize([ocr(bal_rows(), header=FULL_HEADER), ocr([["Руководитель", "Иванов"]])], rules) - assert len(result.documents) == 1 and [w.code for w in result.file_warnings] == ["TABLE_SKIPPED"] - - -# --- P1-6: у DISCLOSURE предупреждений шапки нет, статус DONE (требование ревью T15.1) --- - - -def test_disclosure_by_hint_empty_header_has_no_warnings(rules: Rules) -> None: - doc = normalize_candidate(ocr([["Текст"]], type_hint=DocumentType.DISCLOSURE), rules).document - assert doc is not None - assert (doc.warnings, doc.status) == ((), DocStatus.DONE) - - -def test_disclosure_by_codes_empty_header_has_no_warnings(rules: Rules) -> None: - rows = [HEAD, ["Пояснение", "5101", "1"], ["Пояснение", "5102", "2"], ["Пояснение", "5100", "3"]] - doc = normalize_candidate(ocr(rows), rules).document - assert doc is not None and doc.document_type is DocumentType.DISCLOSURE - assert (doc.warnings, doc.status) == ((), DocStatus.DONE) - - -# --- ROW_SPLIT: предупреждение документа без locator (место — locator документа, спека «Шаг 8 — предупреждения») --- - - -def test_row_split_has_no_locator(rules: Rules) -> None: - rows = bal_rows() - rows[1] = ["Запасы и дебиторы", "1210\n1230", "60\n40"] - doc = normalize_candidate(ocr(rows, header=FULL_HEADER, type_hint=DocumentType.BALANCE), rules).document - assert doc is not None - split = [w for w in doc.warnings if w.code == "ROW_SPLIT"] - assert len(split) == 1 and split[0].locator is None and split[0].severity is Severity.INFO - - -# --- строка «в тысячах рублей» над таблицей и уверенность даты отчёта у Excel --- - - -def test_units_row_above_table_is_not_header_and_excel_date_is_confident(rules: Rules) -> None: - rows = [["Бухгалтерский баланс"], ["в тысячах рублей"], *bal_rows()] - cand = excel([("Баланс", rows)], header=FULL_HEADER, type_hint=DocumentType.BALANCE) - doc = normalize_candidate(cand, rules).document - assert doc is not None - assert doc.header.report_date.confidence == FULL_CONFIDENCE and doc.header.report_date.value is not None - assert [c.column_index for c in doc.columns] == [2] - assert (doc.status, [w.code for w in doc.warnings]) == (DocStatus.DONE, []) - - -# --- порядок предупреждений документа: адаптера, раскладки, строк, единиц, TYPE_CONFLICT, шапки --- - - -def test_document_warnings_order(rules: Rules) -> None: - adapter = make_warning("NUMBER_FORMAT_UNPARSED", "формат", rules) - rows = [["Наименование", "Код", "Итого"], *bal_rows()[1:]] - rows[1] = ["Запасы и дебиторы", "1210\n1230", "60,5\n40,5"] - cand = excel([("Баланс", rows)], header=hdr(units="тыс. руб."), type_hint=DocumentType.INCOME, warnings=[adapter]) - doc = normalize_candidate(cand, rules).document - assert doc is not None - codes = [w.code for w in doc.warnings] - expected = [ - "NUMBER_FORMAT_UNPARSED", - "COLUMN_WITHOUT_DATE", - "ROW_SPLIT", - "FRACTIONAL_IN_THOUSANDS", - "TYPE_CONFLICT", - "NO_ORGANIZATION_NO_INN", - ] - assert [c for c in codes if c in expected] == [c for c in expected if c in codes] - assert codes[0] == "NUMBER_FORMAT_UNPARSED" and codes.index("TYPE_CONFLICT") < codes.index("NO_ORGANIZATION_NO_INN") - - -# --- числа правее таблицы формы (решение владельца 18.10.2026): не колонки сумм, статус не портят --- - - -def test_numbers_right_of_table_do_not_break_status_in_repeated_segments(rules: Rules) -> None: - rows: list[list[str | None]] = [[*r, None] for r in bal_rows()] - rows[6][3] = "7" # формула аналитика правее таблицы, в строке БАЛАНС (1600) - rows[9][3] = "8" - rows.insert(7, [*HEAD, None]) # повторная шапка второго сегмента (страница 2 формы) - cand = ocr(rows, header=FULL_HEADER, type_hint=DocumentType.BALANCE) - doc = normalize_candidate(cand, rules).document - assert doc is not None and doc.status is DocStatus.DONE - assert [c.column_index for c in doc.columns] == [2] - assert [(w.code, w.severity) for w in doc.warnings] == [("NUMBERS_OUTSIDE_TABLE", Severity.INFO)] - assert all(len(line.values) == 1 for line in doc.lines) diff --git a/api/arm_statements/tests/test_recompute.py b/api/arm_statements/tests/test_recompute.py deleted file mode 100644 index 7e9d3c0..0000000 --- a/api/arm_statements/tests/test_recompute.py +++ /dev/null @@ -1,118 +0,0 @@ -from dataclasses import replace -from datetime import date -from decimal import Decimal as D - -from arm_statements import StoredColumn, StoredForm, StoredLine, StoredValue, choose_edition, recompute -from arm_statements.loader import Rules -from arm_statements.model import DocStatus, DocumentType, Edition, FormVariant, LineStatus, PeriodKind -from arm_statements.structure import edition_by_date - -OK, DETAIL = LineStatus.OK, LineStatus.DETAIL -COL = StoredColumn("c1", 2, date(2025, 9, 30), PeriodKind.ABSOLUTE) - - -def sl( # noqa: PLR0913 -- конструктор тестовых строк - key: str, - row: int | None, - code: str | None, - amount: str | None, - *, - status: LineStatus = OK, - conf: float = 100.0, - code_fixed: bool = False, - value_conf: float = 100.0, - value_fixed: bool = False, - recognized: bool = True, -) -> StoredLine: - value = StoredValue("c1", None if amount is None else D(amount), amount, value_conf, recognized, value_fixed) - return StoredLine(key, row, status, code, conf, None, code_fixed, None, (value,)) - - -def form(*lines: StoredLine, document_type: DocumentType = DocumentType.BALANCE) -> StoredForm: - return StoredForm(document_type, date(2025, 9, 30), True, True, (), False, (), (COL,), tuple(lines)) - - -def test_recompute_after_edit(rules: Rules) -> None: - result = recompute( - form( - sl("l1210", 0, "1210", "100"), - sl("l1230", 1, "1230", "300", value_conf=60.0, value_fixed=True), - sl("l1250", 2, "1250", "50", conf=40.0, code_fixed=True), - sl("l1260", None, "1260", "50"), - sl("ldet", 3, None, "7", status=DETAIL, value_conf=60.0), - sl("l1200", 4, "1200", "500"), - ), - rules, - ) - # 1200 = 100 + 300 + 50 + 50 (с 1250 и строкой правки 1260) — сходится; 1600 нет — «пустая цель» - assert [(f.relation, f.column_key, f.column_index) for f in result.checks.relations_failed] == [ - ("1600 = 1100 + 1200", "c1", 2) - ] - assert result.value_needs_review[("l1230", "c1")] is False - assert result.value_needs_review[("ldet", "c1")] is False - assert result.line_needs_review["l1250"] is False - assert result.detail_parents == {"ldet": "l1250"} - assert (result.edition, result.form_variant, result.status) == ( - Edition.FSBU4, - FormVariant.FULL, - DocStatus.NEEDS_CHECK, - ) - - -def test_code_change_switches_variant(rules: Rules) -> None: - simplified = form( - sl("a", 0, "1150", "50"), sl("b", 1, "1600", "50"), sl("c", 2, "1300", "50"), sl("d", 3, "1700", "50") - ) - assert recompute(simplified, rules).form_variant is FormVariant.SIMPLIFIED - edited = form( - sl("a", 0, "1100", "50", code_fixed=True), - sl("b", 1, "1600", "50"), - sl("c", 2, "1300", "50"), - sl("d", 3, "1700", "50"), - ) - result = recompute(edited, rules) - assert result.form_variant is FormVariant.FULL - assert result.status is DocStatus.NEEDS_CHECK - - -def test_edition_equals_choose_edition(rules: Rules) -> None: - stored = form(sl("a", 0, "1110", "10"), sl("b", 1, "1120", "5", conf=60.0), sl("c", 2, "1130", "5")) - pairs = [("1110", True), ("1120", False), ("1130", True)] - expected = choose_edition(DocumentType.BALANCE, pairs, date(2025, 9, 30), rules) - assert recompute(stored, rules).edition is expected is Edition.RAS_66N - - -def test_accepted_and_empty_edit_line(rules: Rules) -> None: - broken = form(sl("a", 0, "1600", "100"), sl("b", 1, "1700", "90"), sl("e", None, "1220", None)) - assert recompute(broken, rules).status is DocStatus.NEEDS_CHECK - assert recompute(replace(broken, accepted=True), rules).status is DocStatus.DONE - - -def test_disclosure_has_complete_maps(rules: Rules) -> None: - """I-3: для расшифровки в словарях ключи всех строк и значений, а не пустые словари.""" - stored = form( - sl("a", 0, "1110", "10"), - sl("d", 1, None, "7", status=DETAIL), - sl("e", None, "1120", None), - document_type=DocumentType.DISCLOSURE, - ) - result = recompute(stored, rules) - assert result.line_needs_review == {"a": False, "d": False, "e": False} - assert result.value_needs_review == {("a", "c1"): False, ("d", "c1"): False, ("e", "c1"): False} - assert result.detail_parents == {"d": "a"} - assert (result.form_variant, result.checks.relations_failed, result.status) == (None, (), DocStatus.DONE) - assert result.edition is edition_by_date(date(2025, 9, 30)) - - -def test_edit_line_without_amount_is_excluded_from_checks_and_variant(rules: Rules) -> None: - """T18.6: пустая строка правки не участвует в §7 и не влияет на вариант.""" - base = [sl("a", 0, "1150", "50"), sl("b", 1, "1600", "50"), sl("c", 2, "1300", "50"), sl("d", 3, "1700", "50")] - with_edit = recompute(form(*base, sl("e", None, "1100", None)), rules) - without = recompute(form(*base), rules) - assert with_edit.form_variant is without.form_variant is FormVariant.SIMPLIFIED - assert with_edit.checks == without.checks - - -def test_detail_without_row_index_has_no_parent(rules: Rules) -> None: - stored = form(sl("a", 0, "1210", "10"), sl("d", None, None, "3", status=DETAIL)) - assert recompute(stored, rules).detail_parents == {"d": None} diff --git a/api/arm_statements/tests/test_review.py b/api/arm_statements/tests/test_review.py deleted file mode 100644 index 96de895..0000000 --- a/api/arm_statements/tests/test_review.py +++ /dev/null @@ -1,139 +0,0 @@ -from datetime import date -from decimal import Decimal as D - -from arm_statements.columns import DocumentDates -from arm_statements.evaluation import EvalLine, EvalValue -from arm_statements.header import parse_header -from arm_statements.loader import Rules -from arm_statements.model import DocumentType, LineStatus -from arm_statements.review import evaluate, review_header -from arm_statements.tests.builders import hdr - -OK, DETAIL, UNREADABLE = LineStatus.OK, LineStatus.DETAIL, LineStatus.UNREADABLE -NO_DATES = DocumentDates(None, None, None, None) - - -def ln( # noqa: PLR0913, PLR0917 -- сигнатура задана планом (Task 15) - code: str | None, - text: str, - amount: str | None, - conf: float = 100.0, - recognized: bool = True, - status: LineStatus = OK, - order: int = 0, -) -> EvalLine: - value = EvalValue("0", None if amount is None else D(amount), text, conf, recognized) - return EvalLine(f"{code}-{order}", order, status, code, 100.0, False, (value,)) - - -def flags(lines: list[EvalLine], doc: DocumentType, rules: Rules) -> dict[str, bool]: - result = evaluate(lines, ["0"], doc, date(2025, 9, 30), rules) - return {key: flag for (key, _), flag in result.value_needs_review.items()} - - -def test_guess_confirmed_by_relation(rules: Rules) -> None: - lines = [ - ln("1210", "100", "100", order=0), - ln("1230", "42831 1", "428311", recognized=False, order=1), - ln("1200", "428411", "428411", order=2), - ] - assert flags(lines, DocumentType.BALANCE, rules)["1230-1"] is False - - -def test_sign_in_curly_bracket_confirmed(rules: Rules) -> None: - lines = [ - ln("2110", "10000000", "10000000", order=0), - ln("2120", "(7744536}", "-7744536", recognized=False, order=1), - ln("2100", "2255464", "2255464", order=2), - ln("2210", "(100)", "-100", order=3), - ln("2220", "(200)", "-200", order=4), - ln("2200", "2255164", "2255164", order=5), - ] - assert flags(lines, DocumentType.INCOME, rules)["2120-1"] is False - - -def test_sign_outside_convention(rules: Rules) -> None: - lines = [ - ln("2110", "100000", "100000", order=0), - ln("2120", "38378", "38378", conf=60.0, order=1), - ln("2100", "61622", "61622", order=2), - ln("2210", "(100)", "-100", order=3), - ln("2220", "(200)", "-200", order=4), - ] - assert flags(lines, DocumentType.INCOME, rules)["2120-1"] is True - - -def test_null_amount_and_unchecked_code(rules: Rules) -> None: - lines = [ln("2120", "{38378)", None, recognized=False, order=0), ln("2900", "15", "15", conf=80.0, order=1)] - result = flags(lines, DocumentType.INCOME, rules) - assert (result["2120-0"], result["2900-1"]) == (True, True) - - -def test_detail_values_never_flagged(rules: Rules) -> None: - lines = [ln("2110", "100", "100", order=0), ln(None, "1 425 130", "1425130", conf=78.0, status=DETAIL, order=1)] - assert flags(lines, DocumentType.INCOME, rules)["None-1"] is False - - -def test_soft_relation_covers_only_exactly(rules: Rules) -> None: - exact = [ - ln("2300", "1000", "1000", order=0), - ln("2410", "200", "200", order=1), - ln("2400", "800", "800", conf=66.0, order=2), - ] - assert flags(exact, DocumentType.INCOME, rules)["2400-2"] is False - rounding = [ - ln("2300", "17000000000", "17000000000", order=0), - ln("2410", "0", "0", order=1), - ln("2400", "17000000009", "17000000009", conf=66.0, order=2), - ] - assert flags(rounding, DocumentType.INCOME, rules)["2400-2"] is True - - -def test_hard_failure_flags_clean_cells(rules: Rules) -> None: - lines = [ln("1210", "100", "100", order=0), ln("1230", "50", "50", order=1), ln("1200", "999", "999", order=2)] - result = evaluate(lines, ["0"], DocumentType.BALANCE, None, rules) - assert result.hard_failures >= 1 - assert all(result.value_needs_review[(f"{c}-{i}", "0")] for i, c in enumerate(["1210", "1230", "1200"])) - accepted = evaluate(lines, ["0"], DocumentType.BALANCE, None, rules, accepted=True) - assert not any(accepted.value_needs_review.values()) - - -def test_unreadable_values(rules: Rules) -> None: - lines = [ln(None, "7", "7", conf=60.0, status=UNREADABLE), ln(None, "5", "5", status=UNREADABLE, order=1)] - result = flags(lines, DocumentType.BALANCE, rules) - assert (result["None-0"], result["None-1"]) == (True, False) - - -def test_report_date_confirmed_or_not(rules: Rules) -> None: - head = parse_header( - hdr(report_date=("30.09.2025", 60.0), organization_name="ООО «Пример»", inn="7700000000", units="тыс. руб."), - rules, - ) - ok_dates = DocumentDates(date(2025, 9, 30), date(2025, 9, 30), date(2025, 9, 30), date(2025, 9, 30)) - reviewed, warnings = review_header(head, ok_dates, rules) - assert (reviewed.needs_review_fields, warnings) == ((), []) - relative_only = DocumentDates(date(2025, 9, 30), None, None, None) - reviewed, warnings = review_header(head, relative_only, rules) - assert reviewed.needs_review_fields == ("report_date",) - found = [(w.code, w.severity.value, w.field) for w in warnings] - assert found == [("REPORT_DATE_UNCERTAIN", "error", "report_date")] - - -def test_header_presence_warnings(rules: Rules) -> None: - cells = hdr(organization_name="ООО «Пример»", report_date="30.09.2025", units="тыс. руб.") - _, no_inn = review_header(parse_header(cells, rules), NO_DATES, rules) - assert [w.code for w in no_inn] == ["INN_NOT_FOUND"] - _, nothing = review_header(parse_header(hdr(), rules), NO_DATES, rules) - assert {w.code for w in nothing} == {"NO_ORGANIZATION_NO_INN", "PERIOD_UNKNOWN", "UNITS_NOT_FOUND"} - - -def test_uncertain_inn_and_unparsed_date(rules: Rules) -> None: - cells = hdr(organization_name=("ООО «Пример»", 70.0), inn=("7700000000", 80.0), report_date="дата?", units="руб.") - dates = DocumentDates(None, None, date(2025, 9, 30), None) - reviewed, warnings = review_header(parse_header(cells, rules), dates, rules) - assert reviewed.needs_review_fields == () - assert [(w.code, w.field) for w in warnings] == [ - ("ORGANIZATION_UNCERTAIN", "organization_name"), - ("INN_UNCERTAIN", "inn"), - ("REPORT_DATE_UNPARSED", "report_date"), - ] diff --git a/api/arm_statements/tests/test_rows.py b/api/arm_statements/tests/test_rows.py deleted file mode 100644 index 87ae28d..0000000 --- a/api/arm_statements/tests/test_rows.py +++ /dev/null @@ -1,115 +0,0 @@ -from typing import Any - -from arm_statements.header import parse_header -from arm_statements.loader import Rules -from arm_statements.model import LineStatus, StatementWarning -from arm_statements.rows import RawLine, collect_raw_lines -from arm_statements.segments import find_layout -from arm_statements.tests.builders import excel, hdr, ocr - -HEAD = ["Наименование показателя", "Код", "На 30.09.2025", "На 31.12.2024"] -OK, UNREADABLE, DETAIL, MISSING = LineStatus.OK, LineStatus.UNREADABLE, LineStatus.DETAIL, LineStatus.MISSING - - -def lines_of(rows: Any, rules: Rules, source: str = "ocr") -> tuple[list[RawLine], list[StatementWarning]]: - cand = ocr(rows) if source == "ocr" else excel([("Баланс", rows)]) - layout = find_layout(cand, parse_header(hdr(), rules), rules) - return collect_raw_lines(cand, layout, rules) - - -def test_junk_code_lines_survive(rules: Rules) -> None: - lines, _ = lines_of( - [ - HEAD, - ["НМА", "1110", "5", "4"], - ["Запасы", "1150", "5", "4"], - ["Мусор", "ИЧ,250", "7", "6"], - ["в том числе: прочее", "", "1", ""], - ["", "44 30\nI |", "", ""], - ["Итого", "1100", "10", "8"], - ], - rules, - ) - assert [(ln.line_status, ln.code) for ln in lines] == [ - (OK, "1110"), - (OK, "1150"), - (UNREADABLE, None), - (DETAIL, None), - (UNREADABLE, None), - (OK, "1100"), - ] - assert lines[3].parent_ref == lines[1].ref == "row:2" - assert [ln.row_index for ln in lines] == [0, 1, 2, 3, 4, 5] - - -def test_lookalike_only_code_cells_are_empty(rules: Rules) -> None: - lines, _ = lines_of( - [ - HEAD, - ["НМА", "1110", "5", "4"], - ["Запасы", "1150", "5", "4"], - ["", "изо :", ("3", 0.0), ""], - ["", "НИ", "I", ""], - ["", "| 140", "2", "1"], - ["Итого", "1100", "12", "9"], - ], - rules, - ) - assert [(ln.line_status, ln.code, ln.code_fixed) for ln in lines] == [ - (OK, "1110", False), - (OK, "1150", False), - (DETAIL, None, False), - (DETAIL, None, False), - (OK, "1140", True), - (OK, "1100", False), - ] - assert [ln.parent_ref for ln in lines[2:4]] == ["row:2", "row:2"] - assert len({ln.ref for ln in lines}) == len(lines) - - -def test_missing_before_first_code_and_name_only_row(rules: Rules) -> None: - lines, _ = lines_of( - [HEAD, ["Выручка", "", "100", "90"], ["Раздел I. Активы", "", "", ""], ["Запасы", "1150", "5", "4"]], rules - ) - assert [ln.line_status for ln in lines] == [MISSING, OK] - - -def test_duplicate_code_both_ok(rules: Rules) -> None: - lines, _ = lines_of([HEAD, ["Налог", "2410", "100", "1"], ["Налог УСН", "2410", "649", "2"]], rules) - assert [(ln.line_status, ln.code) for ln in lines] == [(OK, "2410"), (OK, "2410")] - - -def test_split_glued_rows(rules: Rules) -> None: - lines, warnings = lines_of( - [HEAD, ["Запасы", "1150", "5", "4"], ["Займы\nКредиторы", "1500\n1700", "917 775\n2 371 333", "1\n2"]], rules - ) - split = lines[1:] - assert [(ln.ref, ln.code, ln.split) for ln in split] == [("row:2#1", "1500", True), ("row:2#2", "1700", True)] - assert [ln.cells[0][1].text for ln in split if ln.cells[0][1] is not None] == ["917 775", "2 371 333"] - assert [ln.name.text for ln in split if ln.name is not None] == ["Займы", "Кредиторы"] - assert [(w.code, w.line_refs) for w in warnings] == [("ROW_SPLIT", ("row:2#1", "row:2#2"))] - - -def test_not_split_when_no_part_in_column_range(rules: Rules) -> None: - lines, warnings = lines_of( - [ - HEAD, - ["НМА", "1110", "5", "4"], - ["Запасы", "1150", "5", "4"], - ["", "44 30\nI |", "1\n2", ""], - ["Итого", "1100", "10", "8"], - ], - rules, - ) - assert [ln.line_status for ln in lines] == [OK, OK, UNREADABLE, OK] and warnings == [] - - -def test_excel_signature_text_is_not_content(rules: Rules) -> None: - rows = [ - ["Наименование", "", "Код", "На 30.09.2025"], - ["Запасы", "", "1150", "5"], - ["Руководитель", "", "", "Иванов"], - ["", "", "", "(расшифровка подписи)"], - ] - lines, _ = lines_of(rows, rules, source="excel") - assert [ln.ref for ln in lines] == ["Баланс!C2"] diff --git a/api/arm_statements/tests/test_segments.py b/api/arm_statements/tests/test_segments.py deleted file mode 100644 index c5c8fa2..0000000 --- a/api/arm_statements/tests/test_segments.py +++ /dev/null @@ -1,259 +0,0 @@ -from decimal import Decimal as D - -from arm_statements.header import parse_header -from arm_statements.loader import Rules -from arm_statements.model import RowRole -from arm_statements.segments import Layout, find_layout -from arm_statements.tests.builders import excel, hdr, ocr, place - -DATES3 = ["На 30 сентября 2025 г.", "На 31 декабря 2024 г.", "На 31 декабря 2023 г."] - - -def test_npz_sheet_with_own_header_maps_by_date(rules: Rules) -> None: - first = [ - place({1: "Наименование показателя", 12: "Код", 16: DATES3[0], 21: DATES3[1], 24: DATES3[2]}), - place({1: "Нематериальные активы", 12: "1110", 16: D("10"), 21: D("9"), 24: D("8")}), - place({1: "Итого по разделу I", 12: "1100", 16: D("10"), 21: D("9"), 24: D("8")}), - ] - sheet4 = [ - place({0: "Наименование показателя", 6: "Код", 8: DATES3[0], 9: DATES3[1], 10: DATES3[2]}), - place({0: "Заемные средства", 6: "1510", 8: D("3"), 9: D("2"), 10: D("1")}), - ] - cand = excel([("Бухгалтерский баланс", first), ("Бухгалтерский баланс. Лист 4", sheet4)]) - layout = find_layout(cand, parse_header(hdr(report_date="30.09.2025"), rules), rules) - assert [c.column_index for c in layout.columns] == [16, 21, 24] - assert layout.segments[1].value_cols == ((8, 16), (9, 21), (10, 24)) - assert not [w for w in layout.warnings if w.code.startswith("SEGMENT_")] - - -def test_repeat_header_mid_table(rules: Rules) -> None: - rows = [ - ["Наименование показателя", "Код", "На 30.09.2025", "На 31.12.2024"], - ["Запасы", "1210", "5", "4"], - ["Наименование показателя", "Код", "На 30.09.2025", "На 31.12.2024"], - ["Итого", "1200", "5", "4"], - ] - layout = find_layout(ocr(rows), parse_header(hdr(), rules), rules) - assert [s.first_row for s in layout.segments] == [0, 2] - assert layout.roles[2] is RowRole.HEADER - - -def test_repeat_header_without_dates_maps_by_order(rules: Rules) -> None: - rows = [ - ["Наименование", "Код", "На 30.09.2025", "На 31.12.2024"], - ["Запасы", "1210", "5", "4"], - ["Наименование", "Код", "отчетн", "прошл"], - ["Итого", "1200", "5", "4"], - ] - layout = find_layout(ocr(rows), parse_header(hdr(), rules), rules) - assert layout.segments[1].value_cols == ((2, 2), (3, 3)) - assert layout.warnings == () - - -def test_wider_segment_is_mismatch(rules: Rules) -> None: - rows = [ - ["Наименование", "Код", "На 30.09.2025", "На 31.12.2024", "На 31.12.2023"], - ["Запасы", "1210", "5", "4", "3"], - ["Наименование", "Код", "графа 3", "графа 4", "графа 5", "графа 6"], - ["Итого", "1200", "5", "4", "3", "2"], - ] - layout = find_layout(ocr(rows), parse_header(hdr(), rules), rules) - (warning,) = [w for w in layout.warnings if w.code == "SEGMENT_COLUMNS_MISMATCH"] - assert warning.severity.value == "error" and "5" in warning.message - assert layout.segments[1].value_cols == ((2, 2), (3, 3), (4, 4)) and layout.segments[1].extra_cols == (5,) - - -def test_second_sheet_single_code_in_other_column(rules: Rules) -> None: - first = [ - place({0: "Наименование", 3: "Код", 4: "За 9 месяцев 2025 г.", 5: "За 9 месяцев 2024 г."}), - place({0: "Выручка", 3: "2110", 4: D("100"), 5: D("90")}), - place({0: "Себестоимость", 3: "2120", 4: D("-60"), 5: D("-50")}), - place({0: "Валовая прибыль", 3: "2100", 4: D("40"), 5: D("40")}), - ] - second = [place({0: "Чистая прибыль", 1: "2400", 2: D("7"), 3: D("6")})] - cand = excel([("ОФР", first), ("ОФР_2", second)]) - layout = find_layout(cand, parse_header(hdr(), rules), rules) - assert layout.segments[1].code_col == 1 - assert layout.segments[1].value_cols == ((2, 4), (3, 5)) - assert not [w for w in layout.warnings if w.code.startswith(("SEGMENT_", "HEADER_"))] - - -def test_header_not_found_anywhere(rules: Rules) -> None: - rows = [["Запасы", "1210", "5", "4"], ["Дебиторы", "1230", "3", "2"], ["Итого", "1200", "8", "6"]] - layout = find_layout(ocr(rows), parse_header(hdr(), rules), rules) - assert [w.code for w in layout.warnings] == ["HEADER_ROW_NOT_FOUND"] - assert [(c.column_index, c.label) for c in layout.columns] == [(2, ""), (3, "")] - - -def test_segment_without_codes(rules: Rules) -> None: - first = [["Наименование", "Код", "На 30.09.2025"], ["Запасы", "1210", "5"], ["Итого", "1200", "5"]] - tail = [["Примечание к форме", "", "7"]] - layout = find_layout(excel([("Баланс", first), ("Хвост", tail)]), parse_header(hdr(), rules), rules) - assert [w.code for w in layout.warnings] == ["SEGMENT_WITHOUT_CODES"] - assert layout.roles[3] is RowRole.DATA and layout.code_found - - -def test_first_segment_without_header_columns_from_first_headed(rules: Rules) -> None: - """P1-18: колонки документа задаёт первый сегмент с заголовком, у первого значения — по порядку.""" - rows = [ - ["Запасы", "1210", "5", "4"], - ["Дебиторы", "1230", "3", "2"], - ["Итого", "1200", "8", "6"], - ["Наименование", "Код", "На 30.09.2025", "На 31.12.2024"], - ["Прочее", "1250", "1", "1"], - ] - layout = find_layout(ocr(rows), parse_header(hdr(), rules), rules) - assert [s.first_row for s in layout.segments] == [0, 3] - assert [c.column_index for c in layout.columns] == [2, 3] - assert layout.segments[0].value_cols == ((2, 2), (3, 3)) - assert layout.warnings == () - - -def test_no_codes_anywhere_has_no_segment_warnings(rules: Rules) -> None: - layout = find_layout(ocr([["Просто", "текст"], ["ещё", "строка"]]), parse_header(hdr(), rules), rules) - assert not layout.code_found and layout.columns == () and layout.warnings == () - assert layout.roles == (RowRole.DATA, RowRole.DATA) - - -def _no_date_noise(layout: Layout) -> None: - assert not [w for w in layout.warnings if w.code == "COLUMN_WITHOUT_DATE"] - - -def test_two_row_header_with_caption_on_top_ocr(rules: Rules) -> None: - """I-1: годы во второй строке шапки — не строка данных, колонки с абсолютными датами.""" - rows = [ - ["Наименование", "Код", "На 30 сентября", "На 31 декабря", "На 31 декабря"], - ["", "", "2025 г.", "2024 г.", "2023 г."], - ["Запасы", "1210", "5", "4", "3"], - ] - layout = find_layout(ocr(rows), parse_header(hdr(), rules), rules) - assert layout.roles[:2] == (RowRole.HEADER, RowRole.HEADER) and layout.roles[2] is RowRole.DATA - assert [c.period_kind.value != "none" for c in layout.columns] == [True, True, True] - _no_date_noise(layout) - - -def test_two_row_header_with_caption_on_top_excel(rules: Rules) -> None: - sheet = [ - place({0: "Наименование", 1: "Код", 2: "На 30 сентября", 3: "На 31 декабря", 4: "На 31 декабря"}), - place({2: "2025 г.", 3: "2024 г.", 4: "2023 г."}), - place({0: "Запасы", 1: "1210", 2: D("5"), 3: D("4"), 4: D("3")}), - ] - layout = find_layout(excel([("Баланс", sheet)]), parse_header(hdr(), rules), rules) - assert layout.roles[1] is RowRole.HEADER and layout.roles[2] is RowRole.DATA - assert [c.period_kind.value != "none" for c in layout.columns] == [True, True, True] - _no_date_noise(layout) - - -def test_repeat_header_in_two_rows(rules: Rules) -> None: - """T10.5: повторная шапка в два ряда использует ту же склейку.""" - rows = [ - ["Наименование", "Код", "На 30 сентября", "На 31 декабря"], - ["", "", "2025 г.", "2024 г."], - ["Запасы", "1210", "5", "4"], - ["Наименование", "Код", "На 30 сентября", "На 31 декабря"], - ["", "", "2025 г.", "2024 г."], - ["Итого", "1200", "5", "4"], - ] - layout = find_layout(ocr(rows), parse_header(hdr(), rules), rules) - assert [s.first_row for s in layout.segments] == [0, 3] - assert layout.roles[4] is RowRole.HEADER and layout.roles[5] is RowRole.DATA - _no_date_noise(layout) - - -def _npz_sheets(second_dates: list[str]) -> list[tuple[str, list[list[str | D | None]]]]: - first = [ - place({1: "Наименование показателя", 12: "Код", 16: DATES3[0], 21: DATES3[1], 24: DATES3[2]}), - place({1: "Нематериальные активы", 12: "1110", 16: D("10"), 21: D("9"), 24: D("8")}), - ] - second = [ - place({0: "Наименование показателя", 6: "Код", 8: second_dates[0], 9: second_dates[1], 10: second_dates[2]}), - place({0: "Заемные средства", 6: "1510", 8: D("3"), 9: D("2"), 10: D("1")}), - ] - return [("Бухгалтерский баланс", first), ("Бухгалтерский баланс. Лист 4", second)] - - -def test_continuation_sheet_with_permuted_dates_maps_by_date_not_order(rules: Rules) -> None: - """M-3: даты листа-продолжения переставлены — колонки ищутся по дате, а не по порядку.""" - permuted = [DATES3[2], DATES3[0], DATES3[1]] # 31.12.2023, 30.09.2025, 31.12.2024 - cand = excel(_npz_sheets(permuted)) - layout = find_layout(cand, parse_header(hdr(report_date="30.09.2025"), rules), rules) - assert layout.segments[1].value_cols == ((8, 24), (9, 16), (10, 21)) - assert not [w for w in layout.warnings if w.code.startswith("SEGMENT_")] - - -def test_continuation_sheet_with_duplicate_date_maps_by_order(rules: Rules) -> None: - """M-3: две колонки на одну дату — сопоставление по порядку.""" - duplicate = [DATES3[0], DATES3[0], DATES3[2]] - cand = excel(_npz_sheets(duplicate)) - layout = find_layout(cand, parse_header(hdr(report_date="30.09.2025"), rules), rules) - assert layout.segments[1].value_cols == ((8, 16), (9, 21), (10, 24)) - - -def test_numbers_right_of_table_in_all_segments_give_one_warning(rules: Rules) -> None: - head: list[str | D | None] = ["Наименование", "Код", "На 30.09.2025", "На 31.12.2024", None] - rows: list[list[str | D | None]] = [ - head, - ["Запасы", "1210", D("5"), D("4"), D("9")], - head, - ["Итого", "1200", D("5"), D("4"), D("8")], - ] - layout = find_layout(excel([("Лист1", rows)]), parse_header(hdr(), rules), rules) - assert [w.code for w in layout.warnings] == ["NUMBERS_OUTSIDE_TABLE"] - assert layout.warnings[0].locator == "Лист1!E2\nЛист1!E4" - assert layout.segments[1].value_cols == ((2, 2), (3, 3)) - - -def test_headerless_continuation_sheet_with_outside_numbers(rules: Rules) -> None: - first: list[list[str | D | None]] = [ - ["Наименование", "", "Код", "На 30.09.2025", "На 31.12.2024", None], - ["Запасы", "", "1210", D("5"), D("4"), D("9")], - ] - second: list[list[str | D | None]] = [["Итого", "", "1200", D("5"), D("4"), D("8")]] - cand = excel([("Лист1", first), ("Лист2", second)]) - layout = find_layout(cand, parse_header(hdr(), rules), rules) - assert [w.code for w in layout.warnings] == ["NUMBERS_OUTSIDE_TABLE"] - assert layout.warnings[0].locator == "Лист1!F2\nЛист2!F1" - assert layout.segments[1].value_cols == ((3, 3), (4, 4)) - assert layout.segments[1].extra_cols == () - - -BALANCE_CAPTIONS = ["Наименование показателя", "Коды", "На 30 июня\n2026 года", "На 31 декабря\n2025 года"] -ASSETS = [["Запасы", "1210", "5", "4"], ["Дебиторская задолженность", "1230", "3", "2"], ["Итого", "1200", "8", "6"]] -LIABILITIES = [["Заемные средства", "1510", "3", "2"], ["Кредиторская задолженность", "1520", "5", "4"]] - - -def test_numbering_with_ocr_noise_right_under_header(rules: Rules) -> None: - """AURORA-1517: нумерация под шапкой со сбоем OCR («/» вместо 1, мусор, протёкшее «года», «V» вместо 4).""" - for numbering in ( - ["/", "2", "3", "4"], - ["• ■ 1", "2", "3", "4"], - ["1", "2", "3гола", "гола\n4"], - ["1", "2", "3", "V"], - ): - layout = find_layout(ocr([BALANCE_CAPTIONS, numbering, *ASSETS]), parse_header(hdr(), rules), rules) - assert layout.roles[1] is RowRole.NUMBERING, numbering - - -def test_numbering_with_ocr_noise_under_repeat_header(rules: Rules) -> None: - """AURORA-1517, «Светлый путь»: «• ■ 1 2 3 4 5» под повторной шапкой второй страницы.""" - rows = [BALANCE_CAPTIONS, ["1", "2", "3", "4"], *ASSETS, BALANCE_CAPTIONS, ["• ■ 1", "2", "3", "4"], *LIABILITIES] - layout = find_layout(ocr(rows), parse_header(hdr(), rules), rules) - assert [s.first_row for s in layout.segments] == [0, 5] - assert layout.roles[6] is RowRole.NUMBERING and layout.roles[7] is RowRole.DATA - - -def test_noisy_numbering_elsewhere_stays_data(rules: Rules) -> None: - """Мягкая проверка — только сразу под шапкой; строка с кодом формы — не нумерация.""" - rows = [BALANCE_CAPTIONS, ["Запасы 1", "1210", "2", "3"], *ASSETS, ["/", "2", "3", "4"], *LIABILITIES] - layout = find_layout(ocr(rows), parse_header(hdr(), rules), rules) - assert layout.roles[1] is RowRole.DATA and layout.roles[5] is RowRole.DATA - - -def test_repeat_header_glued_with_numbering(rules: Rules) -> None: - """AURORA-1517, «Будамшинское»: повторная шапка, в каждой подписи которой снизу номер столбца.""" - glued = ["Наименование показателя\n1", "Коды\n2", "На 30 июня\n2026 года\n3", "На 31 декабря\n2025 года\n4"] - rows = [BALANCE_CAPTIONS, ["1", "2", "3", "4"], *ASSETS, glued, *LIABILITIES] - layout = find_layout(ocr(rows), parse_header(hdr(), rules), rules) - assert [s.first_row for s in layout.segments] == [0, 5] - assert layout.roles[5] is RowRole.HEADER and layout.roles[6] is RowRole.DATA - assert layout.segments[1].value_cols == ((2, 2), (3, 3)) diff --git a/api/arm_statements/tests/test_status.py b/api/arm_statements/tests/test_status.py deleted file mode 100644 index 0952f37..0000000 --- a/api/arm_statements/tests/test_status.py +++ /dev/null @@ -1,128 +0,0 @@ -from dataclasses import replace -from datetime import date -from decimal import Decimal as D - -from arm_statements import status_input_from_contract, to_contract -from arm_statements.header import parse_header -from arm_statements.issues import make_warning -from arm_statements.loader import Rules -from arm_statements.model import ( - DocStatus, - DocumentType, - EnvelopeStatus, - FormVariant, - HeaderField, - LineStatus, - PeriodKind, - Severity, - StatementChecks, - StatementColumn, - StatementDocument, - StatementLine, - StatementValue, -) -from arm_statements.status import ( - StatusInput, - StatusLine, - StatusValue, - document_status, - envelope_status, - status_input, -) -from arm_statements.tests.builders import hdr - -OK, DETAIL, UNREADABLE = LineStatus.OK, LineStatus.DETAIL, LineStatus.UNREADABLE -DONE, CHECK = DocStatus.DONE, DocStatus.NEEDS_CHECK -BASE = StatusInput( - document_type=DocumentType.BALANCE, - layout_found=True, - has_organization_or_inn=True, - period_known=True, - needs_review_fields=(), - lines=(StatusLine(OK, False),), - values=(StatusValue(OK, False),), - hard_failures=0, - accepted=False, - warning_severities=(Severity.INFO,), -) - - -def test_clean_form_done() -> None: - assert document_status(BASE) is DONE - - -def test_unreadable_line_blocks() -> None: - assert document_status(replace(BASE, lines=(StatusLine(OK, False), StatusLine(UNREADABLE, False)))) is CHECK - - -def test_detail_value_does_not_block() -> None: - lines = (StatusLine(OK, False), StatusLine(DETAIL, False)) - values = (StatusValue(OK, False), StatusValue(DETAIL, True)) - assert document_status(replace(BASE, lines=lines, values=values)) is DONE - - -def test_blocking_conditions() -> None: - hard = 2 - assert document_status(replace(BASE, values=(StatusValue(OK, True),))) is CHECK - assert document_status(replace(BASE, lines=(StatusLine(OK, True),))) is CHECK - assert document_status(replace(BASE, needs_review_fields=("report_date",))) is CHECK - assert document_status(replace(BASE, hard_failures=hard)) is CHECK - assert document_status(replace(BASE, hard_failures=hard, accepted=True)) is DONE - assert document_status(replace(BASE, warning_severities=(Severity.ERROR,))) is CHECK - assert document_status(replace(BASE, layout_found=False)) is CHECK - assert document_status(replace(BASE, has_organization_or_inn=False)) is CHECK - assert document_status(replace(BASE, period_known=False)) is CHECK - - -def test_disclosure_always_done() -> None: - disclosure = replace(BASE, document_type=DocumentType.DISCLOSURE, layout_found=False, lines=(), values=()) - assert document_status(disclosure) is DONE - - -def test_envelope(rules: Rules) -> None: - assert envelope_status([DONE, CHECK], []) is EnvelopeStatus.NEEDS_CHECK - assert envelope_status([DONE], []) is EnvelopeStatus.DONE - lost = make_warning("TABLE_LOST", "таблица без колонки кодов", rules, locator="file:a.json;table:1") - assert envelope_status([DONE], [lost]) is EnvelopeStatus.NEEDS_CHECK - assert envelope_status([], []) is EnvelopeStatus.ERROR - - -def test_status_input_from_document(rules: Rules) -> None: - head = parse_header(hdr(organization_name="ООО «Пример»", units="тыс. руб."), rules) - column = StatementColumn(2, "На 30.09.2025", 100.0, date(2025, 9, 30), PeriodKind.ABSOLUTE) - value = StatementValue(2, D("5"), None, "5", 60.0, True, needs_review=True) - detail = StatementLine(1, "row:1", DETAIL, None, None, None, "в т. ч.", 100.0, "row:0", False, (value,)) - line = StatementLine( - 0, "row:0", OK, "1210", "1210", 100.0, "Запасы", 100.0, None, False, (replace(value, needs_review=False),) - ) - doc = StatementDocument( - DocumentType.BALANCE, - FormVariant.FULL, - "file:t.json;table:0", - head, - (column,), - (line, detail), - StatementChecks((), ()), - DocStatus.DONE, - (), - ) - data = status_input(doc) - assert (data.layout_found, data.has_organization_or_inn, data.period_known, data.hard_failures) == ( - True, - True, - True, - 0, - ) - assert data.values == (StatusValue(OK, False), StatusValue(DETAIL, True)) - assert document_status(data) is DONE - empty = replace(doc, columns=(), lines=(), checks=None) - assert (status_input(empty).layout_found, status_input(empty).period_known) == (False, False) - - -def test_empty_string_organization_and_inn_mean_absent(rules: Rules) -> None: - """M-1: пустая строка в шапке — «нет»; совпадает с расчётом по контракту.""" - empty = HeaderField("", 100.0, "") - head = replace(parse_header(hdr(), rules), organization_name=empty, inn=empty) - doc = StatementDocument(DocumentType.BALANCE, None, "l", head, (), (), None, DONE, ()) - assert status_input(doc).has_organization_or_inn is False - assert status_input_from_contract(to_contract(doc)).has_organization_or_inn is False diff --git a/api/arm_statements/tests/test_structure.py b/api/arm_statements/tests/test_structure.py deleted file mode 100644 index 3fe0f26..0000000 --- a/api/arm_statements/tests/test_structure.py +++ /dev/null @@ -1,106 +0,0 @@ -from datetime import date - -from arm_statements.evaluation import EvalLine -from arm_statements.loader import Rules -from arm_statements.model import DocumentType, Edition, FormVariant, LineStatus -from arm_statements.structure import choose_edition, confirm_codes - -BAL = DocumentType.BALANCE -Q3_2025 = date(2025, 9, 30) -FULL_CONF = 100.0 - - -def seq(*items: tuple[str, float]) -> list[EvalLine]: - return [EvalLine(f"r{i}", i, LineStatus.OK, code, conf, False, ()) for i, (code, conf) in enumerate(items)] - - -def unconfirmed(lines: list[EvalLine], rules: Rules, effective: date | None = Q3_2025) -> set[str]: - return set(confirm_codes(lines, BAL, FormVariant.FULL, effective, rules).unconfirmed) - - -def test_uncertain_code_in_place(rules: Rules) -> None: - assert unconfirmed(seq(("1220", 100.0), ("1230", 50.0), ("1240", 100.0)), rules) == set() - - -def test_swapped_uncertain_neighbours(rules: Rules) -> None: - lines = seq(("1210", 100.0), ("1230", 50.0), ("1220", 50.0), ("1240", 100.0)) - assert unconfirmed(lines, rules) == {"r1", "r2"} - - -def test_conflicting_anchors(rules: Rules) -> None: - lines = seq(("1210", 100.0), ("1240", 100.0), ("1230", 100.0), ("1250", 50.0)) - assert unconfirmed(lines, rules) == set() - - -def test_legal_repeat_1220_fsbu4(rules: Rules) -> None: - lines = seq(("1215", 100.0), ("1220", 100.0), ("1220", 50.0), ("1230", 100.0)) - assert unconfirmed(lines, rules) == set() - - -def test_repeat_outside_orders(rules: Rules) -> None: - lines = seq(("1220", 100.0), ("1230", 100.0), ("1230", 50.0), ("1240", 100.0)) - assert unconfirmed(lines, rules) == {"r2"} - - -def test_66n_blank_in_2025(rules: Rules) -> None: - lines = seq(("1110", 100.0), ("1120", 60.0), ("1130", 100.0)) - result = confirm_codes(lines, BAL, FormVariant.FULL, Q3_2025, rules) - assert (result.edition, result.unconfirmed) == (Edition.RAS_66N, frozenset()) - - -def test_tie_resolved_by_date(rules: Rules) -> None: - lines = seq(("1210", 100.0), ("1230", 100.0)) - assert confirm_codes(lines, BAL, FormVariant.FULL, Q3_2025, rules).edition is Edition.FSBU4 - assert confirm_codes(lines, BAL, FormVariant.FULL, date(2024, 12, 31), rules).edition is Edition.RAS_66N - - -def test_choose_edition_public(rules: Rules) -> None: - blank_66n = [("1110", True), ("1120", False), ("1130", True)] - assert choose_edition(BAL, blank_66n, Q3_2025, rules) is Edition.RAS_66N - tie = [("1210", True), ("1230", True)] - assert choose_edition(BAL, tie, Q3_2025, rules) is Edition.FSBU4 - assert choose_edition(BAL, tie, date(2024, 12, 31), rules) is Edition.RAS_66N - assert choose_edition(BAL, tie, None, rules) is None - assert choose_edition(BAL, [], Q3_2025, rules) is Edition.FSBU4 - assert choose_edition(DocumentType.DISCLOSURE, [("5101", True)], date(2024, 12, 31), rules) is Edition.RAS_66N - - -def test_choose_edition_equals_confirm_codes(rules: Rules) -> None: - lines = seq(("1110", 100.0), ("1120", 60.0), ("1130", 100.0)) - pairs = [(ln.line_code or "", (ln.code_confidence or 0.0) >= FULL_CONF) for ln in lines] - expected = confirm_codes(lines, BAL, FormVariant.FULL, Q3_2025, rules).edition - assert choose_edition(BAL, pairs, Q3_2025, rules, variant=FormVariant.FULL) is expected - - -def test_confident_nonstandard_code_not_flagged(rules: Rules) -> None: - assert unconfirmed(seq(("1230", 100.0), ("1231", 100.0), ("11101", 100.0)), rules) == set() - - -def test_subline_follows_confirmed_parent(rules: Rules) -> None: - lines = seq(("1220", 100.0), ("1230", 50.0), ("12301", 60.0), ("1240", 100.0)) - assert unconfirmed(lines, rules) == set() - orphan = seq(("1220", 100.0), ("12301", 60.0), ("1240", 100.0)) - assert unconfirmed(orphan, rules) == {"r1"} - - -def test_tie_without_date_union_of_editions(rules: Rules) -> None: - lines = seq(("1210", 100.0), ("1230", 50.0)) - result = confirm_codes(lines, BAL, FormVariant.FULL, None, rules) - assert result.edition is None - assert result.unconfirmed == frozenset() - - -def test_disclosure_has_no_unconfirmed(rules: Rules) -> None: - lines = seq(("5101", 50.0)) - result = confirm_codes(lines, DocumentType.DISCLOSURE, None, Q3_2025, rules) - assert (result.edition, result.unconfirmed) == (Edition.FSBU4, frozenset()) - - -def test_uncertain_code_outside_order_unconfirmed(rules: Rules) -> None: - assert unconfirmed(seq(("1230", 100.0), ("9999", 50.0), ("1240", 100.0)), rules) == {"r1"} - - -def test_performance_realistic_size(rules: Rules) -> None: - order = rules.code_orders.sequence(Edition.FSBU4, BAL, FormVariant.FULL) - lines = seq(*[(code, 50.0) for code in order] * 1) - assert confirm_codes(lines, BAL, FormVariant.FULL, Q3_2025, rules).unconfirmed == frozenset() diff --git a/api/arm_statements/tests/test_text_codes.py b/api/arm_statements/tests/test_text_codes.py deleted file mode 100644 index 77c94d3..0000000 --- a/api/arm_statements/tests/test_text_codes.py +++ /dev/null @@ -1,74 +0,0 @@ -import pytest - -from arm_statements.codes import is_code_cell, majority_section, parse_code, sheet_joins -from arm_statements.loader import CodeSection, Rules -from arm_statements.model import DocumentType -from arm_statements.text import fix_lookalikes, is_disguised_digit, normalize_caption, normalize_label - - -@pytest.mark.parametrize( - ("raw", "code", "fixed"), - [ - ("1220 ,", "1220", False), - ("| 140", "1140", True), - ("I1О0", "1100", True), - ("1110.0", "1110", False), - ("12301", "12301", False), - ("3450", "3450", False), - ("2024", None, False), - ("1050", None, False), - ("7450", None, False), - ("ИЧ,250", None, False), - ("изо :", None, False), - ("НИ", None, False), - ("1500\n1700", None, False), - ("2", None, False), - ], -) -def test_parse_code(rules: Rules, raw: str, code: str | None, fixed: bool) -> None: - assert parse_code(raw, rules) == (code, fixed) - - -def test_multiline_code_cell(rules: Rules) -> None: - assert is_code_cell("1500\n1700", rules) - assert not is_code_cell("44 30\nI |", rules) - - -def test_lookalikes_only_near_digit(rules: Rules) -> None: - assert fix_lookalikes("2ОО5", rules.lookalikes) == "2005" - assert fix_lookalikes("2|0|2|4|", rules.lookalikes).split() == ["2", "0", "2", "4"] - assert fix_lookalikes("Основные средства", rules.lookalikes) == "Основные средства" - assert fix_lookalikes("З", rules.lookalikes) == "З" - assert fix_lookalikes("З", rules.lookalikes, anywhere=True) == "3" - - -def test_disguised_digit(rules: Rules) -> None: - assert is_disguised_digit("I", rules.lookalikes) and is_disguised_digit("| О", rules.lookalikes) - assert not is_disguised_digit("Иванов", rules.lookalikes) - assert not is_disguised_digit("(расшифровка подписи)", rules.lookalikes) - - -def test_caption_normalization() -> None: - assert normalize_caption("Код строки") == "код строки" - assert normalize_caption("Изменение, %") == "изменение %" - assert normalize_caption("Наименование\nпоказателя") == "наименование показателя" - - -def test_label_normalization_keeps_punctuation(rules: Rules) -> None: - assert normalize_label("Местонахождение (адрес)") == "местонахождение (адрес)" - assert normalize_label("по ОКОПФ / ОКФС") == "по окопф / окфс" - assert normalize_label("Форма 0710001 с.2") == "форма 0710001 с.2" - assert normalize_label("Отчёт о финансовых\nрезультатах") == "отчет о финансовых результатах" - assert normalize_label("за­дол­жен—ность") == "задолжен ность" - assert normalize_label(None) == "" - for key, patterns in rules.headers.labels.items(): - assert patterns, key - assert any(p.search(normalize_label("Местонахождение (адрес)")) for p in rules.headers.labels["address"]) - assert any(p.search(normalize_label("по ОКОПФ / ОКФС")) for p in rules.headers.labels["okopf_okfs"]) - - -def test_majority_section_and_join(rules: Rules) -> None: - assert majority_section(["1110", "1150", "2110"], rules) is CodeSection.BALANCE - assert sheet_joins(DocumentType.INCOME, ["2400"], 1, 3, rules) - assert not sheet_joins(DocumentType.BALANCE, ["1110", "1150", "1100"], 4, 3, rules) - assert not sheet_joins(DocumentType.BALANCE, ["2110"], 3, 3, rules) diff --git a/api/arm_statements/tests/test_version.py b/api/arm_statements/tests/test_version.py deleted file mode 100644 index 817baec..0000000 --- a/api/arm_statements/tests/test_version.py +++ /dev/null @@ -1,21 +0,0 @@ -from pathlib import Path - -from arm_statements.version import statements_version - -FULL_HASH = "abc1234def5678abc1234def5678abc1234def56" - - -def test_version_absent_file_is_none(tmp_path: Path) -> None: - assert statements_version(tmp_path / "VERSION") is None - - -def test_version_read_from_file(tmp_path: Path) -> None: - marker = tmp_path / "VERSION" - marker.write_text(f"{FULL_HASH}\n", encoding="utf-8") - assert statements_version(marker) == FULL_HASH - - -def test_version_empty_file_is_none(tmp_path: Path) -> None: - marker = tmp_path / "VERSION" - marker.write_text(" \n", encoding="utf-8") - assert statements_version(marker) is None diff --git a/api/arm_statements/text.py b/api/arm_statements/text.py deleted file mode 100644 index e4dd601..0000000 --- a/api/arm_statements/text.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Очистка OCR-текста: буквы-двойники, нормализация подписей и меток (§11.1 п. 1, 6).""" - -import re -from collections.abc import Mapping - -# первая строка из цифр, пробелов, `|`, точек, запятых и двоеточий: `|` в ней — разделитель клеток формы -_CELL_SEPARATOR_LINE = re.compile(r"[\s\d|•·.:,]*") -_INVISIBLE = re.compile("[\u00ad\u200b\ufeff]") -_SEPARATORS = re.compile(r"[\s\-\u2010-\u2015\u2212]+") -_NON_CAPTION = re.compile(r"[^\w%]|_") - - -def replace_lookalikes(text: str, lookalikes: Mapping[str, str], *, anywhere: bool = False) -> str: - """Двойники → цифры вплотную к цифре (несколько проходов); `anywhere` — без условия соседства.""" - if not lookalikes: - return text - table = str.maketrans(dict(lookalikes)) - if anywhere: - return text.translate(table) - chars = re.escape("".join(lookalikes)) - near_digit = re.compile(rf"(?<=\d)[{chars}]|[{chars}](?=\d)") - previous = None - while previous != text: # «2ОО5» требует двух проходов - previous = text - text = near_digit.sub(lambda match: match.group(0).translate(table), text) - return text - - -def fix_lookalikes(text: str, lookalikes: Mapping[str, str], *, anywhere: bool = False) -> str: - """NBSP → пробел; `|` — разделитель клеток, если первая строка из цифр и пунктуации; двойники → цифры.""" - text = text.replace("\xa0", " ") - if _CELL_SEPARATOR_LINE.fullmatch(text.split("\n", 1)[0]): - text = text.replace("|", " ") - return replace_lookalikes(text, lookalikes, anywhere=anywhere) - - -def has_digit(text: str | None) -> bool: - return text is not None and any(char.isdigit() for char in text) - - -def is_disguised_digit(text: str, lookalikes: Mapping[str, str]) -> bool: - """После замены двойников везде есть цифра и нет других букв («I», «| О» — да; «Иванов» — нет).""" - fixed = fix_lookalikes(text, lookalikes, anywhere=True) - return has_digit(fixed) and not any(char.isalpha() for char in fixed) - - -def normalize_caption(text: str | None) -> str: - """Нижний регистр, `ё → е`, всё кроме букв, цифр и `%` → пробел, пробелы схлопнуты.""" - if text is None: - return "" - lowered = text.lower().replace("ё", "е") - return " ".join(_NON_CAPTION.sub(" ", lowered).split()) - - -def normalize_label(text: str | None) -> str: - """Метки шапки: невидимые удалены, нижний регистр, `ё → е`, пробелы и дефисы/тире → пробел; пунктуация сохранена.""" - if text is None: - return "" - lowered = _INVISIBLE.sub("", text).lower().replace("ё", "е") - return _SEPARATORS.sub(" ", lowered).strip() diff --git a/api/arm_statements/version.py b/api/arm_statements/version.py deleted file mode 100644 index d9e2ad5..0000000 --- a/api/arm_statements/version.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Маркер версии копии пакета: файл VERSION пишет только sync-statements в хосте.""" - -from pathlib import Path - -_DEFAULT_MARKER = Path(__file__).parent / "VERSION" - - -def statements_version(path: Path | None = None) -> str | None: - """Вернуть версию копии (хеш коммита) или None, если файла нет или он пуст.""" - marker = _DEFAULT_MARKER if path is None else path - if not marker.is_file(): - return None - text = marker.read_text(encoding="utf-8").strip() - return text or None diff --git a/api/pyproject.toml b/api/pyproject.toml index 1e463a1..c3bdf82 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -8,7 +8,7 @@ target-version = "py311" select = ["E", "F", "I", "UP", "B", "PL"] [tool.ruff.lint.isort] -known-first-party = ["src", "arm_statements"] +known-first-party = ["src"] [tool.mypy] python_version = "3.11" @@ -39,18 +39,7 @@ ignore_missing_imports = true module = ["src.parser", "src.parser.*"] disallow_untyped_defs = true -# Копия общего кода (sync-statements, не править) — в строгом режиме, как в репозитории arm-statements. -# `strict = true` в overrides mypy (2.x) применяет глобально, а не к модулю, поэтому флаги семейства strict перечислены явно. +# Установленная библиотека не поставляет py.typed. [[tool.mypy.overrides]] module = ["arm_statements", "arm_statements.*"] -disallow_any_generics = true -disallow_subclassing_any = true -disallow_untyped_calls = true -disallow_untyped_defs = true -disallow_incomplete_defs = true -disallow_untyped_decorators = true -warn_return_any = true -warn_unused_ignores = true -no_implicit_reexport = true -strict_equality = true -extra_checks = true +ignore_missing_imports = true diff --git a/api/pytest.ini b/api/pytest.ini index 5f08f31..cf530e1 100644 --- a/api/pytest.ini +++ b/api/pytest.ini @@ -2,7 +2,6 @@ pythonpath = . testpaths = tests - arm_statements/tests python_files = test_*.py tests*.py diff --git a/api/requirements-dev.txt b/api/requirements-dev.txt index 59f37d2..573caca 100644 --- a/api/requirements-dev.txt +++ b/api/requirements-dev.txt @@ -1,6 +1,6 @@ # Инструменты разработки и проверок. Платформенные зависимости — в requirements.txt (не меняем). # Нижние границы мажорных версий: воспроизводимо, но без конфликта с пинами платформы. -# ruff, mypy, types-PyYAML — те же версии, что в arm-statements (копия пакета проверяется конфигом хоста) +# Инструменты проверки кода сервиса -r requirements.txt ruff==0.16.10 mypy==2.4.0 diff --git a/api/requirements.txt b/api/requirements.txt index 4050601..e86100c 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -43,6 +43,7 @@ Pygments==2.20.0 PyNaCl==1.5.0 python-dotenv==1.2.2 python-multipart==0.0.32 +pylib_ai_arm_statements==0.1.1 pyxlsb==1.0.10 PyYAML==6.0.3 rich==15.0.0 diff --git a/api/rules/README.md b/api/rules/README.md index 9acdd21..c43a1d3 100644 --- a/api/rules/README.md +++ b/api/rules/README.md @@ -17,7 +17,7 @@ AST-линтер `tests/unit/parser/test_no_coordinates.py`. | `data_types.yaml` | `header`, `standalone_header` (тип данных отдельной ячейкой без метки), `row`, `currency_codes`, `currency_aliases` | «Выводимые данные» -> `BU` / `NU` / `SUM`; строки «Вал.» / «Кол.», «Кол-во», «Кол-во, <единица>» -> `CURRENCY` / `QUANTITY`; метки колонки «Показатели» (`indicator_column`: «Кол.», «Кол. доп.» -> `QUANTITY`); коды валют ISO 4217 (строка, равная коду, — валютная); написания валюты в шапке (`руб.` -> `RUB`; ключ алиаса обязан быть в `currency_codes`) | Словари форм ББ/ОФР (`statement_*.yaml` в прежней версии) удалены из сервиса: их держит пакет `arm_statements` -(`api/arm_statements/rules/`), сервис их не читает. +(дистрибутив `pylib_ai_arm_statements`), сервис их напрямую не читает. ## Как писать паттерны diff --git a/api/src/contract/common.py b/api/src/contract/common.py index 5556128..511acc2 100644 --- a/api/src/contract/common.py +++ b/api/src/contract/common.py @@ -209,7 +209,7 @@ LaxErrorCode = Annotated[ErrorCode, Field(strict=False)] class ServiceInfo(ContractModel): """Кто ответил: `service {name, version, statements_version?}`. - `statements_version` — коммит arm-statements, из которого разложена копия общего кода; обязателен в файле + `statements_version` — версия библиотеки arm_statements; обязателен в файле результата, если в нём есть документ формы (проверяет `RecognitionResult`). """ diff --git a/api/src/parser/__init__.py b/api/src/parser/__init__.py index 16f5f49..7045635 100644 --- a/api/src/parser/__init__.py +++ b/api/src/parser/__init__.py @@ -13,6 +13,7 @@ from pathlib import Path from typing import Protocol, runtime_checkable from arm_statements import RulesError, default_rules, statements_version + from src.contract import ErrorCode, FileType, MessageError, Metrics, OsvDocument, ResultStatus, StatementDocument from src.core.config import Settings from src.parser.rules import DEFAULT_RULES_DIR, load_rules @@ -63,14 +64,14 @@ class NullParser: def require_statements_version(path: Path | None = None) -> str: - """Версия копии `arm_statements` (коммит arm-statements) для `service.statements_version`. + """Версия библиотеки `arm_statements` для `service.statements_version`. - Копию раскладывает `sync-statements`; без файла `VERSION` сервис не стартует: результат с формами без версии общего - кода контракт отвергает (P4-16). `path` — файл версии (в тестах), по умолчанию `arm_statements/VERSION`. + Библиотека читает VERSION или метаданные установленного пакета. + `path` позволяет явно указать файл версии в тестах. Без версии сервис не стартует. """ version = statements_version(path) if version is None: - raise RuntimeError("копия arm_statements без VERSION: разложите её sync-statements") + raise RuntimeError("библиотека arm_statements без VERSION и версии пакета: выполните make install") return version diff --git a/api/src/parser/book.py b/api/src/parser/book.py index becfe08..44fde07 100644 --- a/api/src/parser/book.py +++ b/api/src/parser/book.py @@ -11,8 +11,6 @@ import time from collections.abc import Callable -from loguru import logger - from arm_statements import ( DocStatus, Severity, @@ -23,6 +21,8 @@ from arm_statements import ( warning_to_contract, ) from arm_statements import Rules as StatementRules +from loguru import logger + from src.contract import ( ErrorCode, FileType, diff --git a/api/src/parser/detect.py b/api/src/parser/detect.py index 5e1359b..982715f 100644 --- a/api/src/parser/detect.py +++ b/api/src/parser/detect.py @@ -16,6 +16,7 @@ from typing import Literal from arm_statements import Rules as StatementRules from arm_statements import default_rules + from src.parser.deadline import Deadline from src.parser.errors import InvalidFormatError from src.parser.grid import Cell, Sheet, column_letter diff --git a/api/src/parser/statements/adapter.py b/api/src/parser/statements/adapter.py index 3998273..a8d15e9 100644 --- a/api/src/parser/statements/adapter.py +++ b/api/src/parser/statements/adapter.py @@ -21,6 +21,7 @@ from arm_statements import ( make_warning, sheet_joins, ) + from src.parser.deadline import Deadline from src.parser.detect import SheetKind from src.parser.grid import Sheet diff --git a/api/src/parser/statements/cells.py b/api/src/parser/statements/cells.py index 2e27332..9ac6be7 100644 --- a/api/src/parser/statements/cells.py +++ b/api/src/parser/statements/cells.py @@ -8,6 +8,7 @@ import datetime from collections.abc import Sequence from arm_statements import Cell as StatementCell + from src.parser.grid import Cell, Sheet, cell_ref from src.parser.statements.number_format import display_amount, general_text diff --git a/api/src/parser/statements/forms.py b/api/src/parser/statements/forms.py index 6ac88c9..122f934 100644 --- a/api/src/parser/statements/forms.py +++ b/api/src/parser/statements/forms.py @@ -25,6 +25,7 @@ from arm_statements import ( type_hint, ) from arm_statements import Rules as StatementRules + from src.parser.grid import Sheet from src.parser.statements.cells import excel_cell, grid_rows diff --git a/api/src/parser/statements/header_cells.py b/api/src/parser/statements/header_cells.py index 8072a92..67577a9 100644 --- a/api/src/parser/statements/header_cells.py +++ b/api/src/parser/statements/header_cells.py @@ -9,6 +9,7 @@ import re from arm_statements import Cell, Evidence, HeaderCells, parse_date from arm_statements import Rules as StatementRules + from src.parser.grid import Sheet from src.parser.statements.cells import excel_cell from src.parser.statements.forms import FormBlock, matches diff --git a/api/src/parser/statements/pipeline.py b/api/src/parser/statements/pipeline.py index ccf85c4..aa6ac48 100644 --- a/api/src/parser/statements/pipeline.py +++ b/api/src/parser/statements/pipeline.py @@ -10,6 +10,7 @@ from dataclasses import dataclass from arm_statements import Rules as StatementRules from arm_statements import StatementWarning, normalize_candidate, to_contract, warning_to_contract + from src.contract import FileWarning, StatementDocument from src.parser.deadline import Deadline from src.parser.detect import SheetKind diff --git a/api/tests/fixtures/test_rules_coverage.py b/api/tests/fixtures/test_rules_coverage.py index 05655db..24589c7 100644 --- a/api/tests/fixtures/test_rules_coverage.py +++ b/api/tests/fixtures/test_rules_coverage.py @@ -17,6 +17,7 @@ from pathlib import Path import pytest +from arm_statements import default_rules from fixtures_helpers import ( osv_dkb_files, osv_fixture_files, @@ -26,7 +27,6 @@ from fixtures_helpers import ( ) from rules_coverage import DICTIONARIES, PATTERNLESS, MatchCounter, iter_patterns, uncovered -from arm_statements import default_rules from src.parser import get_parser from src.parser.book import BookParser from src.parser.osv import OsvParser diff --git a/api/tests/fixtures/test_statements_adapter_real.py b/api/tests/fixtures/test_statements_adapter_real.py index 97f61e0..dae1d0b 100644 --- a/api/tests/fixtures/test_statements_adapter_real.py +++ b/api/tests/fixtures/test_statements_adapter_real.py @@ -3,8 +3,8 @@ from pathlib import Path import pytest - from arm_statements import DocumentType, default_rules + from src.parser.deadline import Deadline from src.parser.detect import is_osv_sheet from src.parser.readers import read_book diff --git a/api/tests/unit/parser/statements/test_adapter.py b/api/tests/unit/parser/statements/test_adapter.py index 0f5b82b..04d02d6 100644 --- a/api/tests/unit/parser/statements/test_adapter.py +++ b/api/tests/unit/parser/statements/test_adapter.py @@ -1,9 +1,9 @@ """Адаптер книги -> `FormCandidate`: план листов, склейка листов-продолжений, предупреждения (план 4, задача 5).""" import pytest +from arm_statements import DocumentType, Severity, Source, default_rules from gridbuilder import at, sheet_from_rows, with_formats -from arm_statements import DocumentType, Severity, Source, default_rules from src.parser.deadline import Deadline from src.parser.detect import SheetKind, is_osv_sheet from src.parser.grid import Sheet diff --git a/api/tests/unit/parser/statements/test_cells.py b/api/tests/unit/parser/statements/test_cells.py index bcca02e..9f4b6e1 100644 --- a/api/tests/unit/parser/statements/test_cells.py +++ b/api/tests/unit/parser/statements/test_cells.py @@ -3,9 +3,9 @@ import datetime from decimal import Decimal +from arm_statements import Cell from gridbuilder import M, at, sheet_from_rows, with_formats -from arm_statements import Cell from src.parser.statements.cells import excel_cell, grid_rows diff --git a/api/tests/unit/parser/statements/test_forms.py b/api/tests/unit/parser/statements/test_forms.py index 36d292d..64f942f 100644 --- a/api/tests/unit/parser/statements/test_forms.py +++ b/api/tests/unit/parser/statements/test_forms.py @@ -1,9 +1,9 @@ """Блоки форм, вид листа и столбцы на словарях `arm_statements` (план 4, задача 4).""" import pytest +from arm_statements import DocumentType, continuation_type, default_rules from gridbuilder import at, sheet_from_rows -from arm_statements import DocumentType, continuation_type, default_rules from src.parser.statements.forms import ( StatementRules, caption_matches, diff --git a/api/tests/unit/parser/statements/test_header_cells.py b/api/tests/unit/parser/statements/test_header_cells.py index d9c35b8..b87431c 100644 --- a/api/tests/unit/parser/statements/test_header_cells.py +++ b/api/tests/unit/parser/statements/test_header_cells.py @@ -1,9 +1,9 @@ """Шапка-ячейки формы на словарях `arm_statements` (план 4, задача 4).""" import pytest +from arm_statements import default_rules, parse_header from gridbuilder import at, sheet_from_rows -from arm_statements import default_rules, parse_header from src.parser.statements.forms import StatementRules, find_blocks from src.parser.statements.header_cells import header_cells diff --git a/api/tests/unit/parser/statements/test_pipeline.py b/api/tests/unit/parser/statements/test_pipeline.py index 1ce7184..f5c257a 100644 --- a/api/tests/unit/parser/statements/test_pipeline.py +++ b/api/tests/unit/parser/statements/test_pipeline.py @@ -1,11 +1,11 @@ """Конвейер форм книги: кандидаты адаптера → пакет → документы и предупреждения файла контракта 2.0.""" import pytest +from arm_statements import Rules as StatementRules +from arm_statements import default_rules from gridbuilder import at, sheet_from_rows from result_contract import assert_valid_result -from arm_statements import Rules as StatementRules -from arm_statements import default_rules from src.contract import FileWarning, Metrics, ResultStatus from src.parser import ParseOutcome from src.parser.deadline import Deadline diff --git a/api/tests/unit/parser/test_book_parser.py b/api/tests/unit/parser/test_book_parser.py index de0d974..02c6397 100644 --- a/api/tests/unit/parser/test_book_parser.py +++ b/api/tests/unit/parser/test_book_parser.py @@ -18,12 +18,12 @@ from typing import Any import openpyxl import pytest +from arm_statements import Rules as StatementRules +from arm_statements import default_rules from gridbuilder import L, M, sheet_from_rows from loguru import logger from result_contract import assert_valid_result -from arm_statements import Rules as StatementRules -from arm_statements import default_rules from src.contract import ErrorCode, FileWarning, OsvDocument, ResultStatus, StatementDocument from src.parser import ParseOutcome, hierarchy from src.parser.book import BookParser diff --git a/api/tests/unit/parser/test_read_limits.py b/api/tests/unit/parser/test_read_limits.py index d925583..f1d8ba7 100644 --- a/api/tests/unit/parser/test_read_limits.py +++ b/api/tests/unit/parser/test_read_limits.py @@ -12,8 +12,8 @@ from typing import Any import openpyxl import pytest - from arm_statements import default_rules + from src.contract import ErrorCode, ResultStatus from src.parser.book import BookParser from src.parser.errors import UnreadableError, UnsupportedFormatError diff --git a/api/tests/unit/test_worker.py b/api/tests/unit/test_worker.py index 35b8f8d..f6ce5dd 100644 --- a/api/tests/unit/test_worker.py +++ b/api/tests/unit/test_worker.py @@ -14,11 +14,11 @@ from types import SimpleNamespace from typing import Any import pytest +from arm_statements import RulesError from jsonschema import Draft202012Validator, FormatChecker from loguru import logger from referencing import Registry, Resource -from arm_statements import RulesError from src.contract import ( NATS_MSG_ID_HEADER, RESULT_SUBJECT, @@ -935,7 +935,7 @@ def test_require_statements_version_reads_the_marker(tmp_path: Path) -> None: marker.write_text("abc1234\n", encoding="utf-8") assert require_statements_version(marker) == "abc1234" - assert require_statements_version() # копия пакета в репозитории размечена sync-statements + assert require_statements_version() # версия установленной библиотеки def test_get_parser_wraps_rules_error_into_runtime_error(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/openspec/changes/use-arm-statements-library/.openspec.yaml b/openspec/changes/use-arm-statements-library/.openspec.yaml new file mode 100644 index 0000000..1f5d280 --- /dev/null +++ b/openspec/changes/use-arm-statements-library/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-10-24 diff --git a/openspec/changes/use-arm-statements-library/design.md b/openspec/changes/use-arm-statements-library/design.md new file mode 100644 index 0000000..36ae262 --- /dev/null +++ b/openspec/changes/use-arm-statements-library/design.md @@ -0,0 +1,61 @@ +# Design + +## Context + +См. `proposal.md`, раздел Why. Переход уже отражён в staged-изменениях сервиса; этот change документирует выполненную работу и оставшиеся проверки. Основные спецификации после отмены предыдущих правок по-прежнему описывают копию исходников. Их изменение вынесено в delta-файлы этого change. + +Пример локальной установки — `../arm-core/Makefile`. Дистрибутив находится в `../pylib_ai_arm_statements`: имя установки `pylib-ai-arm-statements`, версия `0.1.1`, имя импорта `arm_statements`. Его API `statements_version(path)` читает VERSION, затем метаданные пакета; явный путь используется тестами и не включает fallback. `default_rules()` загружает словари из установленной библиотеки. + +## Goals / Non-Goals + +**Goals:** единый установленный источник кода и словарей для разработки и контейнера; повторная установка локальных изменений без смены версии; проверка версии при старте; сохранение адаптеров и проверок сервиса. + +**Non-Goals:** редактирование исходников библиотеки, библиотечных тестов, внешнего CI/Compose и канонических схем. Денежные расчёты остаются Decimal, единицы (рубли / тысячи рублей) не меняются. ORM и консолидация не затрагиваются. + +## Decisions + +1. **Локальная установка через uv.** `make venv` фильтрует `^(raisa_|pylib_ai)` из `api/requirements.txt`, устанавливает оставшиеся зависимости и dev-инструменты, затем ставит `STATEMENTS_DIR`. `make install` создаёт окружение Python 3.11 при необходимости и использует `--reinstall-package pylib-ai-arm-statements`, чтобы подтянуть изменения при прежней версии. Путь по умолчанию совпадает с соседним checkout; переопределение поддерживает другую раскладку workspace. Альтернативы: editable-установка делает окружение зависимым от живых исходников; копирование возвращает проблему синхронизации; индекс недоступен для этого пакета. Pin в requirements документирует ожидаемую версию, но локальная установка не проверяет совпадение VERSION с pin автоматически. + +2. **Удаление vendored-кода и его проверок.** `api/arm_statements/` удаляется целиком вместе с тестами, rules и VERSION. `statements-check` и зависимость `check-full` от него удаляются. Pytest собирает `tests`, mypy проверяет `src`; Ruff считает только `src` first-party, поэтому импорты библиотеки перегруппированы в адаптерах и тестах. В mypy строгий override для копии заменён `ignore_missing_imports`, поскольку дистрибутив не поставляет `py.typed`. Тесты сервиса остаются; библиотечные тесты принадлежат библиотеке. Альтернатива сохранения двух копий допускает затенение установленного пакета локальной папкой. + +3. **Версия через API библиотеки.** `require_statements_version` продолжает вызывать `statements_version(path)` и запрещает старт при отсутствии версии. Меняются документация и RuntimeError: `библиотека arm_statements без VERSION и версии пакета: выполните make install`. VERSION сохраняет приоритет, метаданные дают fallback для установленного дистрибутива; прямое чтение метаданных в сервисе дублировало бы логику библиотеки. Тип поля контракта остаётся непустой строкой, JSON-схемы не редактируются. + +4. **Именованный контекст BuildKit.** `make docker-build` передаёт `--build-context arm_statements="$(STATEMENTS_DIR)"`. Dockerfile подключает его через временный bind mount на шаге RUN и устанавливает `/tmp/arm_statements` вместе с отфильтрованными requirements. `rw` нужен сборочным инструментам для временных файлов; изменения mount не записываются обратно на хост и не становятся содержимым образа. Установленный пакет и его package data остаются в образе. Альтернатива расширения основного контекста на весь workspace меняет правила COPY и состав контекста; копирование checkout в слой добавляет ненужные исходники. Прямым вызовам Docker, внешним CI и Compose потребуется тот же именованный контекст; их конфигурация в этой задаче не менялась. + +5. **Документация отражает библиотечную поставку.** README, CLAUDE.md, `api/rules/README.md`, комментарии в requirements-dev, проверке версии, тесте worker и модели `Service` обновлены. Архивные OpenSpec change и копии JSON-схем не переписываются. Внешнее описание поля как коммита требует согласования отдельным change в docs. + +## Risks / Trade-offs + +- [Нет локального checkout или неверный STATEMENTS_DIR] → установка завершается ошибкой; путь и команда переустановки описаны в README. +- [Отличие локальной версии от pin] → перед сборкой проверять метаданные дистрибутива и `statements_version()`; автоматическая сверка не добавлялась. +- [Версия 0.1.1 не идентифицирует локальные изменения исходников] → переустановка принудительная, воспроизводимый релиз требует дисциплины версий библиотеки; Git SHA больше не является обязательным форматом поля. +- [Нет BuildKit или не передан именованный контекст] → использовать `make docker-build`; в других системах сборки явно передавать контекст. +- [Нет маркера типов у библиотеки] → mypy проверяет код сервиса, поведение интеграции подтверждают тесты адаптеров. +- [Описание JSON-схемы ещё говорит о коммите] → не нарушать идентичность копии контракта; согласовать описание в docs отдельно. + +## Migration Plan + +1. Подготовить checkout библиотеки версии 0.1.1; выполнить `make venv` либо `make install` в существующем окружении. +2. Проверить, что `arm_statements.__file__` указывает на site-packages, версия равна 0.1.1, `default_rules()` загружает данные, `get_parser()` создаёт BookParser. +3. Выполнить lint, тесты адаптеров/версии и golden/fixtures-проверки на доступных реальных книгах. Перед релизом собрать и проверить контейнер с локальным контекстом. +4. Для отката восстановить удалённую копию и прежние Makefile/Dockerfile/конфиги одним откатом изменения; не оставлять обе модели поставки одновременно. + +## Validation Evidence + +Результаты предыдущего выполнения в этой сессии (не повторная проверка в рамках proposal): + +- Локальная установка завершилась: `pylib-ai-arm-statements==0.1.1`; импорт из site-packages, библиотека сообщает `0.1.1`, словари загружаются, создаётся BookParser. +- `make lint` прошёл (Ruff, format, mypy). +- `tests/unit/parser`: 1206 passed, 5 skipped; три теста проверки версии/ошибки загрузки словарей в `test_worker.py` прошли. +- Полный unit-прогон остановлен после зависания на worker-тестах; успешный полный прогон не заявляется. Для запуска тестов потребовалось переопределить некорректное внешнее значение DEBUG через `DEBUG=false`. +- Docker-сборка не проверена: доступ к `/var/run/docker.sock` запрещён. Реальные fixtures/golden и интеграционные NATS/S3-проверки отдельно не выполнялись. + +### Проверки при продолжении apply (24.10.2026) + +- Реальные формы: 27 passed; сравнение форм с golden-снимками: 27 passed. Снимки не перезаписывались. +- `make venv` и повторный `make install` с абсолютным STATEMENTS_DIR прошли; библиотека 0.1.1 импортируется из site-packages. +- `make test-integration` вне песочницы: 10 passed, включая реальные формы и версию библиотеки в результате. +- Полный unit-прогон вне песочницы: 1733 passed, 1 failed за 4.78 s. В песочнице worker-тесты зависают; вне неё это не воспроизводится. +- Единственная ошибка unit-прогона — идентичность `api/contracts/recognition-result.schema.json` актуальному файлу docs: в docs уже изменены два текстовых описания (библиотека и источник её версии), структура схемы прежняя. Синхронизация копии ожидает решения пользователя по границе задачи. + +- Docker-сборка завершилась успешно; запуск образа с `--network none` без mount исходников подтвердил импорт из site-packages, версию 0.1.1 и загрузку словарей. `make lint` повторно прошёл. diff --git a/openspec/changes/use-arm-statements-library/proposal.md b/openspec/changes/use-arm-statements-library/proposal.md new file mode 100644 index 0000000..5cbb468 --- /dev/null +++ b/openspec/changes/use-arm-statements-library/proposal.md @@ -0,0 +1,39 @@ +# Proposal + +## Why + +`arm_statements` поставляется как устанавливаемая библиотека, отсутствующая в PyPI. Копирование её исходников, словарей и тестов в сервис через `sync-statements` больше не соответствует способу поставки; сервис должен устанавливать локальный дистрибутив и сообщать его версию. + +## What Changes + +- Зафиксировать `pylib_ai_arm_statements==0.1.1` в `api/requirements.txt`, сохранив Python-импорты `arm_statements`. +- В `make venv` пропускать строки `pylib_ai*` при установке из индекса, как и внутренние `raisa_*`; библиотеку устанавливать из `../pylib_ai_arm_statements` через `uv`. +- Добавить `make install` для установки или переустановки библиотеки; путь переопределяется `STATEMENTS_DIR`. +- Удалить `api/arm_statements/` целиком, включая словари, VERSION, тесты и golden-файлы библиотеки. Удалить `statements-check` и его вызов из `check-full`. +- Проверять и тестировать только код сервиса: убрать библиотеку из путей pytest/mypy, считать её сторонней в Ruff, обновить порядок импортов и обработку отсутствующих типов. +- Получать `service.statements_version` через API библиотеки (VERSION либо метаданные установленного дистрибутива); при отсутствии версии останавливать старт с актуальной диагностикой. +- **BREAKING для сборки:** передавать исходники библиотеки в Docker через именованный контекст BuildKit `arm_statements`; обычный `docker build .` без него больше не поддерживается. +- Обновить README, CLAUDE.md и комментарии о зависимости: библиотека, установка и обновление через `make install` вместо синхронизации копии. + +## Capabilities + +### New Capabilities + +Нет; установка и сборка относятся к существующей capability сервиса. + +### Modified Capabilities + +- `tabular-recognition-service`: локальная установка и Docker-сборка библиотеки, версия библиотеки в результате и при старте. +- `statement-parsing`: выполнение шагов 2–8 установленной библиотекой, обновление её словарей переустановкой вместо синхронизации копии. + +## Impact + +Срез 2, сопровождение ББ/ОФР из Excel (срез 2b по `../docs/context/09-delivery-plan.md`); алгоритмы остаются по §11–12 и §7 `05-domain-rules.md`. Новых алгоритмов и допущений по открытым вопросам нет. + +Изменяется только репозиторий `arm-tabular-parser`: Makefile, Dockerfile, зависимости и конфигурация инструментов в `api/`, документация, адаптеры (порядок импортов), сообщение проверки версии. `../pylib_ai_arm_statements` — существующий источник дистрибутива, `../arm-core` — пример установки; их код не меняется. Таблицы и поля DBML v5.17 не затрагиваются, миграций нет. + +Текущие основные спецификации требуют коммит копии и `sync-statements`; delta-спецификации описывают замену этих требований. Копии JSON-контрактов сохраняются байт в байт. Их описание версии как коммита остаётся вопросом согласования в отдельном change `../docs`, не локальной правкой схемы. + +## Non-goals + +Публикация в PyPI/Nexus; изменение API библиотеки, правил разбора, сумм, единиц измерения, транспорта NATS/S3 и версии JSON-контракта; изменение arm-core, Compose или внешнего CI. Основные спецификации и архивы OpenSpec в этом предложении не редактируются. diff --git a/openspec/changes/use-arm-statements-library/specs/statement-parsing/spec.md b/openspec/changes/use-arm-statements-library/specs/statement-parsing/spec.md new file mode 100644 index 0000000..62f3da3 --- /dev/null +++ b/openspec/changes/use-arm-statements-library/specs/statement-parsing/spec.md @@ -0,0 +1,50 @@ +# Spec Delta + +## MODIFIED Requirements + +### Requirement: Словари форм, координаты, golden +Словари форм (заголовки и ОКУД, периоды, соотношения §7, единицы, перечни кодов) SHALL жить в установленной библиотеке `arm_statements` и в +tabular-parser MUST NOT дублироваться: файлы `api/rules/statement_*.yaml` удалены. Код `src/parser/statements/**` и +`src/parser/book.py` MUST NOT содержать литералов координат (линтер `test_no_coordinates`); `ref` ячейки собирается из +индексов функцией `grid.cell_ref`. Снимки `../fixtures/golden/expected_statement_parse/**` SHALL покрывать все файлы +`statements-xlsx/**` и обновляться только `make golden-update`; `service.statements_version` — волатильное поле снимка. +Property-тест по манифесту `tests/statements_manifest.yaml` SHALL проверять тип, ИНН, ожидаемый статус, сходимость +жёстких соотношений для полных форм и, где задано, множество кодов предупреждений `error` документа. + +#### Scenario: Новый синоним без релиза кода +- **WHEN** в словарь заголовков библиотеки `arm_statements` добавлен синоним заголовка «Баланс предприятия» и библиотека + переустановлена командой `make install` +- **THEN** лист классифицирован как `BALANCE` без изменения кода tabular-parser; собственного словаря форм в сервисе нет + +#### Scenario: Мёртвая запись словаря +- **WHEN** запись словаря форм не срабатывает ни на одном файле +- **THEN** это проверяет тест покрытия словарей библиотеки `arm_statements` (внутри пакета, на его данных); тест покрытия + tabular-parser словари форм не охватывает + +#### Scenario: Регрессия по снимкам +- **WHEN** результат разбора любого файла `statements-xlsx/**` отличается от снимка +- **THEN** golden-тест падает с diff; обновление — только `make golden-update` и отдельный коммит в fixtures + +### Requirement: Адаптер Excel шага 1 +tabular-parser SHALL оставить у себя чтение книги (лимиты `PARSE_MAX_BYTES`, `PARSE_TIMEOUT_SECONDS`, формат по +сигнатуре, лимиты zip и объединённых ячеек, `UNREADABLE`), классификацию листов (ОСВ — отдельный путь; форма; +продолжение; прочее) и адаптер «блок формы на листе → `FormCandidate`» общего кода; шаги 2–8 по формам SHALL +выполняться установленной библиотекой `arm_statements`. Адаптер SHALL: выделять блоки форм на листе по заголовку или ОКУД (без +кодов между ними блоки сливаются, заголовок после кодов — новая форма); собирать шапку каждого блока в его границах +(организация, ИНН без ИНН аудитора, ОКПО, ОКОПФ, ОКФС, единицы, дата, адрес) как `Cell` с `confidence = 100`; +заполнять у каждой ячейки `text` по числовому формату (tp#42) и `value`, `ref = <лист>!` (имя листа в одинарных +кавычках по правилу Excel, если в нём есть что-то кроме букв, цифр и `_`); дописывать +листы-продолжения в сетку открытой формы до раскладки (маркер «Форма … с.2» или позиция листа; проверки типа, +диапазона кодов и, при трёх и более кодах, числа колонок — как `join_blocker`), записывая границы листов в +`segment_breaks`. Несовпадение формата с `file_type` — предупреждение файла `FILE_TYPE_MISMATCH` (`info`). Лист без +признаков ОСВ и формы, без `type_hint` и кодов — предупреждение файла `SHEET_OTHER` (`info`, `locator = sheet:<имя>`), +документа нет. + +#### Scenario: Книга ББ + ОФР +- **WHEN** книга содержит лист «Баланс» (ОКУД 0710001), лист «Баланс (2)» с маркером «Форма 0710001 с.2» и лист «ОФР» +- **THEN** адаптер выдаёт два кандидата: `BALANCE` с `locator = sheet:Баланс,Баланс (2)` и одной сеткой из двух листов + (`segment_breaks` — первая строка второго листа) и `INCOME` с `locator = sheet:ОФР` + +#### Scenario: Ссылки ячеек +- **WHEN** код строки стоит в ячейке C27 листа «Баланс» +- **THEN** `ref` строки — `Баланс!C27`; у листа «Бухгалтерский баланс. Лист 1» — `'Бухгалтерский баланс. Лист 1'!C27` diff --git a/openspec/changes/use-arm-statements-library/specs/tabular-recognition-service/spec.md b/openspec/changes/use-arm-statements-library/specs/tabular-recognition-service/spec.md new file mode 100644 index 0000000..913440e --- /dev/null +++ b/openspec/changes/use-arm-statements-library/specs/tabular-recognition-service/spec.md @@ -0,0 +1,105 @@ +# Spec Delta + +## MODIFIED Requirements + +### Requirement: Результат по контракту +После подтверждения сервис SHALL прочитать объект `s3.key` из бакета `s3.bucket`, вызвать парсер, записать файл результата +(по `recognition-result.schema.json` v2.0, `schema_version = "2.0"`) в `results/{file_id}/{job_id}/result.json` и +опубликовать конверт результата (по `recognition-result-message.schema.json` v2.0, `result_schema_version = "2.0"`) с +заголовком `Nats-Msg-Id = result::`. Ack `processing` SHALL нести `schema_version = "2.0"`; запрос +`recognition-request` остаётся v1.0. `service.statements_version` файла результата SHALL быть версией библиотеки +`arm_statements` (VERSION или метаданные установленного дистрибутива; без версии сервис не стартует) и записываться при наличии `statementDocument`. +`metrics.warnings` конверта SHALL содержать предупреждения уровня файла структурами `{code, severity, message, locator?}`, +не строками. Статус конверта — по общему правилу: `done` ⇔ все документы `done` ∧ нет предупреждений файла уровня +`error`. При неудаче публикации сервис SHALL повторить её до трёх раз с паузой, затем залогировать ошибку. + +#### Scenario: Запрос с заглушкой парсера (Срез 0) +- **WHEN** воркеру явно передана заглушка `NullParser` (только тесты воркера; реализация по умолчанию — реальный парсер), + запрос подтверждён и объект существует в `uploads/` +- **THEN** в `files.tabular_parsed` появляется конверт `status = error`, `error.code = NOT_IMPLEMENTED`, `job_id`/`file_id` + из запроса, `metrics.duration_ms ≥ 0`; файл результата может отсутствовать + +#### Scenario: Запрос с реальным парсером +- **WHEN** запрос подтверждён и объект в `uploads/` существует, но не является книгой Excel (сигнатура не XLS/XLSX/XLSB) +- **THEN** в `files.tabular_parsed` появляется конверт `status = error`, `error.code = UNSUPPORTED_FORMAT` (повреждённая + книга — `UNREADABLE`), `job_id`/`file_id` из запроса, `metrics.duration_ms ≥ 0`; файл результата может отсутствовать; + корректная ОСВ в том же потоке даёт `done` (см. «Парсер — заменяемый интерфейс») + +#### Scenario: Результат и ack версии 2.0 +- **WHEN** обработан запрос `schema_version = "1.0"` с книгой, содержащей баланс +- **THEN** ack `processing` и конверт результата несут `schema_version = "2.0"`, `result_schema_version = "2.0"`, файл + результата проходит схему v2.0, `service.statements_version` равен версии библиотеки `arm_statements` + +#### Scenario: Предупреждение файла структурой +- **WHEN** в книге лист «Примечания» без признаков ОСВ и формы +- **THEN** `metrics.warnings` конверта содержит `{code: "SHEET_OTHER", severity: "info", message, locator: "sheet:Примечания"}` + +#### Scenario: Повтор публикации +- **WHEN** первая публикация конверта результата завершается ошибкой соединения, вторая — успешно +- **THEN** конверт опубликован один раз с тем же `Nats-Msg-Id`, обработка не повторяется + +### Requirement: Парсер — заменяемый интерфейс +Разбор файла SHALL выполняться через единый интерфейс «байты + тип файла → документы/метрики/ошибка», не знающий о +NATS и S3; воркер MUST NOT зависеть от конкретной реализации. Реализация по умолчанию (`get_parser()`) SHALL быть +оркестратором книги `BookParser`: он читает книгу один раз, классифицирует листы по содержимому и отдаёт листы ОСВ +конвейеру спеки `osv-parsing`, листы форм и их продолжения — конвейеру спеки `statement-parsing` (адаптер Excel и установленная библиотека + `arm_statements`); прочие листы дают предупреждение файла `SHEET_OTHER`. Файл результата SHALL писаться с +`schema_version = "2.0"`; документы книги — ОСВ в порядке листов, затем формы в порядке листов. Заглушка `NullParser` +(`error NOT_IMPLEMENTED`) SHALL использоваться только в тестах воркера и MUST NOT быть реализацией по умолчанию. +Реализация по умолчанию SHALL создаваться один раз при старте приложения (словари ОСВ читаются и проверяются там же, +словари форм — `default_rules()` пакета). Каталог `api/rules/` SHALL содержать словари ОСВ (`osv_headers`, `osv_groups`, +`periods`, `data_types`, `osv_checks`) и README; словарей форм ББ/ОФР в сервисе нет — они часть библиотеки `arm_statements`. + +#### Scenario: Подмена реализации в тесте +- **WHEN** в тесте воркеру передан парсер, возвращающий один `osvDocument` +- **THEN** файл результата с этим документом записан в S3, конверт имеет `status = done` и `result_s3` на этот файл + +#### Scenario: Реальная ОСВ через воркер +- **WHEN** в интеграционном тесте в S3 лежит `rb/2024-FY/ОСВ_66 за 2024.xls` и опубликован запрос `file_type = XLS` +- **THEN** после ack публикуется конверт `done` с `result_schema_version = "2.0"`, файл результата проходит схему v2.0 и + содержит один `osvDocument` со `status = done` и `account_code_raw = "66"` + +#### Scenario: Реальные формы через воркер +- **WHEN** в интеграционном тесте в S3 лежит книга `statements-xlsx/npz/2025-Q3/…xlsx` и опубликован запрос `file_type = XLSX` +- **THEN** после ack публикуется конверт `done` с `result_schema_version = "2.0"`, файл результата проходит схему v2.0 и + содержит два `statementDocument` (`BALANCE`, `INCOME`) со строками `line_status`, `ref`, `checks` и пустыми + `checks.relations_failed`; расхождения итогов в пределах округления (`1600 = 1700` на 1.94 / 9.02 / 2.48, `2100`/`2200`/ + `2300`) — в `soft_relations_failed` (OQ-138); `service.statements_version` равен версии библиотеки + +#### Scenario: Книга без ОСВ и форм +- **WHEN** в S3 лежит книга Excel, ни один лист которой не имеет признаков ОСВ или формы +- **THEN** публикуется конверт `error`, `error.code = INVALID_DOCUMENT`, сообщение называет число просмотренных листов + +#### Scenario: Сервис без версии общего кода не стартует +- **WHEN** у библиотеки `arm_statements` нет ни VERSION, ни версии установленного дистрибутива +- **THEN** `require_statements_version()` бросает `RuntimeError` с текстом «библиотека arm_statements без VERSION и версии пакета: выполните make install», сервис не + стартует + +#### Scenario: Установленный дистрибутив без VERSION +- **WHEN** библиотека установлена как `pylib_ai_arm_statements==0.1.1`, файл VERSION отсутствует, а метаданные дистрибутива доступны +- **THEN** сервис стартует и указывает `service.statements_version = "0.1.1"` при обработке реальной книги `statements-xlsx/npz/2025-Q3/…xlsx` + +## ADDED Requirements + +### Requirement: Локальная поставка библиотеки форм +Окружение сервиса SHALL использовать установленную библиотеку `pylib_ai_arm_statements==0.1.1` с импортом `arm_statements`. `make venv` SHALL пропускать строки `pylib_ai*` и `raisa_*` при установке из requirements через индекс и устанавливать библиотеку из `STATEMENTS_DIR` (по умолчанию `../pylib_ai_arm_statements`). `make install` SHALL создавать окружение при необходимости и переустанавливать библиотеку из этого пути. + +#### Scenario: Подготовка окружения +- **WHEN** разработчик выполняет `make venv` при доступной локальной библиотеке +- **THEN** runtime- и dev-зависимости установлены в `api/.venv`, библиотека импортируется из установленного пакета; запрос на скачивание `pylib_ai*` из PyPI не выполняется + +#### Scenario: Переустановка после изменения библиотеки +- **WHEN** разработчик выполняет `make install STATEMENTS_DIR=<путь-к-библиотеке>` после изменения её исходников без смены версии +- **THEN** библиотека переустановлена из указанного пути в `api/.venv` и изменения доступны сервису + +### Requirement: Библиотека форм в контейнерном образе +`make docker-build` SHALL передавать `STATEMENTS_DIR` именованным контекстом BuildKit `arm_statements`. Сборка SHALL устанавливать библиотеку из этого контекста вместе с публичными зависимостями, без загрузки `pylib_ai*` из PyPI. Образ SHALL содержать установленный пакет и его словари, доступные при запуске без исходной папки хоста. + +#### Scenario: Сборка и запуск с локальной библиотекой +- **WHEN** образ собран через `make docker-build` при доступном BuildKit и локальной библиотеке +- **THEN** внутри образа импортируется `arm_statements`, доступны словари и версия `0.1.1`; исходная папка хоста при запуске не требуется + +#### Scenario: Прямая сборка +- **WHEN** сборка запускается напрямую командой `docker build` +- **THEN** вызывающая сторона SHALL передать `--build-context arm_statements=<путь-к-библиотеке>`; отсутствие контекста не заменяется загрузкой библиотеки из PyPI + diff --git a/openspec/changes/use-arm-statements-library/tasks.md b/openspec/changes/use-arm-statements-library/tasks.md new file mode 100644 index 0000000..f3042b2 --- /dev/null +++ b/openspec/changes/use-arm-statements-library/tasks.md @@ -0,0 +1,36 @@ +# Tasks + +Change ретроспективно фиксирует работу в `arm-tabular-parser`. Отметки [x] основаны на текущих изменениях и проверках предыдущих шагов этой сессии; оставшиеся проверки не считаются выполненными. Каждая задача рассчитана не более чем на половину дня. + +## 1. Тесты и исходные ожидания + +- [x] 1.1 Сохранить тесты адаптеров `api/tests/unit/parser/` и проверки версии/ошибки словарей в `api/tests/unit/test_worker.py`, обновить комментарий об установленной версии. Проверка: 1206 parser-тестов и 3 выбранных worker-теста прошли; 5 parser-тестов пропущены. +- [x] 1.2 Проверить реальные формы по `api/tests/statements_manifest.yaml`, `api/tests/fixtures/test_statements_adapter_real.py` и golden-снимкам `../fixtures/golden/expected_statement_parse/` с установленной библиотекой. Критерий: тесты реальных форм проходят без необъяснённых расхождений; существующие снимки автоматически не перезаписывать. Проверено: 27 тестов форм и 27 golden-тестов прошли, снимки не менялись. + +## 2. Локальная установка и удаление копии + +- [x] 2.1 Обновить `Makefile` и `api/requirements.txt`: pin 0.1.1, фильтр `raisa_*`/`pylib_ai*`, STATEMENTS_DIR, установка через `make venv`, переустановка через `make install`. Проверка: dry-run обеих целей соответствует design, локальный `make install` установил дистрибутив 0.1.1. +- [x] 2.2 Удалить `api/arm_statements/` со словарями, VERSION и библиотечными тестами; удалить `statements-check` и его вызов в `Makefile`. Проверка: каталога нет, импорт указывает на site-packages, `check-full` больше не запускает sync-statements. +- [x] 2.3 Обновить `api/pytest.ini`, `api/pyproject.toml`, комментарий `api/requirements-dev.txt`; перегруппировать сторонние импорты в `api/src/parser/` и соответствующих тестах. Проверка: pytest не собирает библиотечные тесты, `make lint` проходит, mypy проверяет src. +- [x] 2.4 Описать установку и обновление в `README.md`, `CLAUDE.md`, `api/rules/README.md`. Проверка: описаны отсутствие PyPI, `STATEMENTS_DIR` и `make install`, отсутствует инструкция синхронизировать копию. +- [x] 2.5 Проверить подготовку окружения полной командой `make venv` и повторную установку библиотеки той же версии через `make install STATEMENTS_DIR=<путь>`. Критерий: обе команды успешны, импорт работает; пакет pylib_ai не запрашивается из индекса. Проверено: make venv и повторный make install с абсолютным STATEMENTS_DIR успешны; импорт из site-packages, версия 0.1.1. + +## 3. Версия библиотеки при старте + +- [x] 3.1 Обновить docstring и диагностику `require_statements_version` в `api/src/parser/__init__.py`, комментарий поля в `api/src/contract/common.py`. Проверка: сохраняется API библиотеки и явный путь для тестов; RuntimeError предлагает `make install`; JSON-схемы не изменены. +- [x] 3.2 Проверить загрузку установленного пакета, его метаданных, словарей и создание парсера. Критерий: импорт из site-packages, версия дистрибутива и `statements_version()` равны 0.1.1, `get_parser()` создаёт BookParser; проверено ранее с `DEBUG=false ENV_FILE=.env.test`. + +## 4. Контейнерная поставка + +- [x] 4.1 Изменить `Makefile`, `Dockerfile`, `README.md`: именованный контекст BuildKit, временный bind mount библиотеки и установка локального пакета, фильтрация requirements. Проверка реализации: `make -n docker-build` передаёт `--build-context arm_statements`, исходники подключаются только на шаге установки; требование к прямым сборкам документировано. +- [x] 4.2 Собрать образ через `make docker-build` и проверить в контейнере импорт, `statements_version()` и `default_rules()` без mount исходников при запуске. Критерий: образ содержит рабочую библиотеку 0.1.1 и словари. Предыдущая попытка заблокирована доступом к Docker socket. Проверено 24.10.2026: make docker-build успешен; docker run --network none без mount подтверждает импорт из site-packages, версию 0.1.1 и загрузку словарей. + +## 5. Итоговые проверки + +- [ ] 5.1 Завершить полный unit-прогон `api/tests/unit` с корректным DEBUG, выяснив зависание worker-тестов. Критерий: зафиксированы итоговый результат и причины пропусков; предыдущий остановленный прогон не считать успешным. +- [ ] 5.2 Выполнить доступные интеграционные проверки NATS/S3 через `make test-integration` и итоговый `make check-full`. Критерий: обработка реальных форм проходит контракт v2.0 и содержит версию библиотеки; lint, тесты и Docker-сборка успешны. + +## Workflow follow-up + +- Основные `openspec/specs/` остаются без правок до отдельной синхронизации/архивирования этого change; архивные changes не переписываются. +- Финальное межрепозиторное согласование провести отдельным change в `../docs`: проверить актуальность 03-architecture и 07-open-questions, согласовать описание `service.statements_version` в каноническом контракте; затем обновить копии контрактов и README согласованно. Эти внешние правки в текущей задаче не выполнялись. -- 2.47.2 From cac2054e8ac5bd9e2fb787d7e5d698c124ba0a0b Mon Sep 17 00:00:00 2001 From: tsygankoviva Date: Mon, 26 Oct 2026 18:56:23 +0300 Subject: [PATCH 2/6] AURORA-1677: Dockerfile fix --- Dockerfile | 8 ++++---- Makefile | 8 +++++++- README.md | 13 ++++++++++++- .../changes/use-arm-statements-library/design.md | 4 ++-- .../specs/tabular-recognition-service/spec.md | 4 ++-- 5 files changed, 27 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 19ed694..4ae40a3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 # Образ для dev и `make check-full`. Контекст сборки — корень репозитория tabular-parser. -# Библиотека передаётся именованным контекстом arm_statements (см. make docker-build). +# Библиотека передаётся именованным контекстом statements (см. make docker-build). # Платформенные пакеты raisa_* (Nexus) в образ не ставятся. FROM python:3.11-slim @@ -10,9 +10,9 @@ ENV PYTHONUNBUFFERED=1 \ WORKDIR /app/api COPY api/requirements.txt ./requirements.txt -RUN --mount=type=bind,from=arm_statements,target=/tmp/arm_statements,rw \ - grep -Ev '^(raisa_|pylib_ai)' requirements.txt > /tmp/req.txt \ - && pip install --no-cache-dir -r /tmp/req.txt /tmp/arm_statements \ +RUN --mount=type=bind,from=statements,target=/tmp/statements,rw \ + grep -v '^raisa_' requirements.txt > /tmp/req.txt \ + && pip install --no-cache-dir -r /tmp/req.txt /tmp/statements \ && rm /tmp/req.txt COPY api/ ./ diff --git a/Makefile b/Makefile index 0311571..aabbad2 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ VENV := api/.venv BIN := $(CURDIR)/$(VENV)/bin STATEMENTS_DIR ?= ../pylib_ai_arm_statements +STATEMENTS_TAG ?= v0.1.1 COMPOSE_FILE ?= ../docs/dev/docker-compose.yml .PHONY: venv install lint test test-integration test-fixtures golden-update golden-check check docker-build check-full up @@ -46,7 +47,12 @@ check: lint test # Сборка образа из корня репозитория docker-build: - docker build --build-context arm_statements="$(STATEMENTS_DIR)" -t tabular-parser . + @release=$$(git -C "$(STATEMENTS_DIR)" rev-parse --verify "refs/tags/$(STATEMENTS_TAG)^{commit}") || \ + { echo "Ошибка: в библиотеке отсутствует тег $(STATEMENTS_TAG); выполните git fetch --tags в $(STATEMENTS_DIR)" >&2; exit 1; }; \ + head=$$(git -C "$(STATEMENTS_DIR)" rev-parse HEAD) || exit 1; \ + [ "$$head" = "$$release" ] || \ + { echo "Ошибка: HEAD библиотеки не совпадает с $(STATEMENTS_TAG)" >&2; exit 1; } + docker build --build-context "statements=$(STATEMENTS_DIR)" -t tabular-parser . # Полная проверка перед PR: быстрые проверки, затем NATS/MinIO из compose (make up), затем образ check-full: check test-integration docker-build diff --git a/README.md b/README.md index 7e35763..0a72f5e 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ openspec/ требования и дельта текущего ср | `make golden-check` | то же без записи; код возврата 1, если снимки расходятся с разбором | | `make check` | `make lint` + `make test` (включая идентичность копии контракта с `../docs/contracts`); с каталогом fixtures ≈ 95 с — из-за тестов на 82 файлах | | `make check-full` | `make check` + `make test-integration` + `docker build`; запускать перед PR (нужен поднятый compose, `make up`) | -| `make docker-build` | собирает образ, передавая локальную библиотеку через `--build-context arm_statements="$(STATEMENTS_DIR)"` (нужен BuildKit) | +| `make docker-build` | собирает образ, передавая локальную библиотеку через `--build-context "statements=$(STATEMENTS_DIR)"` (нужен BuildKit) | | `make up` | `docker compose -f ../docs/dev/docker-compose.yml up -d --wait postgres nats minio` | Интеграционные тесты (`make test-integration`, `make check-full`) поднимают сервис в процессе теста и вступают в queue group `tabular-parser` на реальной шине: на шине compose не должно работать другого экземпляра tabular-parser (и наоборот: на время теста in-process воркер ответит ошибкой разбора (`UNSUPPORTED_FORMAT` / `UNREADABLE`) на реальные запросы других производителей). @@ -544,3 +544,14 @@ ENV_FILE=.env.test .venv/bin/python -m src.parser.cli <файл> [--file-type XL `SERVICE_NAME` — имя сервиса в конвертах контракта (`service.name`); допустимо только `tabular-parser`, иное значение — ошибка конфигурации при старте. Queue group подписки NATS — константа `tabular-parser` в `src/core/nats.py`, настройкой не меняется. + +Сборка Docker использует BuildKit-контекст `statements` из `STATEMENTS_DIR` +(по умолчанию `../pylib_ai_arm_statements`). Другой путь: +`make docker-build STATEMENTS_DIR=/путь/к/библиотеке`. Библиотека устанавливается +из исходников вместе с requirements; её версия должна соответствовать закреплённой +в requirements. Остальные зависимости загружаются из пакетного индекса. + +Перед локальной сборкой `make docker-build` проверяет совпадение HEAD библиотеки +с тегом `STATEMENTS_TAG` (по умолчанию `v0.1.1`). Получить тег: +`git -C ../pylib_ai_arm_statements fetch origin --tags`. +Незакоммиченные изменения разрешены и попадают в сборку из `STATEMENTS_DIR`. diff --git a/openspec/changes/use-arm-statements-library/design.md b/openspec/changes/use-arm-statements-library/design.md index 36ae262..973af52 100644 --- a/openspec/changes/use-arm-statements-library/design.md +++ b/openspec/changes/use-arm-statements-library/design.md @@ -20,14 +20,14 @@ 3. **Версия через API библиотеки.** `require_statements_version` продолжает вызывать `statements_version(path)` и запрещает старт при отсутствии версии. Меняются документация и RuntimeError: `библиотека arm_statements без VERSION и версии пакета: выполните make install`. VERSION сохраняет приоритет, метаданные дают fallback для установленного дистрибутива; прямое чтение метаданных в сервисе дублировало бы логику библиотеки. Тип поля контракта остаётся непустой строкой, JSON-схемы не редактируются. -4. **Именованный контекст BuildKit.** `make docker-build` передаёт `--build-context arm_statements="$(STATEMENTS_DIR)"`. Dockerfile подключает его через временный bind mount на шаге RUN и устанавливает `/tmp/arm_statements` вместе с отфильтрованными requirements. `rw` нужен сборочным инструментам для временных файлов; изменения mount не записываются обратно на хост и не становятся содержимым образа. Установленный пакет и его package data остаются в образе. Альтернатива расширения основного контекста на весь workspace меняет правила COPY и состав контекста; копирование checkout в слой добавляет ненужные исходники. Прямым вызовам Docker, внешним CI и Compose потребуется тот же именованный контекст; их конфигурация в этой задаче не менялась. +4. **Именованный контекст BuildKit.** `make docker-build` передаёт `--build-context statements="$(STATEMENTS_DIR)"`. Dockerfile подключает его через временный bind mount на шаге RUN и устанавливает `/tmp/statements` вместе с отфильтрованными requirements. `rw` нужен сборочным инструментам для временных файлов; изменения mount не записываются обратно на хост и не становятся содержимым образа. Установленный пакет и его package data остаются в образе. Альтернатива расширения основного контекста на весь workspace меняет правила COPY и состав контекста; копирование checkout в слой добавляет ненужные исходники. Прямым вызовам Docker, внешним CI и Compose потребуется тот же именованный контекст; их конфигурация в этой задаче не менялась. 5. **Документация отражает библиотечную поставку.** README, CLAUDE.md, `api/rules/README.md`, комментарии в requirements-dev, проверке версии, тесте worker и модели `Service` обновлены. Архивные OpenSpec change и копии JSON-схем не переписываются. Внешнее описание поля как коммита требует согласования отдельным change в docs. ## Risks / Trade-offs - [Нет локального checkout или неверный STATEMENTS_DIR] → установка завершается ошибкой; путь и команда переустановки описаны в README. -- [Отличие локальной версии от pin] → перед сборкой проверять метаданные дистрибутива и `statements_version()`; автоматическая сверка не добавлялась. +- [Отличие локальной версии от pin] → перед сборкой проверять метаданные дистрибутива и `statements_version()`; при Docker-сборке pip автоматически проверяет соответствие локальной версии pin из requirements. - [Версия 0.1.1 не идентифицирует локальные изменения исходников] → переустановка принудительная, воспроизводимый релиз требует дисциплины версий библиотеки; Git SHA больше не является обязательным форматом поля. - [Нет BuildKit или не передан именованный контекст] → использовать `make docker-build`; в других системах сборки явно передавать контекст. - [Нет маркера типов у библиотеки] → mypy проверяет код сервиса, поведение интеграции подтверждают тесты адаптеров. diff --git a/openspec/changes/use-arm-statements-library/specs/tabular-recognition-service/spec.md b/openspec/changes/use-arm-statements-library/specs/tabular-recognition-service/spec.md index 913440e..38012ce 100644 --- a/openspec/changes/use-arm-statements-library/specs/tabular-recognition-service/spec.md +++ b/openspec/changes/use-arm-statements-library/specs/tabular-recognition-service/spec.md @@ -93,7 +93,7 @@ NATS и S3; воркер MUST NOT зависеть от конкретной р - **THEN** библиотека переустановлена из указанного пути в `api/.venv` и изменения доступны сервису ### Requirement: Библиотека форм в контейнерном образе -`make docker-build` SHALL передавать `STATEMENTS_DIR` именованным контекстом BuildKit `arm_statements`. Сборка SHALL устанавливать библиотеку из этого контекста вместе с публичными зависимостями, без загрузки `pylib_ai*` из PyPI. Образ SHALL содержать установленный пакет и его словари, доступные при запуске без исходной папки хоста. +`make docker-build` SHALL передавать `STATEMENTS_DIR` именованным контекстом BuildKit `statements`. Сборка SHALL устанавливать библиотеку из этого контекста вместе с публичными зависимостями, без загрузки `pylib_ai*` из PyPI. Образ SHALL содержать установленный пакет и его словари, доступные при запуске без исходной папки хоста. #### Scenario: Сборка и запуск с локальной библиотекой - **WHEN** образ собран через `make docker-build` при доступном BuildKit и локальной библиотеке @@ -101,5 +101,5 @@ NATS и S3; воркер MUST NOT зависеть от конкретной р #### Scenario: Прямая сборка - **WHEN** сборка запускается напрямую командой `docker build` -- **THEN** вызывающая сторона SHALL передать `--build-context arm_statements=<путь-к-библиотеке>`; отсутствие контекста не заменяется загрузкой библиотеки из PyPI +- **THEN** вызывающая сторона SHALL передать `--build-context statements=<путь-к-библиотеке>`; отсутствие контекста не заменяется загрузкой библиотеки из PyPI -- 2.47.2 From cf0b55f8422f5de272c7868cd1c5dd36fde60b61 Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Tue, 27 Oct 2026 16:33:27 +0400 Subject: [PATCH 3/6] =?UTF-8?q?=D0=9A=D0=BE=D0=BF=D0=B8=D1=8F=20recognitio?= =?UTF-8?q?n-result.schema.json=20=D0=B8=D0=B7=20docs:=20=D0=BE=D0=BF?= =?UTF-8?q?=D0=B8=D1=81=D0=B0=D0=BD=D0=B8=D1=8F=20=D0=BF=D0=BE=D0=B4=20?= =?UTF-8?q?=D0=B1=D0=B8=D0=B1=D0=BB=D0=B8=D0=BE=D1=82=D0=B5=D0=BA=D1=83=20?= =?UTF-8?q?(AURORA-1677)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- api/contracts/recognition-result.schema.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/contracts/recognition-result.schema.json b/api/contracts/recognition-result.schema.json index 19531dd..7590744 100644 --- a/api/contracts/recognition-result.schema.json +++ b/api/contracts/recognition-result.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "arm-ka/recognition-result/v2", "title": "Результат распознавания файла (tabular-parser / scan-gateway), контракт v2.0", - "description": "Один файл (stored_file) → один job → N документов. Сервисы не знают БД: только структура файла. Формы ББ/ОФР нормализуются общим кодом arm_statements (05-domain-rules.md §11): статусы строк, needs_review, проверки §7, статус документа и структурные предупреждения приходят готовыми; ядро хранит их как пришли и пересчитывает тем же кодом только после правок аналитика. Нормализация счетов ОСВ, резолв контрагентов, сверка ИНН и периода — в arm-core.", + "description": "Один файл (stored_file) → один job → N документов. Сервисы не знают БД: только структура файла. Формы ББ/ОФР нормализуются библиотекой arm_statements (05-domain-rules.md §11): статусы строк, needs_review, проверки §7, статус документа и структурные предупреждения приходят готовыми; ядро хранит их как пришли и пересчитывает тем же кодом только после правок аналитика. Нормализация счетов ОСВ, резолв контрагентов, сверка ИНН и периода — в arm-core.", "$comment": "Версионирование: schema_version = MAJOR.MINOR. MINOR (2.1, 2.2, …) — необязательные поля и новые значения enum; MAJOR (3.0) — удаление/переименование полей, смена типов или семантики. Потребитель принимает известный MAJOR и любой MINOR. Новый MINOR раскатывается сначала на потребителей (ядро и сервисы обновляют копии схем раньше, чем производитель начнёт слать новый MINOR), потому что закрытые enum отвергают незнакомые значения. Новый MAJOR — порядок «сначала ядро»: ядро на время перехода принимает и прежний, и новый MAJOR (два маппера), сервисы переходят после выкладки ядра. Changelog: 1.0 — исходный контракт; 1.1 — osvDocument.rows[].extra; 1.2 — DISCLOSURE, statementDocument.checks, границы периода ОСВ; 2.0 (RSHB/arm-ka-docs#116, change unified-statement-normalization) — line_code nullable и line_status (ok | unreadable | missing | detail), row_index, ref, parent_ref, line_code_text, needs_review у значения и строки, values[].text, header.<поле>_text и <поле>_confidence у всех полей шапки, header.needs_review_fields, columns[].period_kind и label_confidence, document.status и document.warnings у всех документов (включая ОСВ), statementDocument.form_variant, metrics.warnings и warnings — структуры с уровнем и привязкой, checks обязателен у BALANCE/INCOME со строками, нарушения с codes, удалён checks.duplicate_line_codes, service.statements_version, header.okei (+ _text, _confidence), смысл is_recognized у OCR отвязан от confidence, statementDocument.form_variant обязателен, OTHER удалён из statementDocument.document_type. Схема 1.2 на окне перехода лежит рядом — recognition-result.v1.schema.json (примеры — examples/v1/); удаляется вместе с маппером 1.x ядра после перехода обоих сервисов.", "type": "object", "required": ["schema_version", "job_id", "file_id", "service", "status", "documents"], @@ -16,7 +16,7 @@ "properties": { "name": { "enum": ["tabular-parser", "scan-gateway"] }, "version": { "type": "string" }, - "statements_version": { "type": "string", "minLength": 1, "description": "коммит репозитория arm-statements, из которого разложена копия общего кода; обязателен, если в documents есть statementDocument" } + "statements_version": { "type": "string", "minLength": 1, "description": "маркер VERSION библиотеки arm_statements, если он есть, иначе версия установленного дистрибутива pylib-ai-arm-statements; обязателен, если в documents есть statementDocument" } } }, "status": { "enum": ["done", "needs_check", "error"], "description": "done ⇔ все документы done и нет предупреждений файла уровня error; error — документов нет" }, -- 2.47.2 From 4a030bfc425618a2ce6416e460e146ccf80c9891 Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Tue, 27 Oct 2026 16:34:39 +0400 Subject: [PATCH 4/6] =?UTF-8?q?=D0=9F=D0=BE=D1=80=D1=8F=D0=B4=D0=BE=D0=BA?= =?UTF-8?q?=20=D0=B8=D0=BC=D0=BF=D0=BE=D1=80=D1=82=D0=BE=D0=B2=20=D0=B2=20?= =?UTF-8?q?=D1=82=D0=B5=D1=81=D1=82=D0=B5=20=D0=B8=D0=B7=20main:=20arm=5Fs?= =?UTF-8?q?tatements=20=E2=80=94=20=D1=81=D1=82=D0=BE=D1=80=D0=BE=D0=BD?= =?UTF-8?q?=D0=BD=D0=B8=D0=B9=20=D0=BF=D0=B0=D0=BA=D0=B5=D1=82=20(AURORA-1?= =?UTF-8?q?677)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- api/tests/unit/parser/test_osv_account_marker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/tests/unit/parser/test_osv_account_marker.py b/api/tests/unit/parser/test_osv_account_marker.py index 72acd90..79eccb9 100644 --- a/api/tests/unit/parser/test_osv_account_marker.py +++ b/api/tests/unit/parser/test_osv_account_marker.py @@ -8,8 +8,8 @@ import io import openpyxl import pytest - from arm_statements import default_rules + from src.contract import OsvDocument from src.parser.book import BookParser from src.parser.osv import OsvParser -- 2.47.2 From bdc8e555ff23b2d333cad9384095b009fca03240 Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Tue, 27 Oct 2026 17:07:13 +0400 Subject: [PATCH 5/6] =?UTF-8?q?statements-version:=20=D1=81=D0=B2=D0=B5?= =?UTF-8?q?=D1=80=D0=BA=D0=B0=20checkout=20=D0=B1=D0=B8=D0=B1=D0=BB=D0=B8?= =?UTF-8?q?=D0=BE=D1=82=D0=B5=D0=BA=D0=B8=20=D1=81=20requirements.txt=20?= =?UTF-8?q?=D0=B2=20make=20venv/install;=20=D1=82=D0=B5=D0=B3=20=E2=80=94?= =?UTF-8?q?=20=D0=B8=D0=B7=20requirements=20(AURORA-1677)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- Makefile | 21 +++++++++++++++++---- README.md | 2 ++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index aabbad2..2614cc6 100644 --- a/Makefile +++ b/Makefile @@ -2,20 +2,33 @@ VENV := api/.venv BIN := $(CURDIR)/$(VENV)/bin STATEMENTS_DIR ?= ../pylib_ai_arm_statements -STATEMENTS_TAG ?= v0.1.1 +# Тег выпуска библиотеки — по версии, закреплённой в api/requirements.txt +STATEMENTS_TAG ?= v$(shell sed -n 's/^pylib_ai_arm_statements==//p' api/requirements.txt) COMPOSE_FILE ?= ../docs/dev/docker-compose.yml -.PHONY: venv install lint test test-integration test-fixtures golden-update golden-check check docker-build check-full up +.PHONY: venv install statements-version lint test test-integration test-fixtures golden-update golden-check check docker-build check-full up + +# Сверка checkout библиотеки с версией из api/requirements.txt (pylib_ai_arm_statements==<версия>): VERSION checkout +# не равен закреплённой — ошибка; checkout не на теге v<версия> — предупреждение (правка библиотеки до выпуска). +# docker-build требует тег строго. +statements-version: + @ver=$$(cat "$(STATEMENTS_DIR)/VERSION" 2>/dev/null) || \ + { echo "Ошибка: нет $(STATEMENTS_DIR)/VERSION — клонируйте RSHB/pylib_ai_arm_statements рядом или задайте STATEMENTS_DIR" >&2; exit 1; }; \ + [ "v$$ver" = "$(STATEMENTS_TAG)" ] || \ + { echo "Ошибка: в $(STATEMENTS_DIR) версия $$ver, в api/requirements.txt — $(STATEMENTS_TAG:v%=%)" >&2; exit 1; }; \ + release=$$(git -C "$(STATEMENTS_DIR)" rev-parse -q --verify "refs/tags/$(STATEMENTS_TAG)^{commit}"); \ + [ -n "$$release" ] && [ "$$release" = "$$(git -C "$(STATEMENTS_DIR)" rev-parse HEAD)" ] || \ + echo "Предупреждение: $(STATEMENTS_DIR) не на теге $(STATEMENTS_TAG) — ставится рабочее дерево библиотеки" # Python 3.11; пропускаем внутренние raisa_* и устанавливаемые локально pylib_ai* -venv: +venv: statements-version cd api && uv venv --python 3.11 --allow-existing .venv cd api && grep -Ev '^(raisa_|pylib_ai)' requirements.txt | uv pip install --python .venv/bin/python -r - cd api && grep -v '^-r' requirements-dev.txt | uv pip install --python .venv/bin/python -r - uv pip install --python "$(BIN)/python" --reinstall-package pylib-ai-arm-statements "$(STATEMENTS_DIR)" # Установить или переустановить библиотеку в окружение backend. -install: +install: statements-version uv venv --python 3.11 --allow-existing "$(VENV)" uv pip install --python "$(BIN)/python" --reinstall-package pylib-ai-arm-statements "$(STATEMENTS_DIR)" diff --git a/README.md b/README.md index 0a72f5e..cb368db 100644 --- a/README.md +++ b/README.md @@ -308,6 +308,8 @@ openpyxl 0,58 с; синтетический лист в 2 млн ячеек (63 Библиотека не публикуется в PyPI. `make venv` устанавливает её из `../pylib_ai_arm_statements`; `make install` переустанавливает её после изменений. Путь можно задать через `STATEMENTS_DIR`. +Перед установкой обе цели сверяют checkout (`make statements-version`): его `VERSION` должен совпадать с версией +в `api/requirements.txt` (иначе ошибка), checkout не на теге `v<версия>` — предупреждение. `make docker-build` требует тег. | Модуль (`api/src/parser/statements/`) | Что делает | |---|---| -- 2.47.2 From f4df39ac2332feb4c7f71d3b8a2137024a23473a Mon Sep 17 00:00:00 2001 From: tsygankoviva Date: Fri, 30 Oct 2026 13:58:52 +0300 Subject: [PATCH 6/6] AURORA-1677: arm_statements 0.1.2 --- api/pyproject.toml | 5 ----- api/requirements.txt | 2 +- api/src/parser/__init__.py | 9 ++++----- api/tests/unit/test_worker.py | 16 +++++++--------- 4 files changed, 12 insertions(+), 20 deletions(-) diff --git a/api/pyproject.toml b/api/pyproject.toml index c3bdf82..28a74f1 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -38,8 +38,3 @@ ignore_missing_imports = true [[tool.mypy.overrides]] module = ["src.parser", "src.parser.*"] disallow_untyped_defs = true - -# Установленная библиотека не поставляет py.typed. -[[tool.mypy.overrides]] -module = ["arm_statements", "arm_statements.*"] -ignore_missing_imports = true diff --git a/api/requirements.txt b/api/requirements.txt index e86100c..8ce7059 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -43,7 +43,7 @@ Pygments==2.20.0 PyNaCl==1.5.0 python-dotenv==1.2.2 python-multipart==0.0.32 -pylib_ai_arm_statements==0.1.1 +pylib_ai_arm_statements==0.1.2 pyxlsb==1.0.10 PyYAML==6.0.3 rich==15.0.0 diff --git a/api/src/parser/__init__.py b/api/src/parser/__init__.py index 7045635..c548f59 100644 --- a/api/src/parser/__init__.py +++ b/api/src/parser/__init__.py @@ -63,15 +63,14 @@ class NullParser: ) -def require_statements_version(path: Path | None = None) -> str: +def require_statements_version() -> str: """Версия библиотеки `arm_statements` для `service.statements_version`. - Библиотека читает VERSION или метаданные установленного пакета. - `path` позволяет явно указать файл версии в тестах. Без версии сервис не стартует. + Библиотека читает метаданные установленного пакета. Без версии сервис не стартует. """ - version = statements_version(path) + version = statements_version() if version is None: - raise RuntimeError("библиотека arm_statements без VERSION и версии пакета: выполните make install") + raise RuntimeError("библиотека arm_statements без версии пакета: выполните make install") return version diff --git a/api/tests/unit/test_worker.py b/api/tests/unit/test_worker.py index f6ce5dd..cbaee4a 100644 --- a/api/tests/unit/test_worker.py +++ b/api/tests/unit/test_worker.py @@ -925,17 +925,15 @@ async def test_worker_with_form_result_has_statements_version_in_result_file() - assert [error.message for error in VALIDATORS["result"].iter_errors(stored)] == [] -def test_require_statements_version_fails_without_file(tmp_path: Path) -> None: - with pytest.raises(RuntimeError, match="VERSION"): - require_statements_version(tmp_path / "VERSION") +def test_require_statements_version_fails_without_metadata(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("src.parser.statements_version", lambda: None) + with pytest.raises(RuntimeError, match="без версии пакета"): + require_statements_version() -def test_require_statements_version_reads_the_marker(tmp_path: Path) -> None: - marker = tmp_path / "VERSION" - marker.write_text("abc1234\n", encoding="utf-8") - - assert require_statements_version(marker) == "abc1234" - assert require_statements_version() # версия установленной библиотеки +def test_require_statements_version_reads_distribution(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("src.parser.statements_version", lambda: "0.1.2") + assert require_statements_version() == "0.1.2" def test_get_parser_wraps_rules_error_into_runtime_error(monkeypatch: pytest.MonkeyPatch) -> None: -- 2.47.2