feat(statements): маппинг форм ББ/ОФР в map_file — meta, колонки, строки, значения, статус (D-1…D-3, D-6)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012vJkmBGKKqMoEMzvsXG3y2
This commit is contained in:
Andrey Epifancev
parent
29f516db3c
commit
1d821b4d3e
140
api/src/repositories/statements.py
Normal file
140
api/src/repositories/statements.py
Normal file
@ -0,0 +1,140 @@
|
||||
"""Таблицы `financial_statement_*` (Срез 3b): вставка формы при маппинге, очистка файла, чтение сумм и флагов значений.
|
||||
Только SQL, без правил: правила — `domain/statement_mapping.py`, `domain/statement_checks.py`."""
|
||||
|
||||
import uuid
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, insert, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from src.domain.statement_mapping import StatementDraft
|
||||
from src.models.files import Document
|
||||
from src.models.statements import (
|
||||
FinancialStatementColumn,
|
||||
FinancialStatementLine,
|
||||
FinancialStatementMeta,
|
||||
FinancialStatementValue,
|
||||
)
|
||||
|
||||
__all__ = ["BATCH", "delete_file_statements", "insert_statement", "statement_amounts", "value_flags"]
|
||||
|
||||
BATCH = 1000 # значений на один INSERT: 8 колонок × 1000 < 32767 параметров asyncpg
|
||||
|
||||
|
||||
async def insert_statement(
|
||||
session: AsyncSession, *, document_id: uuid.UUID, job_id: uuid.UUID, draft: StatementDraft
|
||||
) -> FinancialStatementMeta:
|
||||
"""Шапка, колонки, строки и значения формы из черновика (без commit); возвращает шапку."""
|
||||
meta = FinancialStatementMeta(
|
||||
id=uuid.uuid4(),
|
||||
document_id=document_id,
|
||||
recognition_job_id=job_id,
|
||||
statement_type=draft.statement_type,
|
||||
**draft.header,
|
||||
)
|
||||
session.add(meta)
|
||||
await session.flush()
|
||||
column_ids: dict[int, uuid.UUID] = {}
|
||||
columns: list[dict[str, Any]] = []
|
||||
for column_index, label, period_date, is_current in draft.columns:
|
||||
column_ids[column_index] = uuid.uuid4()
|
||||
columns.append(
|
||||
{
|
||||
"id": column_ids[column_index],
|
||||
"meta_id": meta.id,
|
||||
"column_index": column_index,
|
||||
"label": label,
|
||||
"period_date": period_date,
|
||||
"is_current": is_current,
|
||||
}
|
||||
)
|
||||
if columns:
|
||||
await session.execute(insert(FinancialStatementColumn).values(columns))
|
||||
line_ids: dict[str, uuid.UUID] = {}
|
||||
lines: list[dict[str, Any]] = []
|
||||
for line in draft.lines:
|
||||
line_ids[line.line_code] = uuid.uuid4()
|
||||
lines.append(
|
||||
{
|
||||
"id": line_ids[line.line_code],
|
||||
"meta_id": meta.id,
|
||||
"line_code": line.line_code,
|
||||
"line_code_original_value": line.line_code,
|
||||
"line_code_confidence": line.line_code_confidence,
|
||||
"line_code_modified": False,
|
||||
"line_name": line.line_name,
|
||||
"line_name_original_value": line.line_name,
|
||||
"line_name_confidence": line.line_name_confidence,
|
||||
"line_name_modified": False,
|
||||
}
|
||||
)
|
||||
for start in range(0, len(lines), BATCH):
|
||||
await session.execute(insert(FinancialStatementLine).values(lines[start : start + BATCH]))
|
||||
values = [
|
||||
{
|
||||
"id": uuid.uuid4(),
|
||||
"line_id": line_ids[value.line_code],
|
||||
"column_id": column_ids[value.column_index],
|
||||
"amount": value.amount,
|
||||
"string_value": value.string_value,
|
||||
"confidence": value.confidence,
|
||||
"is_recognized": value.is_recognized,
|
||||
"is_modified": False,
|
||||
}
|
||||
for value in draft.values
|
||||
]
|
||||
for start in range(0, len(values), BATCH):
|
||||
await session.execute(insert(FinancialStatementValue).values(values[start : start + BATCH]))
|
||||
return meta
|
||||
|
||||
|
||||
async def delete_file_statements(session: AsyncSession, file_id: uuid.UUID) -> None:
|
||||
"""Значения, строки, колонки и шапки форм всех документов файла (в порядке FK)."""
|
||||
documents = select(Document.id).where(Document.stored_file_id == file_id).scalar_subquery()
|
||||
metas = select(FinancialStatementMeta.id).where(FinancialStatementMeta.document_id.in_(documents)).scalar_subquery()
|
||||
lines = select(FinancialStatementLine.id).where(FinancialStatementLine.meta_id.in_(metas)).scalar_subquery()
|
||||
await session.execute(delete(FinancialStatementValue).where(FinancialStatementValue.line_id.in_(lines)))
|
||||
await session.execute(delete(FinancialStatementLine).where(FinancialStatementLine.meta_id.in_(metas)))
|
||||
await session.execute(delete(FinancialStatementColumn).where(FinancialStatementColumn.meta_id.in_(metas)))
|
||||
await session.execute(delete(FinancialStatementMeta).where(FinancialStatementMeta.document_id.in_(documents)))
|
||||
|
||||
|
||||
async def statement_amounts(
|
||||
session: AsyncSession, meta_id: uuid.UUID
|
||||
) -> tuple[list[int], list[tuple[str, dict[int, Decimal | None]]]]:
|
||||
"""Индексы колонок и `(код строки, {колонка: сумма})` по возрастанию кода — вход `run_checks`."""
|
||||
columns = (
|
||||
await session.execute(
|
||||
select(FinancialStatementColumn.id, FinancialStatementColumn.column_index)
|
||||
.where(FinancialStatementColumn.meta_id == meta_id)
|
||||
.order_by(FinancialStatementColumn.column_index)
|
||||
)
|
||||
).all()
|
||||
index_by_id = {column_id: index for column_id, index in columns}
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(FinancialStatementLine.line_code, FinancialStatementValue.column_id, FinancialStatementValue.amount)
|
||||
.join(FinancialStatementValue, FinancialStatementValue.line_id == FinancialStatementLine.id)
|
||||
.where(FinancialStatementLine.meta_id == meta_id)
|
||||
.order_by(FinancialStatementLine.line_code)
|
||||
)
|
||||
).all()
|
||||
by_code: dict[str, dict[int, Decimal | None]] = {}
|
||||
for code, column_id, amount in rows:
|
||||
by_code.setdefault(code or "", {})[index_by_id[column_id]] = amount
|
||||
return [index for _, index in columns], list(by_code.items())
|
||||
|
||||
|
||||
async def value_flags(session: AsyncSession, meta_id: uuid.UUID) -> list[tuple[bool, bool | None, Decimal | None]]:
|
||||
"""`(is_modified, is_recognized, confidence)` всех значений формы — вход `values_ok`."""
|
||||
rows = await session.execute(
|
||||
select(
|
||||
FinancialStatementValue.is_modified,
|
||||
FinancialStatementValue.is_recognized,
|
||||
FinancialStatementValue.confidence,
|
||||
)
|
||||
.join(FinancialStatementLine, FinancialStatementLine.id == FinancialStatementValue.line_id)
|
||||
.where(FinancialStatementLine.meta_id == meta_id)
|
||||
)
|
||||
return [(bool(modified), recognized, confidence) for modified, recognized, confidence in rows]
|
||||
@ -11,10 +11,10 @@ PROCESSING` у файла, чьё ТЕКУЩЕЕ задание (`current_recogn
|
||||
3. транзакция маппинга: задание и файл перечитываются под блокировкой (порядок «задание → файл», как у inbox,
|
||||
вахтёра и публикатора, — без взаимоблокировок); файл уже не `PROCESSING` или задание уже не текущее (результат
|
||||
старого задания, Review Focus 1) — ничего не меняется. Иначе `map_file`: очистка документов и строк файла,
|
||||
документы и строки, `group_schema`, предупреждения, `PROCESSED`, автозамена прежних версий (`supersede_previous`,
|
||||
D-10, аудит `FILE_REPLACE`).
|
||||
документы и строки (листы ОСВ, затем формы ББ/ОФР Среза 3b и расшифровки), `group_schema` (только листы ОСВ),
|
||||
предупреждения, `PROCESSED`, автозамена прежних версий (`supersede_previous`, D-10, аудит `FILE_REPLACE`).
|
||||
|
||||
Ошибки (D-2): неповторяемые (`MappingFatal`: ключ вне папки, объекта нет, JSON не проходит схему, нет документов ОСВ)
|
||||
Ошибки (D-2): неповторяемые (`MappingFatal`: ключ вне папки, объекта нет, JSON не проходит схему, нет документов)
|
||||
— сразу файл `ERROR` и `metrics.mapping_error`; прочие — откат транзакции маппинга, в отдельной транзакции
|
||||
`metrics.mapping_attempts + 1`, на `MAPPING_MAX_ATTEMPTS` — `ERROR`; следующая попытка — после паузы цикла (R-15).
|
||||
`mapping_error` виден пользователю: в нём категория (схема — число ошибок и путь первой; БД/S3 — тип исключения и число
|
||||
@ -37,7 +37,7 @@ from pydantic import ValidationError
|
||||
from sqlalchemy import Integer, Select, delete, func, insert, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from src.contract.result import OsvDocument, RecognitionResult
|
||||
from src.contract.result import OsvDocument, RecognitionResult, StatementDocument
|
||||
from src.core.config import Settings
|
||||
from src.core.storage import ObjectNotFoundError, ObjectStorage
|
||||
from src.domain.osv_mapping import (
|
||||
@ -48,6 +48,8 @@ from src.domain.osv_mapping import (
|
||||
document_status,
|
||||
opening_matches,
|
||||
)
|
||||
from src.domain.statement_checks import load_check_rules, run_checks
|
||||
from src.domain.statement_mapping import StatementDraft, build_statement, statement_status, values_ok
|
||||
from src.models.accounting import ChartOfAccount
|
||||
from src.models.directories import LegalEntity
|
||||
from src.models.enums import (
|
||||
@ -64,6 +66,7 @@ from src.models.enums import (
|
||||
)
|
||||
from src.models.files import Document, RecognitionJob, S3Object, StoredFile
|
||||
from src.models.osv import OsvEntry
|
||||
from src.repositories import statements as statements_repo
|
||||
from src.repositories.counterparties import CounterpartyIndex
|
||||
from src.services import consolidation_documents
|
||||
from src.services.audit import AuditService
|
||||
@ -74,6 +77,7 @@ __all__ = [
|
||||
"MAPPING_ATTEMPTS_KEY",
|
||||
"MAPPING_ERROR_KEY",
|
||||
"MAPPING_WARNINGS_KEY",
|
||||
"NO_DOCUMENTS",
|
||||
"Candidate",
|
||||
"MappingFatal",
|
||||
"map_file",
|
||||
@ -89,6 +93,7 @@ MAPPING_ERROR_KEY = "mapping_error" # причина `ERROR` файла по м
|
||||
MAPPING_WARNINGS_KEY = "mapping_warnings" # предупреждения маппинга файла, список строк (D-7, R-6)
|
||||
ENTRY_BATCH_SIZE = 1000 # строк `osv_entry` на один INSERT: 23 колонки × 1000 < 32767 параметров asyncpg
|
||||
MAPPING_ERROR_MAX_LENGTH = 2000
|
||||
NO_DOCUMENTS = "в результате нет документов"
|
||||
RESULTS_PREFIX = "results"
|
||||
MAPPABLE_JOB_STATUSES = (RecognitionStatus.done, RecognitionStatus.needs_check)
|
||||
CUMULATIVE_CHECK_QUARTERS = frozenset({PeriodQuarter.Q2, PeriodQuarter.Q3, PeriodQuarter.Q4})
|
||||
@ -206,8 +211,8 @@ async def _load_result(storage: ObjectStorage, candidate: Candidate) -> Recognit
|
||||
raise MappingFatal(
|
||||
f"файл результата относится к другому заданию: job_id={result.job_id}, file_id={result.file_id}"
|
||||
)
|
||||
if not _osv_documents(result):
|
||||
raise MappingFatal("в результате нет документов ОСВ")
|
||||
if not result.documents:
|
||||
raise MappingFatal(NO_DOCUMENTS)
|
||||
return result
|
||||
|
||||
|
||||
@ -261,6 +266,7 @@ def _entry_values(
|
||||
async def _clear_file(session: AsyncSession, file_id: uuid.UUID) -> None:
|
||||
"""D-3: строки и документы файла от прерванной попытки (файл в `PROCESSING` правок аналитика не содержит)."""
|
||||
documents = select(Document.id).where(Document.stored_file_id == file_id).scalar_subquery()
|
||||
await statements_repo.delete_file_statements(session, file_id)
|
||||
await session.execute(delete(OsvEntry).where(OsvEntry.document_id.in_(documents)))
|
||||
await session.execute(delete(Document).where(Document.stored_file_id == file_id))
|
||||
|
||||
@ -330,6 +336,98 @@ def _split_duplicates(sheets: Sequence[OsvDocument]) -> tuple[list[OsvDocument],
|
||||
return kept, by_locator, warnings
|
||||
|
||||
|
||||
def _map_statement_docs( # noqa: PLR0913 - все входы именованные
|
||||
statement_docs: Sequence[StatementDocument],
|
||||
*,
|
||||
file: StoredFile,
|
||||
owner: LegalEntity,
|
||||
owner_names: frozenset[str],
|
||||
now: datetime,
|
||||
) -> tuple[list[tuple[Document, StatementDraft]], list[Document], list[str], set[DocumentType]]:
|
||||
"""Документы форм ББ/ОФР и расшифровок (spec «Документ формы ББ/ОФР», «Повтор формы в файле и прочие документы»;
|
||||
D-2, D-3). Возвращает формы (документ + черновик; статус уточняется после вставки значений), документы
|
||||
`DISCLOSURE`/`OTHER` (по одному на тип, `locator` через запятую, `RECOGNIZED`), предупреждения и типы форм,
|
||||
у которых был повтор в файле (R-1: первый документ такого типа — `NEEDS_CHECK`)."""
|
||||
forms: list[tuple[Document, StatementDraft]] = []
|
||||
seen_forms: set[DocumentType] = set()
|
||||
duplicated: set[DocumentType] = set()
|
||||
other_locators: dict[DocumentType, list[str]] = {}
|
||||
warnings: list[str] = []
|
||||
for doc in statement_docs:
|
||||
doc_type = DocumentType(doc.document_type)
|
||||
if doc_type in (DocumentType.DISCLOSURE, DocumentType.OTHER):
|
||||
other_locators.setdefault(doc_type, []).append(doc.locator)
|
||||
continue
|
||||
if doc_type in seen_forms:
|
||||
warnings.append(f"{doc.locator}: вторая форма {doc_type.value} в файле пропущена (OQ-77)")
|
||||
duplicated.add(doc_type)
|
||||
continue
|
||||
seen_forms.add(doc_type)
|
||||
draft = build_statement(doc)
|
||||
warnings.extend(draft.warnings)
|
||||
check = counterparty_check(doc.header.inn, doc.header.organization_name, owner.inn, owner_names)
|
||||
document = Document(
|
||||
id=uuid.uuid4(),
|
||||
stored_file_id=file.id,
|
||||
legal_entity_id=file.legal_entity_id,
|
||||
document_type=doc_type,
|
||||
osv_account_code=None,
|
||||
period_year=draft.period_year,
|
||||
period_quarter=draft.period_quarter,
|
||||
is_cumulative=None,
|
||||
reporting_date=draft.period_date,
|
||||
locator=doc.locator,
|
||||
status=DocumentStatus.NEEDS_CHECK, # уточняется после вставки значений (нужны проверки по суммам)
|
||||
period_check=DocumentCheckResult.UNKNOWN,
|
||||
counterparty_check=check,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
forms.append((document, draft))
|
||||
others = [
|
||||
Document(
|
||||
id=uuid.uuid4(),
|
||||
stored_file_id=file.id,
|
||||
legal_entity_id=file.legal_entity_id,
|
||||
document_type=doc_type,
|
||||
locator=",".join(locators),
|
||||
status=DocumentStatus.RECOGNIZED,
|
||||
period_check=DocumentCheckResult.UNKNOWN,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
for doc_type, locators in other_locators.items()
|
||||
]
|
||||
return forms, others, warnings, duplicated
|
||||
|
||||
|
||||
async def _insert_statements(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
job: RecognitionJob,
|
||||
forms: Sequence[tuple[Document, StatementDraft]],
|
||||
duplicated: set[DocumentType],
|
||||
job_done: bool,
|
||||
) -> None:
|
||||
"""Шапка, колонки, строки и значения каждой формы; статус документа по D-6 (акцепта при маппинге нет)."""
|
||||
if not forms:
|
||||
return
|
||||
rules = load_check_rules()
|
||||
for document, draft in forms:
|
||||
meta = await statements_repo.insert_statement(session, document_id=document.id, job_id=job.id, draft=draft)
|
||||
column_indexes, amounts = await statements_repo.statement_amounts(session, meta.id)
|
||||
report = run_checks(document.document_type.value, amounts, column_indexes, rules)
|
||||
document.status = statement_status(
|
||||
job_done=job_done,
|
||||
checks_passed=not report.failed,
|
||||
counterparty=document.counterparty_check or DocumentCheckResult.UNKNOWN,
|
||||
period_known=document.period_year is not None and document.period_quarter is not None,
|
||||
values_ok=values_ok(await statements_repo.value_flags(session, meta.id)),
|
||||
has_lines=bool(draft.lines),
|
||||
warnings=[*draft.warnings, *(["повтор формы"] if document.document_type in duplicated else [])],
|
||||
)
|
||||
|
||||
|
||||
async def map_file( # noqa: PLR0913 - все входы маппинга файла именованные
|
||||
session: AsyncSession,
|
||||
*,
|
||||
@ -342,13 +440,15 @@ async def map_file( # noqa: PLR0913 - все входы маппинга фай
|
||||
) -> list[str]:
|
||||
"""Маппинг результата в транзакции вызывающего (D-3, D-6…D-10); возвращает предупреждения маппинга файла.
|
||||
|
||||
Очистка прежних документов/строк файла → документ на каждый лист ОСВ (повтор счёта — предупреждение) → строки
|
||||
пакетами по `ENTRY_BATCH_SIZE` → `job.group_schema`, `metrics.mapping_warnings` → файл `PROCESSED` (только из
|
||||
Очистка прежних документов/строк файла → документ на каждый лист ОСВ (повтор счёта — предупреждение) и на каждую
|
||||
форму ББ/ОФР (`_map_statement_docs`) → строки ОСВ пакетами по `ENTRY_BATCH_SIZE` → вставка форм и их статус →
|
||||
`job.group_schema`, `metrics.mapping_warnings` → файл `PROCESSED` (только из
|
||||
`PROCESSING` и только для текущего задания) → автозамена прежних версий (`supersede_previous`). Без commit.
|
||||
"""
|
||||
if not result.documents:
|
||||
raise MappingFatal(NO_DOCUMENTS)
|
||||
sheets = _osv_documents(result)
|
||||
if not sheets:
|
||||
raise MappingFatal("в результате нет документов ОСВ")
|
||||
statement_docs = [doc for doc in result.documents if isinstance(doc, StatementDocument)]
|
||||
owner = await session.get(LegalEntity, file.legal_entity_id)
|
||||
if owner is None:
|
||||
raise MappingFatal(f"ЮЛ-владелец файла не найден: {file.legal_entity_id}")
|
||||
@ -422,6 +522,13 @@ async def map_file( # noqa: PLR0913 - все входы маппинга фай
|
||||
for entry in mapped.entries
|
||||
)
|
||||
|
||||
forms, other_documents, form_warnings, duplicated = _map_statement_docs(
|
||||
statement_docs, file=file, owner=owner, owner_names=owner_names, now=now
|
||||
)
|
||||
warnings.extend(form_warnings)
|
||||
documents.extend(other_documents)
|
||||
documents.extend(document for document, _ in forms)
|
||||
|
||||
session.add_all(documents)
|
||||
await session.flush()
|
||||
# T7.2: родитель раньше потомка на любой границе пачки (self-FK `parent_entry_id`) — не полагаемся на порядок строк
|
||||
@ -430,6 +537,8 @@ async def map_file( # noqa: PLR0913 - все входы маппинга фай
|
||||
for start in range(0, len(entries), ENTRY_BATCH_SIZE):
|
||||
await session.execute(insert(OsvEntry).values(entries[start : start + ENTRY_BATCH_SIZE]))
|
||||
|
||||
await _insert_statements(session, job=job, forms=forms, duplicated=duplicated, job_done=job_done)
|
||||
|
||||
job.group_schema = [_group_schema(sheet) for sheet in sheets]
|
||||
job.metrics = {**_metrics(job), MAPPING_WARNINGS_KEY: warnings}
|
||||
await session.flush()
|
||||
|
||||
@ -31,7 +31,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from src.contract.result import RecognitionResult
|
||||
from src.domain.osv_mapping import normalize_account_code
|
||||
from src.models.accounting import ChartOfAccount
|
||||
from src.models.enums import FileStatus, RecognitionStatus
|
||||
from src.models.enums import DocumentType, FileStatus, RecognitionStatus
|
||||
from src.models.files import Document, StoredFile
|
||||
from src.models.osv import OsvEntry
|
||||
from src.repositories.counterparties import CounterpartyIndex
|
||||
@ -148,7 +148,12 @@ async def test_all_golden_snapshots_mapped(seeded: AsyncSession, fixtures_dir: P
|
||||
documents = {
|
||||
document.locator: document
|
||||
for document in (
|
||||
await session.execute(select(Document).where(Document.stored_file_id == file_id))
|
||||
await session.execute(
|
||||
# формы ББ/ОФР смешанных снимков (s12) маппятся с Среза 3b — здесь сверяются только листы ОСВ
|
||||
select(Document).where(
|
||||
Document.stored_file_id == file_id, Document.document_type == DocumentType.OSV
|
||||
)
|
||||
)
|
||||
).scalars()
|
||||
}
|
||||
sheets = [sheet for sheet in raw["documents"] if sheet["document_type"] == "OSV"]
|
||||
|
||||
204
api/tests/test_mapping_statements.py
Normal file
204
api/tests/test_mapping_statements.py
Normal file
@ -0,0 +1,204 @@
|
||||
"""Маппинг форм ББ/ОФР (spec osv-result-mapping «Цикл маппинга…», «Документ формы ББ/ОФР», «Повтор формы в файле и
|
||||
прочие документы», «Статус документа формы»; design D-1…D-3, D-6) на golden-снимках fixtures."""
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from src.models.directories import LegalEntity
|
||||
from src.models.enums import DocumentCheckResult, DocumentStatus, DocumentType, FileStatus, PeriodQuarter
|
||||
from src.models.files import Document, StoredFile
|
||||
from src.models.statements import (
|
||||
FinancialStatementColumn,
|
||||
FinancialStatementLine,
|
||||
FinancialStatementMeta,
|
||||
FinancialStatementValue,
|
||||
)
|
||||
from tests.factories import make_legal_entity
|
||||
from tests.fixtures_helpers import GOLDEN_STATEMENTS, OCR_EXPECTED, map_golden_snapshot, ocr_result
|
||||
|
||||
pytestmark = [pytest.mark.integration, pytest.mark.fixtures]
|
||||
|
||||
MEZH = GOLDEN_STATEMENTS / "mezheninovskaya" / "2025-Q3"
|
||||
NPZ = GOLDEN_STATEMENTS / "npz" / "2025-Q3"
|
||||
SYNTH = GOLDEN_STATEMENTS / "synthetic"
|
||||
MEZH_BALANCE_LINES = 37
|
||||
NPZ_BALANCE_LINES = 132
|
||||
NPZ_INCOME_LINES = 43
|
||||
DISCLOSURE_LOCATORS_COMMAS = 4 # пять `DISCLOSURE` снимка st06 → пять locator через запятую
|
||||
MIN_EXCEL_SNAPSHOTS = 25
|
||||
MIN_MAPPED_SNAPSHOTS = 27
|
||||
|
||||
|
||||
async def _meta(session: AsyncSession, document: Document) -> FinancialStatementMeta:
|
||||
stmt = select(FinancialStatementMeta).where(FinancialStatementMeta.document_id == document.id)
|
||||
return (await session.execute(stmt)).scalar_one()
|
||||
|
||||
|
||||
async def _count(session: AsyncSession, model: type, meta_id: uuid.UUID) -> int:
|
||||
if model is FinancialStatementValue:
|
||||
stmt = (
|
||||
select(func.count())
|
||||
.select_from(FinancialStatementValue)
|
||||
.join(FinancialStatementLine, FinancialStatementLine.id == FinancialStatementValue.line_id)
|
||||
.where(FinancialStatementLine.meta_id == meta_id)
|
||||
)
|
||||
else:
|
||||
stmt = select(func.count()).select_from(model).where(model.meta_id == meta_id) # type: ignore[attr-defined]
|
||||
return (await session.execute(stmt)).scalar_one()
|
||||
|
||||
|
||||
async def _owner(session: AsyncSession, path: Path) -> LegalEntity:
|
||||
"""Владелец файла с ИНН шапки формы (MATCH), чтобы чистые формы могли быть RECOGNIZED."""
|
||||
data = json.loads(path.read_text("utf-8"))
|
||||
inn = next((d["header"].get("inn") for d in data["documents"] if d.get("header", {}).get("inn")), None)
|
||||
return await make_legal_entity(session, name="ООО Владелец", inn=inn)
|
||||
|
||||
|
||||
async def test_file_with_forms_only(seeded: AsyncSession, fixtures_dir: Path) -> None:
|
||||
path = fixtures_dir / MEZH / "1_Форма1_бух_09.json"
|
||||
owner = await _owner(seeded, path)
|
||||
documents = await map_golden_snapshot(seeded, entity=owner, path=path, uploaded_by=None)
|
||||
assert [d.document_type for d in documents] == [DocumentType.BALANCE]
|
||||
balance = documents[0]
|
||||
assert (balance.period_year, balance.period_quarter) == (2025, PeriodQuarter.Q3)
|
||||
assert balance.reporting_date is not None and balance.reporting_date.isoformat() == "2025-09-30"
|
||||
assert balance.osv_account_code is None and balance.is_cumulative is None
|
||||
meta = await _meta(seeded, balance)
|
||||
lines = await _count(seeded, FinancialStatementLine, meta.id)
|
||||
columns = await _count(seeded, FinancialStatementColumn, meta.id)
|
||||
assert lines == MEZH_BALANCE_LINES
|
||||
assert await _count(seeded, FinancialStatementValue, meta.id) == lines * columns
|
||||
file = await seeded.get(StoredFile, balance.stored_file_id)
|
||||
assert file is not None and file.status is FileStatus.PROCESSED
|
||||
|
||||
|
||||
async def test_clean_income_recognized(seeded: AsyncSession, fixtures_dir: Path) -> None:
|
||||
path = fixtures_dir / MEZH / "1_Форма2_бух_09.json"
|
||||
owner = await _owner(seeded, path)
|
||||
(income,) = await map_golden_snapshot(seeded, entity=owner, path=path, uploaded_by=None)
|
||||
assert income.document_type is DocumentType.INCOME
|
||||
assert income.counterparty_check is DocumentCheckResult.MATCH
|
||||
assert income.status is DocumentStatus.RECOGNIZED
|
||||
|
||||
|
||||
async def test_book_with_two_forms(seeded: AsyncSession, fixtures_dir: Path) -> None:
|
||||
path = next((fixtures_dir / NPZ).glob("*.json"))
|
||||
owner = await _owner(seeded, path)
|
||||
documents = await map_golden_snapshot(seeded, entity=owner, path=path, uploaded_by=None)
|
||||
by_type = {d.document_type: d for d in documents}
|
||||
assert set(by_type) == {DocumentType.BALANCE, DocumentType.INCOME}
|
||||
balance_meta = await _meta(seeded, by_type[DocumentType.BALANCE])
|
||||
income_meta = await _meta(seeded, by_type[DocumentType.INCOME])
|
||||
assert await _count(seeded, FinancialStatementLine, balance_meta.id) == NPZ_BALANCE_LINES
|
||||
assert await _count(seeded, FinancialStatementLine, income_meta.id) == NPZ_INCOME_LINES
|
||||
for document in documents:
|
||||
assert (document.period_year, document.period_quarter) == (2025, PeriodQuarter.Q3)
|
||||
|
||||
|
||||
async def test_disclosures_merged(seeded: AsyncSession, fixtures_dir: Path) -> None:
|
||||
owner = await make_legal_entity(seeded, name="ООО Синтетика")
|
||||
path = fixtures_dir / SYNTH / "st06_header_variants.json"
|
||||
documents = await map_golden_snapshot(seeded, entity=owner, path=path, uploaded_by=None)
|
||||
types = [d.document_type for d in documents]
|
||||
assert types.count(DocumentType.DISCLOSURE) == 1 and DocumentType.INCOME in types
|
||||
disclosure = next(d for d in documents if d.document_type is DocumentType.DISCLOSURE)
|
||||
assert disclosure.status is DocumentStatus.RECOGNIZED
|
||||
assert disclosure.locator.count(",") == DISCLOSURE_LOCATORS_COMMAS
|
||||
|
||||
|
||||
async def test_form_without_codes(seeded: AsyncSession, fixtures_dir: Path) -> None:
|
||||
owner = await make_legal_entity(seeded, name="ООО Без кодов")
|
||||
path = fixtures_dir / SYNTH / "st03_no_codes.json"
|
||||
(balance,) = await map_golden_snapshot(seeded, entity=owner, path=path, uploaded_by=None)
|
||||
assert balance.status is DocumentStatus.NEEDS_CHECK
|
||||
meta = await _meta(seeded, balance)
|
||||
assert await _count(seeded, FinancialStatementLine, meta.id) == 0
|
||||
# в снимке st03_no_codes колонок нет (`columns = []`): только шапка (расхождение со spec «и колонками»)
|
||||
assert await _count(seeded, FinancialStatementColumn, meta.id) == 0
|
||||
|
||||
|
||||
async def test_mixed_book_osv_and_balance(seeded: AsyncSession, fixtures_dir: Path) -> None:
|
||||
owner = await make_legal_entity(seeded, name="ООО Смешанная")
|
||||
path = fixtures_dir / SYNTH / "st04_mixed_book.json"
|
||||
documents = await map_golden_snapshot(seeded, entity=owner, path=path, uploaded_by=None)
|
||||
assert {d.document_type for d in documents} == {DocumentType.OSV, DocumentType.BALANCE}
|
||||
|
||||
|
||||
async def test_ocr_form_without_report_date(seeded: AsyncSession, fixtures_dir: Path) -> None:
|
||||
owner = await make_legal_entity(seeded, name="ООО Дорстроймонтаж")
|
||||
documents = await map_golden_snapshot(
|
||||
seeded,
|
||||
entity=owner,
|
||||
path=fixtures_dir / OCR_EXPECTED / "aep_6.json",
|
||||
uploaded_by=None,
|
||||
result_builder=ocr_result,
|
||||
)
|
||||
balance = next(d for d in documents if d.document_type is DocumentType.BALANCE)
|
||||
assert (balance.period_year, balance.period_quarter) == (2025, PeriodQuarter.Q2)
|
||||
meta = await _meta(seeded, balance)
|
||||
assert meta.report_date is None and meta.inn is None
|
||||
current = (
|
||||
await seeded.execute(
|
||||
select(FinancialStatementColumn.period_date).where(
|
||||
FinancialStatementColumn.meta_id == meta.id, FinancialStatementColumn.is_current.is_(True)
|
||||
)
|
||||
)
|
||||
).scalar_one()
|
||||
assert current is not None and current.isoformat() == "2025-06-30"
|
||||
assert balance.status is DocumentStatus.NEEDS_CHECK
|
||||
|
||||
|
||||
async def test_ocr_noisy_form_mismatch(seeded: AsyncSession, fixtures_dir: Path) -> None:
|
||||
owner = await make_legal_entity(seeded, name="ООО Другое", inn="7700000000")
|
||||
documents = await map_golden_snapshot(
|
||||
seeded,
|
||||
entity=owner,
|
||||
path=fixtures_dir / OCR_EXPECTED / "aep_3.json",
|
||||
uploaded_by=None,
|
||||
result_builder=ocr_result,
|
||||
)
|
||||
balance = next(d for d in documents if d.document_type is DocumentType.BALANCE)
|
||||
assert balance.counterparty_check is DocumentCheckResult.MISMATCH
|
||||
assert balance.status is DocumentStatus.NEEDS_CHECK
|
||||
|
||||
|
||||
async def test_all_statement_snapshots_map(seeded: AsyncSession, fixtures_dir: Path) -> None:
|
||||
"""Все golden форм (Excel и OCR, кроме ответов-ошибок) маппятся без исключений; у каждой формы значения на все
|
||||
ячейки."""
|
||||
owner = await make_legal_entity(seeded, name="ООО Все формы")
|
||||
excel = sorted((fixtures_dir / GOLDEN_STATEMENTS).rglob("*.json"))
|
||||
ocr = sorted((fixtures_dir / OCR_EXPECTED).glob("*.json"))
|
||||
assert len(excel) >= MIN_EXCEL_SNAPSHOTS
|
||||
mapped = 0
|
||||
for path in excel:
|
||||
await map_golden_snapshot(seeded, entity=owner, path=path, uploaded_by=None)
|
||||
mapped += 1
|
||||
for path in ocr:
|
||||
if json.loads(path.read_text("utf-8"))["status"] == "error":
|
||||
continue
|
||||
await map_golden_snapshot(seeded, entity=owner, path=path, uploaded_by=None, result_builder=ocr_result)
|
||||
mapped += 1
|
||||
assert mapped >= MIN_MAPPED_SNAPSHOTS
|
||||
metas = (await seeded.execute(select(FinancialStatementMeta.id))).scalars().all()
|
||||
for meta_id in metas:
|
||||
lines = await _count(seeded, FinancialStatementLine, meta_id)
|
||||
columns = await _count(seeded, FinancialStatementColumn, meta_id)
|
||||
assert await _count(seeded, FinancialStatementValue, meta_id) == lines * columns
|
||||
|
||||
|
||||
async def test_duplicate_form_in_file_skipped(seeded: AsyncSession, fixtures_dir: Path, tmp_path: Path) -> None:
|
||||
"""R-1, spec «Повтор формы в файле»: вторая форма того же типа не создаётся, первая — `NEEDS_CHECK` даже чистая."""
|
||||
source = fixtures_dir / MEZH / "1_Форма2_бух_09.json"
|
||||
owner = await _owner(seeded, source)
|
||||
data = json.loads(source.read_text("utf-8"))
|
||||
data["documents"].append(data["documents"][0] | {"locator": "sheet:повтор"})
|
||||
path = tmp_path / "duplicate.json"
|
||||
path.write_text(json.dumps(data, ensure_ascii=False), "utf-8")
|
||||
documents = await map_golden_snapshot(seeded, entity=owner, path=path, uploaded_by=None)
|
||||
assert [d.document_type for d in documents] == [DocumentType.INCOME]
|
||||
assert documents[0].status is DocumentStatus.NEEDS_CHECK # без повтора тот же снимок — RECOGNIZED
|
||||
@ -25,6 +25,7 @@ from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from src.contract import ack_msg_id, result_msg_id
|
||||
from src.contract.result import RecognitionResult
|
||||
from src.core.config import Settings, get_settings
|
||||
from src.models.access import AppUser
|
||||
from src.models.accounting import ChartOfAccount
|
||||
@ -47,12 +48,14 @@ from src.models.enums import (
|
||||
)
|
||||
from src.models.files import Document, RecognitionJob, StoredFile
|
||||
from src.models.osv import OsvEntry
|
||||
from src.repositories.counterparties import CounterpartyIndex
|
||||
from src.services import result_mapping as rm
|
||||
from src.services.recognition_exchange import expire_deadlines, handle_message
|
||||
from src.services.result_mapping import (
|
||||
MAPPING_ATTEMPTS_KEY,
|
||||
MAPPING_ERROR_KEY,
|
||||
MAPPING_WARNINGS_KEY,
|
||||
NO_DOCUMENTS,
|
||||
Candidate,
|
||||
MappingFatal,
|
||||
process_once,
|
||||
@ -335,32 +338,31 @@ async def test_missing_result_object_is_error(committed_sessionmaker: async_sess
|
||||
assert "не найден" in (await _job(maker, job_id)).metrics[MAPPING_ERROR_KEY]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("documents", "status", "message"),
|
||||
[
|
||||
([], "needs_check", "схем"), # контракт требует documents ≥ 1 — результат не проходит схему
|
||||
(
|
||||
[{"document_type": "BALANCE", "locator": "pages:1", "header": {}, "columns": [], "lines": []}],
|
||||
"needs_check",
|
||||
"нет документов ОСВ",
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_result_without_osv_documents_is_error(
|
||||
committed_sessionmaker: async_sessionmaker[AsyncSession], documents: list[Any], status: str, message: str
|
||||
):
|
||||
"""Review Focus 2: результат без документов ОСВ → файл `ERROR` «нет документов ОСВ», не исключение цикла."""
|
||||
async def test_result_with_empty_documents_is_error(committed_sessionmaker: async_sessionmaker[AsyncSession]):
|
||||
"""Результат без документов не проходит схему контракта (documents ≥ 1) → файл `ERROR`, не исключение цикла."""
|
||||
maker, storage = committed_sessionmaker, FakeStorage()
|
||||
result = _minimal() | {"status": status, "documents": documents}
|
||||
result = _minimal() | {"status": "needs_check", "documents": []}
|
||||
file_id, job_id, _ = await _prepare(maker, storage, result, job_status=RecognitionStatus.needs_check)
|
||||
|
||||
assert await process_once(maker, storage, _settings()) is True
|
||||
|
||||
assert (await _file(maker, file_id)).status is FileStatus.ERROR
|
||||
assert message in (await _job(maker, job_id)).metrics[MAPPING_ERROR_KEY]
|
||||
assert "схем" in (await _job(maker, job_id)).metrics[MAPPING_ERROR_KEY]
|
||||
assert await _documents(maker, file_id) == []
|
||||
|
||||
|
||||
async def test_map_file_without_documents_is_fatal(seeded: AsyncSession):
|
||||
"""`map_file` на результате без документов (схема такого не пропускает — защита) → `NO_DOCUMENTS`."""
|
||||
entity = await make_legal_entity(seeded)
|
||||
file, job = await make_stored_file(
|
||||
seeded, entity=entity, status=FileStatus.PROCESSING, job_status=RecognitionStatus.done
|
||||
)
|
||||
result = RecognitionResult.model_construct(documents=[])
|
||||
index = await CounterpartyIndex.load(seeded)
|
||||
with pytest.raises(MappingFatal, match=NO_DOCUMENTS):
|
||||
await rm.map_file(seeded, job=job, file=file, result=result, chart={}, index=index, now=datetime.now(UTC))
|
||||
|
||||
|
||||
async def test_mapping_failure_retries_then_error(
|
||||
committed_sessionmaker: async_sessionmaker[AsyncSession], monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user