feat(statements): единая нормализация ББ/ОФР, часть Б — ядро (#131) #132
@ -1,18 +1,60 @@
|
||||
"""Черновик записи `document_issue` (DBML v5.19) — чистые данные без ORM.
|
||||
"""Черновик записи `document_issue` (DBML v5.19) — чистые данные без ORM — и проверки ядра уровня `info`.
|
||||
|
||||
Маппер `1.x` пишет только `MAPPING_WARNING` (P3-5, P3-7): предупреждения черновика формы, листа ОСВ и повтор формы
|
||||
в файле; проверки §7 и прочие коды ядра в `1.x` не пишутся. Запись в БД — `repositories/statements.add_issues`.
|
||||
в файле; проверки §7 и прочие коды ядра в `1.x` не пишутся. Маппер `2.0` и пересчёт после правки пишут проверки ядра
|
||||
`CODE_NOT_IN_DICTIONARY`, `CODE_NOT_IN_EDITION`, `NAME_MISMATCH`, `STATEMENTS_VERSION_MISMATCH` (`core_info_issues`,
|
||||
`version_issue`) — одним кодом (P3-39). Запись в БД — `repositories/statements.add_issues`.
|
||||
"""
|
||||
|
||||
import re
|
||||
import uuid
|
||||
from collections.abc import Collection, Hashable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from decimal import Decimal
|
||||
from typing import NamedTuple
|
||||
|
||||
from src.models.enums import DocumentIssueKind, IssueSeverity, IssueSource
|
||||
from src.domain.statement_lines import LineCatalog
|
||||
from src.domain.statement_status import NameCheckLine, name_mismatches
|
||||
from src.models.enums import DocumentIssueKind, IssueSeverity, IssueSource, StatementLineStatus
|
||||
from src.models.statements import FinancialStatementLine
|
||||
|
||||
__all__ = ["COLUMN_CLEARS", "HEADER_CLEARS", "MAPPING_WARNING_CODE", "IssueDraft", "mapping_warning"]
|
||||
__all__ = [
|
||||
"CODE_NOT_IN_DICTIONARY",
|
||||
"CODE_NOT_IN_EDITION",
|
||||
"COLUMN_CLEARS",
|
||||
"HEADER_CLEARS",
|
||||
"MAPPING_WARNING_CODE",
|
||||
"NAME_MISMATCH",
|
||||
"RECOMPUTED_CORE_CODES",
|
||||
"RELATION_FAILED",
|
||||
"RELATION_SOFT",
|
||||
"RESULT_STATUS_INCONSISTENT",
|
||||
"STATEMENTS_VERSION_MISMATCH",
|
||||
"CoreInfo",
|
||||
"IssueDraft",
|
||||
"code_issues",
|
||||
"core_info_issues",
|
||||
"mapping_warning",
|
||||
"name_check_lines",
|
||||
"name_issues",
|
||||
"relation_message",
|
||||
"result_status_inconsistent",
|
||||
"version_issue",
|
||||
]
|
||||
|
||||
MAPPING_WARNING_CODE = "MAPPING_WARNING"
|
||||
RELATION_FAILED = "RELATION_FAILED"
|
||||
RELATION_SOFT = "RELATION_SOFT"
|
||||
CODE_NOT_IN_DICTIONARY = "CODE_NOT_IN_DICTIONARY"
|
||||
CODE_NOT_IN_EDITION = "CODE_NOT_IN_EDITION"
|
||||
NAME_MISMATCH = "NAME_MISMATCH"
|
||||
STATEMENTS_VERSION_MISMATCH = "STATEMENTS_VERSION_MISMATCH"
|
||||
RESULT_STATUS_INCONSISTENT = "RESULT_STATUS_INCONSISTENT"
|
||||
# записи `core`, которые пересчёт заменяет целиком (P3-8); прочие (`MAPPING_WARNING`, `RESULT_STATUS_INCONSISTENT`,
|
||||
# `LEGACY_WARNING`) — факт приёма, их пересчёт сохраняет
|
||||
RECOMPUTED_CORE_CODES = (CODE_NOT_IN_DICTIONARY, CODE_NOT_IN_EDITION, NAME_MISMATCH, STATEMENTS_VERSION_MISMATCH)
|
||||
_CHECKED_CODE = re.compile(r"\d{4}", re.ASCII) # R-1 (Р-40): проверяются только четырёхзначные коды
|
||||
_EMPTY = "пусто"
|
||||
|
||||
# предупреждения сервиса, которые снимает правка поля шапки (в том числе подтверждение даты тем же значением) —
|
||||
# spec arm-core «Записи document_issue»
|
||||
@ -53,3 +95,134 @@ def mapping_warning(message: str) -> IssueDraft:
|
||||
message=message,
|
||||
source=IssueSource.core,
|
||||
)
|
||||
|
||||
|
||||
def _core_info(code: str, message: str, line_ids: tuple[uuid.UUID, ...] = ()) -> IssueDraft:
|
||||
return IssueDraft(
|
||||
kind=DocumentIssueKind.warning,
|
||||
code=code,
|
||||
severity=IssueSeverity.info,
|
||||
message=message,
|
||||
source=IssueSource.core,
|
||||
line_ids=line_ids,
|
||||
)
|
||||
|
||||
|
||||
def relation_message(relation: str, expected: Decimal | None, actual: Decimal | None) -> str:
|
||||
"""Сообщение нарушения §7; `None` (цель или все члены пусты, P1-21) — «пусто»."""
|
||||
left = _EMPTY if expected is None else str(expected)
|
||||
right = _EMPTY if actual is None else str(actual)
|
||||
return f"не сошлось {relation}: {left} ≠ {right}"
|
||||
|
||||
|
||||
def result_status_inconsistent(service_status: str) -> IssueDraft:
|
||||
"""`RESULT_STATUS_INCONSISTENT` (`info`, `core`, у документа): сервис прислал `done`, общая функция статуса пакета
|
||||
по содержимому документа даёт иное — документ считается `needs_check`."""
|
||||
return _core_info(
|
||||
RESULT_STATUS_INCONSISTENT,
|
||||
f"сервис прислал статус {service_status}, а по содержимому документа он needs_check",
|
||||
)
|
||||
|
||||
|
||||
def code_issues(
|
||||
lines: Sequence[FinancialStatementLine],
|
||||
*,
|
||||
in_dictionary: Mapping[str, bool],
|
||||
in_edition: Mapping[str, bool] | None,
|
||||
) -> list[IssueDraft]:
|
||||
"""`CODE_NOT_IN_DICTIONARY` / `CODE_NOT_IN_EDITION` — по записи на код документа (`line_ids` — все строки `ok` с
|
||||
этим кодом), только для кодов из ровно четырёх цифр (R-1, Р-40). `in_dictionary` — код есть в словаре имён редакции,
|
||||
`in_edition` — код из перечня редакции и варианта (`None` — редакция или вариант не определены, не проверяется)."""
|
||||
by_code: dict[str, list[FinancialStatementLine]] = {}
|
||||
for line in lines:
|
||||
if line.line_status is StatementLineStatus.ok and line.line_code and _CHECKED_CODE.fullmatch(line.line_code):
|
||||
by_code.setdefault(line.line_code, []).append(line)
|
||||
drafts: list[IssueDraft] = []
|
||||
for code in sorted(by_code):
|
||||
ids = tuple(
|
||||
line.id for line in sorted(by_code[code], key=lambda item: (item.row_index is None, item.row_index or 0))
|
||||
)
|
||||
if not in_dictionary.get(code, True):
|
||||
drafts.append(_core_info(CODE_NOT_IN_DICTIONARY, f"кода {code} нет в словаре наименований редакции", ids))
|
||||
if in_edition is not None and not in_edition.get(code, True):
|
||||
drafts.append(_core_info(CODE_NOT_IN_EDITION, f"код {code} не входит в перечень кодов редакции формы", ids))
|
||||
return drafts
|
||||
|
||||
|
||||
def name_issues(lines: Sequence[FinancialStatementLine], mismatched: Sequence[Hashable]) -> list[IssueDraft]:
|
||||
"""`NAME_MISMATCH` — по записи на строку из `name_mismatches`."""
|
||||
by_id = {line.id: line for line in lines}
|
||||
drafts: list[IssueDraft] = []
|
||||
for key in mismatched:
|
||||
line = by_id[key] # type: ignore[index]
|
||||
drafts.append(
|
||||
_core_info(
|
||||
NAME_MISMATCH,
|
||||
f"наименование «{line.line_name or ''}» не совпало со справочным для кода {line.line_code}",
|
||||
(line.id,),
|
||||
)
|
||||
)
|
||||
return drafts
|
||||
|
||||
|
||||
def version_issue(job_version: str | None, engine_version: str | None) -> IssueDraft | None:
|
||||
"""`STATEMENTS_VERSION_MISMATCH`: версия копии пакета у задания известна и не равна версии копии ядра."""
|
||||
if job_version is None or job_version == engine_version:
|
||||
return None
|
||||
return _core_info(
|
||||
STATEMENTS_VERSION_MISMATCH,
|
||||
f"форма распознана общим кодом версии {job_version}, в ядре версия {engine_version or 'не определена'}",
|
||||
)
|
||||
|
||||
|
||||
def name_check_lines(lines: Sequence[FinancialStatementLine]) -> list[NameCheckLine]:
|
||||
"""Вход проверки наименований из строк БД: закреплённый код (`code_fixed`) — правленый или подтверждённый."""
|
||||
return [
|
||||
NameCheckLine(
|
||||
key=line.id,
|
||||
line_status=line.line_status.value,
|
||||
line_code=line.line_code,
|
||||
line_code_confidence=line.line_code_confidence,
|
||||
line_name=line.line_name,
|
||||
code_fixed=bool(line.line_code_modified) or bool(line.is_confirmed),
|
||||
)
|
||||
for line in lines
|
||||
]
|
||||
|
||||
|
||||
class CoreInfo(NamedTuple):
|
||||
"""Проверки ядра уровня `info` по строкам формы: записи и ключи строк с несовпавшим наименованием."""
|
||||
|
||||
drafts: list[IssueDraft]
|
||||
mismatched: list[Hashable]
|
||||
|
||||
|
||||
def core_info_issues(
|
||||
lines: Sequence[FinancialStatementLine],
|
||||
*,
|
||||
form: str,
|
||||
edition: str | None,
|
||||
order: Collection[str] | None,
|
||||
catalog: LineCatalog,
|
||||
) -> CoreInfo:
|
||||
"""Проверки ядра уровня `info` (приём `2.0` и пересчёт после правки — одно правило, P3-39).
|
||||
|
||||
`edition` — редакция точки входа пакета (`"66n"` / `"fsbu4"`, `None` — не определена: `CODE_NOT_IN_*` не пишутся);
|
||||
`order` — перечень кодов редакции и варианта формы (`None` — редакция или вариант не определены:
|
||||
`CODE_NOT_IN_EDITION` не пишется). `NAME_MISMATCH` не зависит от редакции (словарь имён — любая редакция, P3-34).
|
||||
Домен пакет не импортирует: редакцию и перечень считает сервис через `StatementsEngine`."""
|
||||
in_dictionary = {
|
||||
line.line_code: catalog.reference_name(edition, form, line.line_code) is not None
|
||||
for line in lines
|
||||
if line.line_code is not None
|
||||
}
|
||||
in_edition: dict[str, bool] | None = None
|
||||
if order is not None:
|
||||
allowed = set(order)
|
||||
in_edition = {line.line_code: line.line_code in allowed for line in lines if line.line_code is not None}
|
||||
mismatched = name_mismatches(name_check_lines(lines), catalog, form)
|
||||
drafts = [
|
||||
*(code_issues(lines, in_dictionary=in_dictionary, in_edition=in_edition) if edition is not None else []),
|
||||
*name_issues(lines, mismatched),
|
||||
]
|
||||
return CoreInfo(drafts, mismatched)
|
||||
|
||||
@ -49,7 +49,7 @@ class ColumnDraft:
|
||||
|
||||
@dataclass
|
||||
class ValueDraft:
|
||||
line_code: str
|
||||
line_code: str | None # `None` у строк `unreadable` / `missing` / `detail` результата 2.0
|
||||
column_index: int
|
||||
amount: Decimal | None
|
||||
string_value: str | None
|
||||
@ -62,9 +62,9 @@ class ValueDraft:
|
||||
|
||||
@dataclass
|
||||
class LineDraft:
|
||||
line_code: str
|
||||
line_code: str | None # `None` у строк `unreadable` / `missing` / `detail` результата 2.0
|
||||
line_code_confidence: Decimal | None
|
||||
line_name: str
|
||||
line_name: str | None
|
||||
line_name_confidence: Decimal | None
|
||||
row_index: int = 0 # индекс строки в `doc.lines` результата — ключ строки формы; повтор кода — свой индекс
|
||||
line_status: str = "ok"
|
||||
@ -72,6 +72,7 @@ class LineDraft:
|
||||
source_ref: str | None = None
|
||||
line_code_text: str | None = None
|
||||
needs_review: bool = False
|
||||
parent_row_index: int | None = None # `row_index` родителя строки `detail` (2.0, по `parent_ref`)
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
299
api/src/domain/statement_mapping_v2.py
Normal file
299
api/src/domain/statement_mapping_v2.py
Normal file
@ -0,0 +1,299 @@
|
||||
"""Чистые функции маппинга формы ББ/ОФР контракта `2.0` (`StatementDocumentV2`) в черновики `financial_statement_*`
|
||||
(spec arm-core «Хранение результата формы v2.0», «Шапка формы — сырой текст и уверенность», «Предупреждения и
|
||||
нарушения документа — document_issue»; P3-9, P3-39…).
|
||||
|
||||
Форма хранится как пришла: строки с их `row_index` и `ref`, повтор кода не склеивается, значение на каждую пару
|
||||
«строка × колонка». Статус документа и проверки ядра `info` здесь не считаются: им нужны id строк после вставки
|
||||
(`services/statement_intake_v2.py`).
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from src.contract.result_v2 import DocumentWarning, StatementColumnV2, StatementDocumentV2, StatementValueV2
|
||||
from src.domain.statement_issues import RELATION_FAILED, RELATION_SOFT, IssueDraft, relation_message
|
||||
from src.domain.statement_mapping import (
|
||||
FULL_CONFIDENCE,
|
||||
ColumnDraft,
|
||||
LineDraft,
|
||||
StatementDraft,
|
||||
ValueDraft,
|
||||
quarter_of,
|
||||
statement_period,
|
||||
)
|
||||
from src.models.enums import (
|
||||
DocumentIssueKind,
|
||||
DocumentType,
|
||||
IssueSeverity,
|
||||
IssueSource,
|
||||
StatementFormVariant,
|
||||
StatementType,
|
||||
)
|
||||
|
||||
__all__ = ["HEADER_SOURCE_FIELDS", "StatementDraftV2", "build_statement_v2", "service_issues"]
|
||||
|
||||
HEADER_SOURCE_FIELDS = (
|
||||
"organization_name",
|
||||
"inn",
|
||||
"report_date",
|
||||
"units",
|
||||
"okud",
|
||||
"okpo",
|
||||
"okopf",
|
||||
"okfs",
|
||||
"okei",
|
||||
"address",
|
||||
)
|
||||
_STATEMENT_TYPES = {DocumentType.BALANCE: StatementType.balance, DocumentType.INCOME: StatementType.income}
|
||||
|
||||
|
||||
@dataclass
|
||||
class StatementDraftV2(StatementDraft):
|
||||
"""Черновик формы `2.0`: то же, что у `1.x`, плюс вариант формы (он же в `header["form_variant"]`)."""
|
||||
|
||||
form_variant: StatementFormVariant | None = None
|
||||
|
||||
|
||||
def _decimal(value: float | None) -> Decimal | None:
|
||||
return Decimal(str(value)) if value is not None else None
|
||||
|
||||
|
||||
def _header_meta(doc: StatementDocumentV2, form_variant: StatementFormVariant | None) -> dict[str, Any]:
|
||||
header = doc.header
|
||||
inn_confidence = _decimal(header.inn_confidence)
|
||||
header_source = {
|
||||
name: {"text": getattr(header, f"{name}_text"), "confidence": getattr(header, f"{name}_confidence")}
|
||||
for name in HEADER_SOURCE_FIELDS
|
||||
}
|
||||
return {
|
||||
"inn": header.inn,
|
||||
"inn_original_value": header.inn,
|
||||
"inn_confidence": inn_confidence,
|
||||
"inn_verified": inn_confidence == FULL_CONFIDENCE if header.inn is not None else None,
|
||||
"inn_modified": False,
|
||||
"organization_name": header.organization_name,
|
||||
"organization_name_original_value": header.organization_name,
|
||||
"organization_name_modified": False,
|
||||
"report_date": header.report_date,
|
||||
"report_date_original_value": header.report_date_text,
|
||||
"report_date_verified": header.report_date is not None,
|
||||
"report_date_modified": False,
|
||||
"units": header.units,
|
||||
"units_original_value": header.units_text,
|
||||
"units_verified": header.units is not None,
|
||||
"units_modified": False,
|
||||
"okud": header.okud,
|
||||
"okpo": header.okpo,
|
||||
"okopf": header.okopf,
|
||||
"okfs": header.okfs,
|
||||
"okei": header.okei,
|
||||
"address": header.address,
|
||||
"header_source": header_source,
|
||||
"needs_review_fields": list(header.needs_review_fields),
|
||||
"form_variant": form_variant,
|
||||
}
|
||||
|
||||
|
||||
def _columns(doc: StatementDocumentV2, warnings: list[str]) -> list[StatementColumnV2]:
|
||||
"""Колонки результата; повтор `column_index` — остаётся первая, о повторе предупреждение (P3-9)."""
|
||||
seen: set[int] = set()
|
||||
result: list[StatementColumnV2] = []
|
||||
for column in doc.columns:
|
||||
if column.column_index in seen:
|
||||
warnings.append(f"{doc.locator}: колонка {column.column_index} повторяется — оставлена первая")
|
||||
continue
|
||||
seen.add(column.column_index)
|
||||
result.append(column)
|
||||
return result
|
||||
|
||||
|
||||
def _column_drafts(source_columns: Sequence[StatementColumnV2]) -> list[ColumnDraft]:
|
||||
"""`is_current` — у колонки с самой поздней датой (при равенстве — с меньшим `column_index`), как у `1.x` (§9.5)."""
|
||||
latest = max((c.period_date for c in source_columns if c.period_date is not None), default=None)
|
||||
first_latest = (
|
||||
min(c.column_index for c in source_columns if c.period_date == latest) if latest is not None else None
|
||||
)
|
||||
return [
|
||||
ColumnDraft(
|
||||
column.column_index,
|
||||
column.label,
|
||||
column.period_date,
|
||||
None if latest is None else column.period_date == latest and column.column_index == first_latest,
|
||||
period_kind=column.period_kind,
|
||||
label_confidence=_decimal(column.label_confidence),
|
||||
)
|
||||
for column in source_columns
|
||||
]
|
||||
|
||||
|
||||
def _lines_and_values(
|
||||
doc: StatementDocumentV2, source_columns: Sequence[StatementColumnV2], warnings: list[str]
|
||||
) -> tuple[list[LineDraft], list[ValueDraft]]:
|
||||
known_columns = {column.column_index for column in source_columns}
|
||||
row_by_ref = {line.ref: line.row_index for line in doc.lines}
|
||||
lines: list[LineDraft] = []
|
||||
values: list[ValueDraft] = []
|
||||
seen_rows: set[int] = set()
|
||||
seen_refs: set[str] = set()
|
||||
for line in doc.lines:
|
||||
if line.row_index in seen_rows or line.ref in seen_refs:
|
||||
warnings.append(
|
||||
f"{doc.locator}: строка row_index={line.row_index}, ref={line.ref} повторяет прежнюю — пропущена"
|
||||
)
|
||||
continue
|
||||
seen_rows.add(line.row_index)
|
||||
seen_refs.add(line.ref)
|
||||
parent: int | None = None
|
||||
if line.line_status == "detail" and line.parent_ref is not None:
|
||||
parent = row_by_ref.get(line.parent_ref)
|
||||
if parent is None:
|
||||
warnings.append(f"{doc.locator}: строка {line.ref}: родитель {line.parent_ref} не найден")
|
||||
lines.append(
|
||||
LineDraft(
|
||||
line.line_code,
|
||||
_decimal(line.line_code_confidence),
|
||||
line.line_name,
|
||||
_decimal(line.line_name_confidence),
|
||||
row_index=line.row_index,
|
||||
line_status=line.line_status,
|
||||
line_code_original_value=line.line_code,
|
||||
source_ref=line.ref,
|
||||
line_code_text=line.line_code_text,
|
||||
needs_review=line.needs_review,
|
||||
parent_row_index=parent,
|
||||
)
|
||||
)
|
||||
by_column: dict[int, StatementValueV2] = {}
|
||||
for cell in line.values:
|
||||
if cell.column_index not in known_columns:
|
||||
warnings.append(
|
||||
f"{doc.locator}: строка {line.ref}: значение колонки {cell.column_index}"
|
||||
" вне колонок формы пропущено"
|
||||
)
|
||||
continue
|
||||
if cell.column_index in by_column:
|
||||
warnings.append(
|
||||
f"{doc.locator}: строка {line.ref}: значение колонки {cell.column_index}"
|
||||
" повторяется — оставлено первое"
|
||||
)
|
||||
continue
|
||||
by_column[cell.column_index] = cell
|
||||
for column in source_columns:
|
||||
cell_value = by_column.get(column.column_index)
|
||||
values.append(
|
||||
ValueDraft(
|
||||
line.line_code,
|
||||
column.column_index,
|
||||
cell_value.amount if cell_value is not None else None,
|
||||
cell_value.string_value if cell_value is not None else None,
|
||||
_decimal(cell_value.confidence) if cell_value is not None else None,
|
||||
cell_value.is_recognized if cell_value is not None else None,
|
||||
row_index=line.row_index,
|
||||
source_text=cell_value.text if cell_value is not None else None,
|
||||
needs_review=cell_value.needs_review if cell_value is not None else False,
|
||||
)
|
||||
)
|
||||
return lines, values
|
||||
|
||||
|
||||
def build_statement_v2(doc: StatementDocumentV2) -> StatementDraftV2:
|
||||
"""Черновик формы `BALANCE` / `INCOME` результата `2.0`: всё, как пришло (спека «Хранение результата формы v2.0»).
|
||||
|
||||
Период — как у `1.x` (§9.5 п. 2): `header.report_date`, иначе самая поздняя дата колонок; квартал — только при конце
|
||||
квартала. Предупреждения черновика (повтор колонки, значения вне колонок, повтор ячейки, повтор строки, потерянный
|
||||
родитель) — строки для `MAPPING_WARNING`, P3-9."""
|
||||
document_type = DocumentType(doc.document_type)
|
||||
warnings: list[str] = []
|
||||
source_columns = _columns(doc, warnings)
|
||||
period_date = statement_period(doc.header.report_date, [column.period_date for column in source_columns])
|
||||
period_year = period_date.year if period_date is not None else None
|
||||
period_quarter = quarter_of(period_date) if period_date is not None else None
|
||||
if period_date is None:
|
||||
warnings.append(f"{doc.locator}: период формы не определён (нет даты отчёта и дат колонок)")
|
||||
elif period_quarter is None:
|
||||
warnings.append(f"{doc.locator}: дата {period_date.isoformat()} не конец квартала — квартал не определён")
|
||||
lines, values = _lines_and_values(doc, source_columns, warnings)
|
||||
form_variant = StatementFormVariant(doc.form_variant) if doc.form_variant is not None else None
|
||||
return StatementDraftV2(
|
||||
document_type=document_type,
|
||||
statement_type=_STATEMENT_TYPES[document_type],
|
||||
locator=doc.locator,
|
||||
period_date=period_date,
|
||||
period_year=period_year,
|
||||
period_quarter=period_quarter,
|
||||
header=_header_meta(doc, form_variant),
|
||||
columns=_column_drafts(source_columns),
|
||||
lines=lines,
|
||||
values=values,
|
||||
warnings=warnings,
|
||||
form_variant=form_variant,
|
||||
)
|
||||
|
||||
|
||||
def _warning_issue(
|
||||
warning: DocumentWarning,
|
||||
*,
|
||||
line_ids_by_ref: Mapping[str, uuid.UUID],
|
||||
column_ids_by_index: Mapping[int, uuid.UUID],
|
||||
) -> IssueDraft:
|
||||
line_ids = tuple(line_ids_by_ref[ref] for ref in (warning.line_refs or ()) if ref in line_ids_by_ref)
|
||||
column_id = column_ids_by_index.get(warning.column_index) if warning.column_index is not None else None
|
||||
return IssueDraft(
|
||||
kind=DocumentIssueKind.warning,
|
||||
code=warning.code,
|
||||
severity=IssueSeverity(warning.severity),
|
||||
message=warning.message,
|
||||
source=IssueSource.service,
|
||||
line_ids=line_ids,
|
||||
column_id=column_id,
|
||||
column_index=warning.column_index if column_id is None else None,
|
||||
field=warning.field,
|
||||
)
|
||||
|
||||
|
||||
def service_issues(
|
||||
doc: StatementDocumentV2,
|
||||
*,
|
||||
line_ids_by_ref: Mapping[str, uuid.UUID],
|
||||
column_ids_by_index: Mapping[int, uuid.UUID],
|
||||
ok_line_ids_by_code: Mapping[str, Sequence[uuid.UUID]],
|
||||
) -> list[IssueDraft]:
|
||||
"""Записи `document_issue` источника `service`: `warnings[]` документа (`line_refs` → `line_ids`; `column_index` →
|
||||
`column_id`, если такая колонка есть в форме, иначе остаётся `column_index`) и `checks` (`relations_failed` —
|
||||
`relation` / `RELATION_FAILED` / `error`, `soft_relations_failed` — `soft` / `RELATION_SOFT` / `info`; `line_ids` —
|
||||
все строки `ok` кодов `codes`; `column_id` — по `column_index` нарушения, ключ колонки сервиса — её номер)."""
|
||||
drafts = [
|
||||
_warning_issue(warning, line_ids_by_ref=line_ids_by_ref, column_ids_by_index=column_ids_by_index)
|
||||
for warning in doc.warnings
|
||||
]
|
||||
if doc.checks is None:
|
||||
return drafts
|
||||
groups = (
|
||||
(doc.checks.relations_failed, DocumentIssueKind.relation, RELATION_FAILED, IssueSeverity.error),
|
||||
(doc.checks.soft_relations_failed, DocumentIssueKind.soft, RELATION_SOFT, IssueSeverity.info),
|
||||
)
|
||||
for failures, kind, code, severity in groups:
|
||||
for failure in failures:
|
||||
line_ids = dict.fromkeys(
|
||||
line_id for form_code in failure.codes for line_id in ok_line_ids_by_code.get(form_code, ())
|
||||
)
|
||||
drafts.append(
|
||||
IssueDraft(
|
||||
kind=kind,
|
||||
code=code,
|
||||
severity=severity,
|
||||
message=relation_message(failure.relation, failure.expected, failure.actual),
|
||||
source=IssueSource.service,
|
||||
line_ids=tuple(line_ids),
|
||||
column_id=column_ids_by_index.get(failure.column_index),
|
||||
column_index=failure.column_index,
|
||||
relation=failure.relation,
|
||||
codes=tuple(failure.codes),
|
||||
expected=failure.expected,
|
||||
actual=failure.actual,
|
||||
)
|
||||
)
|
||||
return drafts
|
||||
@ -118,6 +118,7 @@ async def insert_statement(
|
||||
"line_code_original_value": line.line_code_original_value,
|
||||
"line_code_confidence": line.line_code_confidence,
|
||||
"line_code_modified": False,
|
||||
"parent_row_index": line.parent_row_index,
|
||||
"needs_review": line.needs_review,
|
||||
"line_name": line.line_name,
|
||||
"line_name_original_value": line.line_name,
|
||||
|
||||
@ -39,6 +39,7 @@ from sqlalchemy import Integer, Select, delete, func, insert, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from src.contract.result import OsvDocument, RecognitionResult, StatementDocument
|
||||
from src.contract.result_v2 import RecognitionResultV2, StatementDocumentV2
|
||||
from src.core.config import Settings
|
||||
from src.core.storage import ObjectNotFoundError, ObjectStorage
|
||||
from src.domain.osv_mapping import (
|
||||
@ -74,6 +75,7 @@ from src.repositories.counterparties import CounterpartyIndex
|
||||
from src.services import consolidation_documents
|
||||
from src.services.audit import AuditService
|
||||
from src.services.file_status import set_file_status_for_job
|
||||
from src.services.statement_intake_v2 import FormV2, OtherV2, insert_forms_v2, map_statement_docs_v2
|
||||
from src.services.statement_recompute import checks_on_intake_1x
|
||||
from src.services.statements_engine import StatementsEngine, default_engine
|
||||
|
||||
@ -198,7 +200,7 @@ def _schema_error_summary(exc: ValidationError) -> str:
|
||||
return f"результат не соответствует схеме recognition-result: ошибок {exc.error_count()}, первая: {where} {kind}"
|
||||
|
||||
|
||||
def _osv_documents(result: RecognitionResult) -> list[OsvDocument]:
|
||||
def _osv_documents(result: RecognitionResult | RecognitionResultV2) -> list[OsvDocument]:
|
||||
return [document for document in result.documents if isinstance(document, OsvDocument)]
|
||||
|
||||
|
||||
@ -446,18 +448,98 @@ async def _insert_statements( # noqa: PLR0913 - все входы именов
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FormsPlan:
|
||||
"""Документы форм/раскрытий результата до вставки: общие поля и черновики по MAJOR."""
|
||||
|
||||
documents: list[Document]
|
||||
warnings: list[str]
|
||||
repeats: dict[uuid.UUID, str]
|
||||
forms_v1: list[tuple[Document, StatementDraft]]
|
||||
forms_v2: list[FormV2]
|
||||
others_v2: list[OtherV2]
|
||||
|
||||
|
||||
def _plan_forms( # noqa: PLR0913 - все входы именованные
|
||||
result: RecognitionResult | RecognitionResultV2,
|
||||
*,
|
||||
job: RecognitionJob,
|
||||
ordinals: Mapping[int, int],
|
||||
file: StoredFile,
|
||||
owner: LegalEntity,
|
||||
owner_names: frozenset[str],
|
||||
now: datetime,
|
||||
) -> _FormsPlan:
|
||||
"""Выбор маппера форм по типу результата: `2.0` (и `job.statements_version` сервиса) или `1.x`."""
|
||||
if isinstance(result, RecognitionResultV2):
|
||||
job.statements_version = result.service.statements_version
|
||||
forms_v2, others_v2, warnings, repeats = map_statement_docs_v2(
|
||||
[doc for doc in result.documents if isinstance(doc, StatementDocumentV2)],
|
||||
ordinals=ordinals,
|
||||
file=file,
|
||||
owner=owner,
|
||||
owner_names=owner_names,
|
||||
now=now,
|
||||
)
|
||||
documents = [*(other.document for other in others_v2), *(form.document for form in forms_v2)]
|
||||
return _FormsPlan(documents, warnings, repeats, [], forms_v2, others_v2)
|
||||
forms_v1, other_documents, warnings, repeats = _map_statement_docs(
|
||||
[doc for doc in result.documents if isinstance(doc, StatementDocument)],
|
||||
ordinals=ordinals,
|
||||
file=file,
|
||||
owner=owner,
|
||||
owner_names=owner_names,
|
||||
now=now,
|
||||
)
|
||||
documents = [*other_documents, *(document for document, _ in forms_v1)]
|
||||
return _FormsPlan(documents, warnings, repeats, forms_v1, [], [])
|
||||
|
||||
|
||||
async def _insert_forms( # noqa: PLR0913 - все входы именованные
|
||||
session: AsyncSession,
|
||||
plan: _FormsPlan,
|
||||
*,
|
||||
job: RecognitionJob,
|
||||
result: RecognitionResult | RecognitionResultV2,
|
||||
job_done: bool,
|
||||
now: datetime,
|
||||
engine: StatementsEngine | None,
|
||||
) -> None:
|
||||
"""Вставка форм: `2.0` — как пришли (`insert_forms_v2`), `1.x` — `_insert_statements`."""
|
||||
if isinstance(result, RecognitionResultV2):
|
||||
await insert_forms_v2(
|
||||
session,
|
||||
job=job,
|
||||
result=result,
|
||||
forms=plan.forms_v2,
|
||||
others=plan.others_v2,
|
||||
repeats=plan.repeats,
|
||||
engine=engine or default_engine(),
|
||||
now=now,
|
||||
)
|
||||
return
|
||||
await _insert_statements(
|
||||
session, job=job, forms=plan.forms_v1, repeats=plan.repeats, job_done=job_done, now=now, engine=engine
|
||||
)
|
||||
|
||||
|
||||
async def map_file( # noqa: PLR0913 - все входы маппинга файла именованные
|
||||
session: AsyncSession,
|
||||
*,
|
||||
job: RecognitionJob,
|
||||
file: StoredFile,
|
||||
result: RecognitionResult,
|
||||
result: RecognitionResult | RecognitionResultV2,
|
||||
chart: Mapping[str, uuid.UUID],
|
||||
index: CounterpartyIndex,
|
||||
now: datetime,
|
||||
engine: StatementsEngine | None = None,
|
||||
) -> list[str]:
|
||||
"""Маппинг результата в транзакции вызывающего (D-3, D-6…D-10); возвращает предупреждения маппинга файла.
|
||||
|
||||
Маппер выбирается по типу результата: `RecognitionResultV2` — формы по контракту `2.0`
|
||||
(`services/statement_intake_v2`, `job.statements_version` = `service.statements_version`), `RecognitionResult` —
|
||||
`1.x` (`_map_statement_docs`).
|
||||
|
||||
Очистка прежних документов/строк файла → документ на каждый лист ОСВ (повтор счёта — предупреждение) и на каждую
|
||||
форму ББ/ОФР (`_map_statement_docs`) → строки ОСВ пакетами по `ENTRY_BATCH_SIZE` → вставка форм и их статус →
|
||||
`job.group_schema`, `metrics.mapping_warnings` → файл `PROCESSED` (только из
|
||||
@ -467,7 +549,6 @@ async def map_file( # noqa: PLR0913 - все входы маппинга фай
|
||||
if not result.documents:
|
||||
raise MappingFatal(NO_DOCUMENTS)
|
||||
sheets = _osv_documents(result)
|
||||
statement_docs = [doc for doc in result.documents if isinstance(doc, StatementDocument)]
|
||||
owner = await session.get(LegalEntity, file.legal_entity_id)
|
||||
if owner is None:
|
||||
raise MappingFatal(f"ЮЛ-владелец файла не найден: {file.legal_entity_id}")
|
||||
@ -545,12 +626,9 @@ async def map_file( # noqa: PLR0913 - все входы маппинга фай
|
||||
for entry in mapped.entries
|
||||
)
|
||||
|
||||
forms, other_documents, form_warnings, repeats = _map_statement_docs(
|
||||
statement_docs, ordinals=ordinals, file=file, owner=owner, owner_names=owner_names, now=now
|
||||
)
|
||||
warnings.extend(form_warnings)
|
||||
documents.extend(other_documents)
|
||||
documents.extend(document for document, _ in forms)
|
||||
plan = _plan_forms(result, job=job, ordinals=ordinals, file=file, owner=owner, owner_names=owner_names, now=now)
|
||||
documents.extend(plan.documents)
|
||||
warnings.extend(plan.warnings)
|
||||
|
||||
session.add_all(documents)
|
||||
await session.flush()
|
||||
@ -562,7 +640,7 @@ async def map_file( # noqa: PLR0913 - все входы маппинга фай
|
||||
for start in range(0, len(entries), ENTRY_BATCH_SIZE):
|
||||
await session.execute(insert(OsvEntry).values(entries[start : start + ENTRY_BATCH_SIZE]))
|
||||
|
||||
await _insert_statements(session, job=job, forms=forms, repeats=repeats, job_done=job_done, now=now)
|
||||
await _insert_forms(session, plan, job=job, result=result, job_done=job_done, now=now, engine=engine)
|
||||
|
||||
job.group_schema = [_group_schema(sheet) for sheet in sheets]
|
||||
job.metrics = {**_metrics(job), MAPPING_WARNINGS_KEY: warnings}
|
||||
@ -899,7 +977,10 @@ async def _register_failure(
|
||||
|
||||
|
||||
async def _map_candidate(
|
||||
session_factory: async_sessionmaker[AsyncSession], candidate: Candidate, result: RecognitionResult, clock: Clock
|
||||
session_factory: async_sessionmaker[AsyncSession],
|
||||
candidate: Candidate,
|
||||
result: RecognitionResult | RecognitionResultV2,
|
||||
clock: Clock,
|
||||
) -> None:
|
||||
"""Шаг 3: транзакция маппинга. Задание и файл перечитываются под блокировкой (порядок «задание → файл»); файл уже
|
||||
не ждёт результата этого задания (его опередили, задание не текущее — Review Focus 1) — ничего не меняется."""
|
||||
|
||||
205
api/src/services/statement_intake_v2.py
Normal file
205
api/src/services/statement_intake_v2.py
Normal file
@ -0,0 +1,205 @@
|
||||
"""Приём документов форм ББ/ОФР/раскрытий из результата `2.0` (spec arm-core «Хранение результата формы v2.0»,
|
||||
«Условие RECOGNIZED при приёме 2.0», «Предупреждения и нарушения документа — document_issue»; Р-10, Р-16, Р-18,
|
||||
Р-19, P3-6, P3-9, P3-15, P3-19, P3-39…).
|
||||
|
||||
`map_statement_docs_v2` создаёт документы и черновики (до вставки), `insert_forms_v2` пишет форму, записи
|
||||
`document_issue` и считает статус. Пересчётов §7, `needs_review` и предупреждений сервиса при приёме нет — они уже в
|
||||
результате; ядро добавляет только проверки уровня `info` (`core_info_issues`) и страховку статуса.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from src.contract.result_v2 import RecognitionResultV2, StatementDocumentV2
|
||||
from src.domain.osv_mapping import counterparty_check
|
||||
from src.domain.statement_issues import (
|
||||
IssueDraft,
|
||||
core_info_issues,
|
||||
mapping_warning,
|
||||
result_status_inconsistent,
|
||||
version_issue,
|
||||
)
|
||||
from src.domain.statement_lines import load_line_catalog
|
||||
from src.domain.statement_mapping_v2 import StatementDraftV2, build_statement_v2, service_issues
|
||||
from src.domain.statement_status import duplicate_period_forms, form_status_v2, has_error_warning
|
||||
from src.domain.stored_form import edition_codes
|
||||
from src.models.directories import LegalEntity
|
||||
from src.models.enums import DocumentCheckResult, DocumentStatus, DocumentType
|
||||
from src.models.files import Document, RecognitionJob, StoredFile
|
||||
from src.repositories import statements as statements_repo
|
||||
from src.services.statements_engine import DocStatus, FormVariant, StatementsEngine
|
||||
from src.services.statements_engine import DocumentType as PackageDocumentType
|
||||
|
||||
__all__ = ["FormV2", "OtherV2", "insert_forms_v2", "map_statement_docs_v2"]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FormV2:
|
||||
"""Форма `BALANCE` / `INCOME` результата: документ БД, черновик и исходный документ результата."""
|
||||
|
||||
document: Document
|
||||
draft: StatementDraftV2
|
||||
source: StatementDocumentV2
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OtherV2:
|
||||
"""Объединённый документ `DISCLOSURE` файла (P3-6) и документы результата, из которых он собран."""
|
||||
|
||||
document: Document
|
||||
sources: tuple[StatementDocumentV2, ...]
|
||||
|
||||
|
||||
def map_statement_docs_v2( # noqa: PLR0913 - все входы именованные
|
||||
statement_docs: Sequence[StatementDocumentV2],
|
||||
*,
|
||||
ordinals: Mapping[int, int],
|
||||
file: StoredFile,
|
||||
owner: LegalEntity,
|
||||
owner_names: frozenset[str],
|
||||
now: datetime,
|
||||
) -> tuple[list[FormV2], list[OtherV2], list[str], dict[uuid.UUID, str]]:
|
||||
"""Документы форм и `DISCLOSURE` результата `2.0`. Возвращает формы, объединённые `DISCLOSURE` (один документ на
|
||||
файл, `locator` через запятую, `file_ordinal` — индекс первого, статус `RECOGNIZED`, P3-6), предупреждения маппинга
|
||||
для `metrics.mapping_warnings` и текст «другая форма» по `id` документов форм одного типа и периода (Р-10, Р-18:
|
||||
хранятся все, каждый `NEEDS_CHECK`). `ordinals` — `id(документ результата)` → индекс в `result.documents`."""
|
||||
forms: list[FormV2] = []
|
||||
disclosures: list[StatementDocumentV2] = []
|
||||
warnings: list[str] = []
|
||||
for doc in statement_docs:
|
||||
doc_type = DocumentType(doc.document_type)
|
||||
if doc_type is DocumentType.DISCLOSURE:
|
||||
disclosures.append(doc)
|
||||
continue
|
||||
draft = build_statement_v2(doc)
|
||||
warnings.extend(draft.warnings)
|
||||
document = Document(
|
||||
id=uuid.uuid4(),
|
||||
stored_file_id=file.id,
|
||||
legal_entity_id=file.legal_entity_id,
|
||||
document_type=doc_type,
|
||||
file_ordinal=ordinals[id(doc)],
|
||||
osv_account_code=None,
|
||||
period_year=draft.period_year,
|
||||
period_quarter=draft.period_quarter,
|
||||
is_cumulative=None,
|
||||
reporting_date=draft.period_date,
|
||||
locator=doc.locator,
|
||||
status=DocumentStatus.NEEDS_CHECK, # уточняется после вставки формы
|
||||
period_check=DocumentCheckResult.UNKNOWN,
|
||||
counterparty_check=counterparty_check(doc.header.inn, doc.header.organization_name, owner.inn, owner_names),
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
forms.append(FormV2(document, draft, doc))
|
||||
duplicated = duplicate_period_forms(
|
||||
(form.document.id, form.document.document_type, form.document.period_year, form.document.period_quarter)
|
||||
for form in forms
|
||||
)
|
||||
repeats: dict[uuid.UUID, str] = {}
|
||||
for form in forms:
|
||||
if form.document.id in duplicated:
|
||||
text = f"в файле есть другая форма {form.document.document_type.value} за тот же период"
|
||||
repeats[form.document.id] = text
|
||||
warnings.append(f"{form.document.locator}: {text}")
|
||||
others: list[OtherV2] = []
|
||||
if disclosures:
|
||||
others.append(
|
||||
OtherV2(
|
||||
Document(
|
||||
id=uuid.uuid4(),
|
||||
stored_file_id=file.id,
|
||||
legal_entity_id=file.legal_entity_id,
|
||||
document_type=DocumentType.DISCLOSURE,
|
||||
file_ordinal=ordinals[id(disclosures[0])],
|
||||
locator=",".join(doc.locator for doc in disclosures),
|
||||
status=DocumentStatus.RECOGNIZED,
|
||||
period_check=DocumentCheckResult.UNKNOWN,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
),
|
||||
tuple(disclosures),
|
||||
)
|
||||
)
|
||||
return forms, others, warnings, repeats
|
||||
|
||||
|
||||
def _file_warnings(result: RecognitionResultV2) -> list[dict[str, object]]:
|
||||
"""`metrics.warnings` файла результата структурами (P3-15)."""
|
||||
if result.metrics is None or result.metrics.warnings is None:
|
||||
return []
|
||||
return [warning.model_dump(mode="json") for warning in result.metrics.warnings]
|
||||
|
||||
|
||||
async def insert_forms_v2( # noqa: PLR0913 - все входы именованные
|
||||
session: AsyncSession,
|
||||
*,
|
||||
job: RecognitionJob,
|
||||
result: RecognitionResultV2,
|
||||
forms: Sequence[FormV2],
|
||||
others: Sequence[OtherV2],
|
||||
repeats: Mapping[uuid.UUID, str],
|
||||
engine: StatementsEngine,
|
||||
now: datetime,
|
||||
) -> None:
|
||||
"""Форма как пришла, `document_issue` и статус каждого документа формы; предупреждения `DISCLOSURE` — в
|
||||
`document_issue` его единственного документа. Без commit."""
|
||||
file_error = has_error_warning(_file_warnings(result))
|
||||
catalog = load_line_catalog()
|
||||
for other in others:
|
||||
drafts: list[IssueDraft] = []
|
||||
for source in other.sources:
|
||||
drafts += service_issues(source, line_ids_by_ref={}, column_ids_by_index={}, ok_line_ids_by_code={})
|
||||
await statements_repo.add_issues(session, other.document.id, drafts, now=now)
|
||||
for form in forms:
|
||||
document, source = form.document, form.source
|
||||
meta = await statements_repo.insert_statement(session, document_id=document.id, job_id=job.id, draft=form.draft)
|
||||
columns, aggregates = await statements_repo.form_aggregate(session, meta.id)
|
||||
lines = await statements_repo.form_lines(session, meta.id)
|
||||
|
||||
package_type = PackageDocumentType(source.document_type)
|
||||
variant = FormVariant(source.form_variant) if source.form_variant is not None else None
|
||||
edition = engine.edition(package_type, edition_codes(source), document.reporting_date, variant)
|
||||
order = (
|
||||
engine.code_order(edition, package_type, variant) if edition is not None and variant is not None else None
|
||||
)
|
||||
info = core_info_issues(
|
||||
lines,
|
||||
form=document.document_type.value,
|
||||
edition=edition.value if edition is not None else None,
|
||||
order=order,
|
||||
catalog=catalog,
|
||||
)
|
||||
engine_done = engine.contract_document_status(source.model_dump(mode="json")) is DocStatus.DONE
|
||||
inconsistent = source.status == "done" and not engine_done
|
||||
|
||||
warnings = [*form.draft.warnings, *([repeats[document.id]] if document.id in repeats else [])]
|
||||
drafts = [
|
||||
*(mapping_warning(text) for text in warnings),
|
||||
*service_issues(
|
||||
source,
|
||||
line_ids_by_ref={line.source_ref: line.id for line in lines if line.source_ref is not None},
|
||||
column_ids_by_index={column.column_index: column.id for column in columns},
|
||||
ok_line_ids_by_code={code: aggregate.line_ids for code, aggregate in aggregates.items()},
|
||||
),
|
||||
*info.drafts,
|
||||
]
|
||||
mismatch = version_issue(job.statements_version, engine.version)
|
||||
if mismatch is not None:
|
||||
drafts.append(mismatch)
|
||||
if inconsistent:
|
||||
drafts.append(result_status_inconsistent(source.status))
|
||||
await statements_repo.add_issues(session, document.id, drafts, now=now)
|
||||
|
||||
document.status = form_status_v2(
|
||||
service_done=source.status == "done" and engine_done,
|
||||
file_error_warning=file_error,
|
||||
counterparty=document.counterparty_check or DocumentCheckResult.UNKNOWN,
|
||||
period_known=document.period_year is not None and document.period_quarter is not None,
|
||||
names_ok=not info.mismatched,
|
||||
duplicate_in_file=document.id in repeats,
|
||||
)
|
||||
@ -6,24 +6,29 @@
|
||||
в приёме результата (B3/B4); экран и подтверждение читают его из `document_issue`.
|
||||
"""
|
||||
|
||||
import re
|
||||
import uuid
|
||||
from collections.abc import Hashable, Mapping, Sequence
|
||||
from decimal import Decimal
|
||||
from collections.abc import Mapping, Sequence
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from src.common.clock import utc_now
|
||||
from src.domain.statement_aggregate import CodeAggregate
|
||||
from src.domain.statement_issues import IssueDraft
|
||||
from src.domain.statement_lines import load_line_catalog
|
||||
from src.domain.statement_status import (
|
||||
NameCheckLine,
|
||||
duplicate_period_forms,
|
||||
form_status_v2,
|
||||
has_error_warning,
|
||||
name_mismatches,
|
||||
from src.domain.statement_issues import (
|
||||
CODE_NOT_IN_DICTIONARY,
|
||||
CODE_NOT_IN_EDITION,
|
||||
NAME_MISMATCH,
|
||||
RECOMPUTED_CORE_CODES,
|
||||
RELATION_FAILED,
|
||||
RELATION_SOFT,
|
||||
STATEMENTS_VERSION_MISMATCH,
|
||||
IssueDraft,
|
||||
code_issues,
|
||||
core_info_issues,
|
||||
name_issues,
|
||||
relation_message,
|
||||
version_issue,
|
||||
)
|
||||
from src.domain.statement_lines import load_line_catalog
|
||||
from src.domain.statement_status import duplicate_period_forms, form_status_v2, has_error_warning
|
||||
from src.domain.stored_form import build_stored_form
|
||||
from src.models.enums import (
|
||||
DocumentCheckResult,
|
||||
@ -35,7 +40,7 @@ from src.models.enums import (
|
||||
StatementLineStatus,
|
||||
)
|
||||
from src.models.files import Document, RecognitionJob
|
||||
from src.models.statements import FinancialStatementColumn, FinancialStatementLine, FinancialStatementMeta
|
||||
from src.models.statements import FinancialStatementColumn, FinancialStatementMeta
|
||||
from src.repositories import statements as repo
|
||||
from src.services.statements_engine import FailedRelation, StatementChecks, StatementsEngine
|
||||
|
||||
@ -49,27 +54,14 @@ __all__ = [
|
||||
"STATEMENTS_VERSION_MISMATCH",
|
||||
"checks_on_intake_1x",
|
||||
"code_issues",
|
||||
"core_info_issues",
|
||||
"name_issues",
|
||||
"recompute_form",
|
||||
"relations_to_issues",
|
||||
"version_issue",
|
||||
]
|
||||
|
||||
RELATION_FAILED = "RELATION_FAILED"
|
||||
RELATION_SOFT = "RELATION_SOFT"
|
||||
CODE_NOT_IN_DICTIONARY = "CODE_NOT_IN_DICTIONARY"
|
||||
CODE_NOT_IN_EDITION = "CODE_NOT_IN_EDITION"
|
||||
NAME_MISMATCH = "NAME_MISMATCH"
|
||||
STATEMENTS_VERSION_MISMATCH = "STATEMENTS_VERSION_MISMATCH"
|
||||
# записи `core`, которые пересчёт заменяет целиком (P3-8); прочие (`MAPPING_WARNING`, `RESULT_STATUS_INCONSISTENT`,
|
||||
# `LEGACY_WARNING`) — факт приёма, их пересчёт сохраняет
|
||||
RECOMPUTED_CORE_CODES = (CODE_NOT_IN_DICTIONARY, CODE_NOT_IN_EDITION, NAME_MISMATCH, STATEMENTS_VERSION_MISMATCH)
|
||||
_CHECKED_CODE = re.compile(r"\d{4}", re.ASCII) # R-1 (Р-40): проверяются только четырёхзначные коды
|
||||
_EMPTY = "пусто"
|
||||
|
||||
|
||||
def _amount_text(value: Decimal | None) -> str:
|
||||
return _EMPTY if value is None else str(value)
|
||||
# коды и проверки ядра `info` живут в `domain/statement_issues` (P3-39); здесь они реэкспортируются (`__all__`)
|
||||
|
||||
|
||||
def _find_column(
|
||||
@ -114,10 +106,7 @@ def relations_to_issues(
|
||||
kind=kind,
|
||||
code=code,
|
||||
severity=severity,
|
||||
message=(
|
||||
f"не сошлось {failure.relation}: "
|
||||
f"{_amount_text(failure.expected)} ≠ {_amount_text(failure.actual)}"
|
||||
),
|
||||
message=relation_message(failure.relation, failure.expected, failure.actual),
|
||||
source=source,
|
||||
line_ids=tuple(line_ids),
|
||||
column_id=column.id if column is not None else None,
|
||||
@ -131,82 +120,6 @@ def relations_to_issues(
|
||||
return drafts
|
||||
|
||||
|
||||
def _core_info(code: str, message: str, line_ids: tuple[uuid.UUID, ...] = ()) -> IssueDraft:
|
||||
return IssueDraft(
|
||||
kind=DocumentIssueKind.warning,
|
||||
code=code,
|
||||
severity=IssueSeverity.info,
|
||||
message=message,
|
||||
source=IssueSource.core,
|
||||
line_ids=line_ids,
|
||||
)
|
||||
|
||||
|
||||
def code_issues(
|
||||
lines: Sequence[FinancialStatementLine],
|
||||
*,
|
||||
in_dictionary: Mapping[str, bool],
|
||||
in_edition: Mapping[str, bool] | None,
|
||||
) -> list[IssueDraft]:
|
||||
"""`CODE_NOT_IN_DICTIONARY` / `CODE_NOT_IN_EDITION` — по записи на код документа (`line_ids` — все строки `ok` с
|
||||
этим кодом), только для кодов из ровно четырёх цифр (R-1, Р-40). `in_dictionary` — код есть в словаре имён редакции,
|
||||
`in_edition` — код из перечня редакции и варианта (`None` — редакция или вариант не определены, не проверяется)."""
|
||||
by_code: dict[str, list[FinancialStatementLine]] = {}
|
||||
for line in lines:
|
||||
if line.line_status is StatementLineStatus.ok and line.line_code and _CHECKED_CODE.fullmatch(line.line_code):
|
||||
by_code.setdefault(line.line_code, []).append(line)
|
||||
drafts: list[IssueDraft] = []
|
||||
for code in sorted(by_code):
|
||||
ids = tuple(
|
||||
line.id for line in sorted(by_code[code], key=lambda item: (item.row_index is None, item.row_index or 0))
|
||||
)
|
||||
if not in_dictionary.get(code, True):
|
||||
drafts.append(_core_info(CODE_NOT_IN_DICTIONARY, f"кода {code} нет в словаре наименований редакции", ids))
|
||||
if in_edition is not None and not in_edition.get(code, True):
|
||||
drafts.append(_core_info(CODE_NOT_IN_EDITION, f"код {code} не входит в перечень кодов редакции формы", ids))
|
||||
return drafts
|
||||
|
||||
|
||||
def name_issues(lines: Sequence[FinancialStatementLine], mismatched: Sequence[Hashable]) -> list[IssueDraft]:
|
||||
"""`NAME_MISMATCH` — по записи на строку из `name_mismatches`."""
|
||||
by_id = {line.id: line for line in lines}
|
||||
drafts: list[IssueDraft] = []
|
||||
for key in mismatched:
|
||||
line = by_id[key] # type: ignore[index]
|
||||
drafts.append(
|
||||
_core_info(
|
||||
NAME_MISMATCH,
|
||||
f"наименование «{line.line_name or ''}» не совпало со справочным для кода {line.line_code}",
|
||||
(line.id,),
|
||||
)
|
||||
)
|
||||
return drafts
|
||||
|
||||
|
||||
def version_issue(job_version: str | None, engine_version: str | None) -> IssueDraft | None:
|
||||
"""`STATEMENTS_VERSION_MISMATCH`: версия копии пакета у задания известна и не равна версии копии ядра."""
|
||||
if job_version is None or job_version == engine_version:
|
||||
return None
|
||||
return _core_info(
|
||||
STATEMENTS_VERSION_MISMATCH,
|
||||
f"форма распознана общим кодом версии {job_version}, в ядре версия {engine_version or 'не определена'}",
|
||||
)
|
||||
|
||||
|
||||
def _name_lines(lines: Sequence[FinancialStatementLine]) -> list[NameCheckLine]:
|
||||
return [
|
||||
NameCheckLine(
|
||||
key=line.id,
|
||||
line_status=line.line_status.value,
|
||||
line_code=line.line_code,
|
||||
line_code_confidence=line.line_code_confidence,
|
||||
line_name=line.line_name,
|
||||
code_fixed=bool(line.line_code_modified) or bool(line.is_confirmed),
|
||||
)
|
||||
for line in lines
|
||||
]
|
||||
|
||||
|
||||
async def recompute_form(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
@ -235,20 +148,14 @@ async def recompute_form(
|
||||
form = document.document_type.value
|
||||
catalog = load_line_catalog()
|
||||
edition = result.edition.value if result.edition is not None else None
|
||||
in_dictionary = {
|
||||
line.line_code: catalog.reference_name(edition, form, line.line_code) is not None
|
||||
for line in lines
|
||||
if line.line_code is not None
|
||||
}
|
||||
in_edition: dict[str, bool] | None = None
|
||||
if result.edition is not None and result.form_variant is not None:
|
||||
order = set(engine.code_order(result.edition, stored.document_type, result.form_variant))
|
||||
in_edition = {line.line_code: line.line_code in order for line in lines if line.line_code is not None}
|
||||
mismatched = name_mismatches(_name_lines(lines), catalog, form)
|
||||
drafts = [
|
||||
*(code_issues(lines, in_dictionary=in_dictionary, in_edition=in_edition) if edition is not None else []),
|
||||
*name_issues(lines, mismatched),
|
||||
]
|
||||
order = (
|
||||
engine.code_order(result.edition, stored.document_type, result.form_variant)
|
||||
if result.edition is not None and result.form_variant is not None
|
||||
else None
|
||||
)
|
||||
info = core_info_issues(lines, form=form, edition=edition, order=order, catalog=catalog)
|
||||
mismatched = info.mismatched
|
||||
drafts = list(info.drafts)
|
||||
mismatch = version_issue(job.statements_version if job is not None else None, engine.version)
|
||||
if mismatch is not None:
|
||||
drafts.append(mismatch)
|
||||
|
||||
@ -32,7 +32,11 @@ from arm_statements import (
|
||||
|
||||
__all__ = [
|
||||
"ArmStatementsEngine",
|
||||
"DocStatus",
|
||||
"DocumentType",
|
||||
"Edition",
|
||||
"FailedRelation",
|
||||
"FormVariant",
|
||||
"PeriodKind",
|
||||
"StatementChecks",
|
||||
"StatementsEngine",
|
||||
|
||||
195
api/tests/factories_v2.py
Normal file
195
api/tests/factories_v2.py
Normal file
@ -0,0 +1,195 @@
|
||||
"""Фабрики результата распознавания контракта `2.0` для тестов маппера форм (B4): базовый документ ББ/ОФР, строки,
|
||||
значения, колонки; `statement_result_v2` собирает `RecognitionResultV2` с переопределениями. Реальный пакет
|
||||
`arm_statements` считает статус и редакцию, поэтому базовая форма «чистая»: уверенности 100, `needs_review = false`.
|
||||
"""
|
||||
|
||||
import copy
|
||||
import uuid
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from src.contract import RecognitionResultV2
|
||||
|
||||
__all__ = [
|
||||
"BALANCE_DATE",
|
||||
"balance_lines",
|
||||
"column_v2",
|
||||
"header_v2",
|
||||
"line_v2",
|
||||
"result_builder",
|
||||
"statement_doc_v2",
|
||||
"statement_result_v2",
|
||||
"value_v2",
|
||||
]
|
||||
|
||||
BALANCE_DATE = "2025-09-30"
|
||||
OWNER_INN = "7700000000"
|
||||
_NO_CHECKS: dict[str, Any] = {"relations_failed": [], "soft_relations_failed": []}
|
||||
|
||||
|
||||
def header_v2(**overrides: Any) -> dict[str, Any]:
|
||||
"""Шапка с ИНН владельца по умолчанию (`counterparty_check = MATCH`), датой 30.09.2025 и тыс. руб."""
|
||||
header: dict[str, Any] = {
|
||||
"form_name": "Бухгалтерский баланс",
|
||||
"organization_name": "ООО Ромашка",
|
||||
"organization_name_text": "ООО Ромашка",
|
||||
"organization_name_confidence": 100,
|
||||
"inn": OWNER_INN,
|
||||
"inn_text": OWNER_INN,
|
||||
"inn_confidence": 100,
|
||||
"report_date": BALANCE_DATE,
|
||||
"report_date_text": "30 сентября 2025 г.",
|
||||
"report_date_confidence": 100,
|
||||
"units": "тыс. руб.",
|
||||
"units_text": "тыс. руб.",
|
||||
"units_confidence": 100,
|
||||
"needs_review_fields": [],
|
||||
}
|
||||
return {**header, **overrides}
|
||||
|
||||
|
||||
def column_v2(
|
||||
index: int = 2, *, date: str | None = BALANCE_DATE, kind: str = "absolute", label: str | None = None
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"column_index": index,
|
||||
"label": label or f"На {date}",
|
||||
"label_confidence": 100,
|
||||
"period_date": date,
|
||||
"period_kind": kind,
|
||||
}
|
||||
|
||||
|
||||
def value_v2( # noqa: PLR0913 - все поля значения контракта
|
||||
column_index: int = 2,
|
||||
amount: str | None = "100",
|
||||
*,
|
||||
text: str | None = None,
|
||||
confidence: float = 100,
|
||||
is_recognized: bool = True,
|
||||
needs_review: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"column_index": column_index,
|
||||
"amount": amount,
|
||||
"text": text if text is not None else amount,
|
||||
"confidence": confidence,
|
||||
"is_recognized": is_recognized,
|
||||
"needs_review": needs_review,
|
||||
}
|
||||
|
||||
|
||||
def line_v2( # noqa: PLR0913 - все поля строки контракта
|
||||
row_index: int,
|
||||
code: str | None,
|
||||
name: str | None = "Строка",
|
||||
*,
|
||||
amount: str | None = "100",
|
||||
status: str = "ok",
|
||||
ref: str | None = None,
|
||||
code_confidence: float | None = 100,
|
||||
name_confidence: float | None = 100,
|
||||
code_text: str | None = None,
|
||||
parent_ref: str | None = None,
|
||||
needs_review: bool = False,
|
||||
values: Sequence[Mapping[str, Any]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Строка результата; без `values` — одно значение `amount` в колонке 2 (`amount=None` — значений нет)."""
|
||||
if values is None:
|
||||
values = [] if amount is None else [value_v2(2, amount)]
|
||||
line: dict[str, Any] = {
|
||||
"row_index": row_index,
|
||||
"ref": ref or f"row:{row_index}",
|
||||
"line_status": status,
|
||||
"line_code": code,
|
||||
"line_code_text": code_text if code_text is not None else code,
|
||||
"line_code_confidence": code_confidence,
|
||||
"line_name": name,
|
||||
"line_name_confidence": name_confidence,
|
||||
"needs_review": needs_review,
|
||||
"values": [dict(value) for value in values],
|
||||
}
|
||||
if parent_ref is not None:
|
||||
line["parent_ref"] = parent_ref
|
||||
return line
|
||||
|
||||
|
||||
def balance_lines(amount: str = "100") -> list[dict[str, Any]]:
|
||||
"""Минимальный чистый баланс (ключевые коды, имена как в словаре `fsbu4`)."""
|
||||
names = [
|
||||
("1210", "Запасы"),
|
||||
("1200", "Итого по разделу II"),
|
||||
("1600", "БАЛАНС"),
|
||||
("1300", "Итого по разделу III"),
|
||||
("1700", "БАЛАНС"),
|
||||
]
|
||||
return [line_v2(index, code, name, amount=amount) for index, (code, name) in enumerate(names)]
|
||||
|
||||
|
||||
def statement_doc_v2( # noqa: PLR0913 - все поля документа контракта
|
||||
document_type: str = "BALANCE",
|
||||
*,
|
||||
lines: Sequence[Mapping[str, Any]] | None = None,
|
||||
columns: Sequence[Mapping[str, Any]] | None = None,
|
||||
header: Mapping[str, Any] | None = None,
|
||||
status: str = "done",
|
||||
form_variant: str | None = "full",
|
||||
warnings: Sequence[Mapping[str, Any]] = (),
|
||||
checks: Mapping[str, Any] | None = None,
|
||||
locator: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Документ формы `2.0`; `checks` по умолчанию пустые (форма со строками, иначе поле не допускается)."""
|
||||
doc_lines = [dict(line) for line in (balance_lines() if lines is None else lines)]
|
||||
doc: dict[str, Any] = {
|
||||
"document_type": document_type,
|
||||
"locator": locator or f"file:test.json;table:{document_type}",
|
||||
"status": status,
|
||||
"form_variant": form_variant,
|
||||
"warnings": [dict(warning) for warning in warnings],
|
||||
"header": dict(header_v2() if header is None else header),
|
||||
"columns": [dict(column) for column in ([column_v2()] if columns is None else columns)],
|
||||
"lines": doc_lines,
|
||||
}
|
||||
if doc_lines and document_type != "DISCLOSURE":
|
||||
doc["checks"] = copy.deepcopy(dict(_NO_CHECKS if checks is None else checks))
|
||||
elif checks is not None:
|
||||
doc["checks"] = copy.deepcopy(dict(checks))
|
||||
return doc
|
||||
|
||||
|
||||
def statement_result_v2( # noqa: PLR0913 - все поля результата
|
||||
documents: Sequence[Mapping[str, Any]] | None = None,
|
||||
*,
|
||||
job_id: uuid.UUID | None = None,
|
||||
file_id: uuid.UUID | None = None,
|
||||
status: str = "done",
|
||||
statements_version: str | None = "abc1234",
|
||||
file_warnings: Sequence[Mapping[str, Any]] = (),
|
||||
overrides: Mapping[str, Any] | None = None,
|
||||
) -> RecognitionResultV2:
|
||||
"""Результат `2.0`: базовый словарь + переопределения верхнего уровня (`overrides`)."""
|
||||
payload: dict[str, Any] = {
|
||||
"schema_version": "2.0",
|
||||
"job_id": str(job_id or uuid.uuid4()),
|
||||
"file_id": str(file_id or uuid.uuid4()),
|
||||
"service": {"name": "tabular-parser", "version": "test"},
|
||||
"status": status,
|
||||
"metrics": {"warnings": [dict(warning) for warning in file_warnings]},
|
||||
"documents": [dict(doc) for doc in (documents if documents is not None else [statement_doc_v2()])],
|
||||
}
|
||||
if statements_version is not None:
|
||||
payload["service"]["statements_version"] = statements_version
|
||||
payload.update(overrides or {})
|
||||
return RecognitionResultV2.model_validate(payload)
|
||||
|
||||
|
||||
def result_builder(
|
||||
documents: Sequence[Mapping[str, Any]] | None = None, **kwargs: Any
|
||||
) -> Callable[..., RecognitionResultV2]:
|
||||
"""`result_builder` для `map_golden_snapshot`: путь игнорируется, `job_id`/`file_id` подставляются."""
|
||||
|
||||
def build(_path: Path, *, job_id: uuid.UUID, file_id: uuid.UUID) -> RecognitionResultV2:
|
||||
return statement_result_v2(documents, job_id=job_id, file_id=file_id, **kwargs)
|
||||
|
||||
return build
|
||||
249
api/tests/test_domain_statement_mapping_v2.py
Normal file
249
api/tests/test_domain_statement_mapping_v2.py
Normal file
@ -0,0 +1,249 @@
|
||||
"""Чистый маппинг формы контракта 2.0 в черновики (spec arm-core «Хранение результата формы v2.0», «Шапка формы»,
|
||||
«Предупреждения и нарушения документа»; P3-9, P3-39): `build_statement_v2`, `service_issues`, `core_info_issues`."""
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
|
||||
from src.contract import RecognitionResultV2, StatementDocumentV2
|
||||
from src.domain.statement_issues import CODE_NOT_IN_DICTIONARY, CODE_NOT_IN_EDITION, NAME_MISMATCH, core_info_issues
|
||||
from src.domain.statement_lines import load_line_catalog
|
||||
from src.domain.statement_mapping_v2 import HEADER_SOURCE_FIELDS, build_statement_v2, service_issues
|
||||
from src.models.enums import (
|
||||
DocumentIssueKind,
|
||||
IssueSeverity,
|
||||
IssueSource,
|
||||
PeriodQuarter,
|
||||
StatementFormVariant,
|
||||
StatementLineStatus,
|
||||
)
|
||||
from src.models.statements import FinancialStatementLine
|
||||
from tests.factories_v2 import column_v2, header_v2, line_v2, statement_doc_v2, statement_result_v2, value_v2
|
||||
|
||||
EXAMPLES = Path(__file__).resolve().parents[1] / "contracts" / "examples"
|
||||
HEADER_FIELDS_COUNT = 10
|
||||
|
||||
|
||||
def _doc(**kwargs: object) -> StatementDocumentV2:
|
||||
doc = statement_result_v2([statement_doc_v2(**kwargs)]).documents[0] # type: ignore[arg-type]
|
||||
assert isinstance(doc, StatementDocumentV2)
|
||||
return doc
|
||||
|
||||
|
||||
def _example(name: str) -> RecognitionResultV2:
|
||||
return RecognitionResultV2.model_validate(json.loads((EXAMPLES / name).read_text("utf-8")))
|
||||
|
||||
|
||||
def test_repeated_code_kept() -> None:
|
||||
"""Две строки `ok` 2410 (пустая и `(649)`) — две строки со своими row_index, ref и значениями."""
|
||||
lines = [
|
||||
line_v2(0, "2410", "Налог на прибыль", amount=None, ref="row:7"),
|
||||
line_v2(1, "2410", "Налог на прибыль", amount="-649", ref="row:9"),
|
||||
]
|
||||
draft = build_statement_v2(_doc(document_type="INCOME", lines=lines))
|
||||
assert [(line.row_index, line.source_ref, line.line_code) for line in draft.lines] == [
|
||||
(0, "row:7", "2410"),
|
||||
(1, "row:9", "2410"),
|
||||
]
|
||||
assert [(v.row_index, v.amount) for v in draft.values] == [(0, None), (1, Decimal("-649"))]
|
||||
assert [line.line_code_original_value for line in draft.lines] == ["2410", "2410"]
|
||||
assert not draft.warnings
|
||||
|
||||
|
||||
def test_line_without_code() -> None:
|
||||
"""`unreadable` с текстом кода: код и его исходное значение `None`, статус, row_index и текст — как в результате."""
|
||||
line = line_v2(7, None, "Финансовые вложения", status="unreadable", code_text="ИЧ,250", code_confidence=25)
|
||||
draft = build_statement_v2(_doc(lines=[line]))
|
||||
(stored,) = draft.lines
|
||||
assert stored.line_code is None and stored.line_code_original_value is None
|
||||
assert (stored.line_status, stored.row_index, stored.line_code_text) == ("unreadable", 7, "ИЧ,250")
|
||||
assert stored.line_code_confidence == Decimal(25)
|
||||
assert draft.values[0].amount == Decimal(100) and draft.values[0].line_code is None
|
||||
|
||||
|
||||
def test_detail_parent_row_index() -> None:
|
||||
"""`parent_row_index` — `row_index` строки с `ref == parent_ref`; потерянный родитель — `None` и предупреждение."""
|
||||
lines = [
|
||||
line_v2(0, "2110", "Выручка", ref="r:a"),
|
||||
line_v2(1, None, "в том числе", status="detail", ref="r:b", parent_ref="r:a", code_confidence=None),
|
||||
line_v2(2, None, "сирота", status="detail", ref="r:c", parent_ref="r:missing", code_confidence=None),
|
||||
]
|
||||
draft = build_statement_v2(_doc(document_type="INCOME", lines=lines))
|
||||
assert [line.parent_row_index for line in draft.lines] == [None, 0, None]
|
||||
assert [line.line_status for line in draft.lines] == ["ok", "detail", "detail"]
|
||||
assert any("r:missing" in warning for warning in draft.warnings)
|
||||
|
||||
|
||||
def test_value_for_every_cell_and_empty_record() -> None:
|
||||
"""Значение на каждую пару «строка × колонка»; отсутствующее — пустая запись с `needs_review = false`."""
|
||||
columns = [column_v2(2), column_v2(3, date="2024-12-31")]
|
||||
lines = [line_v2(0, "1210", "Запасы", values=[value_v2(2, "100", text="1 00", needs_review=True)])]
|
||||
draft = build_statement_v2(_doc(lines=lines, columns=columns))
|
||||
first, empty = draft.values
|
||||
assert (first.column_index, first.amount, first.source_text, first.needs_review) == (2, Decimal(100), "1 00", True)
|
||||
assert (empty.column_index, empty.amount, empty.source_text, empty.confidence) == (3, None, None, None)
|
||||
assert empty.needs_review is False and empty.is_recognized is None
|
||||
|
||||
|
||||
def test_columns_carry_period_kind_and_current_flag() -> None:
|
||||
columns = [
|
||||
column_v2(2, kind="relative:current"),
|
||||
column_v2(3, date="2024-12-31", kind="relative:prev_year_end"),
|
||||
column_v2(4, date=None, kind="none", label="Примечание"),
|
||||
]
|
||||
draft = build_statement_v2(_doc(columns=columns, lines=[line_v2(0, "1210")]))
|
||||
assert [(c.column_index, c.period_kind, c.is_current) for c in draft.columns] == [
|
||||
(2, "relative:current", True),
|
||||
(3, "relative:prev_year_end", False),
|
||||
(4, "none", False),
|
||||
]
|
||||
assert all(c.label_confidence == Decimal(100) for c in draft.columns)
|
||||
|
||||
|
||||
def test_duplicate_column_and_cells_first_kept_with_warning() -> None:
|
||||
"""P3-9: повтор `column_index` колонки и ячейки, значение вне колонок — первое остаётся, предупреждения."""
|
||||
columns = [column_v2(2), column_v2(2, date="2024-12-31")]
|
||||
values = [value_v2(2, "100"), value_v2(2, "999"), value_v2(9, "5")]
|
||||
draft = build_statement_v2(_doc(columns=columns, lines=[line_v2(0, "1210", values=values)]))
|
||||
assert len(draft.columns) == 1 and draft.columns[0].period_date is not None
|
||||
assert [v.amount for v in draft.values] == [Decimal(100)]
|
||||
assert len(draft.warnings) == 3 # noqa: PLR2004 - колонка, повтор ячейки, ячейка вне колонок
|
||||
|
||||
|
||||
def test_header_source_and_original_values() -> None:
|
||||
"""`header_source` — `{text, confidence}` для 10 полей; `*_original_value` — сырой текст / значение как пришло."""
|
||||
header = header_v2(
|
||||
organization_name="ООО «Ромашка»",
|
||||
organization_name_text="ООО «Ромашкa»",
|
||||
organization_name_confidence=67,
|
||||
inn_text="77 00000000",
|
||||
inn_confidence=80,
|
||||
report_date_text="30 сентября 2025 г.",
|
||||
units_text="тыс.руб",
|
||||
needs_review_fields=["organization_name"],
|
||||
okei="384",
|
||||
okei_text="384",
|
||||
)
|
||||
draft = build_statement_v2(_doc(header=header))
|
||||
meta = draft.header
|
||||
source = meta["header_source"]
|
||||
assert list(source) == list(HEADER_SOURCE_FIELDS) and len(source) == HEADER_FIELDS_COUNT
|
||||
assert source["organization_name"] == {"text": "ООО «Ромашкa»", "confidence": 67}
|
||||
assert source["inn"] == {"text": "77 00000000", "confidence": 80}
|
||||
assert meta["inn_original_value"] == "7700000000" and meta["inn_confidence"] == Decimal(80)
|
||||
assert meta["inn_verified"] is False
|
||||
assert meta["report_date_original_value"] == "30 сентября 2025 г."
|
||||
assert meta["units_original_value"] == "тыс.руб" and meta["units"] == "тыс. руб."
|
||||
assert meta["organization_name_original_value"] == "ООО «Ромашка»" and meta["organization_name_modified"] is False
|
||||
assert meta["needs_review_fields"] == ["organization_name"] and meta["okei"] == "384"
|
||||
assert meta["form_variant"] is StatementFormVariant.full and draft.form_variant is StatementFormVariant.full
|
||||
|
||||
|
||||
def test_period_from_report_date() -> None:
|
||||
draft = build_statement_v2(_doc())
|
||||
assert (draft.period_year, draft.period_quarter) == (2025, PeriodQuarter.Q3)
|
||||
mid = build_statement_v2(_doc(header=header_v2(report_date="2025-08-15"), columns=[column_v2(2, date=None)]))
|
||||
assert (mid.period_year, mid.period_quarter) == (2025, None)
|
||||
assert any("не конец квартала" in warning for warning in mid.warnings)
|
||||
|
||||
|
||||
def test_example_minimal() -> None:
|
||||
"""Пример docs `result-statement-minimal.json`: `unreadable` без кода, текст кода и документ `needs_check`."""
|
||||
result = _example("result-statement-minimal.json")
|
||||
doc = result.documents[0]
|
||||
assert isinstance(doc, StatementDocumentV2) and doc.status == "needs_check"
|
||||
draft = build_statement_v2(doc)
|
||||
by_row = {line.row_index: line for line in draft.lines}
|
||||
assert by_row[2].line_status == "unreadable" and by_row[2].line_code is None
|
||||
assert by_row[2].line_code_text == "ИЧ,250" and by_row[2].source_ref == "row:3"
|
||||
assert [line.source_ref for line in draft.lines][5:7] == ["row:14#1", "row:14#2"]
|
||||
assert not draft.warnings
|
||||
|
||||
|
||||
def test_example_excel() -> None:
|
||||
"""Пример docs `result-statement-excel.json`: `detail` с родителем, `DISCLOSURE` без строк."""
|
||||
result = _example("result-statement-excel.json")
|
||||
balance, income, disclosure = result.documents
|
||||
assert isinstance(income, StatementDocumentV2) and isinstance(balance, StatementDocumentV2)
|
||||
draft = build_statement_v2(income)
|
||||
detail = next(line for line in draft.lines if line.line_status == "detail")
|
||||
assert detail.parent_row_index == 0 and detail.line_code is None
|
||||
assert isinstance(disclosure, StatementDocumentV2) and not disclosure.lines
|
||||
|
||||
|
||||
def test_service_issues_warnings_and_checks() -> None:
|
||||
"""`line_refs` → `line_ids`; `column_index` → `column_id`; `relation` / `soft` с `codes`, `expected`, `actual`."""
|
||||
ids = {"r0": uuid.uuid4(), "r1": uuid.uuid4()}
|
||||
column = uuid.uuid4()
|
||||
checks = {
|
||||
"relations_failed": [
|
||||
{
|
||||
"column_index": 2,
|
||||
"relation": "1600 = 1700",
|
||||
"codes": ["1600", "1700"],
|
||||
"expected": None,
|
||||
"actual": "5.00",
|
||||
}
|
||||
],
|
||||
"soft_relations_failed": [
|
||||
{"column_index": 2, "relation": "1110 = 11101", "codes": ["1110"], "expected": "9.00", "actual": "8.00"}
|
||||
],
|
||||
}
|
||||
warnings = [
|
||||
{"code": "ROW_SPLIT", "severity": "info", "message": "разлеплена", "line_refs": ["r0", "nope", "r1"]},
|
||||
{"code": "COLUMN_DATE_UNCERTAIN", "severity": "error", "message": "дата", "column_index": 2},
|
||||
{"code": "COLUMN_SKIPPED", "severity": "info", "message": "пропущена", "column_index": 5},
|
||||
{"code": "ORGANIZATION_UNCERTAIN", "severity": "info", "message": "орг", "field": "organization_name"},
|
||||
]
|
||||
doc = _doc(lines=[line_v2(0, "1600", ref="r0")], warnings=warnings, checks=checks)
|
||||
ok_ids = {"1600": (ids["r0"],), "1700": (ids["r1"],)}
|
||||
drafts = service_issues(doc, line_ids_by_ref=ids, column_ids_by_index={2: column}, ok_line_ids_by_code=ok_ids)
|
||||
split, uncertain, skipped, org, hard, soft = drafts
|
||||
assert all(d.source is IssueSource.service for d in drafts)
|
||||
assert split.line_ids == (ids["r0"], ids["r1"]) and split.kind is DocumentIssueKind.warning
|
||||
assert (uncertain.column_id, uncertain.column_index, uncertain.severity) == (column, None, IssueSeverity.error)
|
||||
assert (skipped.column_id, skipped.column_index) == (None, 5)
|
||||
assert org.field == "organization_name"
|
||||
assert (hard.kind, hard.code, hard.severity) == (DocumentIssueKind.relation, "RELATION_FAILED", IssueSeverity.error)
|
||||
assert hard.expected is None and hard.actual == Decimal("5.00") and hard.codes == ("1600", "1700")
|
||||
assert hard.line_ids == (ids["r0"], ids["r1"]) and hard.column_id == column and hard.column_index == 2 # noqa: PLR2004
|
||||
assert (soft.kind, soft.code, soft.severity) == (DocumentIssueKind.soft, "RELATION_SOFT", IssueSeverity.info)
|
||||
assert "пусто ≠ 5.00" in hard.message
|
||||
|
||||
|
||||
def _db_line(code: str, name: str, confidence: int | None, row: int) -> FinancialStatementLine:
|
||||
return FinancialStatementLine(
|
||||
id=uuid.uuid4(),
|
||||
meta_id=uuid.uuid4(),
|
||||
line_code=code,
|
||||
line_name=name,
|
||||
line_code_confidence=None if confidence is None else Decimal(confidence),
|
||||
line_code_modified=False,
|
||||
is_confirmed=False,
|
||||
row_index=row,
|
||||
line_status=StatementLineStatus.ok,
|
||||
)
|
||||
|
||||
|
||||
def test_core_info_issues_rules() -> None:
|
||||
"""Четырёхзначные коды вне словаря/перечня — по записи на код; пятизначные не проверяются (R-1); без редакции
|
||||
`CODE_NOT_IN_*` нет, `NAME_MISMATCH` остаётся; без перечня нет `CODE_NOT_IN_EDITION`."""
|
||||
catalog = load_line_catalog()
|
||||
lines = [
|
||||
_db_line("1999", "Чужая строка", 100, 0),
|
||||
_db_line("1999", "Чужая строка ещё", 100, 1),
|
||||
_db_line("11101", "Расшифровка", 100, 2),
|
||||
_db_line("1210", "Нематериальные активы", 50, 3),
|
||||
]
|
||||
info = core_info_issues(lines, form="BALANCE", edition="fsbu4", order=("1110", "1210"), catalog=catalog)
|
||||
by_code = {d.code: d for d in info.drafts}
|
||||
assert by_code[CODE_NOT_IN_DICTIONARY].line_ids == (lines[0].id, lines[1].id)
|
||||
assert by_code[CODE_NOT_IN_EDITION].line_ids == (lines[0].id, lines[1].id)
|
||||
assert [d.code for d in info.drafts].count(CODE_NOT_IN_DICTIONARY) == 1 # пятизначный 11101 не проверяется
|
||||
assert by_code[NAME_MISMATCH].line_ids == (lines[3].id,) and info.mismatched == [lines[3].id]
|
||||
|
||||
no_edition = core_info_issues(lines, form="BALANCE", edition=None, order=None, catalog=catalog)
|
||||
assert {d.code for d in no_edition.drafts} == {NAME_MISMATCH}
|
||||
no_order = core_info_issues(lines, form="BALANCE", edition="fsbu4", order=None, catalog=catalog)
|
||||
assert {d.code for d in no_order.drafts} == {CODE_NOT_IN_DICTIONARY, NAME_MISMATCH}
|
||||
432
api/tests/test_mapping_statements_v2.py
Normal file
432
api/tests/test_mapping_statements_v2.py
Normal file
@ -0,0 +1,432 @@
|
||||
"""Маппер результата 2.0 для форм ББ/ОФР/раскрытий (spec arm-core «Хранение результата формы v2.0», «Условие
|
||||
RECOGNIZED при приёме 2.0», «Предупреждения и нарушения документа — document_issue»; Р-10, Р-16, Р-18, Р-19, P3-6, P3-9,
|
||||
P3-15, P3-39): реальный пакет `arm_statements`, `map_file` на БД теста."""
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from collections.abc import Sequence
|
||||
from decimal import Decimal
|
||||
from http import HTTPStatus
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from src.common.clock import utc_now
|
||||
from src.contract import RecognitionResultV2
|
||||
from src.models.directories import LegalEntity, RegionalOffice
|
||||
from src.models.enums import DocumentIssueKind, DocumentStatus, DocumentType, IssueSource, PeriodQuarter
|
||||
from src.models.files import Document, RecognitionJob, StoredFile
|
||||
from src.models.statements import (
|
||||
DocumentIssue,
|
||||
FinancialStatementColumn,
|
||||
FinancialStatementLine,
|
||||
FinancialStatementMeta,
|
||||
FinancialStatementValue,
|
||||
)
|
||||
from src.services.auto_confirm import auto_confirm
|
||||
from src.services.statements_engine import DocumentType as DocumentType_
|
||||
from src.services.statements_engine import Edition, FormVariant, default_engine
|
||||
from tests.conftest import AuthClientFactory
|
||||
from tests.factories import make_company_group, make_legal_entity
|
||||
from tests.factories_v2 import (
|
||||
OWNER_INN,
|
||||
balance_lines,
|
||||
column_v2,
|
||||
header_v2,
|
||||
line_v2,
|
||||
result_builder,
|
||||
statement_doc_v2,
|
||||
value_v2,
|
||||
)
|
||||
from tests.fixtures_helpers import map_golden_snapshot
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
EXAMPLES = Path(__file__).resolve().parents[1] / "contracts" / "examples"
|
||||
KA_RF = "ka_rf@armka.local"
|
||||
ROSTOV = "Ростовский региональный филиал"
|
||||
V2_PATH = Path("result-v2.json")
|
||||
HEADER_FIELDS_COUNT = 10
|
||||
ERROR_JSON_FAILED = {"code": "JSON_FAILED", "severity": "error", "message": "потерян json"}
|
||||
ERROR_TABLE_PARSE = {"code": "TABLE_PARSE_ERROR", "severity": "error", "message": "таблица не разобрана"}
|
||||
INCOME_LINES = [line_v2(0, "2110", "Выручка"), line_v2(1, "2100", "Валовая прибыль (убыток)")]
|
||||
|
||||
|
||||
async def _entity(session: AsyncSession, *, inn: str = OWNER_INN, rostov: bool = False) -> LegalEntity:
|
||||
if not rostov:
|
||||
return await make_legal_entity(session, name="ООО Владелец", inn=inn)
|
||||
office = (await session.execute(select(RegionalOffice).where(RegionalOffice.name == ROSTOV))).scalar_one()
|
||||
group = await make_company_group(session, name="ГК Владелец", office=office)
|
||||
return await make_legal_entity(session, name="ООО Владелец", inn=inn, group=group)
|
||||
|
||||
|
||||
async def _map(
|
||||
session: AsyncSession, entity: LegalEntity, documents: Sequence[dict[str, Any]], **kwargs: Any
|
||||
) -> list[Document]:
|
||||
"""Документы файла по `file_ordinal`."""
|
||||
await map_golden_snapshot(
|
||||
session, entity=entity, path=V2_PATH, uploaded_by=None, result_builder=result_builder(documents, **kwargs)
|
||||
)
|
||||
stored = (await session.execute(select(StoredFile).where(StoredFile.legal_entity_id == entity.id))).scalars().all()
|
||||
rows = await session.execute(
|
||||
select(Document)
|
||||
.where(Document.stored_file_id.in_([file.id for file in stored]))
|
||||
.order_by(Document.file_ordinal)
|
||||
)
|
||||
return list(rows.scalars())
|
||||
|
||||
|
||||
async def _issues(session: AsyncSession, document: Document) -> list[DocumentIssue]:
|
||||
stmt = select(DocumentIssue).where(DocumentIssue.document_id == document.id).order_by(DocumentIssue.created_at)
|
||||
return list((await session.execute(stmt)).scalars())
|
||||
|
||||
|
||||
async def _meta(session: AsyncSession, document: Document) -> FinancialStatementMeta:
|
||||
stmt = select(FinancialStatementMeta).where(FinancialStatementMeta.document_id == document.id)
|
||||
return (await session.execute(stmt)).scalar_one()
|
||||
|
||||
|
||||
async def _lines(session: AsyncSession, document: Document) -> list[FinancialStatementLine]:
|
||||
meta = await _meta(session, document)
|
||||
stmt = (
|
||||
select(FinancialStatementLine)
|
||||
.where(FinancialStatementLine.meta_id == meta.id)
|
||||
.order_by(FinancialStatementLine.row_index)
|
||||
)
|
||||
return list((await session.execute(stmt)).scalars())
|
||||
|
||||
|
||||
async def _confirm_clean(session: AsyncSession, documents: Sequence[Document]) -> None:
|
||||
file = await session.get(StoredFile, documents[0].stored_file_id)
|
||||
assert file is not None
|
||||
await auto_confirm(session, file=file, documents=documents, now=utc_now())
|
||||
|
||||
|
||||
def _example_builder(name: str): # noqa: ANN202 - фабрика `result_builder` golden-хелпера
|
||||
def build(_path: Path, *, job_id: uuid.UUID, file_id: uuid.UUID) -> RecognitionResultV2:
|
||||
payload = json.loads((EXAMPLES / name).read_text("utf-8"))
|
||||
return RecognitionResultV2.model_validate({**payload, "job_id": str(job_id), "file_id": str(file_id)})
|
||||
|
||||
return build
|
||||
|
||||
|
||||
async def test_example_minimal(seeded: AsyncSession) -> None:
|
||||
"""Пример docs `result-statement-minimal.json`: `unreadable` без кода, `ROW_SPLIT` и `ORGANIZATION_UNCERTAIN`."""
|
||||
entity = await _entity(seeded)
|
||||
await map_golden_snapshot(
|
||||
seeded,
|
||||
entity=entity,
|
||||
path=V2_PATH,
|
||||
uploaded_by=None,
|
||||
result_builder=_example_builder("result-statement-minimal.json"),
|
||||
)
|
||||
(balance,) = (await seeded.execute(select(Document).where(Document.legal_entity_id == entity.id))).scalars()
|
||||
assert balance.status is DocumentStatus.NEEDS_CHECK and balance.file_ordinal == 0
|
||||
lines = await _lines(seeded, balance)
|
||||
unreadable = next(line for line in lines if line.line_status.value == "unreadable")
|
||||
assert unreadable.line_code is None and unreadable.line_code_original_value is None
|
||||
assert unreadable.line_code_text == "ИЧ,250" and unreadable.row_index == 2 # noqa: PLR2004
|
||||
issues = {issue.code: issue for issue in await _issues(seeded, balance)}
|
||||
split = issues["ROW_SPLIT"]
|
||||
split_refs = {line.id for line in lines if line.source_ref in ("row:14#1", "row:14#2")}
|
||||
assert set(split.line_ids or ()) == {str(line_id) for line_id in split_refs} and len(split_refs) == 2 # noqa: PLR2004
|
||||
assert issues["ORGANIZATION_UNCERTAIN"].field == "organization_name"
|
||||
assert all(issue.source is IssueSource.service for issue in (split, issues["ORGANIZATION_UNCERTAIN"]))
|
||||
|
||||
|
||||
async def test_example_excel(seeded: AsyncSession) -> None:
|
||||
"""Пример docs `result-statement-excel.json`: `BALANCE` с `RELATION_SOFT`, `INCOME` с `detail` и пустой целью,
|
||||
один `DISCLOSURE` без строк."""
|
||||
entity = await _entity(seeded)
|
||||
await map_golden_snapshot(
|
||||
seeded,
|
||||
entity=entity,
|
||||
path=V2_PATH,
|
||||
uploaded_by=None,
|
||||
result_builder=_example_builder("result-statement-excel.json"),
|
||||
)
|
||||
documents = list(
|
||||
(await seeded.execute(select(Document).where(Document.legal_entity_id == entity.id).order_by("file_ordinal")))
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
balance, income, disclosure = documents
|
||||
assert [d.document_type for d in documents] == [DocumentType.BALANCE, DocumentType.INCOME, DocumentType.DISCLOSURE]
|
||||
soft = next(issue for issue in await _issues(seeded, balance) if issue.kind is DocumentIssueKind.soft)
|
||||
assert (soft.code, soft.relation, soft.codes) == ("RELATION_SOFT", "1110 = 11101", ["1110", "11101"])
|
||||
assert soft.expected == Decimal("905786.99") and soft.actual == Decimal("900000.00") and soft.column_id is not None
|
||||
hard = next(issue for issue in await _issues(seeded, income) if issue.kind is DocumentIssueKind.relation)
|
||||
assert hard.expected is None and hard.actual == Decimal("400.00")
|
||||
lines = await _lines(seeded, income)
|
||||
detail = next(line for line in lines if line.line_status.value == "detail")
|
||||
assert detail.line_code is None and detail.parent_row_index == lines[0].row_index == 0
|
||||
assert disclosure.status is DocumentStatus.RECOGNIZED and disclosure.file_ordinal == 2 # noqa: PLR2004
|
||||
assert not await _lines_count(seeded, disclosure)
|
||||
|
||||
|
||||
async def _lines_count(session: AsyncSession, document: Document) -> int:
|
||||
meta = (
|
||||
await session.execute(
|
||||
select(FinancialStatementMeta.id).where(FinancialStatementMeta.document_id == document.id)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
return 0 if meta is None else len(await _lines(session, document))
|
||||
|
||||
|
||||
async def test_repeated_code_kept(seeded: AsyncSession) -> None:
|
||||
"""Две строки `ok` 2410 (пустая и `(649)`) — две строки со своими `row_index`, `source_ref` и значениями."""
|
||||
lines = [
|
||||
line_v2(0, "2410", "Налог на прибыль", amount=None, ref="row:7"),
|
||||
line_v2(1, "2410", "Налог на прибыль", amount="-649", ref="row:9"),
|
||||
]
|
||||
entity = await _entity(seeded)
|
||||
(income,) = await _map(seeded, entity, [statement_doc_v2("INCOME", lines=lines)])
|
||||
stored = await _lines(seeded, income)
|
||||
assert [(line.row_index, line.source_ref, line.line_code) for line in stored] == [
|
||||
(0, "row:7", "2410"),
|
||||
(1, "row:9", "2410"),
|
||||
]
|
||||
amounts = {
|
||||
value.line_id: value.amount
|
||||
for value in (await seeded.execute(select(FinancialStatementValue))).scalars()
|
||||
if value.line_id in {line.id for line in stored}
|
||||
}
|
||||
assert amounts[stored[0].id] is None and amounts[stored[1].id] == Decimal(-649)
|
||||
|
||||
|
||||
async def test_line_without_code(seeded: AsyncSession) -> None:
|
||||
line = line_v2(5, None, "Финансовые вложения", status="unreadable", code_text="ИЧ,250", code_confidence=25)
|
||||
(balance,) = await _map(seeded, await _entity(seeded), [statement_doc_v2(lines=[line])])
|
||||
(stored,) = await _lines(seeded, balance)
|
||||
assert stored.line_code is None and stored.line_code_original_value is None
|
||||
assert stored.line_status.value == "unreadable" and stored.row_index == 5 # noqa: PLR2004
|
||||
assert stored.line_code_text == "ИЧ,250" and stored.is_confirmed is False
|
||||
|
||||
|
||||
async def test_two_balances_same_period(seeded: AsyncSession) -> None:
|
||||
"""Review Focus 1: два `BALANCE` одного периода — разные `file_ordinal`, оба `NEEDS_CHECK`, не `CONFIRMED`."""
|
||||
entity = await _entity(seeded)
|
||||
first, second = await _map(seeded, entity, [statement_doc_v2(), statement_doc_v2()])
|
||||
assert (first.file_ordinal, second.file_ordinal) == (0, 1)
|
||||
assert first.status is second.status is DocumentStatus.NEEDS_CHECK
|
||||
await _confirm_clean(seeded, [first, second])
|
||||
assert first.status is second.status is DocumentStatus.NEEDS_CHECK
|
||||
warning = [i.message for i in await _issues(seeded, first) if i.code == "MAPPING_WARNING"]
|
||||
assert any("другая форма BALANCE за тот же период" in message for message in warning)
|
||||
|
||||
|
||||
async def test_two_balances_other_periods(seeded: AsyncSession) -> None:
|
||||
"""Баланс на 30.09.2025 и на 31.12.2024 — правило Р-18 их не касается, каждый по своим условиям."""
|
||||
entity = await _entity(seeded)
|
||||
docs = [
|
||||
statement_doc_v2(),
|
||||
statement_doc_v2(header=header_v2(report_date="2024-12-31"), columns=[column_v2(2, date="2024-12-31")]),
|
||||
]
|
||||
first, second = await _map(seeded, entity, docs)
|
||||
assert first.status is second.status is DocumentStatus.RECOGNIZED
|
||||
assert (first.period_quarter, second.period_quarter) == (PeriodQuarter.Q3, PeriodQuarter.Q4)
|
||||
|
||||
|
||||
async def test_neighbor_needs_check(seeded: AsyncSession) -> None:
|
||||
"""Р-19: `needs_check` соседнего документа не мешает: ББ `done` автоподтверждён, ОФР `needs_check` — нет."""
|
||||
entity = await _entity(seeded)
|
||||
docs = [statement_doc_v2("BALANCE"), statement_doc_v2("INCOME", lines=INCOME_LINES, status="needs_check")]
|
||||
balance, income = await _map(seeded, entity, docs)
|
||||
assert (balance.status, income.status) == (DocumentStatus.RECOGNIZED, DocumentStatus.NEEDS_CHECK)
|
||||
await _confirm_clean(seeded, [balance, income])
|
||||
assert (balance.status, income.status) == (DocumentStatus.CONFIRMED, DocumentStatus.NEEDS_CHECK)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("warning", [ERROR_JSON_FAILED, ERROR_TABLE_PARSE])
|
||||
async def test_file_error_warning_blocks_all(seeded: AsyncSession, warning: dict[str, str]) -> None:
|
||||
"""P3-15: предупреждение файла уровня `error` в `metrics.warnings` результата — все формы файла `NEEDS_CHECK`."""
|
||||
entity = await _entity(seeded)
|
||||
docs = [statement_doc_v2("BALANCE"), statement_doc_v2("INCOME", lines=INCOME_LINES)]
|
||||
balance, income = await _map(seeded, entity, docs, file_warnings=[warning])
|
||||
assert balance.status is income.status is DocumentStatus.NEEDS_CHECK
|
||||
|
||||
|
||||
async def test_info_file_warning_does_not_block(seeded: AsyncSession) -> None:
|
||||
info = {"code": "SOMETHING", "severity": "info", "message": "к сведению"}
|
||||
(balance,) = await _map(seeded, await _entity(seeded), [statement_doc_v2()], file_warnings=[info])
|
||||
assert balance.status is DocumentStatus.RECOGNIZED
|
||||
|
||||
|
||||
NAME_CASES = [
|
||||
("1120", "Нематериальные активы", 50, True), # подмена кода на соседа
|
||||
("1220", "НДС, начисленный по отгрузке", 50, False), # законный повтор с другим именем (fsbu4)
|
||||
("1200", "Итого по разделу I", 50, True), # подмена итога раздела
|
||||
("1500", "Итого по разделу V1", 56, False), # хвост OCR после номера раздела
|
||||
("1120", "Нематериальные активы", 100, False), # уверенный код не проверяется
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("code", "name", "confidence", "mismatch"), NAME_CASES)
|
||||
async def test_name_mismatch(seeded: AsyncSession, code: str, name: str, confidence: int, mismatch: bool) -> None:
|
||||
lines = [*balance_lines(), line_v2(10, code, name, code_confidence=confidence)]
|
||||
(balance,) = await _map(seeded, await _entity(seeded), [statement_doc_v2(lines=lines)])
|
||||
flagged = [i for i in await _issues(seeded, balance) if i.code == "NAME_MISMATCH"]
|
||||
stored = await _lines(seeded, balance)
|
||||
if mismatch:
|
||||
assert [i.line_ids for i in flagged] == [[str(stored[-1].id)]]
|
||||
assert flagged[0].source is IssueSource.core and balance.status is DocumentStatus.NEEDS_CHECK
|
||||
else:
|
||||
assert not flagged
|
||||
|
||||
|
||||
async def test_done_with_unreadable_without_values(seeded: AsyncSession) -> None:
|
||||
"""Сервис прислал `done`, но есть `unreadable` без значений — функция статуса пакета не согласна."""
|
||||
lines = [*balance_lines(), line_v2(9, None, "Строка", status="unreadable", amount=None, code_text="???")]
|
||||
(balance,) = await _map(seeded, await _entity(seeded), [statement_doc_v2(lines=lines, status="done")])
|
||||
assert balance.status is DocumentStatus.NEEDS_CHECK
|
||||
inconsistent = [i for i in await _issues(seeded, balance) if i.code == "RESULT_STATUS_INCONSISTENT"]
|
||||
assert len(inconsistent) == 1 and inconsistent[0].source is IssueSource.core
|
||||
assert inconsistent[0].line_ids is None and inconsistent[0].kind is DocumentIssueKind.warning
|
||||
await _confirm_clean(seeded, [balance])
|
||||
assert balance.status is DocumentStatus.NEEDS_CHECK
|
||||
|
||||
|
||||
async def test_done_with_unreadable_with_amount(seeded: AsyncSession) -> None:
|
||||
lines = [*balance_lines(), line_v2(9, None, "Строка", status="unreadable", amount="50", code_text="???")]
|
||||
(balance,) = await _map(seeded, await _entity(seeded), [statement_doc_v2(lines=lines, status="done")])
|
||||
assert balance.status is DocumentStatus.NEEDS_CHECK
|
||||
assert [i.code for i in await _issues(seeded, balance) if i.code == "RESULT_STATUS_INCONSISTENT"]
|
||||
await _confirm_clean(seeded, [balance])
|
||||
assert balance.status is DocumentStatus.NEEDS_CHECK
|
||||
|
||||
|
||||
async def test_needs_check_without_inconsistency_record(seeded: AsyncSession) -> None:
|
||||
"""`needs_check` сервиса: документ `NEEDS_CHECK`, `RESULT_STATUS_INCONSISTENT` не пишется."""
|
||||
(balance,) = await _map(seeded, await _entity(seeded), [statement_doc_v2(status="needs_check")])
|
||||
assert balance.status is DocumentStatus.NEEDS_CHECK
|
||||
assert not [i for i in await _issues(seeded, balance) if i.code == "RESULT_STATUS_INCONSISTENT"]
|
||||
|
||||
|
||||
async def test_report_date_not_quarter_end(seeded: AsyncSession, auth_client: AuthClientFactory) -> None:
|
||||
"""Дата 15.08.2025: квартал не определён, `NEEDS_CHECK`, подтверждение — 409 `PERIOD_REQUIRED`."""
|
||||
entity = await _entity(seeded, rostov=True)
|
||||
docs = [statement_doc_v2(header=header_v2(report_date="2025-08-15"), columns=[column_v2(2, date="2025-08-15")])]
|
||||
(balance,) = await _map(seeded, entity, docs)
|
||||
assert balance.status is DocumentStatus.NEEDS_CHECK
|
||||
assert (balance.period_year, balance.period_quarter) == (2025, None)
|
||||
await _confirm_clean(seeded, [balance])
|
||||
assert balance.status is DocumentStatus.NEEDS_CHECK
|
||||
client = await auth_client(KA_RF)
|
||||
blocked = await client.post(f"/documents/{balance.id}/confirm")
|
||||
assert blocked.status_code == HTTPStatus.CONFLICT and blocked.json()["detail"]["code"] == "PERIOD_REQUIRED"
|
||||
|
||||
|
||||
async def test_66n_blank_in_2025(seeded: AsyncSession) -> None:
|
||||
"""Бланк 66н, сданный в 2025 году: редакция документа — `66n` по подпоследовательности кодов, `CODE_NOT_IN_EDITION`
|
||||
у 1120 нет (в перечне `fsbu4` его нет)."""
|
||||
names = {"1110": "Нематериальные активы", "1120": "Результаты исследований и разработок"}
|
||||
codes = ["1110", "1120", "1130", "1140", "1150", "1160", "1170", "1180"]
|
||||
lines = [line_v2(index, code, names.get(code, "Строка")) for index, code in enumerate(codes)]
|
||||
(balance,) = await _map(seeded, await _entity(seeded), [statement_doc_v2(lines=lines)])
|
||||
stored = {line.line_code: line.id for line in await _lines(seeded, balance)}
|
||||
pairs = [(code, True) for code in codes]
|
||||
edition = default_engine().edition(DocumentType_.BALANCE, pairs, balance.reporting_date, FormVariant.FULL)
|
||||
assert edition is Edition.RAS_66N
|
||||
info = [i for i in await _issues(seeded, balance) if i.code == "CODE_NOT_IN_EDITION"]
|
||||
assert str(stored["1120"]) not in {line_id for issue in info for line_id in issue.line_ids or ()}
|
||||
|
||||
|
||||
async def test_code_not_in_edition_for_fsbu4_form(seeded: AsyncSession) -> None:
|
||||
"""Контроль к 66н: код вне перечня выбранной редакции даёт одну запись на код документа (R-1: только 4 цифры)."""
|
||||
lines = [*balance_lines(), line_v2(10, "9999", "Чужая"), line_v2(11, "99999", "Подстрока Excel")]
|
||||
(balance,) = await _map(seeded, await _entity(seeded), [statement_doc_v2(lines=lines)])
|
||||
stored = {line.line_code: line.id for line in await _lines(seeded, balance)}
|
||||
core = [issue for issue in await _issues(seeded, balance) if issue.source is IssueSource.core]
|
||||
by_code = {(issue.code, tuple(issue.line_ids or ())) for issue in core}
|
||||
assert ("CODE_NOT_IN_DICTIONARY", (str(stored["9999"]),)) in by_code
|
||||
assert ("CODE_NOT_IN_EDITION", (str(stored["9999"]),)) in by_code
|
||||
assert not any(str(stored["99999"]) in line_ids for _, line_ids in by_code)
|
||||
|
||||
|
||||
async def test_column_skipped_issue(seeded: AsyncSession) -> None:
|
||||
"""`COLUMN_SKIPPED` с `column_index = 5`, колонки 5 нет — `column_index == 5`, `column_id` пуст; для имеющейся
|
||||
колонки — наоборот."""
|
||||
warnings = [
|
||||
{"code": "COLUMN_SKIPPED", "severity": "info", "message": "колонка пропущена", "column_index": 5},
|
||||
{"code": "COLUMN_DATE_UNCERTAIN", "severity": "info", "message": "дата", "column_index": 2},
|
||||
]
|
||||
(balance,) = await _map(seeded, await _entity(seeded), [statement_doc_v2(warnings=warnings)])
|
||||
issues = {i.code: i for i in await _issues(seeded, balance)}
|
||||
assert (issues["COLUMN_SKIPPED"].column_index, issues["COLUMN_SKIPPED"].column_id) == (5, None)
|
||||
column = (await seeded.execute(select(FinancialStatementColumn))).scalars().one()
|
||||
uncertain = issues["COLUMN_DATE_UNCERTAIN"]
|
||||
assert (uncertain.column_id, uncertain.column_index) == (column.id, None)
|
||||
|
||||
|
||||
async def test_header_source_hint(seeded: AsyncSession, auth_client: AuthClientFactory) -> None:
|
||||
"""`organization_name_text` / `_confidence` — в `header_source` и в `GET …/statement` (`header.source`)."""
|
||||
header = header_v2(organization_name_text="ООО «Ромашкa»", organization_name_confidence=67)
|
||||
entity = await _entity(seeded, rostov=True)
|
||||
(balance,) = await _map(seeded, entity, [statement_doc_v2(header=header)])
|
||||
meta = await _meta(seeded, balance)
|
||||
assert len(meta.header_source) == HEADER_FIELDS_COUNT
|
||||
assert meta.header_source["organization_name"] == {"text": "ООО «Ромашкa»", "confidence": 67}
|
||||
assert meta.inn_original_value == OWNER_INN and meta.organization_name_original_value == "ООО Ромашка"
|
||||
assert meta.form_variant is not None and meta.form_variant.value == "full"
|
||||
body = (await (await auth_client(KA_RF)).get(f"/documents/{balance.id}/statement")).json()
|
||||
assert body["header"]["source"]["organization_name"] == {"text": "ООО «Ромашкa»", "confidence": "67.00"}
|
||||
|
||||
|
||||
async def test_statements_version_and_ordinals(seeded: AsyncSession) -> None:
|
||||
"""`recognition_job.statements_version` из `service.statements_version`; `BALANCE`, `INCOME` — ordinal 0, 1;
|
||||
отличие от версии копии ядра — `STATEMENTS_VERSION_MISMATCH` (`info`, `core`)."""
|
||||
entity = await _entity(seeded)
|
||||
docs = [statement_doc_v2("BALANCE"), statement_doc_v2("INCOME", lines=INCOME_LINES)]
|
||||
balance, income = await _map(seeded, entity, docs, statements_version="abc1234")
|
||||
assert (balance.file_ordinal, income.file_ordinal) == (0, 1)
|
||||
job = (
|
||||
await seeded.execute(select(RecognitionJob).where(RecognitionJob.file_id == balance.stored_file_id))
|
||||
).scalar_one()
|
||||
assert job.statements_version == "abc1234"
|
||||
mismatch = [i for i in await _issues(seeded, balance) if i.code == "STATEMENTS_VERSION_MISMATCH"]
|
||||
assert len(mismatch) == 1 and mismatch[0].source is IssueSource.core
|
||||
|
||||
|
||||
async def test_disclosure_one_document_with_all_warnings(seeded: AsyncSession) -> None:
|
||||
"""P3-6: несколько `DISCLOSURE` — один документ (ordinal первого, `locator` через запятую); предупреждения всех — в
|
||||
его `document_issue`; он `RECOGNIZED`."""
|
||||
first = statement_doc_v2("DISCLOSURE", lines=[], columns=[], form_variant=None, locator="loc:1")
|
||||
first["warnings"] = [{"code": "FIRST_W", "severity": "info", "message": "первое"}]
|
||||
second = statement_doc_v2("DISCLOSURE", lines=[], columns=[], form_variant=None, locator="loc:2")
|
||||
second["warnings"] = [{"code": "SECOND_W", "severity": "info", "message": "второе"}]
|
||||
entity = await _entity(seeded)
|
||||
balance, disclosure = await _map(seeded, entity, [statement_doc_v2(), first, second])
|
||||
assert balance.file_ordinal == 0 and disclosure.file_ordinal == 1
|
||||
assert disclosure.document_type is DocumentType.DISCLOSURE and disclosure.locator == "loc:1,loc:2"
|
||||
assert disclosure.status is DocumentStatus.RECOGNIZED
|
||||
assert {i.code for i in await _issues(seeded, disclosure)} == {"FIRST_W", "SECOND_W"}
|
||||
documents = (await seeded.execute(select(Document).where(Document.stored_file_id == balance.stored_file_id))).all()
|
||||
assert len(documents) == 2 # noqa: PLR2004
|
||||
|
||||
|
||||
async def test_value_cells_and_empty_records(seeded: AsyncSession) -> None:
|
||||
"""Значение на каждую пару «строка × колонка»; сырой текст, `needs_review` и пустая запись без `needs_review`."""
|
||||
columns = [column_v2(2), column_v2(3, date="2024-12-31")]
|
||||
lines = [
|
||||
line_v2(0, "1210", "Запасы", values=[value_v2(2, "428311", text="42831 1", is_recognized=False)]),
|
||||
line_v2(1, "1600", "БАЛАНС", values=[value_v2(2, "5", needs_review=True), value_v2(3, "4")]),
|
||||
]
|
||||
(balance,) = await _map(seeded, await _entity(seeded), [statement_doc_v2(lines=lines, columns=columns)])
|
||||
stored = await _lines(seeded, balance)
|
||||
rows = (
|
||||
await seeded.execute(
|
||||
select(FinancialStatementValue, FinancialStatementColumn.column_index)
|
||||
.join(FinancialStatementColumn, FinancialStatementColumn.id == FinancialStatementValue.column_id)
|
||||
.where(FinancialStatementValue.line_id.in_([line.id for line in stored]))
|
||||
)
|
||||
).all()
|
||||
cells = {(value.line_id, index): value for value, index in rows}
|
||||
assert len(cells) == 4 # noqa: PLR2004
|
||||
typed = cells[(stored[0].id, 2)]
|
||||
assert (typed.amount, typed.source_text, typed.is_recognized) == (Decimal(428311), "42831 1", False)
|
||||
empty = cells[(stored[0].id, 3)]
|
||||
assert (empty.amount, empty.source_text, empty.needs_review) == (None, None, False)
|
||||
assert cells[(stored[1].id, 2)].needs_review is True
|
||||
@ -47,3 +47,10 @@ Goals и Non-goals — см. `proposal.md`.
|
||||
| 2026-10-18 | **P3-36 (B3).** `checks_on_intake_1x` живёт в `statement_recompute.py`; движок передаётся в `_insert_statements` необязательным параметром (по умолчанию `default_engine()`), `map_file` сигнатуру не меняет. Приём `1.x` пишет `meta.form_variant` и записи `relation` / `soft` (`core`), но не `CODE_NOT_IN_*` / `NAME_MISMATCH` (P3-7) и не трогает `needs_review` (A9); условие D-6 «проверки сошлись» — `not result.checks.relations_failed` (нарушение с `expected = NULL` — жёсткое) | задание B3, спека «Пересчёт при приёме»; расхождение с прежним `run_checks` (его `failed`) возможно только на формах с пустой целью, где пакет строже | форма `1.x` с пустой целью получит `NEEDS_CHECK` вместо `RECOGNIZED` |
|
||||
| 2026-10-18 | **P3-37 (fix1, T2.1).** Аудит `lines[]` довносит смену `parent_row_index` после пересчёта: у правленных строк значение заменяется итоговым, у остальных появляется запись `line:<id>` с одним ключом `parent_row_index` (`{"parent_row_index": старый}` → `{…: новый}`) | родителей пишет `recompute_form` уже после снимка строк правки; полный снимок для не правленных строк не нужен | в событии `RECOGNIZED_DATA_EDIT` у не правленных строк неполный набор ключей |
|
||||
| 2026-10-18 | **P3-38 (fix1, T1.1/T1.2/T2.4/T2.6/T2.8).** Паритет редакции приёма и пересчёта для строки с `line_code_confidence < 100` и `code_fixed` подтверждён тестом (расхождения нет, код не менялся); `edition_codes` пропускает строку `ok` без кода; `column_index` нарушения — общим помощником `issue_column_index` (запись, иначе колонка по `column_id`) в ответе и в `detail` отказа; `_unconfirmed_needs_review` и присвоения `needs_review` в `_apply_item` удалены (флаг пишет только пересчёт, он всегда идёт после правки строк); неизвестный `relative:*` в `_resolve_relative_columns` пропускается без ошибки | замечания ревью B1+B2 | колонка с неизвестным `relative:*` не следует за датой отчёта |
|
||||
| 2026-10-18 | **P3-39 (B4).** Проверки ядра `info` (`CODE_NOT_IN_DICTIONARY`, `CODE_NOT_IN_EDITION`, `NAME_MISMATCH`, `STATEMENTS_VERSION_MISMATCH`, `RESULT_STATUS_INCONSISTENT`, `code_issues`, `name_issues`, `version_issue`, `name_check_lines`) вынесены из `services/statement_recompute.py` в `domain/statement_issues.py`; единая функция `core_info_issues(lines, *, form, edition: str \| None, order: Collection[str] \| None, catalog) -> CoreInfo(drafts, mismatched)` используется приёмом `2.0` и `recompute_form` (поведение пересчёта не изменилось, P3-34). Домен пакет не импортирует: редакцию (`engine.edition`) и перечень кодов (`engine.code_order`) считает сервис; `statement_recompute` реэкспортирует прежние имена. Вместо сигнатуры плана `core_info_issues(*, lines, edition, form_variant, catalog, engine)` | домен не должен знать `StatementsEngine`/`arm_statements` (D-1); одно правило для приёма и пересчёта | нет; импортёры прежних имён из `statement_recompute` работают |
|
||||
| 2026-10-18 | **P3-40 (B4).** Приём форм `2.0` — отдельный сервис `services/statement_intake_v2.py` (`map_statement_docs_v2`, `insert_forms_v2`); `map_file` выбирает маппер по типу результата через `_plan_forms` / `_insert_forms` (иначе функция превышала лимит ruff по операторам) и получил необязательный параметр `engine`. `_load_result` и тип в `process_once` остаются под `1.x` до B5 | размер `result_mapping.py` и лимиты линтера; граница B4/B5 из брифов | нет |
|
||||
| 2026-10-18 | **P3-41 (B4).** `DISCLOSURE` в `2.0` — по-прежнему один документ на файл со статусом `RECOGNIZED` (D-3), `file_ordinal` первого; предупреждения всех объединённых документов результата пишутся в его `document_issue` источником `service`. Статус сервиса и предупреждения файла уровня `error` на него не влияют | спека условие `RECOGNIZED` относится к формам ББ/ОФР; у `DISCLOSURE` нет данных | `DISCLOSURE` с `needs_check` сервиса автоподтверждается (данных нет — подтверждать нечего) |
|
||||
| 2026-10-18 | **P3-42 (B4).** `STATEMENTS_VERSION_MISMATCH` пишется и при приёме `2.0` (`version_issue(job.statements_version, engine.version)`), а не только при пересчёте после правки | иначе запись появлялась бы только после первой правки; один набор записей `core` у приёма и пересчёта | лишний `info` на формах, принятых копией иной версии |
|
||||
| 2026-10-18 | **P3-43 (B4).** Результат с повтором `row_index` или `ref` у строк: вторая и далее строка пропускается с `MAPPING_WARNING`; `detail` с `parent_ref`, которого нет, хранится с `parent_row_index = null` и тем же предупреждением | уникальность `(meta_id, source_ref)` и ключа строки при вставке иначе роняет приём (файл `ERROR` после попыток); схема уникальности `ref`/`row_index` не проверяет | потерянная строка дубля видна только в предупреждении |
|
||||
| 2026-10-18 | **P3-44 (B4).** Запись `relation` / `soft` из `checks` результата несёт `column_id` (колонка с таким `column_index`, если есть) и всегда `column_index` (как P3-33); запись `warning` с `column_index` — либо `column_id` (колонка есть), либо `column_index` (колонки нет) | спека «Пропущенная колонка» и ответ API, читающий индекс из записи | нет |
|
||||
| 2026-10-18 | **P3-45 (B4).** `is_current` колонок `2.0` — как у `1.x` (колонка с самой поздней датой, при равенстве меньший `column_index`; нет дат — `NULL`), в том числе у колонок `relative:*` с датой; значение, которого нет в результате, пишется с `is_recognized = NULL`, остальными полями пустыми и `needs_review = false` | §9.5 п. 2; спека «Хранение результата формы v2.0» | колонка `relative:*` без даты не станет текущей |
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user