From c4afe4bc0bf436b29ee28adf7f075701c0733255 Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Sat, 3 Oct 2026 23:01:23 +0400 Subject: [PATCH] =?UTF-8?q?feat:=20pydantic-=D0=BC=D0=BE=D0=B4=D0=B5=D0=BB?= =?UTF-8?q?=D0=B8=20=D0=BA=D0=BE=D0=BD=D1=82=D1=80=D0=B0=D0=BA=D1=82=D0=B0?= =?UTF-8?q?=20=D1=80=D0=B0=D1=81=D0=BF=D0=BE=D0=B7=D0=BD=D0=B0=D0=B2=D0=B0?= =?UTF-8?q?=D0=BD=D0=B8=D1=8F=20v1.0=20=D0=B8=20=D1=82=D0=B5=D1=81=D1=82?= =?UTF-8?q?=D1=8B=20=D1=81=D0=BE=D0=B3=D0=BB=D0=B0=D1=81=D0=BE=D0=B2=D0=B0?= =?UTF-8?q?=D0=BD=D0=BD=D0=BE=D1=81=D1=82=D0=B8=20=D1=81=D0=BE=20=D1=81?= =?UTF-8?q?=D1=85=D0=B5=D0=BC=D0=B0=D0=BC=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5.1 --- api/src/contract/__init__.py | 63 ++++ api/src/contract/common.py | 212 +++++++++++++ api/src/contract/message.py | 63 ++++ api/src/contract/request.py | 27 ++ api/src/contract/result.py | 187 ++++++++++++ api/src/contract/subjects.py | 45 +++ api/tests/test_contract.py | 562 +++++++++++++++++++++++++++++++++++ 7 files changed, 1159 insertions(+) create mode 100644 api/src/contract/__init__.py create mode 100644 api/src/contract/common.py create mode 100644 api/src/contract/message.py create mode 100644 api/src/contract/request.py create mode 100644 api/src/contract/result.py create mode 100644 api/src/contract/subjects.py create mode 100644 api/tests/test_contract.py diff --git a/api/src/contract/__init__.py b/api/src/contract/__init__.py new file mode 100644 index 0000000..79922fd --- /dev/null +++ b/api/src/contract/__init__.py @@ -0,0 +1,63 @@ +"""Контракт обмена с сервисами распознавания v1.0 (схемы — `api/contracts/`, копии `docs/contracts/`). + +Публичный API: pydantic-модели запроса, конверта и файла результата, subject'ы NATS, идентификаторы сообщений. +""" + +from src.contract.common import ( + CONTRACT_VERSION, + ContractModel, + ErrorCode, + Metrics, + S3Ref, + ServiceInfo, + ServiceName, +) +from src.contract.message import MessageError, MessageStatus, RecognitionResultMessage +from src.contract.request import FileType, RecognitionRequest +from src.contract.result import ( + OsvDocument, + RecognitionResult, + ResultStatus, + StatementDocument, +) +from src.contract.subjects import ( + IMAGE_REQUEST_SUBJECT, + NATS_MSG_ID_HEADER, + REQUEST_SUBJECTS, + RESULT_SUBJECT, + TABULAR_REQUEST_SUBJECT, + Pipeline, + ack_msg_id, + request_msg_id, + request_subject, + result_msg_id, +) + +__all__ = [ + "CONTRACT_VERSION", + "IMAGE_REQUEST_SUBJECT", + "NATS_MSG_ID_HEADER", + "REQUEST_SUBJECTS", + "RESULT_SUBJECT", + "TABULAR_REQUEST_SUBJECT", + "ContractModel", + "ErrorCode", + "FileType", + "MessageError", + "MessageStatus", + "Metrics", + "OsvDocument", + "Pipeline", + "RecognitionRequest", + "RecognitionResult", + "RecognitionResultMessage", + "ResultStatus", + "S3Ref", + "ServiceInfo", + "ServiceName", + "StatementDocument", + "ack_msg_id", + "request_msg_id", + "request_subject", + "result_msg_id", +] diff --git a/api/src/contract/common.py b/api/src/contract/common.py new file mode 100644 index 0000000..6d14baf --- /dev/null +++ b/api/src/contract/common.py @@ -0,0 +1,212 @@ +"""Общие основы моделей контракта v1.0: базовый класс, версия, типы UUID / деньги / время, метрики и ошибки. + +Модели повторяют JSON Schema (`api/contracts/`) буквально: схемы не содержат `additionalProperties: false` +(кроме значений `columns` ОСВ), поэтому незнакомые поля принимаются и отбрасываются (`extra="ignore"`) — +так обратно совместимый MINOR-релиз (необязательные поля) не ломает потребителя. Типы строгие (`strict=True`): +JSON Schema не приводит `"1"` к числу, а число — к строке; поля со строковой записью (uuid, date-time, date, деньги) +разбирают строки явно. +""" + +import re +import uuid +from datetime import UTC, date, datetime +from decimal import Decimal +from enum import StrEnum +from typing import Annotated, Any, Literal, TypeVar + +from pydantic import ( + AfterValidator, + AwareDatetime, + BaseModel, + BeforeValidator, + ConfigDict, + Field, + PlainSerializer, + PlainValidator, + WithJsonSchema, + model_validator, +) + +CONTRACT_VERSION = "1.0" + +# Шаблоны версий — буквально из схем +SCHEMA_VERSION_PATTERN = r"^1\.\d+$" # свой MAJOR, любой MINOR +RESULT_SCHEMA_VERSION_PATTERN = r"^\d+\.\d+$" # recognition-result-message.result_schema_version + +T = TypeVar("T") + + +class _NotNull: + """Метка поля, необязательного в схеме, но не допускающего явный `null` (в схеме тип без `null`).""" + + +NOT_NULL = _NotNull() +# Необязательное поле без `null`: отсутствует (None в модели) или значение типа T; явный null во входе — ошибка +OptionalNotNull = Annotated[T | None, NOT_NULL] + + +class ContractModel(BaseModel): + """База моделей контракта: неизменяемые, строгие, незнакомые поля отбрасываются.""" + + model_config = ConfigDict(extra="ignore", strict=True, frozen=True) + + @model_validator(mode="before") + @classmethod + def _reject_explicit_null(cls, data: Any) -> Any: + """Схема не допускает `null`, где тип без `null`: `{"result_s3": null}` — ошибка, а не «нет поля».""" + if isinstance(data, dict): + for name, field in cls.model_fields.items(): + key = field.alias or name + if NOT_NULL in field.metadata and key in data and data[key] is None: + raise ValueError(f"поле {key!r} не допускает null (в схеме контракта нет null в типе)") + return data + + def to_wire(self) -> dict[str, Any]: + """Тело для NATS / файла S3: JSON-словарь только из заданных полей (uuid, время, деньги — строки).""" + return self.model_dump(mode="json", exclude_unset=True) + + +# --- UUID ----------------------------------------------------------------------------------------------------------- + +_UUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}", re.ASCII) + + +def _require_canonical_uuid(value: object) -> object: + """Строка канонического вида 8-4-4-4-12 (как `format: uuid` схемы) либо готовый `uuid.UUID`.""" + if isinstance(value, uuid.UUID) or (isinstance(value, str) and _UUID_RE.fullmatch(value)): + return value + raise ValueError("ожидается uuid строкой вида xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx") + + +ContractUUID = Annotated[uuid.UUID, BeforeValidator(_require_canonical_uuid), Field(strict=False)] + +# --- время ---------------------------------------------------------------------------------------------------------- + +_DATE_RE = re.compile(r"\d{4}-\d{2}-\d{2}", re.ASCII) + + +def _require_text_or_datetime(value: object) -> object: + if isinstance(value, str | datetime): + return value + raise ValueError("ожидается строка date-time (RFC 3339) с часовым поясом") + + +def _to_utc(value: datetime) -> datetime: + return value.astimezone(UTC) + + +# `reply_deadline`: срок ответа, UTC. Часовой пояс обязателен (aware); смещение приводится к UTC +UtcDatetime = Annotated[ + AwareDatetime, + BeforeValidator(_require_text_or_datetime), + AfterValidator(_to_utc), + Field(strict=False), +] + + +def _require_iso_date(value: object) -> object: + if isinstance(value, date) and not isinstance(value, datetime): + return value + if isinstance(value, str) and _DATE_RE.fullmatch(value): + return value + raise ValueError("ожидается дата строкой YYYY-MM-DD") + + +ContractDate = Annotated[date, BeforeValidator(_require_iso_date), Field(strict=False)] + +# --- деньги --------------------------------------------------------------------------------------------------------- + +MONEY_PATTERN = r"^-?\d+(\.\d{1,2})?$" # буквально из `$defs.money` +_MONEY_RE = re.compile(r"-?\d+(\.\d{1,2})?", re.ASCII) + + +def _parse_money(value: object) -> Decimal | None: + """Сумма — строка по шаблону схемы (копейки не теряются) или `null`; `float` / `int` — ошибка.""" + if value is None: + return None + if isinstance(value, Decimal): + text = format(value, "f") if value.is_finite() else "" + elif isinstance(value, str): + text = value + else: + raise ValueError( + "сумма должна быть строкой вида '106311116.46' (float и int недопустимы, чтобы не терять копейки)" + ) + if not _MONEY_RE.fullmatch(text): + raise ValueError(f"сумма {text!r} не соответствует шаблону {MONEY_PATTERN} (до двух знаков после точки)") + return Decimal(text) + + +def _dump_money(value: Decimal | None) -> str | None: + return None if value is None else format(value, "f") + + +# В модели — `Decimal`; в JSON — строка ровно в той записи, как пришла («1100», «-12.30») +Money = Annotated[ + Decimal | None, + PlainValidator(_parse_money), + PlainSerializer(_dump_money, return_type=str | None, when_used="json"), + WithJsonSchema({"type": ["string", "null"], "pattern": MONEY_PATTERN}), +] + +# --- скаляры -------------------------------------------------------------------------------------------------------- + +NonEmptyStr = Annotated[str, Field(min_length=1)] +SchemaVersion = Annotated[str, Field(pattern=SCHEMA_VERSION_PATTERN)] +ResultSchemaVersion = Annotated[str, Field(pattern=RESULT_SCHEMA_VERSION_PATTERN)] +# Уверенность распознавания, проценты (число 0–100) +Confidence = Annotated[float, Field(ge=0, le=100)] | None + +# --- общие объекты -------------------------------------------------------------------------------------------------- + + +class ServiceName(StrEnum): + TABULAR_PARSER = "tabular-parser" + SCAN_GATEWAY = "scan-gateway" + + +class ErrorCode(StrEnum): + UNSUPPORTED_FORMAT = "UNSUPPORTED_FORMAT" + UNREADABLE = "UNREADABLE" + TIMEOUT = "TIMEOUT" + SCHEMA_MISMATCH = "SCHEMA_MISMATCH" + NOT_IMPLEMENTED = "NOT_IMPLEMENTED" + FILE_READ_ERROR = "FILE_READ_ERROR" + INVALID_FORMAT = "INVALID_FORMAT" + INVALID_DOCUMENT = "INVALID_DOCUMENT" + RECOGNITION_FAILED = "RECOGNITION_FAILED" + UNKNOWN_ERROR = "UNKNOWN_ERROR" + + +# Строгий режим модели не принимает значение enum строкой: поле разбирает строку само (как JSON Schema `enum`) +LaxServiceName = Annotated[ServiceName, Field(strict=False)] +LaxErrorCode = Annotated[ErrorCode, Field(strict=False)] + + +class ServiceInfo(ContractModel): + """Кто ответил: `service {name, version}`.""" + + name: LaxServiceName + version: str + + +class S3Ref(ContractModel): + """Указатель на объект S3: `{bucket, key}`.""" + + bucket: NonEmptyStr + key: NonEmptyStr + + +class Metrics(ContractModel): + """Метрики обработки; все поля необязательны.""" + + duration_ms: OptionalNotNull[int] = None + pages: OptionalNotNull[int] = None + rows: OptionalNotNull[int] = None + fields_total: OptionalNotNull[int] = None + fields_low_confidence: OptionalNotNull[int] = None + fields_unrecognized: OptionalNotNull[int] = None + warnings: OptionalNotNull[list[str]] = None + + +DataType = Literal["BU", "NU", "SUM", "CURRENCY", "QUANTITY"] diff --git a/api/src/contract/message.py b/api/src/contract/message.py new file mode 100644 index 0000000..828b109 --- /dev/null +++ b/api/src/contract/message.py @@ -0,0 +1,63 @@ +"""Конверт сообщения на `recognition.result`: `recognition-result-message.schema.json` v1.0. + +`processing` — ack «взял в работу» (до чтения файла из S3); `done | needs_check | error` — итог. +Условная обязательность — буквально из `allOf` схемы: для `done | needs_check` обязательны `result_s3` и +`result_schema_version`; для `error` — `error`. Других правил нет (например, `error` при `done` схема не запрещает). +""" + +from enum import StrEnum +from typing import Annotated, Self + +from pydantic import Field, model_validator + +from src.contract.common import ( + ContractModel, + ContractUUID, + LaxErrorCode, + Metrics, + OptionalNotNull, + ResultSchemaVersion, + S3Ref, + SchemaVersion, + ServiceInfo, +) + + +class MessageStatus(StrEnum): + PROCESSING = "processing" + DONE = "done" + NEEDS_CHECK = "needs_check" + ERROR = "error" + + +LaxMessageStatus = Annotated[MessageStatus, Field(strict=False)] + + +class MessageError(ContractModel): + """`error {code, message}` конверта: оба поля обязательны.""" + + code: LaxErrorCode + message: str + + +class RecognitionResultMessage(ContractModel): + schema_version: SchemaVersion + job_id: ContractUUID # recognition_job.id + file_id: ContractUUID # stored_file.id + status: LaxMessageStatus + service: ServiceInfo + result_s3: OptionalNotNull[S3Ref] = None # results/{file_id}/{job_id}/result.json + result_schema_version: OptionalNotNull[ResultSchemaVersion] = None # schema_version файла результата + error: OptionalNotNull[MessageError] = None + raw_s3_key: str | None = None # scan-gateway: raw/{job_id}.json; в БД не регистрируется + metrics: OptionalNotNull[Metrics] = None + + @model_validator(mode="after") + def _check_status_requirements(self) -> Self: + if self.status in (MessageStatus.DONE, MessageStatus.NEEDS_CHECK): + missing = [name for name in ("result_s3", "result_schema_version") if getattr(self, name) is None] + if missing: + raise ValueError(f"при status = {self.status.value} обязательны поля: {', '.join(missing)}") + elif self.status is MessageStatus.ERROR and self.error is None: + raise ValueError("при status = error обязательно поле error") + return self diff --git a/api/src/contract/request.py b/api/src/contract/request.py new file mode 100644 index 0000000..c5eb3f8 --- /dev/null +++ b/api/src/contract/request.py @@ -0,0 +1,27 @@ +"""Запрос распознавания файла (arm-core → tabular-parser / scan-gateway): `recognition-request.schema.json` v1.0. + +Формируется из `recognition_job ⋈ stored_file ⋈ s3_object` (spec recognition-job-exchange); один файл — один job. +Subject — по `pipeline` (`request_subject`); заголовок `Nats-Msg-Id = :` +(`request_msg_id`) — не поле тела. +""" + +from typing import Annotated, Literal + +from pydantic import Field + +from src.contract.common import ContractModel, ContractUUID, NonEmptyStr, S3Ref, SchemaVersion, UtcDatetime +from src.contract.subjects import Pipeline + +FileType = Literal["XLS", "XLSX", "XLSB", "PDF", "PNG", "JPG"] + + +class RecognitionRequest(ContractModel): + schema_version: SchemaVersion + job_id: ContractUUID # recognition_job.id + file_id: ContractUUID # stored_file.id + pipeline: Pipeline + file_type: FileType + original_name: NonEmptyStr # имя файла, как загрузил пользователь + s3: S3Ref # исходный файл в роли uploads + reply_deadline: UtcDatetime # срок ответа, UTC + attempt: Annotated[int, Field(ge=1)] # номер попытки, с 1 diff --git a/api/src/contract/result.py b/api/src/contract/result.py new file mode 100644 index 0000000..6e37dea --- /dev/null +++ b/api/src/contract/result.py @@ -0,0 +1,187 @@ +"""Файл результата в S3 (`results/{file_id}/{job_id}/result.json`): `recognition-result.schema.json` v1.0. + +Один файл → один job → N документов (ОСВ по счёту / ББ / ОФР / прочее). В Срезе 0 — модели и валидация примеров; +маппинг в таблицы — Срез 2. Суммы — `Decimal` в модели и строка в JSON (`"106311116.46"`, без `float`). +Нормализация счетов, резолв контрагентов и проверки — в arm-core, не здесь. +""" + +from enum import StrEnum +from typing import Annotated, Literal, Self + +from pydantic import Field, model_validator + +from src.contract.common import ( + Confidence, + ContractDate, + ContractModel, + ContractUUID, + DataType, + LaxErrorCode, + Metrics, + Money, + OptionalNotNull, + SchemaVersion, + ServiceInfo, +) + +__all__ = [ + "OsvChecks", + "OsvDocument", + "OsvGroup", + "OsvHeader", + "OsvRow", + "RecognitionResult", + "ResultError", + "ResultStatus", + "StatementColumn", + "StatementDocument", + "StatementHeader", + "StatementLine", + "StatementValue", +] + + +class ResultStatus(StrEnum): + DONE = "done" + NEEDS_CHECK = "needs_check" + ERROR = "error" + + +LaxResultStatus = Annotated[ResultStatus, Field(strict=False)] + + +class ResultError(ContractModel): + """`error` файла результата: в отличие от конверта, `code` и `message` необязательны.""" + + code: OptionalNotNull[LaxErrorCode] = None + message: OptionalNotNull[str] = None + + +# --- ОСВ ------------------------------------------------------------------------------------------------------------ + + +class OsvHeader(ContractModel): + organization_name: str # как в r1 файла + organization_inn: str | None = None + account_code_raw: str # «60», «58.03» + period_text: str # «1 квартал 2025 г.» + period_year: int | None = None + period_quarter: Literal["Q1", "Q2", "Q3", "Q4"] | None = None + is_cumulative: bool | None = None # по тексту; ядро может переопределить по данным (OQ-02) + data_type: DataType | None = None + currency: str | None = None + header_variant: OptionalNotNull[Literal[1, 2]] = None + + +class OsvGroup(ContractModel): + """Группировка из шапки (порядок — после первого ACCOUNT).""" + + type: Literal["ACCOUNT", "DEPARTMENT", "COUNTERPARTY", "CONTRACT", "OTHER"] + name: OptionalNotNull[str] = None # исходный текст из шапки + has_inn: OptionalNotNull[bool] = None + + +class OsvRow(ContractModel): + row_number: int + level: int + parent_row_number: int | None = None + row_value: str + group_type: Literal["ACCOUNT", "DEPARTMENT", "COUNTERPARTY", "CONTRACT", "OTHER", "TOTAL"] + group_name: str | None = None + account_code_raw: str | None = None # код текущей ветки, как в файле (60.1) + counterparty_name: str | None = None + counterparty_inn: str | None = None + data_type: DataType | None = None + opening_debit: Money = None + opening_credit: Money = None + turnover_debit: Money = None + turnover_credit: Money = None + closing_debit: Money = None + closing_credit: Money = None + + +class OsvChecks(ContractModel): + total_row_matches_root: OptionalNotNull[bool] = None + balance_identity_violations: OptionalNotNull[list[int]] = None # row_number, где opening−turnover≠closing + parent_sum_violations: OptionalNotNull[list[int]] = None + + +class OsvDocument(ContractModel): + document_type: Literal["OSV"] + locator: str # «sheet:Лист_1» + header: OsvHeader + group_schema: list[OsvGroup] + columns: OptionalNotNull[dict[str, str]] = None # диагностика: какие колонки приняты за шесть сумм и ИНН + rows: list[OsvRow] + checks: OptionalNotNull[OsvChecks] = None + + +# --- ББ / ОФР / прочее ---------------------------------------------------------------------------------------------- + + +class StatementHeader(ContractModel): + form_name: OptionalNotNull[str] = None + organization_name: str | None = None + organization_name_confidence: Confidence = None + inn: str | None = None + inn_confidence: Confidence = None + report_date_text: str | None = None + report_date: ContractDate | None = None + units: str | None = None # «тыс. руб.» + okud: str | None = None + okpo: str | None = None + okopf: str | None = None + okfs: str | None = None + address: str | None = None + + +class StatementColumn(ContractModel): + column_index: int + label: str # «На 30 сентября 2025 г.» + period_date: ContractDate | None = None + + +class StatementValue(ContractModel): + column_index: int + amount: Money = None + string_value: str | None = None + confidence: Confidence = None + is_recognized: OptionalNotNull[bool] = None + + +class StatementLine(ContractModel): + line_code: str + line_code_confidence: Confidence = None + line_name: str + line_name_confidence: Confidence = None + values: list[StatementValue] + + +class StatementDocument(ContractModel): + document_type: Literal["BALANCE", "INCOME", "OTHER"] + locator: str # «pages:1-2» + header: StatementHeader + columns: list[StatementColumn] + lines: list[StatementLine] + + +ResultDocument = Annotated[OsvDocument | StatementDocument, Field(discriminator="document_type")] + + +class RecognitionResult(ContractModel): + schema_version: SchemaVersion + job_id: ContractUUID + file_id: ContractUUID + service: ServiceInfo + status: LaxResultStatus + error: ResultError | None = None + metrics: OptionalNotNull[Metrics] = None + raw_s3_key: str | None = None # scan-gateway: raw/{job_id}.json + documents: list[ResultDocument] + + @model_validator(mode="after") + def _check_documents_for_success(self) -> Self: + """`if status in (done, needs_check) then documents.minItems = 1` — буквально из схемы.""" + if self.status in (ResultStatus.DONE, ResultStatus.NEEDS_CHECK) and not self.documents: + raise ValueError(f"при status = {self.status.value} список documents не может быть пустым") + return self diff --git a/api/src/contract/subjects.py b/api/src/contract/subjects.py new file mode 100644 index 0000000..be58d01 --- /dev/null +++ b/api/src/contract/subjects.py @@ -0,0 +1,45 @@ +"""Subject'ы NATS, заголовок дедупликации и идентификаторы сообщений контракта v1.0. + +Источник: `docs/context/03-architecture.md`, «Контракт обмена через NATS»; схема recognition-request +(`Nats-Msg-Id = :`) и recognition-result-message +(`ack::` / `result::`). +""" + +import uuid +from collections.abc import Mapping +from typing import Literal, get_args + +Pipeline = Literal["tabular", "image"] + +NATS_MSG_ID_HEADER = "Nats-Msg-Id" +RESULT_SUBJECT = "recognition.result" +TABULAR_REQUEST_SUBJECT = "recognition.tabular.request" +IMAGE_REQUEST_SUBJECT = "recognition.image.request" + +REQUEST_SUBJECTS: Mapping[str, str] = { + "tabular": TABULAR_REQUEST_SUBJECT, + "image": IMAGE_REQUEST_SUBJECT, +} + + +def request_subject(pipeline: str) -> str: + """Subject запроса по pipeline: `tabular` → `recognition.tabular.request`, `image` → `recognition.image.request`.""" + try: + return REQUEST_SUBJECTS[pipeline] + except KeyError: + raise ValueError(f"неизвестный pipeline {pipeline!r}: допустимы {', '.join(get_args(Pipeline))}") from None + + +def request_msg_id(job_id: uuid.UUID | str, attempt: int) -> str: + """`Nats-Msg-Id` запроса: `:`.""" + return f"{job_id}:{attempt}" + + +def ack_msg_id(job_id: uuid.UUID | str, attempt: int) -> str: + """`Nats-Msg-Id` ack (`status = processing`) на `recognition.result`: `ack::`.""" + return f"ack:{job_id}:{attempt}" + + +def result_msg_id(job_id: uuid.UUID | str, attempt: int) -> str: + """`Nats-Msg-Id` результата (`done | needs_check | error`) на `recognition.result`: `result::`.""" + return f"result:{job_id}:{attempt}" diff --git a/api/tests/test_contract.py b/api/tests/test_contract.py new file mode 100644 index 0000000..c992588 --- /dev/null +++ b/api/tests/test_contract.py @@ -0,0 +1,562 @@ +"""Контракт обмена с сервисами распознавания v1.0 (spec recognition-job-exchange, design D-6). + +Быстрые тесты без БД: вендоренные схемы `api/contracts/` идентичны `../docs/contracts/`; каждый пример проходит и +JSON Schema, и pydantic-модель (с round-trip); модели и схемы выносят один и тот же вердикт на негативных кейсах. +""" + +import copy +import json +import uuid +from collections.abc import Callable +from datetime import UTC, datetime +from decimal import Decimal +from pathlib import Path +from typing import Any + +import pytest +from jsonschema import Draft202012Validator, FormatChecker +from pydantic import ValidationError +from referencing import Registry, Resource + +from scripts.dbml_parity import default_dbml_path +from src.contract import ( + CONTRACT_VERSION, + NATS_MSG_ID_HEADER, + RESULT_SUBJECT, + ContractModel, + MessageStatus, + RecognitionRequest, + RecognitionResult, + RecognitionResultMessage, + ack_msg_id, + request_msg_id, + request_subject, + result_msg_id, +) + +CONTRACTS_DIR = Path(__file__).resolve().parents[1] / "contracts" +EXAMPLES_DIR = CONTRACTS_DIR / "examples" +SCHEMA_FILES = { + "request": "recognition-request.schema.json", + "message": "recognition-result-message.schema.json", + "result": "recognition-result.schema.json", +} +MODELS: dict[str, type[ContractModel]] = { + "request": RecognitionRequest, + "message": RecognitionResultMessage, + "result": RecognitionResult, +} +EXAMPLE_KIND = { + "request-image.json": "request", + "request-tabular.json": "request", + "result-message-done.json": "message", + "result-message-error.json": "message", + "result-message-processing.json": "message", + "result-error.json": "result", + "result-osv-minimal.json": "result", + "result-statement-minimal.json": "result", +} +EXAMPLES_COUNT = 8 + + +def _load(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def _example(name: str) -> dict[str, Any]: + data = _load(EXAMPLES_DIR / name) + assert isinstance(data, dict) + return data + + +def _build_validators() -> dict[str, Draft202012Validator]: + """Валидаторы по трём схемам; `$ref` между схемами разрешаются локальным реестром (без сети).""" + schemas = {kind: _load(CONTRACTS_DIR / file_name) for kind, file_name in SCHEMA_FILES.items()} + registry: Registry = Registry() + for schema in schemas.values(): + registry = registry.with_resource(schema["$id"], Resource.from_contents(schema)) + return { + kind: Draft202012Validator(schema, registry=registry, format_checker=FormatChecker()) + for kind, schema in schemas.items() + } + + +VALIDATORS = _build_validators() + + +def _schema_ok(kind: str, payload: dict[str, Any]) -> bool: + return VALIDATORS[kind].is_valid(payload) + + +def _model_ok(kind: str, payload: dict[str, Any]) -> bool: + try: + MODELS[kind].model_validate(payload) + except ValidationError: + return False + return True + + +# --- идентичность вендоренных копий ------------------------------------------------------------------------------- + + +def _docs_contracts_dir() -> Path: + """`docs/contracts` рядом с `arm-core` или из `ARM_KA_DOCS` (тот же поиск, что у DBML).""" + return default_dbml_path().parents[1] / "contracts" + + +def test_vendored_contracts_are_identical_to_docs() -> None: + docs_dir = _docs_contracts_dir() + if not docs_dir.is_dir(): + pytest.skip( + "docs недоступен: тест идентичности контракта пропущен (задайте ARM_KA_DOCS или положите docs рядом)" + ) + docs_files = {p.relative_to(docs_dir).as_posix() for p in docs_dir.rglob("*.json")} + own_files = {p.relative_to(CONTRACTS_DIR).as_posix() for p in CONTRACTS_DIR.rglob("*.json")} + assert own_files == docs_files, ( + f"набор файлов контракта разошёлся: только в api/contracts: {sorted(own_files - docs_files)}, " + f"только в docs/contracts: {sorted(docs_files - own_files)}" + ) + for name in sorted(docs_files): + assert (CONTRACTS_DIR / name).read_bytes() == (docs_dir / name).read_bytes(), ( + f"файл {name} в api/contracts отличается от docs/contracts: скопируйте заново (cp), не правьте руками" + ) + + +def test_example_table_covers_all_example_files() -> None: + files = {p.name for p in EXAMPLES_DIR.glob("*.json")} + assert files == set(EXAMPLE_KIND) + assert len(files) == EXAMPLES_COUNT + + +# --- примеры: JSON Schema, модель, round-trip --------------------------------------------------------------------- + + +@pytest.mark.parametrize("name", sorted(EXAMPLE_KIND)) +def test_example_passes_json_schema(name: str) -> None: + VALIDATORS[EXAMPLE_KIND[name]].validate(_example(name)) + + +@pytest.mark.parametrize("name", sorted(EXAMPLE_KIND)) +def test_example_passes_model_and_round_trips(name: str) -> None: + kind = EXAMPLE_KIND[name] + original = _example(name) + model = MODELS[kind].model_validate(original) + dumped = model.model_dump(mode="json", exclude_unset=True) + # Ничего не потеряно и не изменено (в том числе суммы-строки: «1100», «-12.30», «0.00») + assert dumped == original + VALIDATORS[kind].validate(dumped) + # to_wire() — то, что уйдёт в NATS/S3: тоже проходит схему и совпадает с исходным примером + wire = model.to_wire() + assert wire == original + VALIDATORS[kind].validate(wire) + # повторный разбор даёт равную модель + assert MODELS[kind].model_validate(wire) == model + + +def test_result_amounts_are_decimal_in_model_and_strings_on_the_wire() -> None: + result = RecognitionResult.model_validate(_example("result-statement-minimal.json")) + values = result.documents[0].lines[1].values # type: ignore[union-attr] + assert values[0].amount == Decimal("5678.00") + assert values[1].amount == Decimal("-12.30") + assert isinstance(values[1].amount, Decimal) + wire = result.to_wire() + assert wire["documents"][0]["lines"][1]["values"][1]["amount"] == "-12.30" + assert wire["documents"][0]["lines"][0]["values"][1]["amount"] == "1100" # без «.00» — как пришло + + +def test_osv_amount_keeps_kopecks() -> None: + payload = _example("result-osv-minimal.json") + payload["documents"][0]["rows"][0]["closing_credit"] = "106311116.46" + result = RecognitionResult.model_validate(payload) + assert result.documents[0].rows[0].closing_credit == Decimal("106311116.46") # type: ignore[union-attr] + wire = result.to_wire() + assert wire["documents"][0]["rows"][0]["closing_credit"] == "106311116.46" + VALIDATORS["result"].validate(wire) + + +def test_models_expose_json_schema() -> None: + for model in MODELS.values(): + assert model.model_json_schema()["type"] == "object" + + +# --- согласованность моделей и схем ------------------------------------------------------------------------------- + + +def _set(path: str, value: Any) -> Callable[[dict[str, Any]], None]: + """Мутация: положить `value` по пути `a.b.0.c` (числа — индексы списков).""" + + def apply(payload: dict[str, Any]) -> None: + *parents, last = path.split(".") + node: Any = payload + for part in parents: + node = node[int(part)] if part.isdigit() else node[part] + if isinstance(node, list): + node[int(last)] = value + else: + node[last] = value + + return apply + + +def _drop(path: str) -> Callable[[dict[str, Any]], None]: + def apply(payload: dict[str, Any]) -> None: + *parents, last = path.split(".") + node: Any = payload + for part in parents: + node = node[int(part)] if part.isdigit() else node[part] + del node[last] + + return apply + + +# (идентификатор, пример-основа, мутация, ожидаемый вердикт «валидно») +CASES: list[tuple[str, str, Callable[[dict[str, Any]], None], bool]] = [ + # --- запрос --- + ("request/base", "request-tabular.json", lambda p: None, True), + ("request/no-job_id", "request-tabular.json", _drop("job_id"), False), + ("request/no-attempt", "request-tabular.json", _drop("attempt"), False), + ("request/attempt-0", "request-tabular.json", _set("attempt", 0), False), + ("request/attempt-2", "request-tabular.json", _set("attempt", 2), True), + ("request/attempt-string", "request-tabular.json", _set("attempt", "1"), False), + ("request/attempt-float", "request-tabular.json", _set("attempt", 1.5), False), + ("request/attempt-bool", "request-tabular.json", _set("attempt", True), False), + ("request/pipeline-unknown", "request-tabular.json", _set("pipeline", "ocr"), False), + ("request/file_type-unknown", "request-tabular.json", _set("file_type", "DOCX"), False), + ("request/schema_version-2.0", "request-tabular.json", _set("schema_version", "2.0"), False), + ("request/schema_version-1.7", "request-tabular.json", _set("schema_version", "1.7"), True), + ("request/schema_version-1", "request-tabular.json", _set("schema_version", "1"), False), + ("request/schema_version-1.0.1", "request-tabular.json", _set("schema_version", "1.0.1"), False), + ("request/job_id-not-uuid", "request-tabular.json", _set("job_id", "not-a-uuid"), False), + ( + "request/job_id-uuid-without-dashes", + "request-tabular.json", + _set("job_id", "3f2a9c1e7b4d4e0a9d512c8e6a1b0f01"), + False, + ), + ("request/file_id-number", "request-tabular.json", _set("file_id", 12345), False), + ("request/original_name-empty", "request-tabular.json", _set("original_name", ""), False), + ("request/s3-no-key", "request-tabular.json", _drop("s3.key"), False), + ("request/s3-empty-bucket", "request-tabular.json", _set("s3.bucket", ""), False), + ("request/s3-null", "request-tabular.json", _set("s3", None), False), + ("request/extra-field", "request-tabular.json", _set("trace_id", "abc"), True), + ("request/extra-field-in-s3", "request-tabular.json", _set("s3.version_id", "v1"), True), + # --- конверт --- + ("message/processing", "result-message-processing.json", lambda p: None, True), + ("message/done", "result-message-done.json", lambda p: None, True), + ("message/status-unknown", "result-message-done.json", _set("status", "queued"), False), + ("message/no-service", "result-message-processing.json", _drop("service"), False), + ("message/service-unknown", "result-message-processing.json", _set("service.name", "other-service"), False), + ("message/schema_version-2.0", "result-message-processing.json", _set("schema_version", "2.0"), False), + ("message/schema_version-1.7", "result-message-processing.json", _set("schema_version", "1.7"), True), + ("message/done-no-result_s3", "result-message-done.json", _drop("result_s3"), False), + ("message/done-no-result_schema_version", "result-message-done.json", _drop("result_schema_version"), False), + ("message/needs_check-ok", "result-message-done.json", _set("status", "needs_check"), True), + ( + "message/needs_check-no-result_s3", + "result-message-done.json", + lambda p: (p.pop("result_s3"), p.update(status="needs_check")), + False, + ), + ("message/result_s3-no-key", "result-message-done.json", _drop("result_s3.key"), False), + ("message/result_s3-null", "result-message-done.json", _set("result_s3", None), False), + ("message/result_schema_version-2.0", "result-message-done.json", _set("result_schema_version", "2.0"), True), + ("message/result_schema_version-bad", "result-message-done.json", _set("result_schema_version", "1"), False), + ("message/error-ok", "result-message-error.json", lambda p: None, True), + ("message/error-no-error", "result-message-error.json", _drop("error"), False), + ("message/error-null", "result-message-error.json", _set("error", None), False), + ("message/error-unknown-code", "result-message-error.json", _set("error.code", "BOOM"), False), + ("message/error-no-message", "result-message-error.json", _drop("error.message"), False), + ("message/processing-no-result_s3", "result-message-processing.json", lambda p: None, True), + ( + "message/done-with-error-block", + "result-message-done.json", + _set("error", {"code": "TIMEOUT", "message": "x"}), + True, + ), + ("message/raw_s3_key-null", "result-message-done.json", _set("raw_s3_key", None), True), + ("message/metrics-warnings-not-strings", "result-message-done.json", _set("metrics.warnings", [1]), False), + ("message/metrics-duration-float", "result-message-done.json", _set("metrics.duration_ms", 1.5), False), + ("message/metrics-extra-field", "result-message-done.json", _set("metrics.cpu", 3), True), + ("message/metrics-rows", "result-message-done.json", _set("metrics.rows", 10), True), + ("message/extra-field", "result-message-done.json", _set("trace_id", "abc"), True), + # --- файл результата --- + ("result/osv-base", "result-osv-minimal.json", lambda p: None, True), + ("result/statement-base", "result-statement-minimal.json", lambda p: None, True), + ("result/error-base", "result-error.json", lambda p: None, True), + ( + "result/amount-float", + "result-statement-minimal.json", + _set("documents.0.lines.0.values.0.amount", 1234.5), + False, + ), + ("result/amount-int", "result-statement-minimal.json", _set("documents.0.lines.0.values.0.amount", 1234), False), + ("result/amount-null", "result-statement-minimal.json", _set("documents.0.lines.0.values.0.amount", None), True), + ( + "result/amount-3-decimals", + "result-statement-minimal.json", + _set("documents.0.lines.0.values.0.amount", "1.234"), + False, + ), + ("result/amount-comma", "result-statement-minimal.json", _set("documents.0.lines.0.values.0.amount", "1,5"), False), + ( + "result/amount-negative", + "result-statement-minimal.json", + _set("documents.0.lines.0.values.0.amount", "-0.5"), + True, + ), + ( + "result/amount-exponent", + "result-statement-minimal.json", + _set("documents.0.lines.0.values.0.amount", "1e3"), + False, + ), + ("result/amount-empty", "result-statement-minimal.json", _set("documents.0.lines.0.values.0.amount", ""), False), + ("result/osv-amount-float", "result-osv-minimal.json", _set("documents.0.rows.0.opening_credit", 1000.0), False), + ("result/osv-amount-missing", "result-osv-minimal.json", _drop("documents.0.rows.0.opening_credit"), True), + ("result/done-empty-documents", "result-osv-minimal.json", _set("documents", []), False), + ( + "result/needs_check-empty-documents", + "result-osv-minimal.json", + lambda p: p.update(status="needs_check", documents=[]), + False, + ), + ("result/error-empty-documents", "result-error.json", lambda p: None, True), + ("result/no-documents", "result-error.json", _drop("documents"), False), + ("result/status-processing", "result-osv-minimal.json", _set("status", "processing"), False), + ("result/error-null", "result-error.json", _set("error", None), True), + ("result/error-empty-object", "result-error.json", _set("error", {}), True), + ("result/error-bad-code", "result-error.json", _set("error.code", "BOOM"), False), + ("result/schema_version-2.0", "result-osv-minimal.json", _set("schema_version", "2.0"), False), + ("result/schema_version-1.3", "result-osv-minimal.json", _set("schema_version", "1.3"), True), + ("result/document_type-unknown", "result-osv-minimal.json", _set("documents.0.document_type", "MEMO"), False), + ("result/document-no-document_type", "result-osv-minimal.json", _drop("documents.0.document_type"), False), + ("result/osv-no-rows", "result-osv-minimal.json", _drop("documents.0.rows"), False), + ("result/osv-no-group_schema", "result-osv-minimal.json", _drop("documents.0.group_schema"), False), + ("result/osv-no-header-period_text", "result-osv-minimal.json", _drop("documents.0.header.period_text"), False), + ("result/osv-header_variant-3", "result-osv-minimal.json", _set("documents.0.header.header_variant", 3), False), + ("result/osv-period_quarter-Q5", "result-osv-minimal.json", _set("documents.0.header.period_quarter", "Q5"), False), + ( + "result/osv-period_quarter-null", + "result-osv-minimal.json", + _set("documents.0.header.period_quarter", None), + True, + ), + ( + "result/osv-header-null-for-non-nullable", + "result-osv-minimal.json", + _set("documents.0.header.header_variant", None), + False, + ), + ( + "result/osv-row-group_type-unknown", + "result-osv-minimal.json", + _set("documents.0.rows.0.group_type", "FOLDER"), + False, + ), + ( + "result/osv-row-group_type-TOTAL", + "result-osv-minimal.json", + _set("documents.0.rows.0.group_type", "TOTAL"), + True, + ), + ("result/osv-row-no-row_number", "result-osv-minimal.json", _drop("documents.0.rows.0.row_number"), False), + ("result/osv-row-level-float", "result-osv-minimal.json", _set("documents.0.rows.0.level", 0.5), False), + ("result/osv-columns-non-string", "result-osv-minimal.json", _set("documents.0.columns.opening_debit", 3), False), + ("result/osv-columns-extra-key", "result-osv-minimal.json", _set("documents.0.columns.inn", "B"), True), + ( + "result/osv-group_schema-bad-type", + "result-osv-minimal.json", + _set("documents.0.group_schema.0.type", "FOLDER"), + False, + ), + ( + "result/osv-checks-violations-not-ints", + "result-osv-minimal.json", + _set("documents.0.checks.parent_sum_violations", ["a"]), + False, + ), + ("result/osv-extra-field", "result-osv-minimal.json", _set("documents.0.debug", {"a": 1}), True), + ("result/statement-no-lines", "result-statement-minimal.json", _drop("documents.0.lines"), False), + ("result/statement-no-columns", "result-statement-minimal.json", _drop("documents.0.columns"), False), + ( + "result/statement-type-INCOME", + "result-statement-minimal.json", + _set("documents.0.document_type", "INCOME"), + True, + ), + ( + "result/statement-type-OSV-with-statement-body", + "result-statement-minimal.json", + _set("documents.0.document_type", "OSV"), + False, + ), + ( + "result/statement-confidence-101", + "result-statement-minimal.json", + _set("documents.0.lines.0.values.0.confidence", 101), + False, + ), + ( + "result/statement-confidence-negative", + "result-statement-minimal.json", + _set("documents.0.lines.0.line_code_confidence", -1), + False, + ), + ( + "result/statement-confidence-null", + "result-statement-minimal.json", + _set("documents.0.lines.0.line_code_confidence", None), + True, + ), + ( + "result/statement-confidence-bool", + "result-statement-minimal.json", + _set("documents.0.lines.0.values.0.confidence", True), + False, + ), + ( + "result/statement-confidence-100.0", + "result-statement-minimal.json", + _set("documents.0.lines.0.values.0.confidence", 100.0), + True, + ), + ( + "result/statement-report_date-bad", + "result-statement-minimal.json", + _set("documents.0.header.report_date", "2025-13-45"), + False, + ), + ( + "result/statement-report_date-null", + "result-statement-minimal.json", + _set("documents.0.header.report_date", None), + True, + ), + ( + "result/statement-value-no-column_index", + "result-statement-minimal.json", + _drop("documents.0.lines.0.values.0.column_index"), + False, + ), + ( + "result/statement-is_recognized-string", + "result-statement-minimal.json", + _set("documents.0.lines.0.values.0.is_recognized", "yes"), + False, + ), + ( + "result/statement-string_value-null", + "result-statement-minimal.json", + _set("documents.0.lines.0.values.0.string_value", None), + True, + ), + ( + "result/statement-period_date-bad", + "result-statement-minimal.json", + _set("documents.0.columns.0.period_date", "30.09.2025"), + False, + ), + ("result/statement-extra-field", "result-statement-minimal.json", _set("documents.0.header.fax", "1"), True), +] + + +@pytest.mark.parametrize(("case_id", "example", "mutate", "expected_valid"), CASES, ids=[c[0] for c in CASES]) +def test_model_and_schema_agree( + case_id: str, example: str, mutate: Callable[[dict[str, Any]], None], expected_valid: bool +) -> None: + kind = EXAMPLE_KIND[example] + payload = copy.deepcopy(_example(example)) + mutate(payload) + schema_verdict = _schema_ok(kind, payload) + model_verdict = _model_ok(kind, payload) + assert schema_verdict == expected_valid, ( + f"{case_id}: JSON Schema вынесла вердикт {schema_verdict}, ожидали {expected_valid}" + ) + assert model_verdict == schema_verdict, ( + f"{case_id}: модель {'приняла' if model_verdict else 'отвергла'}, схема " + f"{'приняла' if schema_verdict else 'отвергла'} — расхождение модели и схемы" + ) + + +def test_consistency_table_covers_every_kind() -> None: + assert {EXAMPLE_KIND[c[1]] for c in CASES} == set(MODELS) + + +# --- то, что модель проверяет строже валидатора схемы без дополнительных библиотек --------------------------------- + + +def test_reply_deadline_must_be_aware_and_is_normalized_to_utc() -> None: + naive = _example("request-tabular.json") + naive["reply_deadline"] = "2026-10-03T12:30:00" + with pytest.raises(ValidationError): + RecognitionRequest.model_validate(naive) + shifted = _example("request-tabular.json") + shifted["reply_deadline"] = "2026-10-03T15:30:00+03:00" + request = RecognitionRequest.model_validate(shifted) + assert request.reply_deadline == datetime(2026, 10, 3, 12, 30, tzinfo=UTC) + assert request.reply_deadline.utcoffset() == UTC.utcoffset(None) + assert request.to_wire()["reply_deadline"] == "2026-10-03T12:30:00Z" + + +def test_uuid_fields_are_uuid_objects() -> None: + request = RecognitionRequest.model_validate(_example("request-tabular.json")) + assert request.job_id == uuid.UUID("3f2a9c1e-7b4d-4e0a-9d51-2c8e6a1b0f01") + assert isinstance(request.file_id, uuid.UUID) + + +def test_message_status_enum_values() -> None: + assert [s.value for s in MessageStatus] == ["processing", "done", "needs_check", "error"] + message = RecognitionResultMessage.model_validate(_example("result-message-done.json")) + assert message.status is MessageStatus.DONE + + +def test_contract_version_constant() -> None: + assert CONTRACT_VERSION == "1.0" + + +def test_python_float_amount_is_rejected_with_clear_error() -> None: + payload = _example("result-statement-minimal.json") + payload["documents"][0]["lines"][0]["values"][0]["amount"] = 1234.5 + with pytest.raises(ValidationError, match="строк"): + RecognitionResult.model_validate(payload) + + +def test_decimal_object_input_is_accepted_and_dumped_as_string() -> None: + """Для кода, собирающего результат из `Decimal` (тесты, будущие производители): объект `Decimal` принимается.""" + payload = _example("result-statement-minimal.json") + payload["documents"][0]["lines"][0]["values"][0]["amount"] = Decimal("1234.50") + result = RecognitionResult.model_validate(payload) + assert result.to_wire()["documents"][0]["lines"][0]["values"][0]["amount"] == "1234.50" + + +# --- subject'ы, заголовок и идентификаторы сообщений --------------------------------------------------------------- + + +def test_request_msg_id() -> None: + job_id = uuid.UUID("3f2a9c1e-7b4d-4e0a-9d51-2c8e6a1b0f01") + assert request_msg_id(job_id, 1) == "3f2a9c1e-7b4d-4e0a-9d51-2c8e6a1b0f01:1" + assert request_msg_id(str(job_id), 3) == "3f2a9c1e-7b4d-4e0a-9d51-2c8e6a1b0f01:3" + + +def test_ack_and_result_msg_ids() -> None: + job_id = uuid.UUID("3f2a9c1e-7b4d-4e0a-9d51-2c8e6a1b0f01") + assert ack_msg_id(job_id, 1) == "ack:3f2a9c1e-7b4d-4e0a-9d51-2c8e6a1b0f01:1" + assert result_msg_id(job_id, 2) == "result:3f2a9c1e-7b4d-4e0a-9d51-2c8e6a1b0f01:2" + + +def test_request_subject_by_pipeline() -> None: + assert request_subject("tabular") == "recognition.tabular.request" + assert request_subject("image") == "recognition.image.request" + assert RESULT_SUBJECT == "recognition.result" + assert NATS_MSG_ID_HEADER == "Nats-Msg-Id" + + +def test_request_subject_rejects_unknown_pipeline() -> None: + with pytest.raises(ValueError, match="pipeline"): + request_subject("ocr") + + +def test_request_subject_matches_example_pipeline() -> None: + for name in ("request-tabular.json", "request-image.json"): + request = RecognitionRequest.model_validate(_example(name)) + assert request_subject(request.pipeline) == f"recognition.{request.pipeline}.request"