diff --git a/CLAUDE.md b/CLAUDE.md index 7fc949f..ca4d3d5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -159,7 +159,7 @@ - NATS subjects и схема ключей S3 — `../docs/context/03-architecture.md`. ## Правила репозитория -- ORM 1:1 с `../docs/model/data-model-v5.18.dbml` (текущая версия; миграции `0001_dbml_v5_13` + `0002_dbml_v5_14` + `0003_dbml_v5_15` + `0004_dbml_v5_16` + `0005_dbml_v5_17` + `0006_dbml_v5_18`); `api/scripts/check_dbml_vs_orm.py` (`make dbml-check`, входит в `check-full`) — обязательная проверка. +- ORM 1:1 с `../docs/model/data-model-v5.19.dbml` (текущая версия; миграции `0001_dbml_v5_13` + `0002_dbml_v5_14` + `0003_dbml_v5_15` + `0004_dbml_v5_16` + `0005_dbml_v5_17` + `0006_dbml_v5_18` + `0007_dbml_v5_19`); `api/scripts/check_dbml_vs_orm.py` (`make dbml-check`, входит в `check-full`) — обязательная проверка. - Алгоритмы — только из `../docs/context/05-domain-rules.md`; номер § — в docstring. - Вычисляемые объекты консолидации — `api/src/domain/`, без ORM-таблиц. - Фронт: пользователь слаб во фронте — для TS/React изменений делай ревью сам и объясняй решения в PR. diff --git a/Makefile b/Makefile index 65ebfcc..70b6548 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ VENV := api/.venv BIN := $(CURDIR)/$(VENV)/bin COMPOSE_FILE ?= ../docs/dev/docker-compose.yml # DBML относительно api/: и в основном workspace, и в worktree .wt/<срез>/ это соседний docs/ -DBML ?= ../../docs/model/data-model-v5.18.dbml +DBML ?= ../../docs/model/data-model-v5.19.dbml .PHONY: venv lint test test-integration e2e dbml-check migration-cycle check check-infra docker-build web-check check-full up migrate seed seed-dev diff --git a/api/alembic/versions/0007_dbml_v5_19.py b/api/alembic/versions/0007_dbml_v5_19.py new file mode 100644 index 0000000..3abb319 --- /dev/null +++ b/api/alembic/versions/0007_dbml_v5_19.py @@ -0,0 +1,192 @@ +"""DBML v5.19: единая нормализация форм ББ/ОФР (RSHB/arm-ka-docs#116, openspec unified-statement-normalization). + +Только добавляет: enum'ы, колонки строк/значений/колонок/шапок форм, `document.file_ordinal`, +`recognition_job.statements_version`, `related_group.crm_id`, таблицу `document_issue`, индексы; §7 по существующим +формам не считает (`document_issue` остаётся пустой — корректность нарушений у форм неочищенной базы не гарантируется +до первой правки). Заполнение по умолчанию (миграция проходит и на непустой базе): +- `row_index` — строки источника по возрастанию `line_code` внутри формы с 0; строки без + `line_code_original_value` (созданные правкой) — NULL; +- `needs_review` значений = НЕ (правлено или (`is_recognized` не false и уверенность 100 / NULL)), `source_text` = + `string_value`; `period_kind` колонок — `absolute` при `period_date`, иначе `none`; +- `header_source` — из `*_original_value` и `inn_confidence` (P3-1), `organization_name_original_value` = + `organization_name`, `organization_name_modified` = false (P3-2); `form_variant` = NULL (набор §7 — полный); +- `file_ordinal` — по `(created_at, id)` внутри файла с 0. +Уникальные индексы создаются после заполнения. Откат (`downgrade`) — задача A2 плана 3. + +Revision ID: 0007_dbml_v5_19 +Revises: 0006_dbml_v5_18 +Create Date: 2026-10-18 +""" + +from collections.abc import Sequence + +from alembic import op +from src.models.base import SCHEMA + +revision: str = "0007_dbml_v5_19" +down_revision: str | Sequence[str] | None = "0006_dbml_v5_18" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +S = SCHEMA + +ENUMS_V5_19: dict[str, tuple[str, ...]] = { + "statement_line_status": ("ok", "unreadable", "missing", "detail"), + "statement_form_variant": ("full", "simplified"), + "document_issue_kind": ("warning", "relation", "soft"), + "issue_severity": ("error", "info"), + "issue_source": ("service", "core"), +} + + +def _create_enums() -> None: + for name, labels in ENUMS_V5_19.items(): + values = ", ".join(f"'{label}'" for label in labels) + op.execute(f"CREATE TYPE {S}.{name} AS ENUM ({values})") + + +def _add_columns() -> None: + statements = [ + # financial_statement_line + f"ALTER TABLE {S}.financial_statement_line" + " ADD COLUMN row_index integer," + " ADD COLUMN source_ref varchar," + f" ADD COLUMN line_status {S}.statement_line_status NOT NULL DEFAULT 'ok'," + " ADD COLUMN line_code_text varchar," + " ADD COLUMN parent_row_index integer," + " ADD COLUMN needs_review boolean NOT NULL DEFAULT false," + " ADD COLUMN is_confirmed boolean NOT NULL DEFAULT false", + # financial_statement_value + f"ALTER TABLE {S}.financial_statement_value" + " ADD COLUMN source_text varchar," + " ADD COLUMN needs_review boolean NOT NULL DEFAULT false," + " ADD COLUMN is_confirmed boolean NOT NULL DEFAULT false," + " ADD COLUMN confirmed_by uuid" + f" CONSTRAINT fk_financial_statement_value_confirmed_by_app_user REFERENCES {S}.app_user (id)," + " ADD COLUMN confirmed_at timestamptz", + # financial_statement_column + f"ALTER TABLE {S}.financial_statement_column" + " ADD COLUMN period_kind varchar, ADD COLUMN label_confidence numeric(5,2)", + # financial_statement_meta + f"ALTER TABLE {S}.financial_statement_meta" + f" ADD COLUMN form_variant {S}.statement_form_variant," + " ADD COLUMN needs_review_fields jsonb," + " ADD COLUMN header_source jsonb," + " ADD COLUMN organization_name_original_value varchar," + " ADD COLUMN organization_name_modified boolean", + # document, recognition_job, related_group + f"ALTER TABLE {S}.document ADD COLUMN file_ordinal smallint NOT NULL DEFAULT 0", + f"ALTER TABLE {S}.recognition_job ADD COLUMN statements_version varchar", + f"ALTER TABLE {S}.related_group ADD COLUMN crm_id varchar CONSTRAINT uq_related_group_crm_id UNIQUE", + ] + for statement in statements: + op.execute(statement) + + +def _backfill() -> None: + # row_index: строки источника по возрастанию line_code внутри формы с 0; созданные правкой — NULL + op.execute( + f""" + UPDATE {S}.financial_statement_line l SET row_index = n.rn FROM ( + SELECT id, row_number() OVER (PARTITION BY meta_id ORDER BY line_code, id) - 1 AS rn + FROM {S}.financial_statement_line WHERE line_code_original_value IS NOT NULL) n WHERE l.id = n.id + """ + ) + op.execute( + f""" + UPDATE {S}.financial_statement_value SET source_text = string_value, + needs_review = NOT ( + is_modified OR (COALESCE(is_recognized, true) AND (confidence IS NULL OR confidence = 100))) + """ + ) + op.execute( + f""" + UPDATE {S}.financial_statement_column + SET period_kind = CASE WHEN period_date IS NULL THEN 'none' ELSE 'absolute' END + """ + ) + op.execute( + f""" + UPDATE {S}.financial_statement_meta SET + header_source = jsonb_build_object( + 'inn', jsonb_build_object('text', inn_original_value, 'confidence', inn_confidence), + 'report_date', jsonb_build_object('text', report_date_original_value, 'confidence', NULL), + 'units', jsonb_build_object('text', units_original_value, 'confidence', NULL), + 'organization_name', jsonb_build_object('text', organization_name, 'confidence', NULL)), + organization_name_original_value = organization_name, + organization_name_modified = false + """ + ) + op.execute( + f""" + UPDATE {S}.document d SET file_ordinal = n.rn FROM ( + SELECT id, row_number() OVER (PARTITION BY stored_file_id ORDER BY created_at, id) - 1 AS rn + FROM {S}.document) n WHERE d.id = n.id + """ + ) + + +def _indexes() -> None: + # старый уникальный индекс (meta_id, line_code) → обычный: повтор кода законен + op.execute(f"DROP INDEX {S}.uq_financial_statement_line_meta_id_line_code") + op.execute( + "CREATE INDEX ix_financial_statement_line_meta_id_line_code " + f"ON {S}.financial_statement_line (meta_id, line_code)" + ) + op.execute( + "CREATE UNIQUE INDEX uq_financial_statement_line_meta_id_source_ref " + f"ON {S}.financial_statement_line (meta_id, source_ref) WHERE source_ref IS NOT NULL" + ) + # уникальность типа — только для не-форм; формы ББ/ОФР одного типа могут повторяться (Р-10) + op.execute(f"DROP INDEX {S}.uq_document_file_type_account") + op.execute( + "CREATE UNIQUE INDEX uq_document_file_type_account " + f"ON {S}.document (stored_file_id, document_type, osv_account_code) NULLS NOT DISTINCT " + "WHERE document_type NOT IN ('BALANCE', 'INCOME')" + ) + op.execute( + f"CREATE UNIQUE INDEX uq_document_stored_file_id_file_ordinal ON {S}.document (stored_file_id, file_ordinal)" + ) + + +def _create_document_issue() -> None: + op.execute( + f""" + CREATE TABLE {S}.document_issue ( + id uuid NOT NULL, + document_id uuid NOT NULL, + kind {S}.document_issue_kind NOT NULL, + code varchar NOT NULL, + severity {S}.issue_severity NOT NULL, + message text NOT NULL, + line_ids jsonb, + column_id uuid, + column_index smallint, + codes jsonb, + field varchar, + source {S}.issue_source NOT NULL, + relation varchar, + expected numeric(18,2), + actual numeric(18,2), + created_at timestamptz NOT NULL, + CONSTRAINT pk_document_issue PRIMARY KEY (id), + CONSTRAINT fk_document_issue_document_id_document FOREIGN KEY (document_id) REFERENCES {S}.document (id), + CONSTRAINT fk_document_issue_column_id_financial_statement_column + FOREIGN KEY (column_id) REFERENCES {S}.financial_statement_column (id) + ) + """ + ) + op.execute(f"CREATE INDEX ix_document_issue_document_id ON {S}.document_issue (document_id)") + + +def upgrade() -> None: + _create_enums() + _add_columns() + _backfill() + _indexes() + _create_document_issue() + + +def downgrade() -> None: + # откат (охранные проверки, потеря данных, журнал) — задача A2 плана 3 + raise NotImplementedError("downgrade 0007_dbml_v5_19 ещё не реализован (задача A2 плана 3)") diff --git a/api/scripts/dbml_parity.py b/api/scripts/dbml_parity.py index 4c8dac5..4ff47a4 100644 --- a/api/scripts/dbml_parity.py +++ b/api/scripts/dbml_parity.py @@ -57,7 +57,7 @@ from sqlalchemy.sql.functions import FunctionElement #: Эталонная схема модели. Имя схемы в БД конфигурируемо (`DB_SCHEMA`), CLI сверяет ORM, загруженную с этим значением. SCHEMA = "dkb" -DBML_FILE_NAME = "data-model-v5.18.dbml" +DBML_FILE_NAME = "data-model-v5.19.dbml" ORM = "ORM" DBML = "DBML" ABSENT = "нет" diff --git a/api/src/models/__init__.py b/api/src/models/__init__.py index 534245d..277ec44 100644 --- a/api/src/models/__init__.py +++ b/api/src/models/__init__.py @@ -1,4 +1,4 @@ -"""ORM-модели. Источник истины — docs/model/data-model-v5.18.dbml; заполняется в Срезе 0. +"""ORM-модели. Источник истины — docs/model/data-model-v5.19.dbml; заполняется в Срезе 0. Импорт всех модулей моделей обязателен: он регистрирует таблицы в `Base.metadata` (create_all, alembic). """ @@ -7,7 +7,7 @@ from . import access, accounting, audit_ex, consolidation, directories, files, k from .base import Base, UUIDPrimaryMixin #: Версия DBML, которой соответствуют модели (сверяется с шапкой DBML в check_dbml_vs_orm). -DBML_VERSION = "5.18" +DBML_VERSION = "5.19" __all__ = [ "DBML_VERSION", diff --git a/api/src/models/directories.py b/api/src/models/directories.py index 7c64d69..d71249c 100644 --- a/api/src/models/directories.py +++ b/api/src/models/directories.py @@ -141,6 +141,8 @@ class RelatedGroup(UUIDPrimaryMixin, Base): kind: Mapped[RelatedGroupKind] = mapped_column(pg_enum("related_group_kind"), nullable=False) name: Mapped[str] = mapped_column(String, nullable=False) created_date: Mapped[datetime.date | None] = mapped_column(Date) + # v5.19: ID ГСК/ГСЗ в ЕЦП CRM; вводится вручную, NULL — не задан + crm_id: Mapped[str | None] = mapped_column(String, unique=True) class RelatedGroupMember(UUIDPrimaryMixin, Base): diff --git a/api/src/models/enums.py b/api/src/models/enums.py index 7279617..1e529a3 100644 --- a/api/src/models/enums.py +++ b/api/src/models/enums.py @@ -1,6 +1,6 @@ -"""Enum'ы модели данных DBML v5.18 (28 штук) и фабрика `pg_enum` для PostgreSQL-колонок. +"""Enum'ы модели данных DBML v5.19 (33 штуки) и фабрика `pg_enum` для PostgreSQL-колонок. -Источник истины — docs/model/data-model-v5.18.dbml (блоки `Enum {}`). Значения — строки DBML буквально, в порядке +Источник истины — docs/model/data-model-v5.19.dbml (блоки `Enum {}`). Значения — строки DBML буквально, в порядке DBML (включая кириллические `"ЮЛ"`/`"ИП"` и значения в разном регистре); `notes` DBML значениями не являются. Имя члена совпадает со значением (все значения DBML — валидные python-идентификаторы, кириллица допустима). Классы — `enum.StrEnum` (подкласс `str`) — удобно для pydantic/JSON; `values_callable` в `pg_enum` в БД пишет `.value`. @@ -136,6 +136,44 @@ class StatementType(enum.StrEnum): income = "income" +class StatementLineStatus(enum.StrEnum): + """DBML `statement_line_status` (v5.19): статус строки формы; `line_code` пуст ⇔ статус не `ok`.""" + + ok = "ok" + unreadable = "unreadable" + missing = "missing" + detail = "detail" + + +class StatementFormVariant(enum.StrEnum): + """DBML `statement_form_variant` (v5.19): вариант формы по структуре кодов (§11.7).""" + + full = "full" + simplified = "simplified" + + +class DocumentIssueKind(enum.StrEnum): + """DBML `document_issue_kind` (v5.19): вид записи `document_issue`.""" + + warning = "warning" + relation = "relation" + soft = "soft" + + +class IssueSeverity(enum.StrEnum): + """DBML `issue_severity` (v5.19): `error` — документ `needs_check`, `info` — на статус не влияет.""" + + error = "error" + info = "info" + + +class IssueSource(enum.StrEnum): + """DBML `issue_source` (v5.19): откуда запись — из результата распознавания или посчитана ядром.""" + + service = "service" + core = "core" + + class OsvGroupType(enum.StrEnum): """DBML `osv_group_type`.""" @@ -324,6 +362,11 @@ ENUMS: dict[str, type[enum.Enum]] = { "s3_bucket_role": S3BucketRole, "recognition_status": RecognitionStatus, "statement_type": StatementType, + "statement_line_status": StatementLineStatus, + "statement_form_variant": StatementFormVariant, + "document_issue_kind": DocumentIssueKind, + "issue_severity": IssueSeverity, + "issue_source": IssueSource, "osv_group_type": OsvGroupType, "counterparty_match_kind": CounterpartyMatchKind, "osv_data_type": OsvDataType, diff --git a/api/src/models/files.py b/api/src/models/files.py index 42a2e25..7006ef1 100644 --- a/api/src/models/files.py +++ b/api/src/models/files.py @@ -1,4 +1,4 @@ -"""Часть 04 DBML v5.18 «Файлы / распознавание»: реестр S3, физические файлы, документы, задания распознавания. +"""Часть 04 DBML v5.19 «Файлы / распознавание»: реестр S3, физические файлы, документы, задания распознавания. Цикл FK `stored_file.current_recognition_job_id` -> `recognition_job` и `recognition_job.file_id` -> `stored_file` разорван через `use_alter=True` на nullable-стороне (`stored_file.current_recognition_job_id`): `create_all` создаёт @@ -91,7 +91,8 @@ class Document(UUIDPrimaryMixin, Base): __tablename__ = "document" __table_args__ = ( - # один ББ / один ОФР / одна ОСВ по счёту на файл: NULL в osv_account_code считается значением (PG15+) + # v5.19: одна ОСВ по счёту / одна расшифровка / один прочий на файл; формы ББ/ОФР одного типа могут + # повторяться (Р-10). NULL в osv_account_code считается значением (PG15+) Index( "uq_document_file_type_account", "stored_file_id", @@ -99,7 +100,10 @@ class Document(UUIDPrimaryMixin, Base): "osv_account_code", unique=True, postgresql_nulls_not_distinct=True, + postgresql_where=text("document_type NOT IN ('BALANCE', 'INCOME')"), ), + # v5.19: номер документа в documents[] результата уникален в файле + Index("uq_document_stored_file_id_file_ordinal", "stored_file_id", "file_ordinal", unique=True), # чек-лист required_document_rule Index( "ix_document_legal_entity_period_status", @@ -130,6 +134,8 @@ class Document(UUIDPrimaryMixin, Base): confirmed_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True)) created_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True)) updated_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True)) + # v5.19: индекс документа в documents[] результата (с 0); несозданный документ номер не занимает + file_ordinal: Mapped[int] = mapped_column(SmallInteger, nullable=False, server_default=text("0")) class RecognitionJob(UUIDPrimaryMixin, Base): @@ -164,3 +170,5 @@ class RecognitionJob(UUIDPrimaryMixin, Base): fields_total: Mapped[int | None] = mapped_column(Integer) fields_low_confidence: Mapped[int | None] = mapped_column(Integer) fields_unrecognized: Mapped[int | None] = mapped_column(Integer) + # v5.19: service.statements_version результата — коммит копии общего кода arm_statements + statements_version: Mapped[str | None] = mapped_column(String) diff --git a/api/src/models/statements.py b/api/src/models/statements.py index 2a1d04d..20d1d09 100644 --- a/api/src/models/statements.py +++ b/api/src/models/statements.py @@ -1,19 +1,29 @@ -"""Часть 06 DBML v5.18 «Бухгалтерская отчётность»: шапка, колонки-периоды, строки и значения форм ББ/ОФР (157н). +"""Часть 06 DBML v5.19 «Бухгалтерская отчётность»: шапка, колонки-периоды, строки и значения форм ББ/ОФР (157н). Первичка не мутируется: `*_original_value` / `original_amount` / `is_modified` — рядом с распознанным значением. -Логические проверки (ТЗ 6.3) и «требует проверки» не хранятся — считаются на лету. +v5.19: предупреждения и нарушения §7 хранятся в `document_issue`, `needs_review` строк и значений — в колонках; +повтор кода в форме законен (строки хранятся как в источнике). """ import datetime import decimal import uuid +from typing import Any -from sqlalchemy import Boolean, Date, DateTime, ForeignKey, Index, Numeric, SmallInteger, String, Text, text +from sqlalchemy import Boolean, Date, DateTime, ForeignKey, Index, Integer, Numeric, SmallInteger, String, Text, text from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.orm import Mapped, mapped_column -from .base import Base, UUIDPrimaryMixin -from .enums import StatementType, pg_enum +from .base import Base, UUIDPrimaryMixin, nullable_jsonb +from .enums import ( + DocumentIssueKind, + IssueSeverity, + IssueSource, + StatementFormVariant, + StatementLineStatus, + StatementType, + pg_enum, +) class FinancialStatementMeta(UUIDPrimaryMixin, Base): @@ -57,6 +67,12 @@ class FinancialStatementMeta(UUIDPrimaryMixin, Base): checks_accepted_by: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("app_user.id")) checks_accepted_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True)) checks_accepted_comment: Mapped[str | None] = mapped_column(Text) + # v5.19: порядок колонок = порядок ADD COLUMN миграции 0007 (check_dbml сверяет порядок) + form_variant: Mapped[StatementFormVariant | None] = mapped_column(pg_enum("statement_form_variant")) + needs_review_fields: Mapped[Any | None] = mapped_column(nullable_jsonb()) + header_source: Mapped[Any | None] = mapped_column(nullable_jsonb()) + organization_name_original_value: Mapped[str | None] = mapped_column(String) + organization_name_modified: Mapped[bool | None] = mapped_column(Boolean) class FinancialStatementColumn(UUIDPrimaryMixin, Base): @@ -74,13 +90,25 @@ class FinancialStatementColumn(UUIDPrimaryMixin, Base): label: Mapped[str | None] = mapped_column(String) period_date: Mapped[datetime.date | None] = mapped_column(Date) is_current: Mapped[bool | None] = mapped_column(Boolean) + period_kind: Mapped[str | None] = mapped_column(String) + label_confidence: Mapped[decimal.Decimal | None] = mapped_column(Numeric(5, 2)) class FinancialStatementLine(UUIDPrimaryMixin, Base): """Строка формы с кодом; порядок, разделы и итоговые строки — шаблон формы на фронте.""" __tablename__ = "financial_statement_line" - __table_args__ = (Index("uq_financial_statement_line_meta_id_line_code", "meta_id", "line_code", unique=True),) + __table_args__ = ( + # v5.19: повтор кода в форме законен — индекс не уникален (чтение «по коду» — агрегат формы) + Index("ix_financial_statement_line_meta_id_line_code", "meta_id", "line_code"), + Index( + "uq_financial_statement_line_meta_id_source_ref", + "meta_id", + "source_ref", + unique=True, + postgresql_where=text("source_ref IS NOT NULL"), + ), + ) meta_id: Mapped[uuid.UUID] = mapped_column( UUID(as_uuid=True), ForeignKey("financial_statement_meta.id"), nullable=False @@ -93,6 +121,16 @@ class FinancialStatementLine(UUIDPrimaryMixin, Base): line_name_original_value: Mapped[str | None] = mapped_column(String) line_name_confidence: Mapped[decimal.Decimal | None] = mapped_column(Numeric(5, 2)) line_name_modified: Mapped[bool | None] = mapped_column(Boolean) + # v5.19: `row_index IS NULL` ⇔ строка создана правкой (шаблонная); у строк из результата всегда задан + row_index: Mapped[int | None] = mapped_column(Integer) + source_ref: Mapped[str | None] = mapped_column(String) + line_status: Mapped[StatementLineStatus] = mapped_column( + pg_enum("statement_line_status"), nullable=False, server_default=text("'ok'") + ) + line_code_text: Mapped[str | None] = mapped_column(String) + parent_row_index: Mapped[int | None] = mapped_column(Integer) + needs_review: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("false")) + is_confirmed: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("false")) class FinancialStatementValue(UUIDPrimaryMixin, Base): @@ -117,3 +155,37 @@ class FinancialStatementValue(UUIDPrimaryMixin, Base): original_amount: Mapped[decimal.Decimal | None] = mapped_column(Numeric(18, 2)) modified_by: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("app_user.id")) modified_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True)) + # v5.19 + source_text: Mapped[str | None] = mapped_column(String) + needs_review: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("false")) + is_confirmed: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("false")) + confirmed_by: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("app_user.id")) + confirmed_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True)) + + +class DocumentIssue(UUIDPrimaryMixin, Base): + """v5.19: предупреждение или нарушение §7 документа с привязкой к строкам, ячейкам и полям шапки. + + Удаляется вместе с документом; при пересчёте после правки `relation` / `soft` заменяются целиком, `warning` + источника `service` сохраняются (кроме снимаемых правкой). `line_ids` — без FK: ядро убирает id удалённой + шаблонной строки само. + """ + + __tablename__ = "document_issue" + __table_args__ = (Index("ix_document_issue_document_id", "document_id"),) + + document_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("document.id"), nullable=False) + kind: Mapped[DocumentIssueKind] = mapped_column(pg_enum("document_issue_kind"), nullable=False) + code: Mapped[str] = mapped_column(String, nullable=False) + severity: Mapped[IssueSeverity] = mapped_column(pg_enum("issue_severity"), nullable=False) + message: Mapped[str] = mapped_column(Text, nullable=False) + line_ids: Mapped[Any | None] = mapped_column(nullable_jsonb()) + column_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("financial_statement_column.id")) + column_index: Mapped[int | None] = mapped_column(SmallInteger) + codes: Mapped[Any | None] = mapped_column(nullable_jsonb()) + field: Mapped[str | None] = mapped_column(String) + source: Mapped[IssueSource] = mapped_column(pg_enum("issue_source"), nullable=False) + relation: Mapped[str | None] = mapped_column(String) + expected: Mapped[decimal.Decimal | None] = mapped_column(Numeric(18, 2)) + actual: Mapped[decimal.Decimal | None] = mapped_column(Numeric(18, 2)) + created_at: Mapped[datetime.datetime] = mapped_column(DateTime(timezone=True), nullable=False) diff --git a/api/src/repositories/statements.py b/api/src/repositories/statements.py index 3d8d8de..ac4debb 100644 --- a/api/src/repositories/statements.py +++ b/api/src/repositories/statements.py @@ -14,6 +14,7 @@ from src.domain.statement_mapping import StatementDraft from src.models.enums import DocumentStatus, DocumentType, FileStatus from src.models.files import Document, StoredFile from src.models.statements import ( + DocumentIssue, FinancialStatementColumn, FinancialStatementLine, FinancialStatementMeta, @@ -107,8 +108,9 @@ async def insert_statement( async def delete_file_statements(session: AsyncSession, file_id: uuid.UUID) -> None: - """Значения, строки, колонки и шапки форм всех документов файла (в порядке FK).""" + """`document_issue`, значения, строки, колонки и шапки форм всех документов файла (в порядке FK).""" documents = select(Document.id).where(Document.stored_file_id == file_id).scalar_subquery() + await session.execute(delete(DocumentIssue).where(DocumentIssue.document_id.in_(documents))) metas = select(FinancialStatementMeta.id).where(FinancialStatementMeta.document_id.in_(documents)).scalar_subquery() lines = select(FinancialStatementLine.id).where(FinancialStatementLine.meta_id.in_(metas)).scalar_subquery() await session.execute(delete(FinancialStatementValue).where(FinancialStatementValue.line_id.in_(lines))) diff --git a/api/src/services/result_mapping.py b/api/src/services/result_mapping.py index 61bdf40..cf3977d 100644 --- a/api/src/services/result_mapping.py +++ b/api/src/services/result_mapping.py @@ -532,6 +532,9 @@ async def map_file( # noqa: PLR0913 - все входы маппинга фай documents.extend(other_documents) documents.extend(document for document, _ in forms) + # v5.19: номер документа в файле уникален; до перехода на индекс в documents[] (задачи A5/A9) — порядок создания + for ordinal, document in enumerate(documents): + document.file_ordinal = ordinal session.add_all(documents) await session.flush() # T7.2: родитель раньше потомка на любой границе пачки (self-FK `parent_entry_id`) — не полагаемся на порядок строк diff --git a/api/tests/factories.py b/api/tests/factories.py index 7de3d6a..c70b270 100644 --- a/api/tests/factories.py +++ b/api/tests/factories.py @@ -54,11 +54,14 @@ from src.models.enums import ( ConsolidationScenario, ConsolidationStatus, DocumentCheckResult, + DocumentIssueKind, DocumentStatus, DocumentType, EntityType, FileStatus, FileType, + IssueSeverity, + IssueSource, OsvDataType, OsvGroupType, PeriodQuarter, @@ -69,6 +72,7 @@ from src.models.enums import ( ) from src.models.files import Document, RecognitionJob, S3Object, StoredFile from src.models.osv import OsvEntry +from src.models.statements import DocumentIssue from src.services.result_mapping import MAPPING_WARNINGS_KEY, _entry_values, _group_schema from tests.conftest import FakeStorage @@ -265,10 +269,17 @@ async def make_document( # noqa: PLR0913 - сигнатура фабрики ф """ assert job.file_id == file.id # noqa: S101 - защита от перепутанных аргументов в тестах now = _now() + # v5.19: file_ordinal уникален в файле — берём следующий свободный + ordinal = ( + await session.execute( + select(func.coalesce(func.max(Document.file_ordinal) + 1, 0)).where(Document.stored_file_id == file.id) + ) + ).scalar_one() document = Document( stored_file_id=file.id, legal_entity_id=file.legal_entity_id, document_type=DocumentType.OSV, + file_ordinal=ordinal, osv_account_code=account, period_year=year, period_quarter=quarter, @@ -282,6 +293,22 @@ async def make_document( # noqa: PLR0913 - сигнатура фабрики ф return document +async def make_document_issue(session: AsyncSession, *, document_id: uuid.UUID, code: str = "MAPPING_WARNING") -> Any: + """Запись `document_issue` (предупреждение ядра уровня `info`) документа (`flush`).""" + issue = DocumentIssue( + document_id=document_id, + kind=DocumentIssueKind.warning, + code=code, + severity=IssueSeverity.info, + message="тестовое предупреждение", + source=IssueSource.core, + created_at=_now(), + ) + session.add(issue) + await session.flush() + return issue + + async def make_result_object( session: AsyncSession, *, job: RecognitionJob, storage: FakeStorage, result: dict[str, Any] ) -> S3Object: @@ -337,6 +364,7 @@ async def make_entries_from_result( stored_file_id=file.id, legal_entity_id=file.legal_entity_id, document_type=DocumentType.OSV, + file_ordinal=len(documents), osv_account_code=fields.osv_account_code, period_year=fields.period_year, period_quarter=fields.period_quarter, diff --git a/api/tests/test_api_files_delete.py b/api/tests/test_api_files_delete.py index 7273dc5..9dd722e 100644 --- a/api/tests/test_api_files_delete.py +++ b/api/tests/test_api_files_delete.py @@ -34,6 +34,7 @@ from src.models.enums import ( from src.models.files import Document, RecognitionJob, S3Object, StoredFile from src.models.osv import OsvEntry from src.models.statements import ( + DocumentIssue, FinancialStatementColumn, FinancialStatementLine, FinancialStatementMeta, @@ -45,6 +46,7 @@ from tests.factories import ( make_company_group, make_consolidation, make_document, + make_document_issue, make_entries_from_result, make_legal_entity, make_result_object, @@ -146,7 +148,13 @@ async def _form_file_with_comment(session: AsyncSession, fixtures_dir: Path) -> async def _assert_form_rows_gone(session: AsyncSession, file_id: uuid.UUID, document_ids: list[uuid.UUID]) -> None: assert await _count(session, StoredFile, StoredFile.id == file_id) == 0 assert await _count(session, Document, Document.stored_file_id == file_id) == 0 - for model in (FinancialStatementMeta, FinancialStatementColumn, FinancialStatementLine, FinancialStatementValue): + for model in ( + DocumentIssue, + FinancialStatementMeta, + FinancialStatementColumn, + FinancialStatementLine, + FinancialStatementValue, + ): assert await _count(session, model) == 0, model.__tablename__ assert await _count(session, Comment, Comment.object_id.in_(document_ids)) == 0 @@ -161,6 +169,8 @@ async def test_delete_file_with_statement_form( file_id = await _form_file_with_comment(seeded, fixtures_dir) document_ids = list((await seeded.execute(select(Document.id).where(Document.stored_file_id == file_id))).scalars()) assert await _count(seeded, FinancialStatementValue) > 0 and await _count(seeded, Comment) == 1 + await make_document_issue(seeded, document_id=document_ids[0]) # v5.19: document_issue уходит вместе с файлом + assert await _count(seeded, DocumentIssue) == 1 if superseded: await seeded.execute(update(StoredFile).where(StoredFile.id == file_id).values(status=FileStatus.SUPERSEDED)) await seeded.flush() diff --git a/api/tests/test_check_dbml_script.py b/api/tests/test_check_dbml_script.py index 502ae5a..dcc623c 100644 --- a/api/tests/test_check_dbml_script.py +++ b/api/tests/test_check_dbml_script.py @@ -140,18 +140,18 @@ def test_enum_values_and_sets_are_compared(dbml_file: Path) -> None: def test_newer_dbml_version_fails_before_comparing_tables(tmp_path: Path, dbml_file: Path) -> None: lines = dbml_file.read_text(encoding="utf-8").splitlines() - newer = tmp_path / "data-model-v5.19.dbml" - # шапка 5.18 + заведомо сломанное тело: если бы таблицы сравнивались, были бы другие строки/ошибка разбора - newer.write_text(lines[0].replace("v5.18", "v5.19") + "\nTable broken {{{\n", encoding="utf-8") - assert run_check(newer) == (1, ["версия DBML 5.19 не совпадает с версией ORM 5.18: обновите модели"]) + newer = tmp_path / "data-model-v5.20.dbml" + # шапка 5.19 + заведомо сломанное тело: если бы таблицы сравнивались, были бы другие строки/ошибка разбора + newer.write_text(lines[0].replace("v5.19", "v5.20") + "\nTable broken {{{\n", encoding="utf-8") + assert run_check(newer) == (1, ["версия DBML 5.20 не совпадает с версией ORM 5.19: обновите модели"]) def test_newer_dbml_version_via_cli(tmp_path: Path, dbml_file: Path) -> None: - newer = tmp_path / "data-model-v5.19.dbml" - newer.write_text(dbml_file.read_text(encoding="utf-8").replace("v5.18", "v5.19", 1), encoding="utf-8") + newer = tmp_path / "data-model-v5.20.dbml" + newer.write_text(dbml_file.read_text(encoding="utf-8").replace("v5.19", "v5.20", 1), encoding="utf-8") result = _run_cli(str(newer)) assert result.returncode == 1 - assert result.stdout == "версия DBML 5.19 не совпадает с версией ORM 5.18: обновите модели\n" + assert result.stdout == "версия DBML 5.20 не совпадает с версией ORM 5.19: обновите модели\n" def test_header_without_version_is_usage_error(tmp_path: Path) -> None: diff --git a/api/tests/test_dbml_parity.py b/api/tests/test_dbml_parity.py index 883ba25..ca63e7b 100644 --- a/api/tests/test_dbml_parity.py +++ b/api/tests/test_dbml_parity.py @@ -193,7 +193,7 @@ def test_nulls_not_distinct_flag_is_compared_both_ways() -> None: """DBML-note `NULLS NOT DISTINCT` требует флаг в ORM-индексе; лишний флаг без note — тоже расхождение.""" metadata = clone_metadata() document = metadata.tables["dkb.document"] - index = next(i for i in document.indexes if i.unique) + index = next(i for i in document.indexes if i.name == "uq_document_file_type_account") assert find_problems(metadata, {"document"}) == [] index.dialect_options["postgresql"]["nulls_not_distinct"] = None # индекс без флага problems = find_problems(metadata, {"document"}) diff --git a/api/tests/test_enums.py b/api/tests/test_enums.py index 6b13db3..66b2738 100644 --- a/api/tests/test_enums.py +++ b/api/tests/test_enums.py @@ -1,4 +1,4 @@ -"""Enum'ы ORM против блоков `Enum {}` DBML v5.18: имена, значения и их порядок (без БД).""" +"""Enum'ы ORM против блоков `Enum {}` DBML v5.19: имена, значения и их порядок (без БД).""" import enum import re @@ -10,12 +10,12 @@ from src.models import DBML_VERSION from src.models.enums import ENUMS, pg_enum from tests.dbml_helpers import dbml_path, load_dbml -EXPECTED_ENUM_COUNT = 28 +EXPECTED_ENUM_COUNT = 33 def test_dbml_version_format() -> None: assert re.fullmatch(r"\d+\.\d+", DBML_VERSION) - assert DBML_VERSION == "5.18" + assert DBML_VERSION == "5.19" assert f"v{DBML_VERSION}" in dbml_path().name diff --git a/api/tests/test_mapping_statements.py b/api/tests/test_mapping_statements.py index f43d832..cc1d426 100644 --- a/api/tests/test_mapping_statements.py +++ b/api/tests/test_mapping_statements.py @@ -22,6 +22,7 @@ from src.models.enums import ( ) from src.models.files import Document, RecognitionJob, StoredFile from src.models.statements import ( + DocumentIssue, FinancialStatementColumn, FinancialStatementLine, FinancialStatementMeta, @@ -29,7 +30,7 @@ from src.models.statements import ( ) from src.repositories.counterparties import CounterpartyIndex from src.services.result_mapping import MAPPING_WARNINGS_KEY, map_file -from tests.factories import make_legal_entity +from tests.factories import make_document_issue, make_legal_entity from tests.fixtures_helpers import GOLDEN_STATEMENTS, OCR_EXPECTED, golden_result, map_golden_snapshot, ocr_result pytestmark = [pytest.mark.integration, pytest.mark.fixtures] @@ -291,6 +292,7 @@ async def test_repeated_form_mapping_is_idempotent(seeded: AsyncSession, fixture documents = await map_golden_snapshot(seeded, entity=owner, path=path, uploaded_by=None) before = await _statement_totals(seeded) file_id = documents[0].stored_file_id + await make_document_issue(seeded, document_id=documents[0].id) # v5.19: остаток прежнего маппинга file = await seeded.get(StoredFile, file_id) assert file is not None and file.current_recognition_job_id is not None job = await seeded.get(RecognitionJob, file.current_recognition_job_id) @@ -311,6 +313,7 @@ async def test_repeated_form_mapping_is_idempotent(seeded: AsyncSession, fixture ) assert await _statement_totals(seeded) == before + assert (await seeded.execute(select(func.count()).select_from(DocumentIssue))).scalar_one() == 0 again = (await seeded.execute(select(Document).where(Document.stored_file_id == file_id))).scalars().all() assert {d.document_type for d in again} == {DocumentType.BALANCE, DocumentType.INCOME} assert {d.id for d in again}.isdisjoint({d.id for d in documents}) # документы пересозданы, не дублированы diff --git a/api/tests/test_migrations.py b/api/tests/test_migrations.py index 020b0fb..0faed3f 100644 --- a/api/tests/test_migrations.py +++ b/api/tests/test_migrations.py @@ -1,4 +1,5 @@ -"""Миграции 0001 (DBML v5.13), 0002 (v5.14), 0003 (v5.15), 0004 (v5.16), 0005 (v5.17), 0006 (v5.18) на PostgreSQL 16: +"""Миграции 0001 (DBML v5.13), 0002 (v5.14), 0003 (v5.15), 0004 (v5.16), 0005 (v5.17), 0006 (v5.18), 0007 (v5.19) +на PostgreSQL 16: полный цикл, откат на шаг, пустой autogenerate, схема = схеме ORM. Spec data-model, требование «Миграции применяются с нуля и откатываются» (сценарии «Полный цикл» и «Автогенерация @@ -15,7 +16,7 @@ import pytest from alembic.autogenerate import compare_metadata from alembic.runtime.migration import MigrationContext from sqlalchemy import Connection, make_url, text -from sqlalchemy.exc import ProgrammingError +from sqlalchemy.exc import IntegrityError, ProgrammingError from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, AsyncSession, create_async_engine from src.models import Base @@ -36,7 +37,8 @@ from tests.test_models_part_07_09 import ( _check_active_lock_is_unique, ) -HEAD = "0006_dbml_v5_18" +HEAD = "0007_dbml_v5_19" +V5_18 = "0006_dbml_v5_18" V5_17 = "0005_dbml_v5_17" V5_16 = "0004_dbml_v5_16" V5_15 = "0003_dbml_v5_15" @@ -620,6 +622,253 @@ async def test_upgrade_0006_backfills_consolidation_id(scratch_db_url: str) -> N await engine.dispose() +async def _seed_v5_18_statements(conn: AsyncConnection, ids: dict[str, uuid.UUID]) -> None: + """Данные на ревизии 0006: ЮЛ, файл, `INCOME` (t0) и `BALANCE` (t1 > t0) с формой ББ; ГСК без `crm_id`.""" + t0 = datetime.datetime(2025, 10, 1, 9, 0, tzinfo=datetime.UTC) + t1 = t0 + datetime.timedelta(minutes=5) + await conn.execute( + text("INSERT INTO dkb.legal_entity (id, inn, name, valid_from) VALUES (:le, '7701234567', 'ООО Ромашка', :t0)"), + {**ids, "t0": t0}, + ) + await conn.execute( + text( + "INSERT INTO dkb.s3_object (id, bucket_role, bucket, key, created_at) " + "VALUES (:s3, 'uploads', 'b', 'k', :t0)" + ), + {**ids, "t0": t0}, + ) + await conn.execute( + text( + "INSERT INTO dkb.stored_file (id, legal_entity_id, file_type, s3_object_id, status) " + "VALUES (:file, :le, 'XLSX', :s3, 'PROCESSED')" + ), + ids, + ) + await conn.execute( + text( + "INSERT INTO dkb.recognition_job (id, file_id, pipeline, request_subject, schema_version, status) " + "VALUES (:job, :file, 'tabular', 'files.tabular_uploaded', '1', 'done')" + ), + ids, + ) + for key, doc_type, created in (("income", "INCOME", t0), ("balance", "BALANCE", t1)): + await conn.execute( + text( + "INSERT INTO dkb.document (id, stored_file_id, legal_entity_id, document_type, status, created_at) " + f"VALUES (:{key}, :file, :le, '{doc_type}', 'NEEDS_CHECK', :created)" + ), + {**ids, "created": created}, + ) + await conn.execute( + text( + "INSERT INTO dkb.financial_statement_meta (id, document_id, recognition_job_id, statement_type, inn, " + "inn_original_value, inn_confidence, report_date_original_value, units_original_value, organization_name) " + "VALUES (:meta, :balance, :job, 'balance', '7701234567', '7701234567', 87, '30.09.2025', 'тыс. руб.', " + "'ООО Ромашка')" + ), + ids, + ) + for key, index, period in (("c1", 0, datetime.date(2025, 9, 30)), ("c2", 1, None)): + await conn.execute( + text( + "INSERT INTO dkb.financial_statement_column (id, meta_id, column_index, period_date) " + "VALUES (:id, :meta, :idx, :period)" + ), + {"id": ids[key], "meta": ids["meta"], "idx": index, "period": period}, + ) + for key, code, original in (("l1600", "1600", "1600"), ("l1100", "1100", "1100"), ("l1250", "1250", None)): + await conn.execute( + text( + "INSERT INTO dkb.financial_statement_line (id, meta_id, line_code, line_code_original_value) " + "VALUES (:id, :meta, :code, :original)" + ), + {"id": ids[key], "meta": ids["meta"], "code": code, "original": original}, + ) + values = [ + # (строка, колонка, сумма, строка-значение, уверенность, распознано, правлено) + ("l1100", "c1", 1, None, 100, True, False), + ("l1100", "c2", 2, None, 80, True, False), + ("l1600", "c1", 3, None, 80, True, True), + ("l1600", "c2", 4, None, None, False, False), + ("l1250", "c1", 5, "12 3", None, None, False), + ] + for line, column, amount, text_value, confidence, recognized, modified in values: + await conn.execute( + text( + "INSERT INTO dkb.financial_statement_value (id, line_id, column_id, amount, string_value, confidence, " + "is_recognized, is_modified) VALUES (:id, :line, :column, :amount, :sv, :conf, :rec, :mod)" + ), + { + "id": uuid.uuid4(), + "line": ids[line], + "column": ids[column], + "amount": amount, + "sv": text_value, + "conf": confidence, + "rec": recognized, + "mod": modified, + }, + ) + await conn.execute( + text("INSERT INTO dkb.company_group (id, code, name, valid_from) VALUES (:grp, 'GK-1', 'ГК', :t0)"), + {**ids, "t0": t0}, + ) + await conn.execute( + text( + "INSERT INTO dkb.related_group (id, code, company_group_id, kind, name) " + "VALUES (:rg, 'GSK-1', :grp, 'gsk', 'ГСК')" + ), + ids, + ) + + +@pytest.mark.integration +async def test_upgrade_0007_backfills_v5_19_columns(scratch_db_url: str) -> None: + """0007: заполнение по умолчанию на данных v5.18 (row_index, needs_review, source_text, period_kind, header_source, + file_ordinal, crm_id); `document_issue` пуста.""" + engine = create_async_engine(scratch_db_url) + ids = { + k: uuid.uuid4() + for k in ( + "le", + "s3", + "file", + "job", + "income", + "balance", + "meta", + "c1", + "c2", + "l1600", + "l1100", + "l1250", + "grp", + "rg", + ) + } + try: + await alembic(scratch_db_url, "upgrade", V5_18) + async with engine.begin() as conn: + await _seed_v5_18_statements(conn, ids) + await alembic(scratch_db_url, "upgrade", "head") + async with engine.connect() as conn: + lines = ( + await conn.execute( + text( + "SELECT line_code, row_index, line_status::text, needs_review, is_confirmed " + "FROM dkb.financial_statement_line" + ) + ) + ).all() + rows = {r.line_code: r for r in lines} + assert rows["1100"].row_index == 0 and rows["1600"].row_index == 1 and rows["1250"].row_index is None + assert {r.line_status for r in rows.values()} == {"ok"} and not any(r.needs_review for r in rows.values()) + values = ( + await conn.execute( + text( + "SELECT v.needs_review, v.source_text, v.string_value, v.is_confirmed FROM " + "dkb.financial_statement_value v JOIN dkb.financial_statement_line l ON l.id = v.line_id " + "ORDER BY v.amount" + ) + ) + ).all() + assert [v.needs_review for v in values] == [False, True, False, True, False] + assert not any(v.is_confirmed for v in values) + value_by_text = {v.string_value: v for v in values if v.string_value} + assert value_by_text["12 3"].source_text == "12 3" + period_kind = dict( + (await conn.execute(text("SELECT column_index, period_kind FROM dkb.financial_statement_column"))).all() + ) + assert period_kind == {0: "absolute", 1: "none"} + meta = (await conn.execute(text("SELECT * FROM dkb.financial_statement_meta"))).one() + assert meta.form_variant is None and meta.needs_review_fields is None + assert meta.header_source == { + "inn": {"text": "7701234567", "confidence": 87}, + "report_date": {"text": "30.09.2025", "confidence": None}, + "units": {"text": "тыс. руб.", "confidence": None}, + "organization_name": {"text": "ООО Ромашка", "confidence": None}, + } + assert meta.organization_name_original_value == "ООО Ромашка" and meta.organization_name_modified is False + ordinals = dict( + (await conn.execute(text("SELECT document_type::text, file_ordinal FROM dkb.document"))).all() + ) + assert ordinals == {"INCOME": 0, "BALANCE": 1} + assert await _scalars(conn, "SELECT crm_id FROM dkb.related_group") == [None] + assert await _scalars(conn, "SELECT count(*) FROM dkb.document_issue") == [0] + finally: + await engine.dispose() + + +async def _insert_document(conn: AsyncConnection, ids: dict[str, uuid.UUID], doc_type: str, ordinal: int) -> None: + await conn.execute( + text( + "INSERT INTO dkb.document (id, stored_file_id, legal_entity_id, document_type, status, file_ordinal) " + f"VALUES (:id, :file, :le, '{doc_type}', 'NEEDS_CHECK', :ordinal)" + ), + {"id": uuid.uuid4(), "file": ids["file"], "le": ids["le"], "ordinal": ordinal}, + ) + + +async def _insert_line( + conn: AsyncConnection, ids: dict[str, uuid.UUID], code: str | None, source_ref: str | None +) -> None: + await conn.execute( + text( + "INSERT INTO dkb.financial_statement_line (id, meta_id, line_code, source_ref) " + "VALUES (:id, :meta, :code, :ref)" + ), + {"id": uuid.uuid4(), "meta": ids["meta"], "code": code, "ref": source_ref}, + ) + + +@pytest.mark.integration +async def test_v5_19_constraints(scratch_db_url: str) -> None: + """Поведение индексов v5.19: формы одного типа, одна расшифровка, повтор кода, `source_ref`, `file_ordinal`.""" + engine = create_async_engine(scratch_db_url) + ids = { + k: uuid.uuid4() + for k in ( + "le", + "s3", + "file", + "job", + "income", + "balance", + "meta", + "c1", + "c2", + "l1600", + "l1100", + "l1250", + "grp", + "rg", + ) + } + try: + await alembic(scratch_db_url, "upgrade", V5_18) + async with engine.begin() as conn: + await _seed_v5_18_statements(conn, ids) + await alembic(scratch_db_url, "upgrade", "head") + async with engine.begin() as conn: + await _insert_document(conn, ids, "BALANCE", 2) # вторая форма того же типа — можно + await _insert_document(conn, ids, "DISCLOSURE", 3) + await _insert_line(conn, ids, "1100", None) # повтор кода — можно + await _insert_line(conn, ids, None, None) + await _insert_line(conn, ids, None, None) # source_ref IS NULL — можно повторять + await _insert_line(conn, ids, "1500", "row:8") + with pytest.raises(IntegrityError): # вторая DISCLOSURE (uq_document_file_type_account) + async with engine.begin() as conn: + await _insert_document(conn, ids, "DISCLOSURE", 4) + with pytest.raises(IntegrityError): # два документа файла с одним file_ordinal + async with engine.begin() as conn: + await _insert_document(conn, ids, "OTHER", 3) + with pytest.raises(IntegrityError): # тот же source_ref в форме + async with engine.begin() as conn: + await _insert_line(conn, ids, "1501", "row:8") + finally: + await engine.dispose() + + @pytest.mark.integration async def test_upgrade_0005_adds_nullable_kpp_columns(scratch_db_url: str) -> None: """0005: `kpp varchar` nullable у `legal_entity` и `legal_entity_history`, существующие ЮЛ — с `kpp IS NULL`; @@ -883,8 +1132,15 @@ def test_offline_sql_creates_schema_before_version_table() -> None: assert schema_at < output.index("CREATE TABLE dkb.alembic_version") assert schema_at < output.index("CREATE TYPE dkb.entity_type AS ENUM ('ЮЛ', 'ИП')") assert f"INSERT INTO dkb.alembic_version (version_num) VALUES ('{PREVIOUS}')" in output - # версия head пишется UPDATE'ами по цепочке 0001 → 0002 → 0003 → 0004 → 0005 → 0006 — офлайн-SQL доходит до head - for new_version, old_version in ((V5_14, PREVIOUS), (V5_15, V5_14), (V5_16, V5_15), (V5_17, V5_16), (HEAD, V5_17)): + # версия head пишется UPDATE'ами по цепочке 0001 → … → 0007 — офлайн-SQL доходит до head + for new_version, old_version in ( + (V5_14, PREVIOUS), + (V5_15, V5_14), + (V5_16, V5_15), + (V5_17, V5_16), + (V5_18, V5_17), + (HEAD, V5_18), + ): assert ( f"UPDATE dkb.alembic_version SET version_num='{new_version}' " f"WHERE dkb.alembic_version.version_num = '{old_version}'" in output diff --git a/api/tests/test_models_part_04_06.py b/api/tests/test_models_part_04_06.py index 99a029b..8688c4e 100644 --- a/api/tests/test_models_part_04_06.py +++ b/api/tests/test_models_part_04_06.py @@ -2,6 +2,7 @@ import datetime import decimal +import itertools import uuid import warnings from dataclasses import dataclass @@ -41,6 +42,7 @@ PART_04_06_TABLES = { "financial_statement_column", "financial_statement_line", "financial_statement_value", + "document_issue", } @@ -85,7 +87,7 @@ def test_fk_cycle_is_broken_by_use_alter_without_warning() -> None: def test_document_unique_index_is_nulls_not_distinct() -> None: - index = next(i for i in orm_tables()["document"].indexes if i.unique) + index = next(i for i in orm_tables()["document"].indexes if i.name == "uq_document_file_type_account") assert index.dialect_options["postgresql"]["nulls_not_distinct"] is True assert [c.name for c in index.columns] == ["stored_file_id", "document_type", "osv_account_code"] @@ -170,8 +172,12 @@ async def _make_file_with_job(session: AsyncSession) -> Chain: return Chain(entity_id=entity.id, file_id=file.id, job_id=job.id) +_FILE_ORDINALS = itertools.count() # v5.19: file_ordinal уникален в файле — тесту нужен любой свободный номер + + def _document(chain: Chain, document_type: DocumentType, account: str | None) -> Document: return Document( + file_ordinal=next(_FILE_ORDINALS), stored_file_id=chain.file_id, legal_entity_id=chain.entity_id, document_type=document_type, @@ -181,16 +187,18 @@ def _document(chain: Chain, document_type: DocumentType, account: str | None) -> async def _check_nulls_not_distinct(session: AsyncSession, chain: Chain) -> None: - """Два документа с одним (stored_file_id, document_type) и osv_account_code = NULL -> IntegrityError.""" - session.add(_document(chain, DocumentType.BALANCE, None)) + """v5.19: два документа с одним (stored_file_id, document_type) и osv_account_code = NULL — `IntegrityError` для + не-форм (DISCLOSURE); формы ББ/ОФР одного типа в файле допустимы (Р-10).""" + session.add(_document(chain, DocumentType.DISCLOSURE, None)) await session.flush() - session.add(_document(chain, DocumentType.BALANCE, None)) + session.add(_document(chain, DocumentType.DISCLOSURE, None)) with pytest.raises(IntegrityError): await session.flush() await session.rollback() - # разные счета ОСВ в одном файле и ББ/ОФР рядом — допустимы + # две формы ББ одного типа, разные счета ОСВ и ОФР рядом — допустимы session.add_all( - [_document(chain, DocumentType.OSV, "58"), _document(chain, DocumentType.OSV, "60")] + [_document(chain, DocumentType.BALANCE, None), _document(chain, DocumentType.BALANCE, None)] + + [_document(chain, DocumentType.OSV, "58"), _document(chain, DocumentType.OSV, "60")] + [_document(chain, DocumentType.INCOME, None)] ) await session.flush() diff --git a/api/tests/test_models_part_07_09.py b/api/tests/test_models_part_07_09.py index d3892a8..2e46db0 100644 --- a/api/tests/test_models_part_07_09.py +++ b/api/tests/test_models_part_07_09.py @@ -48,8 +48,8 @@ PART_07_09_TABLES = { "required_document_rule", } ALL_TABLES = PART_01_03_TABLES | PART_04_06_TABLES | PART_07_09_TABLES -EXPECTED_TABLE_COUNT = 43 -EXPECTED_ENUM_COUNT = 28 +EXPECTED_TABLE_COUNT = 44 +EXPECTED_ENUM_COUNT = 33 def test_part_07_09_tables_are_new() -> None: @@ -277,6 +277,10 @@ def test_all_nullable_jsonb_columns_are_listed() -> None: "recognition_job.group_schema", "recognition_job.metrics", "osv_entry.original_values", + "financial_statement_meta.needs_review_fields", + "financial_statement_meta.header_source", + "document_issue.line_ids", + "document_issue.codes", }