feat(models): DBML v5.19 — колонки форм, document_issue, file_ordinal, crm_id ГСК (миграция 0007)
Модели и upgrade ревизии 0007_dbml_v5_19: пять enum'ов, колонки строк/значений/колонок/шапок форм, document.file_ordinal (уникален в файле, уникальность типа только для не-форм), recognition_job.statements_version, related_group.crm_id, таблица document_issue; заполнение по умолчанию (row_index, needs_review, header_source и др.). delete_file_statements удаляет document_issue; make_document берёт следующий file_ordinal; маппер пока нумерует документы по порядку создания. downgrade — заглушка (задача A2). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W
This commit is contained in:
Andrey Epifancev
parent
ed99f3299a
commit
19d8bbbd59
@ -159,7 +159,7 @@
|
||||
- NATS subjects и схема ключей S3 — `../docs/context/03-architecture.md`.
|
||||
|
||||
## Правила репозитория
|
||||
- ORM 1:1 с `../docs/model/data-model-v5.18.dbml` (текущая версия; миграции `0001_dbml_v5_13` + `0002_dbml_v5_14` + `0003_dbml_v5_15` + `0004_dbml_v5_16` + `0005_dbml_v5_17` + `0006_dbml_v5_18`); `api/scripts/check_dbml_vs_orm.py` (`make dbml-check`, входит в `check-full`) — обязательная проверка.
|
||||
- ORM 1:1 с `../docs/model/data-model-v5.19.dbml` (текущая версия; миграции `0001_dbml_v5_13` + `0002_dbml_v5_14` + `0003_dbml_v5_15` + `0004_dbml_v5_16` + `0005_dbml_v5_17` + `0006_dbml_v5_18` + `0007_dbml_v5_19`); `api/scripts/check_dbml_vs_orm.py` (`make dbml-check`, входит в `check-full`) — обязательная проверка.
|
||||
- Алгоритмы — только из `../docs/context/05-domain-rules.md`; номер § — в docstring.
|
||||
- Вычисляемые объекты консолидации — `api/src/domain/`, без ORM-таблиц.
|
||||
- Фронт: пользователь слаб во фронте — для TS/React изменений делай ревью сам и объясняй решения в PR.
|
||||
|
||||
2
Makefile
2
Makefile
@ -3,7 +3,7 @@ VENV := api/.venv
|
||||
BIN := $(CURDIR)/$(VENV)/bin
|
||||
COMPOSE_FILE ?= ../docs/dev/docker-compose.yml
|
||||
# DBML относительно api/: и в основном workspace, и в worktree .wt/<срез>/ это соседний docs/
|
||||
DBML ?= ../../docs/model/data-model-v5.18.dbml
|
||||
DBML ?= ../../docs/model/data-model-v5.19.dbml
|
||||
|
||||
.PHONY: venv lint test test-integration e2e dbml-check migration-cycle check check-infra docker-build web-check check-full up migrate seed seed-dev
|
||||
|
||||
|
||||
192
api/alembic/versions/0007_dbml_v5_19.py
Normal file
192
api/alembic/versions/0007_dbml_v5_19.py
Normal file
@ -0,0 +1,192 @@
|
||||
"""DBML v5.19: единая нормализация форм ББ/ОФР (RSHB/arm-ka-docs#116, openspec unified-statement-normalization).
|
||||
|
||||
Только добавляет: enum'ы, колонки строк/значений/колонок/шапок форм, `document.file_ordinal`,
|
||||
`recognition_job.statements_version`, `related_group.crm_id`, таблицу `document_issue`, индексы; §7 по существующим
|
||||
формам не считает (`document_issue` остаётся пустой — корректность нарушений у форм неочищенной базы не гарантируется
|
||||
до первой правки). Заполнение по умолчанию (миграция проходит и на непустой базе):
|
||||
- `row_index` — строки источника по возрастанию `line_code` внутри формы с 0; строки без
|
||||
`line_code_original_value` (созданные правкой) — NULL;
|
||||
- `needs_review` значений = НЕ (правлено или (`is_recognized` не false и уверенность 100 / NULL)), `source_text` =
|
||||
`string_value`; `period_kind` колонок — `absolute` при `period_date`, иначе `none`;
|
||||
- `header_source` — из `*_original_value` и `inn_confidence` (P3-1), `organization_name_original_value` =
|
||||
`organization_name`, `organization_name_modified` = false (P3-2); `form_variant` = NULL (набор §7 — полный);
|
||||
- `file_ordinal` — по `(created_at, id)` внутри файла с 0.
|
||||
Уникальные индексы создаются после заполнения. Откат (`downgrade`) — задача A2 плана 3.
|
||||
|
||||
Revision ID: 0007_dbml_v5_19
|
||||
Revises: 0006_dbml_v5_18
|
||||
Create Date: 2026-10-18
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from alembic import op
|
||||
from src.models.base import SCHEMA
|
||||
|
||||
revision: str = "0007_dbml_v5_19"
|
||||
down_revision: str | Sequence[str] | None = "0006_dbml_v5_18"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
S = SCHEMA
|
||||
|
||||
ENUMS_V5_19: dict[str, tuple[str, ...]] = {
|
||||
"statement_line_status": ("ok", "unreadable", "missing", "detail"),
|
||||
"statement_form_variant": ("full", "simplified"),
|
||||
"document_issue_kind": ("warning", "relation", "soft"),
|
||||
"issue_severity": ("error", "info"),
|
||||
"issue_source": ("service", "core"),
|
||||
}
|
||||
|
||||
|
||||
def _create_enums() -> None:
|
||||
for name, labels in ENUMS_V5_19.items():
|
||||
values = ", ".join(f"'{label}'" for label in labels)
|
||||
op.execute(f"CREATE TYPE {S}.{name} AS ENUM ({values})")
|
||||
|
||||
|
||||
def _add_columns() -> None:
|
||||
statements = [
|
||||
# financial_statement_line
|
||||
f"ALTER TABLE {S}.financial_statement_line"
|
||||
" ADD COLUMN row_index integer,"
|
||||
" ADD COLUMN source_ref varchar,"
|
||||
f" ADD COLUMN line_status {S}.statement_line_status NOT NULL DEFAULT 'ok',"
|
||||
" ADD COLUMN line_code_text varchar,"
|
||||
" ADD COLUMN parent_row_index integer,"
|
||||
" ADD COLUMN needs_review boolean NOT NULL DEFAULT false,"
|
||||
" ADD COLUMN is_confirmed boolean NOT NULL DEFAULT false",
|
||||
# financial_statement_value
|
||||
f"ALTER TABLE {S}.financial_statement_value"
|
||||
" ADD COLUMN source_text varchar,"
|
||||
" ADD COLUMN needs_review boolean NOT NULL DEFAULT false,"
|
||||
" ADD COLUMN is_confirmed boolean NOT NULL DEFAULT false,"
|
||||
" ADD COLUMN confirmed_by uuid"
|
||||
f" CONSTRAINT fk_financial_statement_value_confirmed_by_app_user REFERENCES {S}.app_user (id),"
|
||||
" ADD COLUMN confirmed_at timestamptz",
|
||||
# financial_statement_column
|
||||
f"ALTER TABLE {S}.financial_statement_column"
|
||||
" ADD COLUMN period_kind varchar, ADD COLUMN label_confidence numeric(5,2)",
|
||||
# financial_statement_meta
|
||||
f"ALTER TABLE {S}.financial_statement_meta"
|
||||
f" ADD COLUMN form_variant {S}.statement_form_variant,"
|
||||
" ADD COLUMN needs_review_fields jsonb,"
|
||||
" ADD COLUMN header_source jsonb,"
|
||||
" ADD COLUMN organization_name_original_value varchar,"
|
||||
" ADD COLUMN organization_name_modified boolean",
|
||||
# document, recognition_job, related_group
|
||||
f"ALTER TABLE {S}.document ADD COLUMN file_ordinal smallint NOT NULL DEFAULT 0",
|
||||
f"ALTER TABLE {S}.recognition_job ADD COLUMN statements_version varchar",
|
||||
f"ALTER TABLE {S}.related_group ADD COLUMN crm_id varchar CONSTRAINT uq_related_group_crm_id UNIQUE",
|
||||
]
|
||||
for statement in statements:
|
||||
op.execute(statement)
|
||||
|
||||
|
||||
def _backfill() -> None:
|
||||
# row_index: строки источника по возрастанию line_code внутри формы с 0; созданные правкой — NULL
|
||||
op.execute(
|
||||
f"""
|
||||
UPDATE {S}.financial_statement_line l SET row_index = n.rn FROM (
|
||||
SELECT id, row_number() OVER (PARTITION BY meta_id ORDER BY line_code, id) - 1 AS rn
|
||||
FROM {S}.financial_statement_line WHERE line_code_original_value IS NOT NULL) n WHERE l.id = n.id
|
||||
"""
|
||||
)
|
||||
op.execute(
|
||||
f"""
|
||||
UPDATE {S}.financial_statement_value SET source_text = string_value,
|
||||
needs_review = NOT (
|
||||
is_modified OR (COALESCE(is_recognized, true) AND (confidence IS NULL OR confidence = 100)))
|
||||
"""
|
||||
)
|
||||
op.execute(
|
||||
f"""
|
||||
UPDATE {S}.financial_statement_column
|
||||
SET period_kind = CASE WHEN period_date IS NULL THEN 'none' ELSE 'absolute' END
|
||||
"""
|
||||
)
|
||||
op.execute(
|
||||
f"""
|
||||
UPDATE {S}.financial_statement_meta SET
|
||||
header_source = jsonb_build_object(
|
||||
'inn', jsonb_build_object('text', inn_original_value, 'confidence', inn_confidence),
|
||||
'report_date', jsonb_build_object('text', report_date_original_value, 'confidence', NULL),
|
||||
'units', jsonb_build_object('text', units_original_value, 'confidence', NULL),
|
||||
'organization_name', jsonb_build_object('text', organization_name, 'confidence', NULL)),
|
||||
organization_name_original_value = organization_name,
|
||||
organization_name_modified = false
|
||||
"""
|
||||
)
|
||||
op.execute(
|
||||
f"""
|
||||
UPDATE {S}.document d SET file_ordinal = n.rn FROM (
|
||||
SELECT id, row_number() OVER (PARTITION BY stored_file_id ORDER BY created_at, id) - 1 AS rn
|
||||
FROM {S}.document) n WHERE d.id = n.id
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def _indexes() -> None:
|
||||
# старый уникальный индекс (meta_id, line_code) → обычный: повтор кода законен
|
||||
op.execute(f"DROP INDEX {S}.uq_financial_statement_line_meta_id_line_code")
|
||||
op.execute(
|
||||
"CREATE INDEX ix_financial_statement_line_meta_id_line_code "
|
||||
f"ON {S}.financial_statement_line (meta_id, line_code)"
|
||||
)
|
||||
op.execute(
|
||||
"CREATE UNIQUE INDEX uq_financial_statement_line_meta_id_source_ref "
|
||||
f"ON {S}.financial_statement_line (meta_id, source_ref) WHERE source_ref IS NOT NULL"
|
||||
)
|
||||
# уникальность типа — только для не-форм; формы ББ/ОФР одного типа могут повторяться (Р-10)
|
||||
op.execute(f"DROP INDEX {S}.uq_document_file_type_account")
|
||||
op.execute(
|
||||
"CREATE UNIQUE INDEX uq_document_file_type_account "
|
||||
f"ON {S}.document (stored_file_id, document_type, osv_account_code) NULLS NOT DISTINCT "
|
||||
"WHERE document_type NOT IN ('BALANCE', 'INCOME')"
|
||||
)
|
||||
op.execute(
|
||||
f"CREATE UNIQUE INDEX uq_document_stored_file_id_file_ordinal ON {S}.document (stored_file_id, file_ordinal)"
|
||||
)
|
||||
|
||||
|
||||
def _create_document_issue() -> None:
|
||||
op.execute(
|
||||
f"""
|
||||
CREATE TABLE {S}.document_issue (
|
||||
id uuid NOT NULL,
|
||||
document_id uuid NOT NULL,
|
||||
kind {S}.document_issue_kind NOT NULL,
|
||||
code varchar NOT NULL,
|
||||
severity {S}.issue_severity NOT NULL,
|
||||
message text NOT NULL,
|
||||
line_ids jsonb,
|
||||
column_id uuid,
|
||||
column_index smallint,
|
||||
codes jsonb,
|
||||
field varchar,
|
||||
source {S}.issue_source NOT NULL,
|
||||
relation varchar,
|
||||
expected numeric(18,2),
|
||||
actual numeric(18,2),
|
||||
created_at timestamptz NOT NULL,
|
||||
CONSTRAINT pk_document_issue PRIMARY KEY (id),
|
||||
CONSTRAINT fk_document_issue_document_id_document FOREIGN KEY (document_id) REFERENCES {S}.document (id),
|
||||
CONSTRAINT fk_document_issue_column_id_financial_statement_column
|
||||
FOREIGN KEY (column_id) REFERENCES {S}.financial_statement_column (id)
|
||||
)
|
||||
"""
|
||||
)
|
||||
op.execute(f"CREATE INDEX ix_document_issue_document_id ON {S}.document_issue (document_id)")
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
_create_enums()
|
||||
_add_columns()
|
||||
_backfill()
|
||||
_indexes()
|
||||
_create_document_issue()
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# откат (охранные проверки, потеря данных, журнал) — задача A2 плана 3
|
||||
raise NotImplementedError("downgrade 0007_dbml_v5_19 ещё не реализован (задача A2 плана 3)")
|
||||
@ -57,7 +57,7 @@ from sqlalchemy.sql.functions import FunctionElement
|
||||
|
||||
#: Эталонная схема модели. Имя схемы в БД конфигурируемо (`DB_SCHEMA`), CLI сверяет ORM, загруженную с этим значением.
|
||||
SCHEMA = "dkb"
|
||||
DBML_FILE_NAME = "data-model-v5.18.dbml"
|
||||
DBML_FILE_NAME = "data-model-v5.19.dbml"
|
||||
ORM = "ORM"
|
||||
DBML = "DBML"
|
||||
ABSENT = "нет"
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
"""ORM-модели. Источник истины — docs/model/data-model-v5.18.dbml; заполняется в Срезе 0.
|
||||
"""ORM-модели. Источник истины — docs/model/data-model-v5.19.dbml; заполняется в Срезе 0.
|
||||
|
||||
Импорт всех модулей моделей обязателен: он регистрирует таблицы в `Base.metadata` (create_all, alembic).
|
||||
"""
|
||||
@ -7,7 +7,7 @@ from . import access, accounting, audit_ex, consolidation, directories, files, k
|
||||
from .base import Base, UUIDPrimaryMixin
|
||||
|
||||
#: Версия DBML, которой соответствуют модели (сверяется с шапкой DBML в check_dbml_vs_orm).
|
||||
DBML_VERSION = "5.18"
|
||||
DBML_VERSION = "5.19"
|
||||
|
||||
__all__ = [
|
||||
"DBML_VERSION",
|
||||
|
||||
@ -141,6 +141,8 @@ class RelatedGroup(UUIDPrimaryMixin, Base):
|
||||
kind: Mapped[RelatedGroupKind] = mapped_column(pg_enum("related_group_kind"), nullable=False)
|
||||
name: Mapped[str] = mapped_column(String, nullable=False)
|
||||
created_date: Mapped[datetime.date | None] = mapped_column(Date)
|
||||
# v5.19: ID ГСК/ГСЗ в ЕЦП CRM; вводится вручную, NULL — не задан
|
||||
crm_id: Mapped[str | None] = mapped_column(String, unique=True)
|
||||
|
||||
|
||||
class RelatedGroupMember(UUIDPrimaryMixin, Base):
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
"""Enum'ы модели данных DBML v5.18 (28 штук) и фабрика `pg_enum` для PostgreSQL-колонок.
|
||||
"""Enum'ы модели данных DBML v5.19 (33 штуки) и фабрика `pg_enum` для PostgreSQL-колонок.
|
||||
|
||||
Источник истины — docs/model/data-model-v5.18.dbml (блоки `Enum {}`). Значения — строки DBML буквально, в порядке
|
||||
Источник истины — docs/model/data-model-v5.19.dbml (блоки `Enum {}`). Значения — строки DBML буквально, в порядке
|
||||
DBML (включая кириллические `"ЮЛ"`/`"ИП"` и значения в разном регистре); `notes` DBML значениями не являются.
|
||||
Имя члена совпадает со значением (все значения DBML — валидные python-идентификаторы, кириллица допустима).
|
||||
Классы — `enum.StrEnum` (подкласс `str`) — удобно для pydantic/JSON; `values_callable` в `pg_enum` в БД пишет `.value`.
|
||||
@ -136,6 +136,44 @@ class StatementType(enum.StrEnum):
|
||||
income = "income"
|
||||
|
||||
|
||||
class StatementLineStatus(enum.StrEnum):
|
||||
"""DBML `statement_line_status` (v5.19): статус строки формы; `line_code` пуст ⇔ статус не `ok`."""
|
||||
|
||||
ok = "ok"
|
||||
unreadable = "unreadable"
|
||||
missing = "missing"
|
||||
detail = "detail"
|
||||
|
||||
|
||||
class StatementFormVariant(enum.StrEnum):
|
||||
"""DBML `statement_form_variant` (v5.19): вариант формы по структуре кодов (§11.7)."""
|
||||
|
||||
full = "full"
|
||||
simplified = "simplified"
|
||||
|
||||
|
||||
class DocumentIssueKind(enum.StrEnum):
|
||||
"""DBML `document_issue_kind` (v5.19): вид записи `document_issue`."""
|
||||
|
||||
warning = "warning"
|
||||
relation = "relation"
|
||||
soft = "soft"
|
||||
|
||||
|
||||
class IssueSeverity(enum.StrEnum):
|
||||
"""DBML `issue_severity` (v5.19): `error` — документ `needs_check`, `info` — на статус не влияет."""
|
||||
|
||||
error = "error"
|
||||
info = "info"
|
||||
|
||||
|
||||
class IssueSource(enum.StrEnum):
|
||||
"""DBML `issue_source` (v5.19): откуда запись — из результата распознавания или посчитана ядром."""
|
||||
|
||||
service = "service"
|
||||
core = "core"
|
||||
|
||||
|
||||
class OsvGroupType(enum.StrEnum):
|
||||
"""DBML `osv_group_type`."""
|
||||
|
||||
@ -324,6 +362,11 @@ ENUMS: dict[str, type[enum.Enum]] = {
|
||||
"s3_bucket_role": S3BucketRole,
|
||||
"recognition_status": RecognitionStatus,
|
||||
"statement_type": StatementType,
|
||||
"statement_line_status": StatementLineStatus,
|
||||
"statement_form_variant": StatementFormVariant,
|
||||
"document_issue_kind": DocumentIssueKind,
|
||||
"issue_severity": IssueSeverity,
|
||||
"issue_source": IssueSource,
|
||||
"osv_group_type": OsvGroupType,
|
||||
"counterparty_match_kind": CounterpartyMatchKind,
|
||||
"osv_data_type": OsvDataType,
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
"""Часть 04 DBML v5.18 «Файлы / распознавание»: реестр S3, физические файлы, документы, задания распознавания.
|
||||
"""Часть 04 DBML v5.19 «Файлы / распознавание»: реестр S3, физические файлы, документы, задания распознавания.
|
||||
|
||||
Цикл FK `stored_file.current_recognition_job_id` -> `recognition_job` и `recognition_job.file_id` -> `stored_file`
|
||||
разорван через `use_alter=True` на nullable-стороне (`stored_file.current_recognition_job_id`): `create_all` создаёт
|
||||
@ -91,7 +91,8 @@ class Document(UUIDPrimaryMixin, Base):
|
||||
|
||||
__tablename__ = "document"
|
||||
__table_args__ = (
|
||||
# один ББ / один ОФР / одна ОСВ по счёту на файл: NULL в osv_account_code считается значением (PG15+)
|
||||
# v5.19: одна ОСВ по счёту / одна расшифровка / один прочий на файл; формы ББ/ОФР одного типа могут
|
||||
# повторяться (Р-10). NULL в osv_account_code считается значением (PG15+)
|
||||
Index(
|
||||
"uq_document_file_type_account",
|
||||
"stored_file_id",
|
||||
@ -99,7 +100,10 @@ class Document(UUIDPrimaryMixin, Base):
|
||||
"osv_account_code",
|
||||
unique=True,
|
||||
postgresql_nulls_not_distinct=True,
|
||||
postgresql_where=text("document_type NOT IN ('BALANCE', 'INCOME')"),
|
||||
),
|
||||
# v5.19: номер документа в documents[] результата уникален в файле
|
||||
Index("uq_document_stored_file_id_file_ordinal", "stored_file_id", "file_ordinal", unique=True),
|
||||
# чек-лист required_document_rule
|
||||
Index(
|
||||
"ix_document_legal_entity_period_status",
|
||||
@ -130,6 +134,8 @@ class Document(UUIDPrimaryMixin, Base):
|
||||
confirmed_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
created_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
updated_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# v5.19: индекс документа в documents[] результата (с 0); несозданный документ номер не занимает
|
||||
file_ordinal: Mapped[int] = mapped_column(SmallInteger, nullable=False, server_default=text("0"))
|
||||
|
||||
|
||||
class RecognitionJob(UUIDPrimaryMixin, Base):
|
||||
@ -164,3 +170,5 @@ class RecognitionJob(UUIDPrimaryMixin, Base):
|
||||
fields_total: Mapped[int | None] = mapped_column(Integer)
|
||||
fields_low_confidence: Mapped[int | None] = mapped_column(Integer)
|
||||
fields_unrecognized: Mapped[int | None] = mapped_column(Integer)
|
||||
# v5.19: service.statements_version результата — коммит копии общего кода arm_statements
|
||||
statements_version: Mapped[str | None] = mapped_column(String)
|
||||
|
||||
@ -1,19 +1,29 @@
|
||||
"""Часть 06 DBML v5.18 «Бухгалтерская отчётность»: шапка, колонки-периоды, строки и значения форм ББ/ОФР (157н).
|
||||
"""Часть 06 DBML v5.19 «Бухгалтерская отчётность»: шапка, колонки-периоды, строки и значения форм ББ/ОФР (157н).
|
||||
|
||||
Первичка не мутируется: `*_original_value` / `original_amount` / `is_modified` — рядом с распознанным значением.
|
||||
Логические проверки (ТЗ 6.3) и «требует проверки» не хранятся — считаются на лету.
|
||||
v5.19: предупреждения и нарушения §7 хранятся в `document_issue`, `needs_review` строк и значений — в колонках;
|
||||
повтор кода в форме законен (строки хранятся как в источнике).
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import decimal
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import Boolean, Date, DateTime, ForeignKey, Index, Numeric, SmallInteger, String, Text, text
|
||||
from sqlalchemy import Boolean, Date, DateTime, ForeignKey, Index, Integer, Numeric, SmallInteger, String, Text, text
|
||||
from sqlalchemy.dialects.postgresql import UUID
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base, UUIDPrimaryMixin
|
||||
from .enums import StatementType, pg_enum
|
||||
from .base import Base, UUIDPrimaryMixin, nullable_jsonb
|
||||
from .enums import (
|
||||
DocumentIssueKind,
|
||||
IssueSeverity,
|
||||
IssueSource,
|
||||
StatementFormVariant,
|
||||
StatementLineStatus,
|
||||
StatementType,
|
||||
pg_enum,
|
||||
)
|
||||
|
||||
|
||||
class FinancialStatementMeta(UUIDPrimaryMixin, Base):
|
||||
@ -57,6 +67,12 @@ class FinancialStatementMeta(UUIDPrimaryMixin, Base):
|
||||
checks_accepted_by: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("app_user.id"))
|
||||
checks_accepted_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
checks_accepted_comment: Mapped[str | None] = mapped_column(Text)
|
||||
# v5.19: порядок колонок = порядок ADD COLUMN миграции 0007 (check_dbml сверяет порядок)
|
||||
form_variant: Mapped[StatementFormVariant | None] = mapped_column(pg_enum("statement_form_variant"))
|
||||
needs_review_fields: Mapped[Any | None] = mapped_column(nullable_jsonb())
|
||||
header_source: Mapped[Any | None] = mapped_column(nullable_jsonb())
|
||||
organization_name_original_value: Mapped[str | None] = mapped_column(String)
|
||||
organization_name_modified: Mapped[bool | None] = mapped_column(Boolean)
|
||||
|
||||
|
||||
class FinancialStatementColumn(UUIDPrimaryMixin, Base):
|
||||
@ -74,13 +90,25 @@ class FinancialStatementColumn(UUIDPrimaryMixin, Base):
|
||||
label: Mapped[str | None] = mapped_column(String)
|
||||
period_date: Mapped[datetime.date | None] = mapped_column(Date)
|
||||
is_current: Mapped[bool | None] = mapped_column(Boolean)
|
||||
period_kind: Mapped[str | None] = mapped_column(String)
|
||||
label_confidence: Mapped[decimal.Decimal | None] = mapped_column(Numeric(5, 2))
|
||||
|
||||
|
||||
class FinancialStatementLine(UUIDPrimaryMixin, Base):
|
||||
"""Строка формы с кодом; порядок, разделы и итоговые строки — шаблон формы на фронте."""
|
||||
|
||||
__tablename__ = "financial_statement_line"
|
||||
__table_args__ = (Index("uq_financial_statement_line_meta_id_line_code", "meta_id", "line_code", unique=True),)
|
||||
__table_args__ = (
|
||||
# v5.19: повтор кода в форме законен — индекс не уникален (чтение «по коду» — агрегат формы)
|
||||
Index("ix_financial_statement_line_meta_id_line_code", "meta_id", "line_code"),
|
||||
Index(
|
||||
"uq_financial_statement_line_meta_id_source_ref",
|
||||
"meta_id",
|
||||
"source_ref",
|
||||
unique=True,
|
||||
postgresql_where=text("source_ref IS NOT NULL"),
|
||||
),
|
||||
)
|
||||
|
||||
meta_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("financial_statement_meta.id"), nullable=False
|
||||
@ -93,6 +121,16 @@ class FinancialStatementLine(UUIDPrimaryMixin, Base):
|
||||
line_name_original_value: Mapped[str | None] = mapped_column(String)
|
||||
line_name_confidence: Mapped[decimal.Decimal | None] = mapped_column(Numeric(5, 2))
|
||||
line_name_modified: Mapped[bool | None] = mapped_column(Boolean)
|
||||
# v5.19: `row_index IS NULL` ⇔ строка создана правкой (шаблонная); у строк из результата всегда задан
|
||||
row_index: Mapped[int | None] = mapped_column(Integer)
|
||||
source_ref: Mapped[str | None] = mapped_column(String)
|
||||
line_status: Mapped[StatementLineStatus] = mapped_column(
|
||||
pg_enum("statement_line_status"), nullable=False, server_default=text("'ok'")
|
||||
)
|
||||
line_code_text: Mapped[str | None] = mapped_column(String)
|
||||
parent_row_index: Mapped[int | None] = mapped_column(Integer)
|
||||
needs_review: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("false"))
|
||||
is_confirmed: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("false"))
|
||||
|
||||
|
||||
class FinancialStatementValue(UUIDPrimaryMixin, Base):
|
||||
@ -117,3 +155,37 @@ class FinancialStatementValue(UUIDPrimaryMixin, Base):
|
||||
original_amount: Mapped[decimal.Decimal | None] = mapped_column(Numeric(18, 2))
|
||||
modified_by: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("app_user.id"))
|
||||
modified_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# v5.19
|
||||
source_text: Mapped[str | None] = mapped_column(String)
|
||||
needs_review: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("false"))
|
||||
is_confirmed: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("false"))
|
||||
confirmed_by: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("app_user.id"))
|
||||
confirmed_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
class DocumentIssue(UUIDPrimaryMixin, Base):
|
||||
"""v5.19: предупреждение или нарушение §7 документа с привязкой к строкам, ячейкам и полям шапки.
|
||||
|
||||
Удаляется вместе с документом; при пересчёте после правки `relation` / `soft` заменяются целиком, `warning`
|
||||
источника `service` сохраняются (кроме снимаемых правкой). `line_ids` — без FK: ядро убирает id удалённой
|
||||
шаблонной строки само.
|
||||
"""
|
||||
|
||||
__tablename__ = "document_issue"
|
||||
__table_args__ = (Index("ix_document_issue_document_id", "document_id"),)
|
||||
|
||||
document_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("document.id"), nullable=False)
|
||||
kind: Mapped[DocumentIssueKind] = mapped_column(pg_enum("document_issue_kind"), nullable=False)
|
||||
code: Mapped[str] = mapped_column(String, nullable=False)
|
||||
severity: Mapped[IssueSeverity] = mapped_column(pg_enum("issue_severity"), nullable=False)
|
||||
message: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
line_ids: Mapped[Any | None] = mapped_column(nullable_jsonb())
|
||||
column_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("financial_statement_column.id"))
|
||||
column_index: Mapped[int | None] = mapped_column(SmallInteger)
|
||||
codes: Mapped[Any | None] = mapped_column(nullable_jsonb())
|
||||
field: Mapped[str | None] = mapped_column(String)
|
||||
source: Mapped[IssueSource] = mapped_column(pg_enum("issue_source"), nullable=False)
|
||||
relation: Mapped[str | None] = mapped_column(String)
|
||||
expected: Mapped[decimal.Decimal | None] = mapped_column(Numeric(18, 2))
|
||||
actual: Mapped[decimal.Decimal | None] = mapped_column(Numeric(18, 2))
|
||||
created_at: Mapped[datetime.datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
@ -14,6 +14,7 @@ from src.domain.statement_mapping import StatementDraft
|
||||
from src.models.enums import DocumentStatus, DocumentType, FileStatus
|
||||
from src.models.files import Document, StoredFile
|
||||
from src.models.statements import (
|
||||
DocumentIssue,
|
||||
FinancialStatementColumn,
|
||||
FinancialStatementLine,
|
||||
FinancialStatementMeta,
|
||||
@ -107,8 +108,9 @@ async def insert_statement(
|
||||
|
||||
|
||||
async def delete_file_statements(session: AsyncSession, file_id: uuid.UUID) -> None:
|
||||
"""Значения, строки, колонки и шапки форм всех документов файла (в порядке FK)."""
|
||||
"""`document_issue`, значения, строки, колонки и шапки форм всех документов файла (в порядке FK)."""
|
||||
documents = select(Document.id).where(Document.stored_file_id == file_id).scalar_subquery()
|
||||
await session.execute(delete(DocumentIssue).where(DocumentIssue.document_id.in_(documents)))
|
||||
metas = select(FinancialStatementMeta.id).where(FinancialStatementMeta.document_id.in_(documents)).scalar_subquery()
|
||||
lines = select(FinancialStatementLine.id).where(FinancialStatementLine.meta_id.in_(metas)).scalar_subquery()
|
||||
await session.execute(delete(FinancialStatementValue).where(FinancialStatementValue.line_id.in_(lines)))
|
||||
|
||||
@ -532,6 +532,9 @@ async def map_file( # noqa: PLR0913 - все входы маппинга фай
|
||||
documents.extend(other_documents)
|
||||
documents.extend(document for document, _ in forms)
|
||||
|
||||
# v5.19: номер документа в файле уникален; до перехода на индекс в documents[] (задачи A5/A9) — порядок создания
|
||||
for ordinal, document in enumerate(documents):
|
||||
document.file_ordinal = ordinal
|
||||
session.add_all(documents)
|
||||
await session.flush()
|
||||
# T7.2: родитель раньше потомка на любой границе пачки (self-FK `parent_entry_id`) — не полагаемся на порядок строк
|
||||
|
||||
@ -54,11 +54,14 @@ from src.models.enums import (
|
||||
ConsolidationScenario,
|
||||
ConsolidationStatus,
|
||||
DocumentCheckResult,
|
||||
DocumentIssueKind,
|
||||
DocumentStatus,
|
||||
DocumentType,
|
||||
EntityType,
|
||||
FileStatus,
|
||||
FileType,
|
||||
IssueSeverity,
|
||||
IssueSource,
|
||||
OsvDataType,
|
||||
OsvGroupType,
|
||||
PeriodQuarter,
|
||||
@ -69,6 +72,7 @@ from src.models.enums import (
|
||||
)
|
||||
from src.models.files import Document, RecognitionJob, S3Object, StoredFile
|
||||
from src.models.osv import OsvEntry
|
||||
from src.models.statements import DocumentIssue
|
||||
from src.services.result_mapping import MAPPING_WARNINGS_KEY, _entry_values, _group_schema
|
||||
from tests.conftest import FakeStorage
|
||||
|
||||
@ -265,10 +269,17 @@ async def make_document( # noqa: PLR0913 - сигнатура фабрики ф
|
||||
"""
|
||||
assert job.file_id == file.id # noqa: S101 - защита от перепутанных аргументов в тестах
|
||||
now = _now()
|
||||
# v5.19: file_ordinal уникален в файле — берём следующий свободный
|
||||
ordinal = (
|
||||
await session.execute(
|
||||
select(func.coalesce(func.max(Document.file_ordinal) + 1, 0)).where(Document.stored_file_id == file.id)
|
||||
)
|
||||
).scalar_one()
|
||||
document = Document(
|
||||
stored_file_id=file.id,
|
||||
legal_entity_id=file.legal_entity_id,
|
||||
document_type=DocumentType.OSV,
|
||||
file_ordinal=ordinal,
|
||||
osv_account_code=account,
|
||||
period_year=year,
|
||||
period_quarter=quarter,
|
||||
@ -282,6 +293,22 @@ async def make_document( # noqa: PLR0913 - сигнатура фабрики ф
|
||||
return document
|
||||
|
||||
|
||||
async def make_document_issue(session: AsyncSession, *, document_id: uuid.UUID, code: str = "MAPPING_WARNING") -> Any:
|
||||
"""Запись `document_issue` (предупреждение ядра уровня `info`) документа (`flush`)."""
|
||||
issue = DocumentIssue(
|
||||
document_id=document_id,
|
||||
kind=DocumentIssueKind.warning,
|
||||
code=code,
|
||||
severity=IssueSeverity.info,
|
||||
message="тестовое предупреждение",
|
||||
source=IssueSource.core,
|
||||
created_at=_now(),
|
||||
)
|
||||
session.add(issue)
|
||||
await session.flush()
|
||||
return issue
|
||||
|
||||
|
||||
async def make_result_object(
|
||||
session: AsyncSession, *, job: RecognitionJob, storage: FakeStorage, result: dict[str, Any]
|
||||
) -> S3Object:
|
||||
@ -337,6 +364,7 @@ async def make_entries_from_result(
|
||||
stored_file_id=file.id,
|
||||
legal_entity_id=file.legal_entity_id,
|
||||
document_type=DocumentType.OSV,
|
||||
file_ordinal=len(documents),
|
||||
osv_account_code=fields.osv_account_code,
|
||||
period_year=fields.period_year,
|
||||
period_quarter=fields.period_quarter,
|
||||
|
||||
@ -34,6 +34,7 @@ from src.models.enums import (
|
||||
from src.models.files import Document, RecognitionJob, S3Object, StoredFile
|
||||
from src.models.osv import OsvEntry
|
||||
from src.models.statements import (
|
||||
DocumentIssue,
|
||||
FinancialStatementColumn,
|
||||
FinancialStatementLine,
|
||||
FinancialStatementMeta,
|
||||
@ -45,6 +46,7 @@ from tests.factories import (
|
||||
make_company_group,
|
||||
make_consolidation,
|
||||
make_document,
|
||||
make_document_issue,
|
||||
make_entries_from_result,
|
||||
make_legal_entity,
|
||||
make_result_object,
|
||||
@ -146,7 +148,13 @@ async def _form_file_with_comment(session: AsyncSession, fixtures_dir: Path) ->
|
||||
async def _assert_form_rows_gone(session: AsyncSession, file_id: uuid.UUID, document_ids: list[uuid.UUID]) -> None:
|
||||
assert await _count(session, StoredFile, StoredFile.id == file_id) == 0
|
||||
assert await _count(session, Document, Document.stored_file_id == file_id) == 0
|
||||
for model in (FinancialStatementMeta, FinancialStatementColumn, FinancialStatementLine, FinancialStatementValue):
|
||||
for model in (
|
||||
DocumentIssue,
|
||||
FinancialStatementMeta,
|
||||
FinancialStatementColumn,
|
||||
FinancialStatementLine,
|
||||
FinancialStatementValue,
|
||||
):
|
||||
assert await _count(session, model) == 0, model.__tablename__
|
||||
assert await _count(session, Comment, Comment.object_id.in_(document_ids)) == 0
|
||||
|
||||
@ -161,6 +169,8 @@ async def test_delete_file_with_statement_form(
|
||||
file_id = await _form_file_with_comment(seeded, fixtures_dir)
|
||||
document_ids = list((await seeded.execute(select(Document.id).where(Document.stored_file_id == file_id))).scalars())
|
||||
assert await _count(seeded, FinancialStatementValue) > 0 and await _count(seeded, Comment) == 1
|
||||
await make_document_issue(seeded, document_id=document_ids[0]) # v5.19: document_issue уходит вместе с файлом
|
||||
assert await _count(seeded, DocumentIssue) == 1
|
||||
if superseded:
|
||||
await seeded.execute(update(StoredFile).where(StoredFile.id == file_id).values(status=FileStatus.SUPERSEDED))
|
||||
await seeded.flush()
|
||||
|
||||
@ -140,18 +140,18 @@ def test_enum_values_and_sets_are_compared(dbml_file: Path) -> None:
|
||||
|
||||
def test_newer_dbml_version_fails_before_comparing_tables(tmp_path: Path, dbml_file: Path) -> None:
|
||||
lines = dbml_file.read_text(encoding="utf-8").splitlines()
|
||||
newer = tmp_path / "data-model-v5.19.dbml"
|
||||
# шапка 5.18 + заведомо сломанное тело: если бы таблицы сравнивались, были бы другие строки/ошибка разбора
|
||||
newer.write_text(lines[0].replace("v5.18", "v5.19") + "\nTable broken {{{\n", encoding="utf-8")
|
||||
assert run_check(newer) == (1, ["версия DBML 5.19 не совпадает с версией ORM 5.18: обновите модели"])
|
||||
newer = tmp_path / "data-model-v5.20.dbml"
|
||||
# шапка 5.19 + заведомо сломанное тело: если бы таблицы сравнивались, были бы другие строки/ошибка разбора
|
||||
newer.write_text(lines[0].replace("v5.19", "v5.20") + "\nTable broken {{{\n", encoding="utf-8")
|
||||
assert run_check(newer) == (1, ["версия DBML 5.20 не совпадает с версией ORM 5.19: обновите модели"])
|
||||
|
||||
|
||||
def test_newer_dbml_version_via_cli(tmp_path: Path, dbml_file: Path) -> None:
|
||||
newer = tmp_path / "data-model-v5.19.dbml"
|
||||
newer.write_text(dbml_file.read_text(encoding="utf-8").replace("v5.18", "v5.19", 1), encoding="utf-8")
|
||||
newer = tmp_path / "data-model-v5.20.dbml"
|
||||
newer.write_text(dbml_file.read_text(encoding="utf-8").replace("v5.19", "v5.20", 1), encoding="utf-8")
|
||||
result = _run_cli(str(newer))
|
||||
assert result.returncode == 1
|
||||
assert result.stdout == "версия DBML 5.19 не совпадает с версией ORM 5.18: обновите модели\n"
|
||||
assert result.stdout == "версия DBML 5.20 не совпадает с версией ORM 5.19: обновите модели\n"
|
||||
|
||||
|
||||
def test_header_without_version_is_usage_error(tmp_path: Path) -> None:
|
||||
|
||||
@ -193,7 +193,7 @@ def test_nulls_not_distinct_flag_is_compared_both_ways() -> None:
|
||||
"""DBML-note `NULLS NOT DISTINCT` требует флаг в ORM-индексе; лишний флаг без note — тоже расхождение."""
|
||||
metadata = clone_metadata()
|
||||
document = metadata.tables["dkb.document"]
|
||||
index = next(i for i in document.indexes if i.unique)
|
||||
index = next(i for i in document.indexes if i.name == "uq_document_file_type_account")
|
||||
assert find_problems(metadata, {"document"}) == []
|
||||
index.dialect_options["postgresql"]["nulls_not_distinct"] = None # индекс без флага
|
||||
problems = find_problems(metadata, {"document"})
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
"""Enum'ы ORM против блоков `Enum {}` DBML v5.18: имена, значения и их порядок (без БД)."""
|
||||
"""Enum'ы ORM против блоков `Enum {}` DBML v5.19: имена, значения и их порядок (без БД)."""
|
||||
|
||||
import enum
|
||||
import re
|
||||
@ -10,12 +10,12 @@ from src.models import DBML_VERSION
|
||||
from src.models.enums import ENUMS, pg_enum
|
||||
from tests.dbml_helpers import dbml_path, load_dbml
|
||||
|
||||
EXPECTED_ENUM_COUNT = 28
|
||||
EXPECTED_ENUM_COUNT = 33
|
||||
|
||||
|
||||
def test_dbml_version_format() -> None:
|
||||
assert re.fullmatch(r"\d+\.\d+", DBML_VERSION)
|
||||
assert DBML_VERSION == "5.18"
|
||||
assert DBML_VERSION == "5.19"
|
||||
assert f"v{DBML_VERSION}" in dbml_path().name
|
||||
|
||||
|
||||
|
||||
@ -22,6 +22,7 @@ from src.models.enums import (
|
||||
)
|
||||
from src.models.files import Document, RecognitionJob, StoredFile
|
||||
from src.models.statements import (
|
||||
DocumentIssue,
|
||||
FinancialStatementColumn,
|
||||
FinancialStatementLine,
|
||||
FinancialStatementMeta,
|
||||
@ -29,7 +30,7 @@ from src.models.statements import (
|
||||
)
|
||||
from src.repositories.counterparties import CounterpartyIndex
|
||||
from src.services.result_mapping import MAPPING_WARNINGS_KEY, map_file
|
||||
from tests.factories import make_legal_entity
|
||||
from tests.factories import make_document_issue, make_legal_entity
|
||||
from tests.fixtures_helpers import GOLDEN_STATEMENTS, OCR_EXPECTED, golden_result, map_golden_snapshot, ocr_result
|
||||
|
||||
pytestmark = [pytest.mark.integration, pytest.mark.fixtures]
|
||||
@ -291,6 +292,7 @@ async def test_repeated_form_mapping_is_idempotent(seeded: AsyncSession, fixture
|
||||
documents = await map_golden_snapshot(seeded, entity=owner, path=path, uploaded_by=None)
|
||||
before = await _statement_totals(seeded)
|
||||
file_id = documents[0].stored_file_id
|
||||
await make_document_issue(seeded, document_id=documents[0].id) # v5.19: остаток прежнего маппинга
|
||||
file = await seeded.get(StoredFile, file_id)
|
||||
assert file is not None and file.current_recognition_job_id is not None
|
||||
job = await seeded.get(RecognitionJob, file.current_recognition_job_id)
|
||||
@ -311,6 +313,7 @@ async def test_repeated_form_mapping_is_idempotent(seeded: AsyncSession, fixture
|
||||
)
|
||||
|
||||
assert await _statement_totals(seeded) == before
|
||||
assert (await seeded.execute(select(func.count()).select_from(DocumentIssue))).scalar_one() == 0
|
||||
again = (await seeded.execute(select(Document).where(Document.stored_file_id == file_id))).scalars().all()
|
||||
assert {d.document_type for d in again} == {DocumentType.BALANCE, DocumentType.INCOME}
|
||||
assert {d.id for d in again}.isdisjoint({d.id for d in documents}) # документы пересозданы, не дублированы
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Миграции 0001 (DBML v5.13), 0002 (v5.14), 0003 (v5.15), 0004 (v5.16), 0005 (v5.17), 0006 (v5.18) на PostgreSQL 16:
|
||||
"""Миграции 0001 (DBML v5.13), 0002 (v5.14), 0003 (v5.15), 0004 (v5.16), 0005 (v5.17), 0006 (v5.18), 0007 (v5.19)
|
||||
на PostgreSQL 16:
|
||||
полный цикл, откат на шаг, пустой autogenerate, схема = схеме ORM.
|
||||
|
||||
Spec data-model, требование «Миграции применяются с нуля и откатываются» (сценарии «Полный цикл» и «Автогенерация
|
||||
@ -15,7 +16,7 @@ import pytest
|
||||
from alembic.autogenerate import compare_metadata
|
||||
from alembic.runtime.migration import MigrationContext
|
||||
from sqlalchemy import Connection, make_url, text
|
||||
from sqlalchemy.exc import ProgrammingError
|
||||
from sqlalchemy.exc import IntegrityError, ProgrammingError
|
||||
from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, AsyncSession, create_async_engine
|
||||
|
||||
from src.models import Base
|
||||
@ -36,7 +37,8 @@ from tests.test_models_part_07_09 import (
|
||||
_check_active_lock_is_unique,
|
||||
)
|
||||
|
||||
HEAD = "0006_dbml_v5_18"
|
||||
HEAD = "0007_dbml_v5_19"
|
||||
V5_18 = "0006_dbml_v5_18"
|
||||
V5_17 = "0005_dbml_v5_17"
|
||||
V5_16 = "0004_dbml_v5_16"
|
||||
V5_15 = "0003_dbml_v5_15"
|
||||
@ -620,6 +622,253 @@ async def test_upgrade_0006_backfills_consolidation_id(scratch_db_url: str) -> N
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def _seed_v5_18_statements(conn: AsyncConnection, ids: dict[str, uuid.UUID]) -> None:
|
||||
"""Данные на ревизии 0006: ЮЛ, файл, `INCOME` (t0) и `BALANCE` (t1 > t0) с формой ББ; ГСК без `crm_id`."""
|
||||
t0 = datetime.datetime(2025, 10, 1, 9, 0, tzinfo=datetime.UTC)
|
||||
t1 = t0 + datetime.timedelta(minutes=5)
|
||||
await conn.execute(
|
||||
text("INSERT INTO dkb.legal_entity (id, inn, name, valid_from) VALUES (:le, '7701234567', 'ООО Ромашка', :t0)"),
|
||||
{**ids, "t0": t0},
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"INSERT INTO dkb.s3_object (id, bucket_role, bucket, key, created_at) "
|
||||
"VALUES (:s3, 'uploads', 'b', 'k', :t0)"
|
||||
),
|
||||
{**ids, "t0": t0},
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"INSERT INTO dkb.stored_file (id, legal_entity_id, file_type, s3_object_id, status) "
|
||||
"VALUES (:file, :le, 'XLSX', :s3, 'PROCESSED')"
|
||||
),
|
||||
ids,
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"INSERT INTO dkb.recognition_job (id, file_id, pipeline, request_subject, schema_version, status) "
|
||||
"VALUES (:job, :file, 'tabular', 'files.tabular_uploaded', '1', 'done')"
|
||||
),
|
||||
ids,
|
||||
)
|
||||
for key, doc_type, created in (("income", "INCOME", t0), ("balance", "BALANCE", t1)):
|
||||
await conn.execute(
|
||||
text(
|
||||
"INSERT INTO dkb.document (id, stored_file_id, legal_entity_id, document_type, status, created_at) "
|
||||
f"VALUES (:{key}, :file, :le, '{doc_type}', 'NEEDS_CHECK', :created)"
|
||||
),
|
||||
{**ids, "created": created},
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"INSERT INTO dkb.financial_statement_meta (id, document_id, recognition_job_id, statement_type, inn, "
|
||||
"inn_original_value, inn_confidence, report_date_original_value, units_original_value, organization_name) "
|
||||
"VALUES (:meta, :balance, :job, 'balance', '7701234567', '7701234567', 87, '30.09.2025', 'тыс. руб.', "
|
||||
"'ООО Ромашка')"
|
||||
),
|
||||
ids,
|
||||
)
|
||||
for key, index, period in (("c1", 0, datetime.date(2025, 9, 30)), ("c2", 1, None)):
|
||||
await conn.execute(
|
||||
text(
|
||||
"INSERT INTO dkb.financial_statement_column (id, meta_id, column_index, period_date) "
|
||||
"VALUES (:id, :meta, :idx, :period)"
|
||||
),
|
||||
{"id": ids[key], "meta": ids["meta"], "idx": index, "period": period},
|
||||
)
|
||||
for key, code, original in (("l1600", "1600", "1600"), ("l1100", "1100", "1100"), ("l1250", "1250", None)):
|
||||
await conn.execute(
|
||||
text(
|
||||
"INSERT INTO dkb.financial_statement_line (id, meta_id, line_code, line_code_original_value) "
|
||||
"VALUES (:id, :meta, :code, :original)"
|
||||
),
|
||||
{"id": ids[key], "meta": ids["meta"], "code": code, "original": original},
|
||||
)
|
||||
values = [
|
||||
# (строка, колонка, сумма, строка-значение, уверенность, распознано, правлено)
|
||||
("l1100", "c1", 1, None, 100, True, False),
|
||||
("l1100", "c2", 2, None, 80, True, False),
|
||||
("l1600", "c1", 3, None, 80, True, True),
|
||||
("l1600", "c2", 4, None, None, False, False),
|
||||
("l1250", "c1", 5, "12 3", None, None, False),
|
||||
]
|
||||
for line, column, amount, text_value, confidence, recognized, modified in values:
|
||||
await conn.execute(
|
||||
text(
|
||||
"INSERT INTO dkb.financial_statement_value (id, line_id, column_id, amount, string_value, confidence, "
|
||||
"is_recognized, is_modified) VALUES (:id, :line, :column, :amount, :sv, :conf, :rec, :mod)"
|
||||
),
|
||||
{
|
||||
"id": uuid.uuid4(),
|
||||
"line": ids[line],
|
||||
"column": ids[column],
|
||||
"amount": amount,
|
||||
"sv": text_value,
|
||||
"conf": confidence,
|
||||
"rec": recognized,
|
||||
"mod": modified,
|
||||
},
|
||||
)
|
||||
await conn.execute(
|
||||
text("INSERT INTO dkb.company_group (id, code, name, valid_from) VALUES (:grp, 'GK-1', 'ГК', :t0)"),
|
||||
{**ids, "t0": t0},
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"INSERT INTO dkb.related_group (id, code, company_group_id, kind, name) "
|
||||
"VALUES (:rg, 'GSK-1', :grp, 'gsk', 'ГСК')"
|
||||
),
|
||||
ids,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
async def test_upgrade_0007_backfills_v5_19_columns(scratch_db_url: str) -> None:
|
||||
"""0007: заполнение по умолчанию на данных v5.18 (row_index, needs_review, source_text, period_kind, header_source,
|
||||
file_ordinal, crm_id); `document_issue` пуста."""
|
||||
engine = create_async_engine(scratch_db_url)
|
||||
ids = {
|
||||
k: uuid.uuid4()
|
||||
for k in (
|
||||
"le",
|
||||
"s3",
|
||||
"file",
|
||||
"job",
|
||||
"income",
|
||||
"balance",
|
||||
"meta",
|
||||
"c1",
|
||||
"c2",
|
||||
"l1600",
|
||||
"l1100",
|
||||
"l1250",
|
||||
"grp",
|
||||
"rg",
|
||||
)
|
||||
}
|
||||
try:
|
||||
await alembic(scratch_db_url, "upgrade", V5_18)
|
||||
async with engine.begin() as conn:
|
||||
await _seed_v5_18_statements(conn, ids)
|
||||
await alembic(scratch_db_url, "upgrade", "head")
|
||||
async with engine.connect() as conn:
|
||||
lines = (
|
||||
await conn.execute(
|
||||
text(
|
||||
"SELECT line_code, row_index, line_status::text, needs_review, is_confirmed "
|
||||
"FROM dkb.financial_statement_line"
|
||||
)
|
||||
)
|
||||
).all()
|
||||
rows = {r.line_code: r for r in lines}
|
||||
assert rows["1100"].row_index == 0 and rows["1600"].row_index == 1 and rows["1250"].row_index is None
|
||||
assert {r.line_status for r in rows.values()} == {"ok"} and not any(r.needs_review for r in rows.values())
|
||||
values = (
|
||||
await conn.execute(
|
||||
text(
|
||||
"SELECT v.needs_review, v.source_text, v.string_value, v.is_confirmed FROM "
|
||||
"dkb.financial_statement_value v JOIN dkb.financial_statement_line l ON l.id = v.line_id "
|
||||
"ORDER BY v.amount"
|
||||
)
|
||||
)
|
||||
).all()
|
||||
assert [v.needs_review for v in values] == [False, True, False, True, False]
|
||||
assert not any(v.is_confirmed for v in values)
|
||||
value_by_text = {v.string_value: v for v in values if v.string_value}
|
||||
assert value_by_text["12 3"].source_text == "12 3"
|
||||
period_kind = dict(
|
||||
(await conn.execute(text("SELECT column_index, period_kind FROM dkb.financial_statement_column"))).all()
|
||||
)
|
||||
assert period_kind == {0: "absolute", 1: "none"}
|
||||
meta = (await conn.execute(text("SELECT * FROM dkb.financial_statement_meta"))).one()
|
||||
assert meta.form_variant is None and meta.needs_review_fields is None
|
||||
assert meta.header_source == {
|
||||
"inn": {"text": "7701234567", "confidence": 87},
|
||||
"report_date": {"text": "30.09.2025", "confidence": None},
|
||||
"units": {"text": "тыс. руб.", "confidence": None},
|
||||
"organization_name": {"text": "ООО Ромашка", "confidence": None},
|
||||
}
|
||||
assert meta.organization_name_original_value == "ООО Ромашка" and meta.organization_name_modified is False
|
||||
ordinals = dict(
|
||||
(await conn.execute(text("SELECT document_type::text, file_ordinal FROM dkb.document"))).all()
|
||||
)
|
||||
assert ordinals == {"INCOME": 0, "BALANCE": 1}
|
||||
assert await _scalars(conn, "SELECT crm_id FROM dkb.related_group") == [None]
|
||||
assert await _scalars(conn, "SELECT count(*) FROM dkb.document_issue") == [0]
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def _insert_document(conn: AsyncConnection, ids: dict[str, uuid.UUID], doc_type: str, ordinal: int) -> None:
|
||||
await conn.execute(
|
||||
text(
|
||||
"INSERT INTO dkb.document (id, stored_file_id, legal_entity_id, document_type, status, file_ordinal) "
|
||||
f"VALUES (:id, :file, :le, '{doc_type}', 'NEEDS_CHECK', :ordinal)"
|
||||
),
|
||||
{"id": uuid.uuid4(), "file": ids["file"], "le": ids["le"], "ordinal": ordinal},
|
||||
)
|
||||
|
||||
|
||||
async def _insert_line(
|
||||
conn: AsyncConnection, ids: dict[str, uuid.UUID], code: str | None, source_ref: str | None
|
||||
) -> None:
|
||||
await conn.execute(
|
||||
text(
|
||||
"INSERT INTO dkb.financial_statement_line (id, meta_id, line_code, source_ref) "
|
||||
"VALUES (:id, :meta, :code, :ref)"
|
||||
),
|
||||
{"id": uuid.uuid4(), "meta": ids["meta"], "code": code, "ref": source_ref},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
async def test_v5_19_constraints(scratch_db_url: str) -> None:
|
||||
"""Поведение индексов v5.19: формы одного типа, одна расшифровка, повтор кода, `source_ref`, `file_ordinal`."""
|
||||
engine = create_async_engine(scratch_db_url)
|
||||
ids = {
|
||||
k: uuid.uuid4()
|
||||
for k in (
|
||||
"le",
|
||||
"s3",
|
||||
"file",
|
||||
"job",
|
||||
"income",
|
||||
"balance",
|
||||
"meta",
|
||||
"c1",
|
||||
"c2",
|
||||
"l1600",
|
||||
"l1100",
|
||||
"l1250",
|
||||
"grp",
|
||||
"rg",
|
||||
)
|
||||
}
|
||||
try:
|
||||
await alembic(scratch_db_url, "upgrade", V5_18)
|
||||
async with engine.begin() as conn:
|
||||
await _seed_v5_18_statements(conn, ids)
|
||||
await alembic(scratch_db_url, "upgrade", "head")
|
||||
async with engine.begin() as conn:
|
||||
await _insert_document(conn, ids, "BALANCE", 2) # вторая форма того же типа — можно
|
||||
await _insert_document(conn, ids, "DISCLOSURE", 3)
|
||||
await _insert_line(conn, ids, "1100", None) # повтор кода — можно
|
||||
await _insert_line(conn, ids, None, None)
|
||||
await _insert_line(conn, ids, None, None) # source_ref IS NULL — можно повторять
|
||||
await _insert_line(conn, ids, "1500", "row:8")
|
||||
with pytest.raises(IntegrityError): # вторая DISCLOSURE (uq_document_file_type_account)
|
||||
async with engine.begin() as conn:
|
||||
await _insert_document(conn, ids, "DISCLOSURE", 4)
|
||||
with pytest.raises(IntegrityError): # два документа файла с одним file_ordinal
|
||||
async with engine.begin() as conn:
|
||||
await _insert_document(conn, ids, "OTHER", 3)
|
||||
with pytest.raises(IntegrityError): # тот же source_ref в форме
|
||||
async with engine.begin() as conn:
|
||||
await _insert_line(conn, ids, "1501", "row:8")
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
async def test_upgrade_0005_adds_nullable_kpp_columns(scratch_db_url: str) -> None:
|
||||
"""0005: `kpp varchar` nullable у `legal_entity` и `legal_entity_history`, существующие ЮЛ — с `kpp IS NULL`;
|
||||
@ -883,8 +1132,15 @@ def test_offline_sql_creates_schema_before_version_table() -> None:
|
||||
assert schema_at < output.index("CREATE TABLE dkb.alembic_version")
|
||||
assert schema_at < output.index("CREATE TYPE dkb.entity_type AS ENUM ('ЮЛ', 'ИП')")
|
||||
assert f"INSERT INTO dkb.alembic_version (version_num) VALUES ('{PREVIOUS}')" in output
|
||||
# версия head пишется UPDATE'ами по цепочке 0001 → 0002 → 0003 → 0004 → 0005 → 0006 — офлайн-SQL доходит до head
|
||||
for new_version, old_version in ((V5_14, PREVIOUS), (V5_15, V5_14), (V5_16, V5_15), (V5_17, V5_16), (HEAD, V5_17)):
|
||||
# версия head пишется UPDATE'ами по цепочке 0001 → … → 0007 — офлайн-SQL доходит до head
|
||||
for new_version, old_version in (
|
||||
(V5_14, PREVIOUS),
|
||||
(V5_15, V5_14),
|
||||
(V5_16, V5_15),
|
||||
(V5_17, V5_16),
|
||||
(V5_18, V5_17),
|
||||
(HEAD, V5_18),
|
||||
):
|
||||
assert (
|
||||
f"UPDATE dkb.alembic_version SET version_num='{new_version}' "
|
||||
f"WHERE dkb.alembic_version.version_num = '{old_version}'" in output
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
|
||||
import datetime
|
||||
import decimal
|
||||
import itertools
|
||||
import uuid
|
||||
import warnings
|
||||
from dataclasses import dataclass
|
||||
@ -41,6 +42,7 @@ PART_04_06_TABLES = {
|
||||
"financial_statement_column",
|
||||
"financial_statement_line",
|
||||
"financial_statement_value",
|
||||
"document_issue",
|
||||
}
|
||||
|
||||
|
||||
@ -85,7 +87,7 @@ def test_fk_cycle_is_broken_by_use_alter_without_warning() -> None:
|
||||
|
||||
|
||||
def test_document_unique_index_is_nulls_not_distinct() -> None:
|
||||
index = next(i for i in orm_tables()["document"].indexes if i.unique)
|
||||
index = next(i for i in orm_tables()["document"].indexes if i.name == "uq_document_file_type_account")
|
||||
assert index.dialect_options["postgresql"]["nulls_not_distinct"] is True
|
||||
assert [c.name for c in index.columns] == ["stored_file_id", "document_type", "osv_account_code"]
|
||||
|
||||
@ -170,8 +172,12 @@ async def _make_file_with_job(session: AsyncSession) -> Chain:
|
||||
return Chain(entity_id=entity.id, file_id=file.id, job_id=job.id)
|
||||
|
||||
|
||||
_FILE_ORDINALS = itertools.count() # v5.19: file_ordinal уникален в файле — тесту нужен любой свободный номер
|
||||
|
||||
|
||||
def _document(chain: Chain, document_type: DocumentType, account: str | None) -> Document:
|
||||
return Document(
|
||||
file_ordinal=next(_FILE_ORDINALS),
|
||||
stored_file_id=chain.file_id,
|
||||
legal_entity_id=chain.entity_id,
|
||||
document_type=document_type,
|
||||
@ -181,16 +187,18 @@ def _document(chain: Chain, document_type: DocumentType, account: str | None) ->
|
||||
|
||||
|
||||
async def _check_nulls_not_distinct(session: AsyncSession, chain: Chain) -> None:
|
||||
"""Два документа с одним (stored_file_id, document_type) и osv_account_code = NULL -> IntegrityError."""
|
||||
session.add(_document(chain, DocumentType.BALANCE, None))
|
||||
"""v5.19: два документа с одним (stored_file_id, document_type) и osv_account_code = NULL — `IntegrityError` для
|
||||
не-форм (DISCLOSURE); формы ББ/ОФР одного типа в файле допустимы (Р-10)."""
|
||||
session.add(_document(chain, DocumentType.DISCLOSURE, None))
|
||||
await session.flush()
|
||||
session.add(_document(chain, DocumentType.BALANCE, None))
|
||||
session.add(_document(chain, DocumentType.DISCLOSURE, None))
|
||||
with pytest.raises(IntegrityError):
|
||||
await session.flush()
|
||||
await session.rollback()
|
||||
# разные счета ОСВ в одном файле и ББ/ОФР рядом — допустимы
|
||||
# две формы ББ одного типа, разные счета ОСВ и ОФР рядом — допустимы
|
||||
session.add_all(
|
||||
[_document(chain, DocumentType.OSV, "58"), _document(chain, DocumentType.OSV, "60")]
|
||||
[_document(chain, DocumentType.BALANCE, None), _document(chain, DocumentType.BALANCE, None)]
|
||||
+ [_document(chain, DocumentType.OSV, "58"), _document(chain, DocumentType.OSV, "60")]
|
||||
+ [_document(chain, DocumentType.INCOME, None)]
|
||||
)
|
||||
await session.flush()
|
||||
|
||||
@ -48,8 +48,8 @@ PART_07_09_TABLES = {
|
||||
"required_document_rule",
|
||||
}
|
||||
ALL_TABLES = PART_01_03_TABLES | PART_04_06_TABLES | PART_07_09_TABLES
|
||||
EXPECTED_TABLE_COUNT = 43
|
||||
EXPECTED_ENUM_COUNT = 28
|
||||
EXPECTED_TABLE_COUNT = 44
|
||||
EXPECTED_ENUM_COUNT = 33
|
||||
|
||||
|
||||
def test_part_07_09_tables_are_new() -> None:
|
||||
@ -277,6 +277,10 @@ def test_all_nullable_jsonb_columns_are_listed() -> None:
|
||||
"recognition_job.group_schema",
|
||||
"recognition_job.metrics",
|
||||
"osv_entry.original_values",
|
||||
"financial_statement_meta.needs_review_fields",
|
||||
"financial_statement_meta.header_source",
|
||||
"document_issue.line_ids",
|
||||
"document_issue.codes",
|
||||
}
|
||||
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user