RSHB/arm-core
9
0
Fork 0
Code Issues 25 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: ORM-модели частей 04–06 DBML v5.13 (файлы, ОСВ, отчётность)

Browse Source 
files.py: s3_object, stored_file, document (NULLS NOT DISTINCT), recognition_job;
FK-цикл stored_file <-> recognition_job разорван через use_alter с явным именем.
osv.py: osv_entry. statements.py: financial_statement_meta/column/line/value.
Тест частей 01–03 не привязан к полному составу metadata.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Andrey Epifancev 2026-10-03 19:22:29 +04:00
parent 24fbe74840
commit 4e363bd00f
5 changed files with 353 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
api/src/models/__init__.py
View File

@ -3,10 +3,20 @@
Импорт всех модулей моделей обязателен: он регистрирует таблицы в `Base.metadata` (create_all, alembic).
"""
from . import access, accounting, directories
from . import access, accounting, directories, files, osv, statements
from .base import Base, UUIDPrimaryMixin
#: Версия DBML, которой соответствуют модели (сверяется с шапкой DBML в check_dbml_vs_orm).
DBML_VERSION = "5.13"
__all__ = ["DBML_VERSION", "Base", "UUIDPrimaryMixin", "access", "accounting", "directories"]
__all__ = [
"DBML_VERSION",
"Base",
"UUIDPrimaryMixin",
"access",
"accounting",
"directories",
"files",
"osv",
"statements",
]

157
api/src/models/files.py Normal file
View File

@ -0,0 +1,157 @@
"""Часть 04 DBML v5.13 «Файлы / распознавание»: реестр S3, физические файлы, документы, задания распознавания.
Цикл FK `stored_file.current_recognition_job_id` -> `recognition_job` и `recognition_job.file_id` -> `stored_file`
разорван через `use_alter=True` на nullable-стороне (`stored_file.current_recognition_job_id`): `create_all` создаёт
таблицы без этого FK и добавляет его отдельным `ALTER TABLE`; имя FK задано явно (то же имя — в миграции 0001).
"""
import datetime
import uuid
from typing import Any
from sqlalchemy import BigInteger, Boolean, Date, DateTime, ForeignKey, Index, Integer, SmallInteger, String, Text, text
from sqlalchemy.dialects.postgresql import JSONB, UUID
from sqlalchemy.orm import Mapped, mapped_column
from .base import Base, UUIDPrimaryMixin
from .enums import (
DocumentCheckResult,
DocumentStatus,
DocumentType,
FileStatus,
FileType,
PeriodQuarter,
RecognitionPipeline,
RecognitionStatus,
S3BucketRole,
pg_enum,
)
class S3Object(UUIDPrimaryMixin, Base):
"""Реестр объектов S3: единая точка ключей для lifecycle/ретеншна (ТЗ 8.4) и дедупа."""
__tablename__ = "s3_object"
__table_args__ = (Index("uq_s3_object_bucket_key", "bucket", "key", unique=True),)
bucket_role: Mapped[S3BucketRole] = mapped_column(pg_enum("s3_bucket_role"), nullable=False)
bucket: Mapped[str] = mapped_column(String, nullable=False)
key: Mapped[str] = mapped_column(String, nullable=False)
content_type: Mapped[str | None] = mapped_column(String)
size_bytes: Mapped[int | None] = mapped_column(BigInteger)
sha256: Mapped[str | None] = mapped_column(String)
version_id: Mapped[str | None] = mapped_column(String)
created_by: Mapped[str | None] = mapped_column(String)
created_at: Mapped[datetime.datetime] = mapped_column(DateTime(timezone=True), nullable=False)
expires_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True))
archived_to: Mapped[str | None] = mapped_column(String)
class StoredFile(UUIDPrimaryMixin, Base):
"""Физический файл одной загрузки; что внутри — `document` (1:N). Первичка не редактируется (ТЗ 8.2)."""
__tablename__ = "stored_file"
__table_args__ = (Index("ix_stored_file_legal_entity_id_status", "legal_entity_id", "status"),)
legal_entity_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("legal_entity.id"), nullable=False
)
file_type: Mapped[FileType] = mapped_column(pg_enum("file_type"), nullable=False)
original_name: Mapped[str | None] = mapped_column(String)
s3_object_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("s3_object.id"), nullable=False)
content_hash: Mapped[str | None] = mapped_column(String)
size_bytes: Mapped[int | None] = mapped_column(BigInteger)
status: Mapped[FileStatus] = mapped_column(pg_enum("file_status"), nullable=False)
version_no: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("1"))
superseded_by_file_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("stored_file.id"))
current_recognition_job_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True),
ForeignKey(
"recognition_job.id",
use_alter=True,
name="fk_stored_file_current_recognition_job_id_recognition_job",
),
)
uploaded_by: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("app_user.id"))
uploaded_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True))
updated_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True))
class Document(UUIDPrimaryMixin, Base):
"""Логический документ внутри файла (ББ / ОФР / ОСВ по счёту / расшифровка); создаётся ядром при маппинге."""
__tablename__ = "document"
__table_args__ = (
# один ББ / один ОФР / одна ОСВ по счёту на файл: NULL в osv_account_code считается значением (PG15+)
Index(
"uq_document_file_type_account",
"stored_file_id",
"document_type",
"osv_account_code",
unique=True,
postgresql_nulls_not_distinct=True,
),
# чек-лист required_document_rule
Index(
"ix_document_legal_entity_period_status",
"legal_entity_id",
"document_type",
"osv_account_code",
"period_year",
"period_quarter",
"status",
),
)
stored_file_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("stored_file.id"), nullable=False)
legal_entity_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("legal_entity.id"), nullable=False
)
document_type: Mapped[DocumentType] = mapped_column(pg_enum("document_type"), nullable=False)
osv_account_code: Mapped[str | None] = mapped_column(String)
period_year: Mapped[int | None] = mapped_column(SmallInteger)
period_quarter: Mapped[PeriodQuarter | None] = mapped_column(pg_enum("period_quarter"))
is_cumulative: Mapped[bool | None] = mapped_column(Boolean)
reporting_date: Mapped[datetime.date | None] = mapped_column(Date)
locator: Mapped[str | None] = mapped_column(String)
status: Mapped[DocumentStatus] = mapped_column(pg_enum("document_status"), nullable=False)
period_check: Mapped[DocumentCheckResult | None] = mapped_column(pg_enum("document_check_result"))
counterparty_check: Mapped[DocumentCheckResult | None] = mapped_column(pg_enum("document_check_result"))
confirmed_by: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("app_user.id"))
confirmed_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True))
created_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True))
updated_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True))
class RecognitionJob(UUIDPrimaryMixin, Base):
"""Единственная точка обмена с NATS: запись = outbox (queued -> published), результат — inbox (дедуп)."""
__tablename__ = "recognition_job"
__table_args__ = (Index("uq_recognition_job_file_id_attempt", "file_id", "attempt", unique=True),)
file_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("stored_file.id"), nullable=False)
pipeline: Mapped[RecognitionPipeline] = mapped_column(pg_enum("recognition_pipeline"), nullable=False)
request_subject: Mapped[str] = mapped_column(String, nullable=False)
request_msg_id: Mapped[str | None] = mapped_column(String, unique=True)
result_msg_id: Mapped[str | None] = mapped_column(String, unique=True)
schema_version: Mapped[str] = mapped_column(String, nullable=False)
service_name: Mapped[str | None] = mapped_column(String)
service_version: Mapped[str | None] = mapped_column(String)
status: Mapped[RecognitionStatus] = mapped_column(pg_enum("recognition_status"), nullable=False)
attempt: Mapped[int] = mapped_column(SmallInteger, nullable=False, server_default=text("1"))
max_attempts: Mapped[int] = mapped_column(SmallInteger, nullable=False, server_default=text("1"))
requested_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True))
published_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True))
started_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True))
finished_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True))
reply_deadline: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True))
external_task_id: Mapped[str | None] = mapped_column(String)
error_code: Mapped[str | None] = mapped_column(String)
error_message: Mapped[str | None] = mapped_column(Text)
result_s3_object_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("s3_object.id"))
result_schema_version: Mapped[str | None] = mapped_column(String)
group_schema: Mapped[Any | None] = mapped_column(JSONB)
metrics: Mapped[Any | None] = mapped_column(JSONB)
fields_total: Mapped[int | None] = mapped_column(Integer)
fields_low_confidence: Mapped[int | None] = mapped_column(Integer)
fields_unrecognized: Mapped[int | None] = mapped_column(Integer)

59
api/src/models/osv.py Normal file
View File

@ -0,0 +1,59 @@
"""Часть 05 DBML v5.13 «ОСВ»: строка оборотно-сальдовой ведомости (ТЗ 6.2, 6.5).
Первичка не мутируется: сырые значения хранятся как пришли, правка аналитика — рядом
(`is_modified`, `original_values`, `modified_by/at` + audit_event(RECOGNIZED_DATA_EDIT)).
"""
import datetime
import decimal
import uuid
from typing import Any
from sqlalchemy import Boolean, DateTime, ForeignKey, Index, Integer, Numeric, SmallInteger, String, text
from sqlalchemy.dialects.postgresql import JSONB, UUID
from sqlalchemy.orm import Mapped, mapped_column
from .base import Base, UUIDPrimaryMixin
from .enums import OsvDataType, OsvGroupType, pg_enum
class OsvEntry(UUIDPrimaryMixin, Base):
"""Строка ОСВ; суммы — рубли, `numeric(18,2)`."""
__tablename__ = "osv_entry"
__table_args__ = (
Index("uq_osv_entry_document_id_row_number", "document_id", "row_number", unique=True),
Index("ix_osv_entry_owner_account_counterparty", "owner_legal_entity_id", "account_id", "counterparty_id"),
)
recognition_job_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("recognition_job.id"), nullable=False
)
document_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("document.id"), nullable=False)
owner_legal_entity_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("legal_entity.id"), nullable=False
)
parent_entry_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("osv_entry.id"))
group_type: Mapped[OsvGroupType | None] = mapped_column(pg_enum("osv_group_type"))
group_name: Mapped[str | None] = mapped_column(String)
row_number: Mapped[int | None] = mapped_column(Integer)
level: Mapped[int | None] = mapped_column(SmallInteger)
row_value: Mapped[str | None] = mapped_column(String)
data_type: Mapped[OsvDataType | None] = mapped_column(pg_enum("osv_data_type"))
account_code: Mapped[str | None] = mapped_column(String)
account_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("chart_of_account.id"))
account_root: Mapped[str | None] = mapped_column(String)
counterparty_name: Mapped[str | None] = mapped_column(String)
counterparty_inn: Mapped[str | None] = mapped_column(String)
counterparty_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("legal_entity.id"))
opening_debit: Mapped[decimal.Decimal | None] = mapped_column(Numeric(18, 2))
opening_credit: Mapped[decimal.Decimal | None] = mapped_column(Numeric(18, 2))
turnover_debit: Mapped[decimal.Decimal | None] = mapped_column(Numeric(18, 2))
turnover_credit: Mapped[decimal.Decimal | None] = mapped_column(Numeric(18, 2))
closing_debit: Mapped[decimal.Decimal | None] = mapped_column(Numeric(18, 2))
closing_credit: Mapped[decimal.Decimal | None] = mapped_column(Numeric(18, 2))
is_modified: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("false"))
original_values: Mapped[Any | None] = mapped_column(JSONB)
modified_by: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("app_user.id"))
modified_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True))
is_excisable: Mapped[bool | None] = mapped_column(Boolean)

119
api/src/models/statements.py Normal file
View File

@ -0,0 +1,119 @@
"""Часть 06 DBML v5.13 «Бухгалтерская отчётность»: шапка, колонки-периоды, строки и значения форм ББ/ОФР (157н).
Первичка не мутируется: `*_original_value` / `original_amount` / `is_modified` — рядом с распознанным значением.
Логические проверки (ТЗ 6.3) и «требует проверки» не хранятся — считаются на лету.
"""
import datetime
import decimal
import uuid
from sqlalchemy import Boolean, Date, DateTime, ForeignKey, Index, Numeric, SmallInteger, String, Text, text
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import Mapped, mapped_column
from .base import Base, UUIDPrimaryMixin
from .enums import StatementType, pg_enum
class FinancialStatementMeta(UUIDPrimaryMixin, Base):
"""Шапка распознанной формы ББ/ОФР; 1:1 с `document`."""
__tablename__ = "financial_statement_meta"
document_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("document.id"), nullable=False, unique=True
)
recognition_job_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("recognition_job.id"), nullable=False
)
statement_type: Mapped[StatementType] = mapped_column(pg_enum("statement_type"), nullable=False)
inn: Mapped[str | None] = mapped_column(String)
inn_original_value: Mapped[str | None] = mapped_column(String)
inn_confidence: Mapped[decimal.Decimal | None] = mapped_column(Numeric(5, 2))
inn_verified: Mapped[bool | None] = mapped_column(Boolean)
inn_modified: Mapped[bool | None] = mapped_column(Boolean)
organization_name: Mapped[str | None] = mapped_column(String)
report_date: Mapped[datetime.date | None] = mapped_column(Date)
report_date_original_value: Mapped[str | None] = mapped_column(String)
report_date_verified: Mapped[bool | None] = mapped_column(Boolean)
report_date_modified: Mapped[bool | None] = mapped_column(Boolean)
units: Mapped[str | None] = mapped_column(String)
units_original_value: Mapped[str | None] = mapped_column(String)
units_verified: Mapped[bool | None] = mapped_column(Boolean)
units_modified: Mapped[bool | None] = mapped_column(Boolean)
currency: Mapped[str | None] = mapped_column(String, server_default=text("'RUB'"))
okpo: Mapped[str | None] = mapped_column(String)
okopf: Mapped[str | None] = mapped_column(String)
okfs: Mapped[str | None] = mapped_column(String)
okud: Mapped[str | None] = mapped_column(String)
okei: Mapped[str | None] = mapped_column(String)
legal_form: Mapped[str | None] = mapped_column(String)
ownership_form: Mapped[str | None] = mapped_column(String)
address: Mapped[str | None] = mapped_column(String)
subject_to_audit: Mapped[bool | None] = mapped_column(Boolean)
reviewed_by: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("app_user.id"))
reviewed_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True))
checks_accepted_by: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("app_user.id"))
checks_accepted_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True))
checks_accepted_comment: Mapped[str | None] = mapped_column(Text)
class FinancialStatementColumn(UUIDPrimaryMixin, Base):
"""Колонка-период формы (2–3 сравнительные даты); `period_date` нужна межформенным проверкам и гудвиллу."""
__tablename__ = "financial_statement_column"
__table_args__ = (
Index("uq_financial_statement_column_meta_id_column_index", "meta_id", "column_index", unique=True),
)
meta_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("financial_statement_meta.id"), nullable=False
)
column_index: Mapped[int] = mapped_column(SmallInteger, nullable=False)
label: Mapped[str | None] = mapped_column(String)
period_date: Mapped[datetime.date | None] = mapped_column(Date)
is_current: Mapped[bool | None] = mapped_column(Boolean)
class FinancialStatementLine(UUIDPrimaryMixin, Base):
"""Строка формы с кодом; порядок, разделы и итоговые строки — шаблон формы на фронте."""
__tablename__ = "financial_statement_line"
__table_args__ = (Index("uq_financial_statement_line_meta_id_line_code", "meta_id", "line_code", unique=True),)
meta_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("financial_statement_meta.id"), nullable=False
)
line_code: Mapped[str | None] = mapped_column(String)
line_code_original_value: Mapped[str | None] = mapped_column(String)
line_code_confidence: Mapped[decimal.Decimal | None] = mapped_column(Numeric(5, 2))
line_code_modified: Mapped[bool | None] = mapped_column(Boolean)
line_name: Mapped[str | None] = mapped_column(String)
line_name_original_value: Mapped[str | None] = mapped_column(String)
line_name_confidence: Mapped[decimal.Decimal | None] = mapped_column(Numeric(5, 2))
line_name_modified: Mapped[bool | None] = mapped_column(Boolean)
class FinancialStatementValue(UUIDPrimaryMixin, Base):
"""Распознанное значение (строка x колонка): сумма, уверенность, признак ручной правки (Прил. 2 ТЗ)."""
__tablename__ = "financial_statement_value"
__table_args__ = (Index("uq_financial_statement_value_line_id_column_id", "line_id", "column_id", unique=True),)
line_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("financial_statement_line.id"), nullable=False
)
column_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True),
ForeignKey("financial_statement_column.id", name="fk_financial_statement_value_column_id_fs_column"),
nullable=False,
)
amount: Mapped[decimal.Decimal | None] = mapped_column(Numeric(18, 2))
string_value: Mapped[str | None] = mapped_column(String)
confidence: Mapped[decimal.Decimal | None] = mapped_column(Numeric(5, 2))
is_recognized: Mapped[bool | None] = mapped_column(Boolean)
is_modified: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("false"))
original_amount: Mapped[decimal.Decimal | None] = mapped_column(Numeric(18, 2))
modified_by: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("app_user.id"))
modified_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True))

14
api/tests/test_models_part_01_03.py
View File

@ -35,7 +35,6 @@ PART_01_03_TABLES = {
"elimination_rule_account",
}
PG_IDENTIFIER_LIMIT = 63
EXPECTED_TABLE_COUNT = 16
def orm_tables() -> dict[str, Table]:
@ -43,8 +42,7 @@ def orm_tables() -> dict[str, Table]:
def test_part_01_03_tables_registered_in_metadata() -> None:
assert set(orm_tables()) == PART_01_03_TABLES
assert len(Base.metadata.sorted_tables) == EXPECTED_TABLE_COUNT # на этом этапе (задачи 7–8 добавят остальные)
assert PART_01_03_TABLES <= set(orm_tables()) # общий состав metadata — в тестах последней по номеру части
assert all(t.schema == "dkb" for t in Base.metadata.sorted_tables)
@ -64,7 +62,7 @@ def test_foreign_keys_resolve_to_registered_tables() -> None:
def test_constraint_names_are_explicit_and_fit_postgres_limit() -> None:
"""Все имена ограничений заданы (конвенцией/явно), без хеш-усечения, и не длиннее 63 символов."""
"""Все имена ограничений ВСЕХ таблиц metadata (любых частей) заданы, без хеш-усечения, и не длиннее 63 символов."""
for table in Base.metadata.sorted_tables:
names = [c.name for c in table.constraints] + [i.name for i in table.indexes]
for name in names:
@ -79,7 +77,7 @@ def test_orm_matches_dbml() -> None:
@pytest.mark.integration
async def test_create_all_on_scratch_db(scratch_db_url: str) -> None:
"""`create_all` на пустой БД: 16 таблиц в схеме dkb, enum-типы, partial unique работают."""
"""`create_all` на пустой БД: таблицы частей 01–03 в схеме dkb, их enum-типы, partial unique работают."""
engine = create_async_engine(scratch_db_url)
try:
async with engine.begin() as conn:
@ -88,7 +86,7 @@ async def test_create_all_on_scratch_db(scratch_db_url: str) -> None:
async with engine.connect() as conn:
tables = await conn.run_sync(lambda c: set(inspect(c).get_table_names(schema="dkb")))
assert tables == PART_01_03_TABLES
assert PART_01_03_TABLES <= tables
enums = {
row[0]
for row in await conn.execute(
@ -98,10 +96,10 @@ async def test_create_all_on_scratch_db(scratch_db_url: str) -> None:
)
)
}
assert enums == {
assert {
"entity_type", "related_group_kind", "account_kind", "account_row_type", "business_segment",
"statement_type", "le_history_event", "permission_object", "permission_action",
} # fmt: skip
} <= enums # fmt: skip
indexdefs = {
row[0]: row[1]
for row in await conn.execute(