diff --git a/api/src/models/__init__.py b/api/src/models/__init__.py index 8071b68..dc6ac3d 100644 --- a/api/src/models/__init__.py +++ b/api/src/models/__init__.py @@ -3,10 +3,20 @@ Импорт всех модулей моделей обязателен: он регистрирует таблицы в `Base.metadata` (create_all, alembic). """ -from . import access, accounting, directories +from . import access, accounting, directories, files, osv, statements from .base import Base, UUIDPrimaryMixin #: Версия DBML, которой соответствуют модели (сверяется с шапкой DBML в check_dbml_vs_orm). DBML_VERSION = "5.13" -__all__ = ["DBML_VERSION", "Base", "UUIDPrimaryMixin", "access", "accounting", "directories"] +__all__ = [ + "DBML_VERSION", + "Base", + "UUIDPrimaryMixin", + "access", + "accounting", + "directories", + "files", + "osv", + "statements", +] diff --git a/api/src/models/files.py b/api/src/models/files.py new file mode 100644 index 0000000..43798b1 --- /dev/null +++ b/api/src/models/files.py @@ -0,0 +1,157 @@ +"""Часть 04 DBML v5.13 «Файлы / распознавание»: реестр S3, физические файлы, документы, задания распознавания. + +Цикл FK `stored_file.current_recognition_job_id` -> `recognition_job` и `recognition_job.file_id` -> `stored_file` +разорван через `use_alter=True` на nullable-стороне (`stored_file.current_recognition_job_id`): `create_all` создаёт +таблицы без этого FK и добавляет его отдельным `ALTER TABLE`; имя FK задано явно (то же имя — в миграции 0001). +""" + +import datetime +import uuid +from typing import Any + +from sqlalchemy import BigInteger, Boolean, Date, DateTime, ForeignKey, Index, Integer, SmallInteger, String, Text, text +from sqlalchemy.dialects.postgresql import JSONB, UUID +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base, UUIDPrimaryMixin +from .enums import ( + DocumentCheckResult, + DocumentStatus, + DocumentType, + FileStatus, + FileType, + PeriodQuarter, + RecognitionPipeline, + RecognitionStatus, + S3BucketRole, + pg_enum, +) + + +class S3Object(UUIDPrimaryMixin, Base): + """Реестр объектов S3: единая точка ключей для lifecycle/ретеншна (ТЗ 8.4) и дедупа.""" + + __tablename__ = "s3_object" + __table_args__ = (Index("uq_s3_object_bucket_key", "bucket", "key", unique=True),) + + bucket_role: Mapped[S3BucketRole] = mapped_column(pg_enum("s3_bucket_role"), nullable=False) + bucket: Mapped[str] = mapped_column(String, nullable=False) + key: Mapped[str] = mapped_column(String, nullable=False) + content_type: Mapped[str | None] = mapped_column(String) + size_bytes: Mapped[int | None] = mapped_column(BigInteger) + sha256: Mapped[str | None] = mapped_column(String) + version_id: Mapped[str | None] = mapped_column(String) + created_by: Mapped[str | None] = mapped_column(String) + created_at: Mapped[datetime.datetime] = mapped_column(DateTime(timezone=True), nullable=False) + expires_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True)) + archived_to: Mapped[str | None] = mapped_column(String) + + +class StoredFile(UUIDPrimaryMixin, Base): + """Физический файл одной загрузки; что внутри — `document` (1:N). Первичка не редактируется (ТЗ 8.2).""" + + __tablename__ = "stored_file" + __table_args__ = (Index("ix_stored_file_legal_entity_id_status", "legal_entity_id", "status"),) + + legal_entity_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("legal_entity.id"), nullable=False + ) + file_type: Mapped[FileType] = mapped_column(pg_enum("file_type"), nullable=False) + original_name: Mapped[str | None] = mapped_column(String) + s3_object_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("s3_object.id"), nullable=False) + content_hash: Mapped[str | None] = mapped_column(String) + size_bytes: Mapped[int | None] = mapped_column(BigInteger) + status: Mapped[FileStatus] = mapped_column(pg_enum("file_status"), nullable=False) + version_no: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("1")) + superseded_by_file_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("stored_file.id")) + current_recognition_job_id: Mapped[uuid.UUID | None] = mapped_column( + UUID(as_uuid=True), + ForeignKey( + "recognition_job.id", + use_alter=True, + name="fk_stored_file_current_recognition_job_id_recognition_job", + ), + ) + uploaded_by: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("app_user.id")) + uploaded_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True)) + updated_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True)) + + +class Document(UUIDPrimaryMixin, Base): + """Логический документ внутри файла (ББ / ОФР / ОСВ по счёту / расшифровка); создаётся ядром при маппинге.""" + + __tablename__ = "document" + __table_args__ = ( + # один ББ / один ОФР / одна ОСВ по счёту на файл: NULL в osv_account_code считается значением (PG15+) + Index( + "uq_document_file_type_account", + "stored_file_id", + "document_type", + "osv_account_code", + unique=True, + postgresql_nulls_not_distinct=True, + ), + # чек-лист required_document_rule + Index( + "ix_document_legal_entity_period_status", + "legal_entity_id", + "document_type", + "osv_account_code", + "period_year", + "period_quarter", + "status", + ), + ) + + stored_file_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("stored_file.id"), nullable=False) + legal_entity_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("legal_entity.id"), nullable=False + ) + document_type: Mapped[DocumentType] = mapped_column(pg_enum("document_type"), nullable=False) + osv_account_code: Mapped[str | None] = mapped_column(String) + period_year: Mapped[int | None] = mapped_column(SmallInteger) + period_quarter: Mapped[PeriodQuarter | None] = mapped_column(pg_enum("period_quarter")) + is_cumulative: Mapped[bool | None] = mapped_column(Boolean) + reporting_date: Mapped[datetime.date | None] = mapped_column(Date) + locator: Mapped[str | None] = mapped_column(String) + status: Mapped[DocumentStatus] = mapped_column(pg_enum("document_status"), nullable=False) + period_check: Mapped[DocumentCheckResult | None] = mapped_column(pg_enum("document_check_result")) + counterparty_check: Mapped[DocumentCheckResult | None] = mapped_column(pg_enum("document_check_result")) + confirmed_by: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("app_user.id")) + confirmed_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True)) + created_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True)) + updated_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True)) + + +class RecognitionJob(UUIDPrimaryMixin, Base): + """Единственная точка обмена с NATS: запись = outbox (queued -> published), результат — inbox (дедуп).""" + + __tablename__ = "recognition_job" + __table_args__ = (Index("uq_recognition_job_file_id_attempt", "file_id", "attempt", unique=True),) + + file_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("stored_file.id"), nullable=False) + pipeline: Mapped[RecognitionPipeline] = mapped_column(pg_enum("recognition_pipeline"), nullable=False) + request_subject: Mapped[str] = mapped_column(String, nullable=False) + request_msg_id: Mapped[str | None] = mapped_column(String, unique=True) + result_msg_id: Mapped[str | None] = mapped_column(String, unique=True) + schema_version: Mapped[str] = mapped_column(String, nullable=False) + service_name: Mapped[str | None] = mapped_column(String) + service_version: Mapped[str | None] = mapped_column(String) + status: Mapped[RecognitionStatus] = mapped_column(pg_enum("recognition_status"), nullable=False) + attempt: Mapped[int] = mapped_column(SmallInteger, nullable=False, server_default=text("1")) + max_attempts: Mapped[int] = mapped_column(SmallInteger, nullable=False, server_default=text("1")) + requested_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True)) + published_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True)) + started_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True)) + finished_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True)) + reply_deadline: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True)) + external_task_id: Mapped[str | None] = mapped_column(String) + error_code: Mapped[str | None] = mapped_column(String) + error_message: Mapped[str | None] = mapped_column(Text) + result_s3_object_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("s3_object.id")) + result_schema_version: Mapped[str | None] = mapped_column(String) + group_schema: Mapped[Any | None] = mapped_column(JSONB) + metrics: Mapped[Any | None] = mapped_column(JSONB) + fields_total: Mapped[int | None] = mapped_column(Integer) + fields_low_confidence: Mapped[int | None] = mapped_column(Integer) + fields_unrecognized: Mapped[int | None] = mapped_column(Integer) diff --git a/api/src/models/osv.py b/api/src/models/osv.py new file mode 100644 index 0000000..697afdc --- /dev/null +++ b/api/src/models/osv.py @@ -0,0 +1,59 @@ +"""Часть 05 DBML v5.13 «ОСВ»: строка оборотно-сальдовой ведомости (ТЗ 6.2, 6.5). + +Первичка не мутируется: сырые значения хранятся как пришли, правка аналитика — рядом +(`is_modified`, `original_values`, `modified_by/at` + audit_event(RECOGNIZED_DATA_EDIT)). +""" + +import datetime +import decimal +import uuid +from typing import Any + +from sqlalchemy import Boolean, DateTime, ForeignKey, Index, Integer, Numeric, SmallInteger, String, text +from sqlalchemy.dialects.postgresql import JSONB, UUID +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base, UUIDPrimaryMixin +from .enums import OsvDataType, OsvGroupType, pg_enum + + +class OsvEntry(UUIDPrimaryMixin, Base): + """Строка ОСВ; суммы — рубли, `numeric(18,2)`.""" + + __tablename__ = "osv_entry" + __table_args__ = ( + Index("uq_osv_entry_document_id_row_number", "document_id", "row_number", unique=True), + Index("ix_osv_entry_owner_account_counterparty", "owner_legal_entity_id", "account_id", "counterparty_id"), + ) + + recognition_job_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("recognition_job.id"), nullable=False + ) + document_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("document.id"), nullable=False) + owner_legal_entity_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("legal_entity.id"), nullable=False + ) + parent_entry_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("osv_entry.id")) + group_type: Mapped[OsvGroupType | None] = mapped_column(pg_enum("osv_group_type")) + group_name: Mapped[str | None] = mapped_column(String) + row_number: Mapped[int | None] = mapped_column(Integer) + level: Mapped[int | None] = mapped_column(SmallInteger) + row_value: Mapped[str | None] = mapped_column(String) + data_type: Mapped[OsvDataType | None] = mapped_column(pg_enum("osv_data_type")) + account_code: Mapped[str | None] = mapped_column(String) + account_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("chart_of_account.id")) + account_root: Mapped[str | None] = mapped_column(String) + counterparty_name: Mapped[str | None] = mapped_column(String) + counterparty_inn: Mapped[str | None] = mapped_column(String) + counterparty_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("legal_entity.id")) + opening_debit: Mapped[decimal.Decimal | None] = mapped_column(Numeric(18, 2)) + opening_credit: Mapped[decimal.Decimal | None] = mapped_column(Numeric(18, 2)) + turnover_debit: Mapped[decimal.Decimal | None] = mapped_column(Numeric(18, 2)) + turnover_credit: Mapped[decimal.Decimal | None] = mapped_column(Numeric(18, 2)) + closing_debit: Mapped[decimal.Decimal | None] = mapped_column(Numeric(18, 2)) + closing_credit: Mapped[decimal.Decimal | None] = mapped_column(Numeric(18, 2)) + is_modified: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("false")) + original_values: Mapped[Any | None] = mapped_column(JSONB) + modified_by: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("app_user.id")) + modified_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True)) + is_excisable: Mapped[bool | None] = mapped_column(Boolean) diff --git a/api/src/models/statements.py b/api/src/models/statements.py new file mode 100644 index 0000000..877396c --- /dev/null +++ b/api/src/models/statements.py @@ -0,0 +1,119 @@ +"""Часть 06 DBML v5.13 «Бухгалтерская отчётность»: шапка, колонки-периоды, строки и значения форм ББ/ОФР (157н). + +Первичка не мутируется: `*_original_value` / `original_amount` / `is_modified` — рядом с распознанным значением. +Логические проверки (ТЗ 6.3) и «требует проверки» не хранятся — считаются на лету. +""" + +import datetime +import decimal +import uuid + +from sqlalchemy import Boolean, Date, DateTime, ForeignKey, Index, Numeric, SmallInteger, String, Text, text +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base, UUIDPrimaryMixin +from .enums import StatementType, pg_enum + + +class FinancialStatementMeta(UUIDPrimaryMixin, Base): + """Шапка распознанной формы ББ/ОФР; 1:1 с `document`.""" + + __tablename__ = "financial_statement_meta" + + document_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("document.id"), nullable=False, unique=True + ) + recognition_job_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("recognition_job.id"), nullable=False + ) + statement_type: Mapped[StatementType] = mapped_column(pg_enum("statement_type"), nullable=False) + inn: Mapped[str | None] = mapped_column(String) + inn_original_value: Mapped[str | None] = mapped_column(String) + inn_confidence: Mapped[decimal.Decimal | None] = mapped_column(Numeric(5, 2)) + inn_verified: Mapped[bool | None] = mapped_column(Boolean) + inn_modified: Mapped[bool | None] = mapped_column(Boolean) + organization_name: Mapped[str | None] = mapped_column(String) + report_date: Mapped[datetime.date | None] = mapped_column(Date) + report_date_original_value: Mapped[str | None] = mapped_column(String) + report_date_verified: Mapped[bool | None] = mapped_column(Boolean) + report_date_modified: Mapped[bool | None] = mapped_column(Boolean) + units: Mapped[str | None] = mapped_column(String) + units_original_value: Mapped[str | None] = mapped_column(String) + units_verified: Mapped[bool | None] = mapped_column(Boolean) + units_modified: Mapped[bool | None] = mapped_column(Boolean) + currency: Mapped[str | None] = mapped_column(String, server_default=text("'RUB'")) + okpo: Mapped[str | None] = mapped_column(String) + okopf: Mapped[str | None] = mapped_column(String) + okfs: Mapped[str | None] = mapped_column(String) + okud: Mapped[str | None] = mapped_column(String) + okei: Mapped[str | None] = mapped_column(String) + legal_form: Mapped[str | None] = mapped_column(String) + ownership_form: Mapped[str | None] = mapped_column(String) + address: Mapped[str | None] = mapped_column(String) + subject_to_audit: Mapped[bool | None] = mapped_column(Boolean) + reviewed_by: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("app_user.id")) + reviewed_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True)) + checks_accepted_by: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("app_user.id")) + checks_accepted_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True)) + checks_accepted_comment: Mapped[str | None] = mapped_column(Text) + + +class FinancialStatementColumn(UUIDPrimaryMixin, Base): + """Колонка-период формы (2–3 сравнительные даты); `period_date` нужна межформенным проверкам и гудвиллу.""" + + __tablename__ = "financial_statement_column" + __table_args__ = ( + Index("uq_financial_statement_column_meta_id_column_index", "meta_id", "column_index", unique=True), + ) + + meta_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("financial_statement_meta.id"), nullable=False + ) + column_index: Mapped[int] = mapped_column(SmallInteger, nullable=False) + label: Mapped[str | None] = mapped_column(String) + period_date: Mapped[datetime.date | None] = mapped_column(Date) + is_current: Mapped[bool | None] = mapped_column(Boolean) + + +class FinancialStatementLine(UUIDPrimaryMixin, Base): + """Строка формы с кодом; порядок, разделы и итоговые строки — шаблон формы на фронте.""" + + __tablename__ = "financial_statement_line" + __table_args__ = (Index("uq_financial_statement_line_meta_id_line_code", "meta_id", "line_code", unique=True),) + + meta_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("financial_statement_meta.id"), nullable=False + ) + line_code: Mapped[str | None] = mapped_column(String) + line_code_original_value: Mapped[str | None] = mapped_column(String) + line_code_confidence: Mapped[decimal.Decimal | None] = mapped_column(Numeric(5, 2)) + line_code_modified: Mapped[bool | None] = mapped_column(Boolean) + line_name: Mapped[str | None] = mapped_column(String) + line_name_original_value: Mapped[str | None] = mapped_column(String) + line_name_confidence: Mapped[decimal.Decimal | None] = mapped_column(Numeric(5, 2)) + line_name_modified: Mapped[bool | None] = mapped_column(Boolean) + + +class FinancialStatementValue(UUIDPrimaryMixin, Base): + """Распознанное значение (строка x колонка): сумма, уверенность, признак ручной правки (Прил. 2 ТЗ).""" + + __tablename__ = "financial_statement_value" + __table_args__ = (Index("uq_financial_statement_value_line_id_column_id", "line_id", "column_id", unique=True),) + + line_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("financial_statement_line.id"), nullable=False + ) + column_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("financial_statement_column.id", name="fk_financial_statement_value_column_id_fs_column"), + nullable=False, + ) + amount: Mapped[decimal.Decimal | None] = mapped_column(Numeric(18, 2)) + string_value: Mapped[str | None] = mapped_column(String) + confidence: Mapped[decimal.Decimal | None] = mapped_column(Numeric(5, 2)) + is_recognized: Mapped[bool | None] = mapped_column(Boolean) + is_modified: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("false")) + original_amount: Mapped[decimal.Decimal | None] = mapped_column(Numeric(18, 2)) + modified_by: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("app_user.id")) + modified_at: Mapped[datetime.datetime | None] = mapped_column(DateTime(timezone=True)) diff --git a/api/tests/test_models_part_01_03.py b/api/tests/test_models_part_01_03.py index c1723f5..040e897 100644 --- a/api/tests/test_models_part_01_03.py +++ b/api/tests/test_models_part_01_03.py @@ -35,7 +35,6 @@ PART_01_03_TABLES = { "elimination_rule_account", } PG_IDENTIFIER_LIMIT = 63 -EXPECTED_TABLE_COUNT = 16 def orm_tables() -> dict[str, Table]: @@ -43,8 +42,7 @@ def orm_tables() -> dict[str, Table]: def test_part_01_03_tables_registered_in_metadata() -> None: - assert set(orm_tables()) == PART_01_03_TABLES - assert len(Base.metadata.sorted_tables) == EXPECTED_TABLE_COUNT # на этом этапе (задачи 7–8 добавят остальные) + assert PART_01_03_TABLES <= set(orm_tables()) # общий состав metadata — в тестах последней по номеру части assert all(t.schema == "dkb" for t in Base.metadata.sorted_tables) @@ -64,7 +62,7 @@ def test_foreign_keys_resolve_to_registered_tables() -> None: def test_constraint_names_are_explicit_and_fit_postgres_limit() -> None: - """Все имена ограничений заданы (конвенцией/явно), без хеш-усечения, и не длиннее 63 символов.""" + """Все имена ограничений ВСЕХ таблиц metadata (любых частей) заданы, без хеш-усечения, и не длиннее 63 символов.""" for table in Base.metadata.sorted_tables: names = [c.name for c in table.constraints] + [i.name for i in table.indexes] for name in names: @@ -79,7 +77,7 @@ def test_orm_matches_dbml() -> None: @pytest.mark.integration async def test_create_all_on_scratch_db(scratch_db_url: str) -> None: - """`create_all` на пустой БД: 16 таблиц в схеме dkb, enum-типы, partial unique работают.""" + """`create_all` на пустой БД: таблицы частей 01–03 в схеме dkb, их enum-типы, partial unique работают.""" engine = create_async_engine(scratch_db_url) try: async with engine.begin() as conn: @@ -88,7 +86,7 @@ async def test_create_all_on_scratch_db(scratch_db_url: str) -> None: async with engine.connect() as conn: tables = await conn.run_sync(lambda c: set(inspect(c).get_table_names(schema="dkb"))) - assert tables == PART_01_03_TABLES + assert PART_01_03_TABLES <= tables enums = { row[0] for row in await conn.execute( @@ -98,10 +96,10 @@ async def test_create_all_on_scratch_db(scratch_db_url: str) -> None: ) ) } - assert enums == { + assert { "entity_type", "related_group_kind", "account_kind", "account_row_type", "business_segment", "statement_type", "le_history_event", "permission_object", "permission_action", - } # fmt: skip + } <= enums # fmt: skip indexdefs = { row[0]: row[1] for row in await conn.execute(