diff --git a/api/tests/dbml_parity.py b/api/tests/dbml_parity.py index 144a8e4..02fa7e1 100644 --- a/api/tests/dbml_parity.py +++ b/api/tests/dbml_parity.py @@ -2,7 +2,8 @@ Публичное API: - `find_problems(metadata, tables)` — список расхождений (пустой = ORM совпадает с DBML) по колонкам (имя, порядок, - тип, nullable, PK, unique, server_default), FK и индексам (состав, уникальность, partial-условие WHERE); + тип, nullable, PK, unique, server_default), FK и индексам (состав, уникальность, partial-условие WHERE, + `NULLS NOT DISTINCT` по note DBML); - `type_problem(orm_type, dbml_type)` — расхождение типа одной колонки (или `None`); - `canonical_default_dbml` / `canonical_default_orm` — приведение default к сопоставимому виду. @@ -42,6 +43,7 @@ from tests.dbml_helpers import load_dbml SCHEMA = "dkb" _NUMERIC_RE = re.compile(r"numeric\((\d+),\s*(\d+)\)") _PARTIAL_RE = re.compile(r"partial\s+UNIQUE\s+WHERE\s+(.+?)(?:\s+—.*)?$", re.IGNORECASE | re.DOTALL) +_NULLS_NOT_DISTINCT_RE = re.compile(r"NULLS\s+NOT\s+DISTINCT", re.IGNORECASE) _SYNONYMS = {"current_timestamp": "now()", "current_timestamp()": "now()"} @@ -212,21 +214,33 @@ def _norm_where(where: object) -> str | None: def _index_problems(orm_table: Table, dbml_table: DbmlTable) -> list[str]: - expected: list[tuple[tuple[str, ...], bool, str | None]] = [] + expected: list[tuple[tuple[str, ...], bool, str | None, bool]] = [] for i in dbml_table.indexes: if i.pk: continue - partial = _PARTIAL_RE.search(str(i.note)) if i.note else None + note = str(i.note) if i.note else "" + partial = _PARTIAL_RE.search(note) where = _norm_where(partial.group(1)) if partial else None - expected.append((tuple(c.name for c in i.subjects), bool(i.unique) or partial is not None, where)) + # note с `NULLS NOT DISTINCT` => уникальный индекс обязан иметь postgresql_nulls_not_distinct=True + nulls_not_distinct = _NULLS_NOT_DISTINCT_RE.search(note) is not None + expected.append( + (tuple(c.name for c in i.subjects), bool(i.unique) or partial is not None, where, nulls_not_distinct) + ) actual = [ - (tuple(c.name for c in i.columns), bool(i.unique), _norm_where(i.dialect_options["postgresql"]["where"])) + ( + tuple(c.name for c in i.columns), + bool(i.unique), + _norm_where(i.dialect_options["postgresql"]["where"]), + bool(i.dialect_options["postgresql"]["nulls_not_distinct"]), + ) for i in orm_table.indexes ] if sorted(actual, key=repr) == sorted(expected, key=repr): return [] - return [f"{dbml_table.name}: индексы (колонки, unique, WHERE): ожидалось {sorted(expected, key=repr)}, " - f"фактически {sorted(actual, key=repr)}"] # fmt: skip + return [ + f"{dbml_table.name}: индексы (колонки, unique, WHERE, NULLS NOT DISTINCT): " + f"ожидалось {sorted(expected, key=repr)}, фактически {sorted(actual, key=repr)}" + ] # --- точка входа ------------------------------------------------------------------------------------------------ diff --git a/api/tests/test_dbml_parity.py b/api/tests/test_dbml_parity.py index 1f7fff3..d610a7e 100644 --- a/api/tests/test_dbml_parity.py +++ b/api/tests/test_dbml_parity.py @@ -148,6 +148,23 @@ def test_missing_extra_and_wrong_where_indexes_are_detected() -> None: assert any("индексы" in p for p in find_problems(metadata, {"regional_office"})) +def test_nulls_not_distinct_flag_is_compared_both_ways() -> None: + """DBML-note `NULLS NOT DISTINCT` требует флаг в ORM-индексе; лишний флаг без note — тоже расхождение.""" + metadata = clone_metadata() + document = metadata.tables["dkb.document"] + index = next(i for i in document.indexes if i.unique) + assert find_problems(metadata, {"document"}) == [] + index.dialect_options["postgresql"]["nulls_not_distinct"] = None # индекс без флага + problems = find_problems(metadata, {"document"}) + assert any("NULLS NOT DISTINCT" in p for p in problems), problems + index.dialect_options["postgresql"]["nulls_not_distinct"] = True + assert find_problems(metadata, {"document"}) == [] + # лишний флаг на индексе, у которого в DBML нет такого note + other = next(i for i in metadata.tables["dkb.s3_object"].indexes if i.unique) + other.dialect_options["postgresql"]["nulls_not_distinct"] = True + assert any("NULLS NOT DISTINCT" in p for p in find_problems(metadata, {"s3_object"})) + + def test_missing_and_extra_foreign_keys_are_detected() -> None: metadata = clone_metadata() app_user = metadata.tables["dkb.app_user"] diff --git a/api/tests/test_models_part_04_06.py b/api/tests/test_models_part_04_06.py new file mode 100644 index 0000000..e1f8440 --- /dev/null +++ b/api/tests/test_models_part_04_06.py @@ -0,0 +1,228 @@ +"""ORM частей 04–06 DBML v5.13 (файлы, ОСВ, отчётность): чётность с DBML без БД и create_all на PostgreSQL.""" + +import datetime +import decimal +import uuid +from dataclasses import dataclass + +import pytest +from sqlalchemy import Table, inspect, select, text +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine + +from src.models import Base +from src.models.directories import LegalEntity +from src.models.enums import ( + DocumentStatus, + DocumentType, + FileStatus, + FileType, + RecognitionPipeline, + RecognitionStatus, + S3BucketRole, +) +from src.models.files import Document, RecognitionJob, S3Object, StoredFile +from src.models.osv import OsvEntry +from tests.dbml_parity import find_problems +from tests.test_models_part_01_03 import PART_01_03_TABLES, PG_IDENTIFIER_LIMIT + +PART_04_06_TABLES = { + # files + "s3_object", + "stored_file", + "document", + "recognition_job", + # osv + "osv_entry", + # statements + "financial_statement_meta", + "financial_statement_column", + "financial_statement_line", + "financial_statement_value", +} +EXPECTED_TABLE_COUNT = 25 # на этом этапе: 16 (части 01–03) + 9 (части 04–06); задача 8 добавит остальные + + +def orm_tables() -> dict[str, Table]: + return {t.name: t for t in Base.metadata.sorted_tables} + + +def test_part_04_06_tables_registered_in_metadata() -> None: + assert set(orm_tables()) == PART_01_03_TABLES | PART_04_06_TABLES + assert len(Base.metadata.sorted_tables) == EXPECTED_TABLE_COUNT + assert all(t.schema == "dkb" for t in Base.metadata.sorted_tables) + + +def test_orm_matches_dbml() -> None: + """Колонки (имя, порядок, тип, nullable, PK, unique, default), FK и индексы (вкл. NND) — как в DBML.""" + assert find_problems(Base.metadata, PART_04_06_TABLES) == [] + + +def test_new_tables_constraint_names_fit_postgres_limit() -> None: + """Имена всех ограничений и индексов новых таблиц заданы и не длиннее 63 символов (FK-цикл — тоже).""" + for name in PART_04_06_TABLES: + table = orm_tables()[name] + for constraint in [*table.constraints, *table.indexes]: + assert constraint.name, f"{name}: безымянное ограничение" + assert len(str(constraint.name)) <= PG_IDENTIFIER_LIMIT, f"{name}: {constraint.name}" + + +def test_fk_cycle_is_broken_by_use_alter_without_warning() -> None: + """`stored_file.current_recognition_job_id` — отложенный FK (use_alter), цикл не мешает `sorted_tables`.""" + stored_file = orm_tables()["stored_file"] + fk = next(f for f in stored_file.foreign_keys if f.parent.name == "current_recognition_job_id") + assert fk.constraint is not None + assert fk.constraint.use_alter is True + assert fk.constraint.name == "fk_stored_file_current_recognition_job_id_recognition_job" + # обратная сторона цикла — обычный FK + job_fk = next(f for f in orm_tables()["recognition_job"].foreign_keys if f.parent.name == "file_id") + assert job_fk.constraint is not None + assert job_fk.constraint.use_alter is False + names = [t.name for t in Base.metadata.sorted_tables] # pytest -W error: SAWarning о цикле стал бы ошибкой + assert names.index("stored_file") < names.index("recognition_job") + + +def test_document_unique_index_is_nulls_not_distinct() -> None: + index = next(i for i in orm_tables()["document"].indexes if i.unique) + assert index.dialect_options["postgresql"]["nulls_not_distinct"] is True + assert [c.name for c in index.columns] == ["stored_file_id", "document_type", "osv_account_code"] + + +@pytest.mark.integration +async def test_create_all_and_behavior_on_scratch_db(scratch_db_url: str) -> None: + """`create_all` частей 01–06 (25 таблиц); NULLS NOT DISTINCT и Decimal работают на реальной PostgreSQL 16.""" + engine = create_async_engine(scratch_db_url) + try: + async with engine.begin() as conn: + await conn.execute(text("CREATE SCHEMA IF NOT EXISTS dkb")) + await conn.run_sync(Base.metadata.create_all) + + async with engine.connect() as conn: + tables = await conn.run_sync(lambda c: set(inspect(c).get_table_names(schema="dkb"))) + assert tables == PART_01_03_TABLES | PART_04_06_TABLES + assert len(tables) == EXPECTED_TABLE_COUNT + fk_names = { + row[0] + for row in await conn.execute( + text("SELECT conname FROM pg_constraint WHERE contype = 'f' AND connamespace = 'dkb'::regnamespace") + ) + } + assert "fk_stored_file_current_recognition_job_id_recognition_job" in fk_names + assert "fk_financial_statement_value_column_id_fs_column" in fk_names + indexdef = ( + await conn.execute( + text( + "SELECT indexdef FROM pg_indexes " + "WHERE schemaname = 'dkb' AND indexname = 'uq_document_file_type_account'" + ) + ) + ).scalar_one() + assert "NULLS NOT DISTINCT" in indexdef + + async with AsyncSession(engine, expire_on_commit=False) as session: + chain = await _make_file_with_job(session) + await session.commit() + await _check_nulls_not_distinct(session, chain) + await _check_decimal_roundtrip(session, chain) + finally: + await engine.dispose() + + +@dataclass(frozen=True) +class Chain: + """Идентификаторы минимальной цепочки FK (простые uuid — объекты ORM истекают после rollback).""" + + entity_id: uuid.UUID + file_id: uuid.UUID + job_id: uuid.UUID + + +async def _make_file_with_job(session: AsyncSession) -> Chain: + """Минимальная цепочка FK: ЮЛ, s3_object, stored_file, recognition_job и замыкание цикла FK.""" + now = datetime.datetime.now(datetime.UTC) + entity = LegalEntity(inn="7700000001", name="ООО Тест", valid_from=now) + s3 = S3Object(bucket_role=S3BucketRole.uploads, bucket="uploads", key=f"{uuid.uuid4()}/osv.xlsx", created_at=now) + session.add_all([entity, s3]) + await session.flush() + file = StoredFile( + legal_entity_id=entity.id, file_type=FileType.XLSX, s3_object_id=s3.id, status=FileStatus.UPLOADED + ) + session.add(file) + await session.flush() + job = RecognitionJob( + file_id=file.id, + pipeline=RecognitionPipeline.tabular, + request_subject="recognition.tabular.request", + schema_version="1", + status=RecognitionStatus.queued, + ) + session.add(job) + await session.flush() + file.current_recognition_job_id = job.id # замыкание цикла через отложенный FK + await session.flush() + await session.refresh(job) + await session.refresh(file) + assert (job.attempt, job.max_attempts) == (1, 1) # server_default + assert file.version_no == 1 # server_default + assert file.current_recognition_job_id == job.id + return Chain(entity_id=entity.id, file_id=file.id, job_id=job.id) + + +def _document(chain: Chain, document_type: DocumentType, account: str | None) -> Document: + return Document( + stored_file_id=chain.file_id, + legal_entity_id=chain.entity_id, + document_type=document_type, + osv_account_code=account, + status=DocumentStatus.RECOGNIZED, + ) + + +async def _check_nulls_not_distinct(session: AsyncSession, chain: Chain) -> None: + """Два документа с одним (stored_file_id, document_type) и osv_account_code = NULL -> IntegrityError.""" + session.add(_document(chain, DocumentType.BALANCE, None)) + await session.flush() + session.add(_document(chain, DocumentType.BALANCE, None)) + with pytest.raises(IntegrityError): + await session.flush() + await session.rollback() + # разные счета ОСВ в одном файле и ББ/ОФР рядом — допустимы + session.add_all( + [_document(chain, DocumentType.OSV, "58"), _document(chain, DocumentType.OSV, "60")] + + [_document(chain, DocumentType.INCOME, None)] + ) + await session.flush() + await session.rollback() + + +async def _check_decimal_roundtrip(session: AsyncSession, chain: Chain) -> None: + """Сумма numeric(18,2) читается обратно как `Decimal` без потери точности (деньги — не float).""" + document = _document(chain, DocumentType.OSV, "62") + session.add(document) + await session.flush() + amount = decimal.Decimal("106311116.46") + entry = OsvEntry( + recognition_job_id=chain.job_id, + document_id=document.id, + owner_legal_entity_id=chain.entity_id, + row_number=1, + closing_debit=amount, + ) + session.add(entry) + await session.flush() + entry_id = entry.id + session.add( + OsvEntry( + recognition_job_id=chain.job_id, + document_id=document.id, + owner_legal_entity_id=chain.entity_id, + parent_entry_id=entry_id, # самоссылка + row_number=2, + ) + ) + await session.commit() + stored = (await session.execute(select(OsvEntry.closing_debit).where(OsvEntry.id == entry_id))).scalar_one() + assert isinstance(stored, decimal.Decimal) + assert stored == amount + is_modified = (await session.execute(select(OsvEntry.is_modified).where(OsvEntry.id == entry_id))).scalar_one() + assert is_modified is False # server_default false