ed5ed43fa6
Явный DDL без импорта src.models: схема dkb, 27 enum-типов (явный CREATE/DROP TYPE), 43 таблицы с индексами (partial unique, NULLS NOT DISTINCT), FK цикла stored_file -> recognition_job отдельным шагом; downgrade — FK цикла, таблицы в обратном порядке, типы. Тесты (integration, scratch-БД, alembic подпроцессом): upgrade -> downgrade -> upgrade, пустой compare_metadata и alembic check, каталог = create_all ORM, поведение partial unique/NULLS NOT DISTINCT/FK-цикла; офлайн-SQL — CREATE SCHEMA первым. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
285 lines
13 KiB
Python
285 lines
13 KiB
Python
"""Миграция 0001 (DBML v5.13) на PostgreSQL 16: полный цикл, пустой autogenerate, схема = схеме ORM.
|
||
|
||
Spec data-model, требование «Миграции применяются с нуля и откатываются» (сценарии «Полный цикл» и «Автогенерация
|
||
ничего не находит»), design D-2. Alembic запускается отдельным процессом (`python -m alembic -x dburl=...`), как из
|
||
CLI: `env.py` вызывает `asyncio.run(...)`, а внутри event loop pytest-asyncio это невозможно; заодно процесс не
|
||
перенастраивает логирование тестов через `fileConfig`. Каждая проверка идёт на своей временной БД (`scratch_db_url`).
|
||
"""
|
||
|
||
import asyncio
|
||
import subprocess
|
||
import sys
|
||
import uuid
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import pytest
|
||
from alembic.autogenerate import compare_metadata
|
||
from alembic.runtime.migration import MigrationContext
|
||
from sqlalchemy import Connection, text
|
||
from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, AsyncSession, create_async_engine
|
||
|
||
from src.models import Base
|
||
from src.models.enums import ENUMS
|
||
from tests.pg_helpers import create_scratch_db, drop_scratch_db
|
||
from tests.test_models_part_01_03 import _check_related_group_member_uniqueness
|
||
from tests.test_models_part_04_06 import _check_nulls_not_distinct, _make_file_with_job
|
||
from tests.test_models_part_07_09 import (
|
||
ALL_TABLES,
|
||
EXPECTED_ENUM_COUNT,
|
||
EXPECTED_TABLE_COUNT,
|
||
_check_active_lock_is_unique,
|
||
)
|
||
|
||
API_DIR = Path(__file__).resolve().parents[1]
|
||
HEAD = "0001_dbml_v5_13"
|
||
VERSION_TABLE = "alembic_version"
|
||
FK_CYCLE = "fk_stored_file_current_recognition_job_id_recognition_job"
|
||
|
||
|
||
def _run_alembic(url: str, *args: str) -> str:
|
||
"""`alembic -x dburl=<url> <args>` из каталога api/; вернуть stdout+stderr, упасть с выводом при ошибке."""
|
||
result = subprocess.run(
|
||
[sys.executable, "-m", "alembic", "-x", f"dburl={url}", *args],
|
||
cwd=API_DIR,
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
output = result.stdout + result.stderr
|
||
assert result.returncode == 0, f"alembic {' '.join(args)} завершился с кодом {result.returncode}:\n{output}"
|
||
return output
|
||
|
||
|
||
async def alembic(url: str, *args: str) -> str:
|
||
return await asyncio.to_thread(_run_alembic, url, *args)
|
||
|
||
|
||
async def _scalars(conn: AsyncConnection, sql: str) -> list[Any]:
|
||
return list((await conn.execute(text(sql))).scalars())
|
||
|
||
|
||
async def tables_in_dkb(conn: AsyncConnection) -> set[str]:
|
||
return set(
|
||
await _scalars(
|
||
conn,
|
||
"SELECT table_name FROM information_schema.tables WHERE table_schema = 'dkb'",
|
||
)
|
||
)
|
||
|
||
|
||
async def enums_in_dkb(conn: AsyncConnection) -> set[str]:
|
||
return set(
|
||
await _scalars(
|
||
conn,
|
||
"SELECT t.typname FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace "
|
||
"WHERE n.nspname = 'dkb' AND t.typtype = 'e'",
|
||
)
|
||
)
|
||
|
||
|
||
async def schema_snapshot(conn: AsyncConnection) -> dict[str, Any]:
|
||
"""Каталог схемы dkb без служебной таблицы alembic: колонки, ограничения, индексы, enum'ы (с порядком значений)."""
|
||
columns = (
|
||
await conn.execute(
|
||
text(
|
||
"SELECT table_name, ordinal_position, column_name, data_type, udt_schema, udt_name, is_nullable, "
|
||
"column_default, numeric_precision, numeric_scale FROM information_schema.columns "
|
||
"WHERE table_schema = 'dkb' AND table_name <> :vt ORDER BY table_name, ordinal_position"
|
||
),
|
||
{"vt": VERSION_TABLE},
|
||
)
|
||
).all()
|
||
constraints = (
|
||
await conn.execute(
|
||
text(
|
||
"SELECT c.conname, c.conrelid::regclass::text, pg_get_constraintdef(c.oid) FROM pg_constraint c "
|
||
"WHERE c.connamespace = 'dkb'::regnamespace AND c.conrelid::regclass::text <> 'dkb.alembic_version' "
|
||
"AND c.contype <> 'n'"
|
||
)
|
||
)
|
||
).all()
|
||
indexes = (
|
||
await conn.execute(
|
||
text("SELECT indexname, indexdef FROM pg_indexes WHERE schemaname = 'dkb' AND tablename <> :vt"),
|
||
{"vt": VERSION_TABLE},
|
||
)
|
||
).all()
|
||
enums = (
|
||
await conn.execute(
|
||
text(
|
||
"SELECT t.typname, array_agg(e.enumlabel ORDER BY e.enumsortorder) FROM pg_type t "
|
||
"JOIN pg_namespace n ON n.oid = t.typnamespace JOIN pg_enum e ON e.enumtypid = t.oid "
|
||
"WHERE n.nspname = 'dkb' GROUP BY t.typname"
|
||
)
|
||
)
|
||
).all()
|
||
return {
|
||
"columns": [tuple(row) for row in columns],
|
||
"constraints": {row[0]: (row[1], row[2]) for row in constraints},
|
||
"indexes": {row[0]: row[1] for row in indexes},
|
||
"enums": {row[0]: list(row[1]) for row in enums},
|
||
}
|
||
|
||
|
||
async def _assert_full_model(conn: AsyncConnection) -> None:
|
||
tables = await tables_in_dkb(conn)
|
||
assert tables - {VERSION_TABLE} == ALL_TABLES
|
||
assert len(tables - {VERSION_TABLE}) == EXPECTED_TABLE_COUNT
|
||
enums = await enums_in_dkb(conn)
|
||
assert enums == set(ENUMS)
|
||
assert len(enums) == EXPECTED_ENUM_COUNT
|
||
assert await _scalars(conn, "SELECT version_num FROM dkb.alembic_version") == [HEAD]
|
||
|
||
|
||
async def _truncate_model_tables(engine: AsyncEngine) -> None:
|
||
async with engine.begin() as conn:
|
||
await conn.execute(text("TRUNCATE " + ", ".join(f"dkb.{table}" for table in sorted(ALL_TABLES)) + " CASCADE"))
|
||
|
||
|
||
@pytest.mark.integration
|
||
async def test_full_cycle_upgrade_downgrade_upgrade(scratch_db_url: str) -> None:
|
||
"""Сценарий «Полный цикл»: upgrade head → downgrade base → upgrade head; после отката схема dkb пуста."""
|
||
engine = create_async_engine(scratch_db_url)
|
||
try:
|
||
await alembic(scratch_db_url, "upgrade", "head")
|
||
async with engine.connect() as conn:
|
||
await _assert_full_model(conn)
|
||
|
||
await alembic(scratch_db_url, "downgrade", "base")
|
||
async with engine.connect() as conn:
|
||
assert await tables_in_dkb(conn) == {VERSION_TABLE} # служебная таблица alembic остаётся, она пуста
|
||
assert await _scalars(conn, "SELECT count(*) FROM dkb.alembic_version") == [0]
|
||
assert await enums_in_dkb(conn) == set()
|
||
# в схеме не осталось других объектов (последовательностей, типов, индексов) кроме alembic_version
|
||
leftovers = await _scalars(
|
||
conn,
|
||
"SELECT relname FROM pg_class WHERE relnamespace = 'dkb'::regnamespace "
|
||
"AND relname NOT IN ('alembic_version', 'alembic_version_pkc')",
|
||
)
|
||
assert leftovers == []
|
||
types = await _scalars(
|
||
conn,
|
||
"SELECT typname FROM pg_type WHERE typnamespace = 'dkb'::regnamespace "
|
||
"AND typname NOT IN ('alembic_version', '_alembic_version')",
|
||
)
|
||
assert types == []
|
||
|
||
await alembic(scratch_db_url, "upgrade", "head")
|
||
async with engine.connect() as conn:
|
||
await _assert_full_model(conn)
|
||
finally:
|
||
await engine.dispose()
|
||
|
||
|
||
@pytest.mark.integration
|
||
async def test_autogenerate_finds_nothing_after_upgrade(scratch_db_url: str) -> None:
|
||
"""Сценарий «Автогенерация ничего не находит»: compare_metadata пуст, `alembic check` не видит операций."""
|
||
await alembic(scratch_db_url, "upgrade", "head")
|
||
|
||
def diff(sync_conn: Connection) -> list[Any]:
|
||
context = MigrationContext.configure(
|
||
sync_conn,
|
||
opts={
|
||
# те же опции сравнения, что в alembic/env.py
|
||
"compare_type": True,
|
||
"compare_server_default": True,
|
||
"include_schemas": True,
|
||
"include_name": lambda name, type_, _parents: name == "dkb" if type_ == "schema" else True,
|
||
"version_table_schema": "dkb",
|
||
},
|
||
)
|
||
return compare_metadata(context, Base.metadata)
|
||
|
||
engine = create_async_engine(scratch_db_url)
|
||
try:
|
||
async with engine.connect() as conn:
|
||
assert await conn.run_sync(diff) == []
|
||
finally:
|
||
await engine.dispose()
|
||
|
||
output = await alembic(scratch_db_url, "check")
|
||
assert "No new upgrade operations detected" in output
|
||
|
||
|
||
@pytest.mark.integration
|
||
async def test_migrated_schema_equals_orm_create_all(scratch_db_url: str) -> None:
|
||
"""Каталог после миграции = каталог `create_all` ORM: колонки, default, ограничения, индексы, enum (имена и DDL)."""
|
||
await alembic(scratch_db_url, "upgrade", "head")
|
||
reference_name = f"armcore_scratch_{uuid.uuid4().hex[:12]}"
|
||
reference_url = await create_scratch_db(reference_name)
|
||
migrated = create_async_engine(scratch_db_url)
|
||
reference = create_async_engine(reference_url)
|
||
try:
|
||
async with reference.begin() as conn:
|
||
await conn.execute(text("CREATE SCHEMA IF NOT EXISTS dkb"))
|
||
await conn.run_sync(Base.metadata.create_all)
|
||
async with migrated.connect() as conn:
|
||
got = await schema_snapshot(conn)
|
||
async with reference.connect() as conn:
|
||
expected = await schema_snapshot(conn)
|
||
assert got["enums"] == expected["enums"]
|
||
assert got["columns"] == expected["columns"]
|
||
assert got["constraints"] == expected["constraints"]
|
||
assert got["indexes"] == expected["indexes"]
|
||
finally:
|
||
await migrated.dispose()
|
||
await reference.dispose()
|
||
await drop_scratch_db(reference_name)
|
||
|
||
|
||
@pytest.mark.integration
|
||
async def test_migrated_schema_behaviour(scratch_db_url: str) -> None:
|
||
"""После миграции: FK-цикл, partial unique, NULLS NOT DISTINCT и server_default работают на реальной БД."""
|
||
await alembic(scratch_db_url, "upgrade", "head")
|
||
engine = create_async_engine(scratch_db_url)
|
||
try:
|
||
async with engine.connect() as conn:
|
||
fk = (
|
||
await conn.execute(
|
||
text(
|
||
"SELECT conrelid::regclass::text, pg_get_constraintdef(oid) FROM pg_constraint "
|
||
"WHERE conname = :name AND connamespace = 'dkb'::regnamespace"
|
||
),
|
||
{"name": FK_CYCLE},
|
||
)
|
||
).one()
|
||
assert fk[0] == "dkb.stored_file"
|
||
assert fk[1] == "FOREIGN KEY (current_recognition_job_id) REFERENCES dkb.recognition_job(id)"
|
||
rows = await conn.execute(
|
||
text(
|
||
"SELECT indexname, indexdef FROM pg_indexes WHERE schemaname = 'dkb' AND indexname IN "
|
||
"('uq_document_file_type_account', 'uq_consolidation_lock_version_active', "
|
||
"'uq_related_group_member_entity_current', 'uq_related_group_member_group_entity_current')"
|
||
)
|
||
)
|
||
indexdefs = {row[0]: row[1] for row in rows}
|
||
assert "NULLS NOT DISTINCT" in indexdefs["uq_document_file_type_account"]
|
||
assert "WHERE (released_at IS NULL)" in indexdefs["uq_consolidation_lock_version_active"]
|
||
for name in ("uq_related_group_member_entity_current", "uq_related_group_member_group_entity_current"):
|
||
assert indexdefs[name].startswith("CREATE UNIQUE INDEX")
|
||
assert "WHERE is_current" in indexdefs[name]
|
||
|
||
# поведенческие проверки из тестов ORM (create_all) — теперь на схеме из миграции; данные между ними
|
||
# очищаются: помощники создают одинаковые справочные записи (ГО, группа G1)
|
||
await _check_related_group_member_uniqueness(engine)
|
||
await _truncate_model_tables(engine)
|
||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||
chain = await _make_file_with_job(session) # замыкает FK-цикл, проверяет server_default
|
||
await session.commit()
|
||
await _check_nulls_not_distinct(session, chain)
|
||
await _truncate_model_tables(engine)
|
||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||
await _check_active_lock_is_unique(session)
|
||
finally:
|
||
await engine.dispose()
|
||
|
||
|
||
def test_offline_sql_creates_schema_before_version_table() -> None:
|
||
"""Офлайн-режим (`upgrade head --sql`): CREATE SCHEMA dkb идёт до таблицы версий и до enum-типов."""
|
||
output = _run_alembic("postgresql+asyncpg://user:pass@localhost/offline", "upgrade", "head", "--sql")
|
||
schema_at = output.index("CREATE SCHEMA IF NOT EXISTS dkb")
|
||
assert schema_at < output.index("CREATE TABLE dkb.alembic_version")
|
||
assert schema_at < output.index("CREATE TYPE dkb.entity_type AS ENUM ('ЮЛ', 'ИП')")
|
||
assert f"INSERT INTO dkb.alembic_version (version_num) VALUES ('{HEAD}')" in output
|