feat: миграция 0001_dbml_v5_13 — полный DDL DBML v5.13 с откатом; тесты цикла и make migration-cycle
Явный DDL без импорта src.models: схема dkb, 27 enum-типов (явный CREATE/DROP TYPE), 43 таблицы с индексами (partial unique, NULLS NOT DISTINCT), FK цикла stored_file -> recognition_job отдельным шагом; downgrade — FK цикла, таблицы в обратном порядке, типы. Тесты (integration, scratch-БД, alembic подпроцессом): upgrade -> downgrade -> upgrade, пустой compare_metadata и alembic check, каталог = create_all ORM, поведение partial unique/NULLS NOT DISTINCT/FK-цикла; офлайн-SQL — CREATE SCHEMA первым. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Andrey Epifancev
parent
3b110b6417
commit
ed5ed43fa6
7
Makefile
7
Makefile
@ -5,7 +5,7 @@ COMPOSE_FILE ?= ../docs/dev/docker-compose.yml
|
||||
# DBML относительно api/: и в основном workspace, и в worktree .wt/<срез>/ это соседний docs/
|
||||
DBML ?= ../../docs/model/data-model-v5.13.dbml
|
||||
|
||||
.PHONY: venv lint test test-integration dbml-check check check-full up migrate seed seed-dev
|
||||
.PHONY: venv lint test test-integration dbml-check migration-cycle check check-full up migrate seed seed-dev
|
||||
|
||||
# Python 3.11 (версия платформы); внутренние пакеты raisa_* вне банка недоступны — пропускаем их
|
||||
venv:
|
||||
@ -28,6 +28,11 @@ test-integration:
|
||||
dbml-check:
|
||||
cd api && $(BIN)/python scripts/check_dbml_vs_orm.py $(DBML)
|
||||
|
||||
# Миграции на временной БД (spec data-model): upgrade head -> downgrade base -> upgrade head, пустой autogenerate,
|
||||
# схема после миграции = create_all ORM. Подключается в check-full (задача 25)
|
||||
migration-cycle:
|
||||
cd api && $(BIN)/pytest -m integration tests/test_migrations.py
|
||||
|
||||
check: lint test
|
||||
|
||||
# TODO (задача 25): добавить фронт (lint/test/build web/), проверку миграций с нуля и откат, актуальность OpenAPI
|
||||
|
||||
1481
api/alembic/versions/0001_dbml_v5_13.py
Normal file
1481
api/alembic/versions/0001_dbml_v5_13.py
Normal file
File diff suppressed because it is too large
Load Diff
@ -7,6 +7,10 @@ target-version = "py311"
|
||||
[tool.ruff.lint]
|
||||
select = ["E", "F", "I", "UP", "B", "PL"]
|
||||
|
||||
# Ревизии alembic — линейный DDL: одна функция upgrade() на десятки create_table/create_index
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
"alembic/versions/*" = ["PLR0915"]
|
||||
|
||||
[tool.ruff.lint.isort]
|
||||
known-first-party = ["src", "scripts"]
|
||||
|
||||
|
||||
284
api/tests/test_migrations.py
Normal file
284
api/tests/test_migrations.py
Normal file
@ -0,0 +1,284 @@
|
||||
"""Миграция 0001 (DBML v5.13) на PostgreSQL 16: полный цикл, пустой autogenerate, схема = схеме ORM.
|
||||
|
||||
Spec data-model, требование «Миграции применяются с нуля и откатываются» (сценарии «Полный цикл» и «Автогенерация
|
||||
ничего не находит»), design D-2. Alembic запускается отдельным процессом (`python -m alembic -x dburl=...`), как из
|
||||
CLI: `env.py` вызывает `asyncio.run(...)`, а внутри event loop pytest-asyncio это невозможно; заодно процесс не
|
||||
перенастраивает логирование тестов через `fileConfig`. Каждая проверка идёт на своей временной БД (`scratch_db_url`).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import subprocess
|
||||
import sys
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from alembic.autogenerate import compare_metadata
|
||||
from alembic.runtime.migration import MigrationContext
|
||||
from sqlalchemy import Connection, text
|
||||
from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, AsyncSession, create_async_engine
|
||||
|
||||
from src.models import Base
|
||||
from src.models.enums import ENUMS
|
||||
from tests.pg_helpers import create_scratch_db, drop_scratch_db
|
||||
from tests.test_models_part_01_03 import _check_related_group_member_uniqueness
|
||||
from tests.test_models_part_04_06 import _check_nulls_not_distinct, _make_file_with_job
|
||||
from tests.test_models_part_07_09 import (
|
||||
ALL_TABLES,
|
||||
EXPECTED_ENUM_COUNT,
|
||||
EXPECTED_TABLE_COUNT,
|
||||
_check_active_lock_is_unique,
|
||||
)
|
||||
|
||||
API_DIR = Path(__file__).resolve().parents[1]
|
||||
HEAD = "0001_dbml_v5_13"
|
||||
VERSION_TABLE = "alembic_version"
|
||||
FK_CYCLE = "fk_stored_file_current_recognition_job_id_recognition_job"
|
||||
|
||||
|
||||
def _run_alembic(url: str, *args: str) -> str:
|
||||
"""`alembic -x dburl=<url> <args>` из каталога api/; вернуть stdout+stderr, упасть с выводом при ошибке."""
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-m", "alembic", "-x", f"dburl={url}", *args],
|
||||
cwd=API_DIR,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
output = result.stdout + result.stderr
|
||||
assert result.returncode == 0, f"alembic {' '.join(args)} завершился с кодом {result.returncode}:\n{output}"
|
||||
return output
|
||||
|
||||
|
||||
async def alembic(url: str, *args: str) -> str:
|
||||
return await asyncio.to_thread(_run_alembic, url, *args)
|
||||
|
||||
|
||||
async def _scalars(conn: AsyncConnection, sql: str) -> list[Any]:
|
||||
return list((await conn.execute(text(sql))).scalars())
|
||||
|
||||
|
||||
async def tables_in_dkb(conn: AsyncConnection) -> set[str]:
|
||||
return set(
|
||||
await _scalars(
|
||||
conn,
|
||||
"SELECT table_name FROM information_schema.tables WHERE table_schema = 'dkb'",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def enums_in_dkb(conn: AsyncConnection) -> set[str]:
|
||||
return set(
|
||||
await _scalars(
|
||||
conn,
|
||||
"SELECT t.typname FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace "
|
||||
"WHERE n.nspname = 'dkb' AND t.typtype = 'e'",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def schema_snapshot(conn: AsyncConnection) -> dict[str, Any]:
|
||||
"""Каталог схемы dkb без служебной таблицы alembic: колонки, ограничения, индексы, enum'ы (с порядком значений)."""
|
||||
columns = (
|
||||
await conn.execute(
|
||||
text(
|
||||
"SELECT table_name, ordinal_position, column_name, data_type, udt_schema, udt_name, is_nullable, "
|
||||
"column_default, numeric_precision, numeric_scale FROM information_schema.columns "
|
||||
"WHERE table_schema = 'dkb' AND table_name <> :vt ORDER BY table_name, ordinal_position"
|
||||
),
|
||||
{"vt": VERSION_TABLE},
|
||||
)
|
||||
).all()
|
||||
constraints = (
|
||||
await conn.execute(
|
||||
text(
|
||||
"SELECT c.conname, c.conrelid::regclass::text, pg_get_constraintdef(c.oid) FROM pg_constraint c "
|
||||
"WHERE c.connamespace = 'dkb'::regnamespace AND c.conrelid::regclass::text <> 'dkb.alembic_version' "
|
||||
"AND c.contype <> 'n'"
|
||||
)
|
||||
)
|
||||
).all()
|
||||
indexes = (
|
||||
await conn.execute(
|
||||
text("SELECT indexname, indexdef FROM pg_indexes WHERE schemaname = 'dkb' AND tablename <> :vt"),
|
||||
{"vt": VERSION_TABLE},
|
||||
)
|
||||
).all()
|
||||
enums = (
|
||||
await conn.execute(
|
||||
text(
|
||||
"SELECT t.typname, array_agg(e.enumlabel ORDER BY e.enumsortorder) FROM pg_type t "
|
||||
"JOIN pg_namespace n ON n.oid = t.typnamespace JOIN pg_enum e ON e.enumtypid = t.oid "
|
||||
"WHERE n.nspname = 'dkb' GROUP BY t.typname"
|
||||
)
|
||||
)
|
||||
).all()
|
||||
return {
|
||||
"columns": [tuple(row) for row in columns],
|
||||
"constraints": {row[0]: (row[1], row[2]) for row in constraints},
|
||||
"indexes": {row[0]: row[1] for row in indexes},
|
||||
"enums": {row[0]: list(row[1]) for row in enums},
|
||||
}
|
||||
|
||||
|
||||
async def _assert_full_model(conn: AsyncConnection) -> None:
|
||||
tables = await tables_in_dkb(conn)
|
||||
assert tables - {VERSION_TABLE} == ALL_TABLES
|
||||
assert len(tables - {VERSION_TABLE}) == EXPECTED_TABLE_COUNT
|
||||
enums = await enums_in_dkb(conn)
|
||||
assert enums == set(ENUMS)
|
||||
assert len(enums) == EXPECTED_ENUM_COUNT
|
||||
assert await _scalars(conn, "SELECT version_num FROM dkb.alembic_version") == [HEAD]
|
||||
|
||||
|
||||
async def _truncate_model_tables(engine: AsyncEngine) -> None:
|
||||
async with engine.begin() as conn:
|
||||
await conn.execute(text("TRUNCATE " + ", ".join(f"dkb.{table}" for table in sorted(ALL_TABLES)) + " CASCADE"))
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
async def test_full_cycle_upgrade_downgrade_upgrade(scratch_db_url: str) -> None:
|
||||
"""Сценарий «Полный цикл»: upgrade head → downgrade base → upgrade head; после отката схема dkb пуста."""
|
||||
engine = create_async_engine(scratch_db_url)
|
||||
try:
|
||||
await alembic(scratch_db_url, "upgrade", "head")
|
||||
async with engine.connect() as conn:
|
||||
await _assert_full_model(conn)
|
||||
|
||||
await alembic(scratch_db_url, "downgrade", "base")
|
||||
async with engine.connect() as conn:
|
||||
assert await tables_in_dkb(conn) == {VERSION_TABLE} # служебная таблица alembic остаётся, она пуста
|
||||
assert await _scalars(conn, "SELECT count(*) FROM dkb.alembic_version") == [0]
|
||||
assert await enums_in_dkb(conn) == set()
|
||||
# в схеме не осталось других объектов (последовательностей, типов, индексов) кроме alembic_version
|
||||
leftovers = await _scalars(
|
||||
conn,
|
||||
"SELECT relname FROM pg_class WHERE relnamespace = 'dkb'::regnamespace "
|
||||
"AND relname NOT IN ('alembic_version', 'alembic_version_pkc')",
|
||||
)
|
||||
assert leftovers == []
|
||||
types = await _scalars(
|
||||
conn,
|
||||
"SELECT typname FROM pg_type WHERE typnamespace = 'dkb'::regnamespace "
|
||||
"AND typname NOT IN ('alembic_version', '_alembic_version')",
|
||||
)
|
||||
assert types == []
|
||||
|
||||
await alembic(scratch_db_url, "upgrade", "head")
|
||||
async with engine.connect() as conn:
|
||||
await _assert_full_model(conn)
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
async def test_autogenerate_finds_nothing_after_upgrade(scratch_db_url: str) -> None:
|
||||
"""Сценарий «Автогенерация ничего не находит»: compare_metadata пуст, `alembic check` не видит операций."""
|
||||
await alembic(scratch_db_url, "upgrade", "head")
|
||||
|
||||
def diff(sync_conn: Connection) -> list[Any]:
|
||||
context = MigrationContext.configure(
|
||||
sync_conn,
|
||||
opts={
|
||||
# те же опции сравнения, что в alembic/env.py
|
||||
"compare_type": True,
|
||||
"compare_server_default": True,
|
||||
"include_schemas": True,
|
||||
"include_name": lambda name, type_, _parents: name == "dkb" if type_ == "schema" else True,
|
||||
"version_table_schema": "dkb",
|
||||
},
|
||||
)
|
||||
return compare_metadata(context, Base.metadata)
|
||||
|
||||
engine = create_async_engine(scratch_db_url)
|
||||
try:
|
||||
async with engine.connect() as conn:
|
||||
assert await conn.run_sync(diff) == []
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
output = await alembic(scratch_db_url, "check")
|
||||
assert "No new upgrade operations detected" in output
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
async def test_migrated_schema_equals_orm_create_all(scratch_db_url: str) -> None:
|
||||
"""Каталог после миграции = каталог `create_all` ORM: колонки, default, ограничения, индексы, enum (имена и DDL)."""
|
||||
await alembic(scratch_db_url, "upgrade", "head")
|
||||
reference_name = f"armcore_scratch_{uuid.uuid4().hex[:12]}"
|
||||
reference_url = await create_scratch_db(reference_name)
|
||||
migrated = create_async_engine(scratch_db_url)
|
||||
reference = create_async_engine(reference_url)
|
||||
try:
|
||||
async with reference.begin() as conn:
|
||||
await conn.execute(text("CREATE SCHEMA IF NOT EXISTS dkb"))
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
async with migrated.connect() as conn:
|
||||
got = await schema_snapshot(conn)
|
||||
async with reference.connect() as conn:
|
||||
expected = await schema_snapshot(conn)
|
||||
assert got["enums"] == expected["enums"]
|
||||
assert got["columns"] == expected["columns"]
|
||||
assert got["constraints"] == expected["constraints"]
|
||||
assert got["indexes"] == expected["indexes"]
|
||||
finally:
|
||||
await migrated.dispose()
|
||||
await reference.dispose()
|
||||
await drop_scratch_db(reference_name)
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
async def test_migrated_schema_behaviour(scratch_db_url: str) -> None:
|
||||
"""После миграции: FK-цикл, partial unique, NULLS NOT DISTINCT и server_default работают на реальной БД."""
|
||||
await alembic(scratch_db_url, "upgrade", "head")
|
||||
engine = create_async_engine(scratch_db_url)
|
||||
try:
|
||||
async with engine.connect() as conn:
|
||||
fk = (
|
||||
await conn.execute(
|
||||
text(
|
||||
"SELECT conrelid::regclass::text, pg_get_constraintdef(oid) FROM pg_constraint "
|
||||
"WHERE conname = :name AND connamespace = 'dkb'::regnamespace"
|
||||
),
|
||||
{"name": FK_CYCLE},
|
||||
)
|
||||
).one()
|
||||
assert fk[0] == "dkb.stored_file"
|
||||
assert fk[1] == "FOREIGN KEY (current_recognition_job_id) REFERENCES dkb.recognition_job(id)"
|
||||
rows = await conn.execute(
|
||||
text(
|
||||
"SELECT indexname, indexdef FROM pg_indexes WHERE schemaname = 'dkb' AND indexname IN "
|
||||
"('uq_document_file_type_account', 'uq_consolidation_lock_version_active', "
|
||||
"'uq_related_group_member_entity_current', 'uq_related_group_member_group_entity_current')"
|
||||
)
|
||||
)
|
||||
indexdefs = {row[0]: row[1] for row in rows}
|
||||
assert "NULLS NOT DISTINCT" in indexdefs["uq_document_file_type_account"]
|
||||
assert "WHERE (released_at IS NULL)" in indexdefs["uq_consolidation_lock_version_active"]
|
||||
for name in ("uq_related_group_member_entity_current", "uq_related_group_member_group_entity_current"):
|
||||
assert indexdefs[name].startswith("CREATE UNIQUE INDEX")
|
||||
assert "WHERE is_current" in indexdefs[name]
|
||||
|
||||
# поведенческие проверки из тестов ORM (create_all) — теперь на схеме из миграции; данные между ними
|
||||
# очищаются: помощники создают одинаковые справочные записи (ГО, группа G1)
|
||||
await _check_related_group_member_uniqueness(engine)
|
||||
await _truncate_model_tables(engine)
|
||||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||||
chain = await _make_file_with_job(session) # замыкает FK-цикл, проверяет server_default
|
||||
await session.commit()
|
||||
await _check_nulls_not_distinct(session, chain)
|
||||
await _truncate_model_tables(engine)
|
||||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||||
await _check_active_lock_is_unique(session)
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def test_offline_sql_creates_schema_before_version_table() -> None:
|
||||
"""Офлайн-режим (`upgrade head --sql`): CREATE SCHEMA dkb идёт до таблицы версий и до enum-типов."""
|
||||
output = _run_alembic("postgresql+asyncpg://user:pass@localhost/offline", "upgrade", "head", "--sql")
|
||||
schema_at = output.index("CREATE SCHEMA IF NOT EXISTS dkb")
|
||||
assert schema_at < output.index("CREATE TABLE dkb.alembic_version")
|
||||
assert schema_at < output.index("CREATE TYPE dkb.entity_type AS ENUM ('ЮЛ', 'ИП')")
|
||||
assert f"INSERT INTO dkb.alembic_version (version_num) VALUES ('{HEAD}')" in output
|
||||
Loading…
x
Reference in New Issue
Block a user