"""Очистка форм ББ/ОФР перед выкладкой v5.19 `scripts/purge_statements.py` (spec arm-core «Очистка форм перед выкладкой»; P3-12, P3-13, Р-24, Р-27, Р-35). Скрипт получает сессию из `transaction` — тесты подставляют `db_session` (транзакция теста), боевую БД не трогают. """ import datetime import uuid from collections.abc import AsyncIterator, Callable from contextlib import AbstractAsyncContextManager, asynccontextmanager import pytest from sqlalchemy import func, select, update from sqlalchemy.ext.asyncio import AsyncSession from scripts import purge_statements from src.models.audit_ex import AuditEvent, Comment, ExSnapshot, ExSnapshotItem from src.models.consolidation import Consolidation, ConsolidationDocument, ConsolidationVersion from src.models.directories import CompanyGroup, LegalEntity from src.models.enums import ( AuditAction, ConsolidationStatus, DocumentStatus, DocumentType, ExSnapshotStatus, FileStatus, PeriodQuarter, StatementType, ) from src.models.files import Document, RecognitionJob, S3Object, StoredFile from src.models.osv import OsvEntry from src.models.statements import ( FinancialStatementColumn, FinancialStatementLine, FinancialStatementMeta, FinancialStatementValue, ) from src.services.primary_files import PrimaryFilesService from tests.factories import ( make_company_group, make_consolidation, make_document, make_legal_entity, make_osv_entry, make_regional_office, make_related_group, make_stored_file, make_user, make_version, ) pytestmark = pytest.mark.integration EXIT_OK, EXIT_REFUSED = 0, 1 DESCRIPTION = "очистка форм перед v5.19" FORM_AND_MIXED_FILES = 2 # событий FILE_DELETE: файл только с формами и смешанный def _transaction(session: AsyncSession) -> Callable[[], AbstractAsyncContextManager[AsyncSession]]: @asynccontextmanager async def factory() -> AsyncIterator[AsyncSession]: yield session return factory async def _run(session: AsyncSession, *argv: str) -> int: return await purge_statements.run(list(argv), transaction=_transaction(session)) async def _count(session: AsyncSession, model: type, *where: object) -> int: return (await session.execute(select(func.count()).select_from(model).where(*where))).scalar_one() class World: """ГК с ЮЛ, инициатор и счётчик номеров версий.""" def __init__(self, session: AsyncSession) -> None: self.session = session self.entity: LegalEntity self.user_id: uuid.UUID self._version_no = 1 async def build(self) -> "World": session = self.session office = await _office(session) group = await make_company_group(session, name="ГК Очистка", office=office) self.group = group self.entity = await make_legal_entity(session, name="ООО Очистка", group=group) self.user = await make_user(session, email="purge@armka.local", office=office) self.consolidation, self.first_version = await make_consolidation( session, group=group, year=2025, quarter=PeriodQuarter.Q1, members=[self.entity], initiator=self.user, status=ConsolidationStatus.INITIATED, ) return self async def version(self, status: ConsolidationStatus): # type: ignore[no-untyped-def] """Ещё одна версия консолидации в статусе `status`.""" self._version_no += 1 return await make_version( self.session, consolidation=self.consolidation, version_no=self._version_no, members=[self.entity], initiator=self.user, status=status, ) async def form_document( self, file: StoredFile, job: RecognitionJob, document_type: DocumentType = DocumentType.BALANCE, *, status: DocumentStatus = DocumentStatus.RECOGNIZED, ) -> Document: """Документ формы с шапкой, колонкой, строкой и значением.""" session = self.session ordinal = ( await session.execute( select(func.coalesce(func.max(Document.file_ordinal) + 1, 0)).where(Document.stored_file_id == file.id) ) ).scalar_one() now = datetime.datetime.now(datetime.UTC) document = Document( stored_file_id=file.id, legal_entity_id=file.legal_entity_id, document_type=document_type, file_ordinal=ordinal, period_year=2025, period_quarter=PeriodQuarter.Q1, status=status, confirmed_by=self.user.id if status is DocumentStatus.CONFIRMED else None, created_at=now, updated_at=now, ) session.add(document) await session.flush() if document_type in (DocumentType.BALANCE, DocumentType.INCOME): meta = FinancialStatementMeta( document_id=document.id, recognition_job_id=job.id, statement_type=StatementType.balance if document_type is DocumentType.BALANCE else StatementType.income, ) session.add(meta) await session.flush() column = FinancialStatementColumn(meta_id=meta.id, column_index=0) line = FinancialStatementLine(meta_id=meta.id, line_code="1600", row_index=1) session.add_all([column, line]) await session.flush() session.add(FinancialStatementValue(line_id=line.id, column_id=column.id, amount=1)) await session.flush() return document async def bind(self, document: Document, version) -> None: # type: ignore[no-untyped-def] self.session.add(ConsolidationDocument(consolidation_version_id=version.id, document_id=document.id)) await self.session.flush() async def comment(self, document: Document) -> None: self.session.add( Comment( object_type="document", object_id=document.id, text="проверить", author_id=self.user.id, created_at=datetime.datetime.now(datetime.UTC), ) ) await self.session.flush() async def snapshot(self, version, *, document: Document | None = None) -> ExSnapshot: # type: ignore[no-untyped-def] """Срез ЕХ версии `version`; при `document` — с `ex_snapshot_item` на документ.""" related = await make_related_group(self.session, group=self.group) now = datetime.datetime.now(datetime.UTC) snapshot = ExSnapshot( related_group_id=related.id, company_group_id=self.group.id, reporting_date=datetime.date(2025, 4, 1), year=2025, quarter=PeriodQuarter.Q1, version_no=1, status=ExSnapshotStatus.ACTUAL, source_version_id=version.id, source_version_no=version.version_no, verified_at=now, written_at=now, ) self.session.add(snapshot) await self.session.flush() if document is not None: self.session.add( ExSnapshotItem(ex_snapshot_id=snapshot.id, item_type="primary_document", document_id=document.id) ) await self.session.flush() return snapshot async def _office(session: AsyncSession): # type: ignore[no-untyped-def] return await make_regional_office(session) @pytest.fixture async def world(seeded: AsyncSession) -> World: return await World(seeded).build() async def _mixed_and_form_only(world: World) -> dict[str, object]: """Файл только с ББ+ОФР (ББ CONFIRMED, привязан к INITIATED), смешанный файл (ОСВ + ББ), файл только с ОСВ.""" session = world.session form_file, form_job = await make_stored_file(session, entity=world.entity, original_name="forms.pdf") balance = await world.form_document(form_file, form_job, DocumentType.BALANCE, status=DocumentStatus.CONFIRMED) income = await world.form_document(form_file, form_job, DocumentType.INCOME) await world.bind(balance, world.first_version) mixed_file, mixed_job = await make_stored_file(session, entity=world.entity, original_name="mixed.xls") osv = await make_document(session, file=mixed_file, job=mixed_job) mixed_balance = await world.form_document(mixed_file, mixed_job, DocumentType.BALANCE) osv_file, osv_job = await make_stored_file(session, entity=world.entity, original_name="osv.xls") await make_document(session, file=osv_file, job=osv_job) return { "form_file": form_file, "form_job": form_job, "balance": balance, "income": income, "mixed_file": mixed_file, "mixed_job": mixed_job, "osv": osv, "mixed_balance": mixed_balance, "osv_file": osv_file, } TABLES = ( StoredFile, Document, RecognitionJob, FinancialStatementMeta, FinancialStatementColumn, FinancialStatementLine, FinancialStatementValue, Comment, ConsolidationDocument, OsvEntry, AuditEvent, ) async def _counts(session: AsyncSession) -> dict[str, int]: return {model.__tablename__: await _count(session, model) for model in TABLES} async def test_dry_run_reports_and_changes_nothing(world: World, capsys: pytest.CaptureFixture[str]) -> None: await _mixed_and_form_only(world) before = await _counts(world.session) code = await _run(world.session) out = capsys.readouterr().out assert code == EXIT_OK assert "файлов только с формами: 1" in out assert "смешанных файлов: 1" in out assert "документов форм: 3" in out assert "подтверждённых документов: 1" in out assert "привязок к неверифицированным версиям: 1" in out assert await _counts(world.session) == before async def test_usage_error(world: World) -> None: assert await _run(world.session, "--unknown") == purge_statements.EXIT_USAGE @pytest.mark.parametrize("status", [ConsolidationStatus.VERIFIED, ConsolidationStatus.ARCHIVED]) async def test_apply_refuses_verified_binding( world: World, capsys: pytest.CaptureFixture[str], status: ConsolidationStatus ) -> None: objects = await _mixed_and_form_only(world) version = await world.version(status) if status is ConsolidationStatus.ARCHIVED: # архивированная версия мешает, только если заморожена в ЕХ await world.snapshot(version) await world.bind(objects["income"], version) # type: ignore[arg-type] before = await _counts(world.session) code = await _run(world.session, "--apply") out = capsys.readouterr() assert code == EXIT_REFUSED assert str(version.id) in out.out + out.err assert status.value in out.out + out.err assert await _counts(world.session) == before async def test_apply_refuses_ex_snapshot_item(world: World, capsys: pytest.CaptureFixture[str]) -> None: objects = await _mixed_and_form_only(world) version = await world.version(ConsolidationStatus.CANCELLED) snapshot = await world.snapshot(version, document=objects["balance"]) # type: ignore[arg-type] before = await _counts(world.session) code = await _run(world.session, "--apply") out = capsys.readouterr() assert code == EXIT_REFUSED assert str(objects["balance"].id) in out.out + out.err # type: ignore[attr-defined] assert str(snapshot.source_version_id) in out.out + out.err assert await _counts(world.session) == before async def test_apply_ignores_unverified_versions(world: World) -> None: session = world.session file, job = await make_stored_file(session, entity=world.entity, original_name="forms.pdf") balance = await world.form_document(file, job, DocumentType.BALANCE) income = await world.form_document(file, job, DocumentType.INCOME) other = await world.form_document(file, job, DocumentType.BALANCE) archived = await world.form_document(file, job, DocumentType.BALANCE) versions = { ConsolidationStatus.VERIFICATION_RF: balance, ConsolidationStatus.REJECTED: income, ConsolidationStatus.CANCELLED: other, ConsolidationStatus.ARCHIVED: archived, # архивная без ex_snapshot (из CANCELLED) } statuses = {} for status, document in versions.items(): version = await world.version(status) await world.bind(document, version) statuses[version.id] = status code = await _run(session, "--apply") assert code == EXIT_OK assert await _count(session, ConsolidationDocument) == 0 assert await _count(session, Document, Document.stored_file_id == file.id) == 0 assert await _count(session, StoredFile, StoredFile.id == file.id) == 0 rows = (await session.execute(select(ConsolidationVersion.id, ConsolidationVersion.status))).all() for version_id, status in rows: if version_id in statuses: assert status is statuses[version_id] async def test_apply_deletes_form_only_file(world: World) -> None: session = world.session file, job = await make_stored_file(session, entity=world.entity, original_name="forms.pdf") balance = await world.form_document(file, job, DocumentType.BALANCE, status=DocumentStatus.CONFIRMED) await world.form_document(file, job, DocumentType.INCOME) await world.form_document(file, job, DocumentType.DISCLOSURE) await world.comment(balance) version = await world.version(ConsolidationStatus.DATA_CORRECTION) await world.bind(balance, version) old, _ = await make_stored_file(session, entity=world.entity, original_name="old.pdf") old.status = FileStatus.SUPERSEDED old.superseded_by_file_id = file.id await session.flush() file_id, s3_id, old_id, version_id = file.id, file.s3_object_id, old.id, version.id before_status = version.status code = await _run(session, "--apply") assert code == EXIT_OK assert await _count(session, StoredFile, StoredFile.id == file_id) == 0 assert await _count(session, Document, Document.stored_file_id == file_id) == 0 for model in (FinancialStatementMeta, FinancialStatementColumn, FinancialStatementLine, FinancialStatementValue): assert await _count(session, model) == 0, model.__tablename__ assert await _count(session, Comment) == 0 assert await _count(session, RecognitionJob, RecognitionJob.file_id == file_id) == 0 old_row = await session.get(StoredFile, old_id, populate_existing=True) assert old_row is not None assert old_row.superseded_by_file_id is None assert old_row.status is FileStatus.SUPERSEDED s3 = await session.get(S3Object, s3_id, populate_existing=True) assert s3 is not None and s3.expires_at is not None assert await _count(session, ConsolidationDocument) == 0 reloaded = await session.get(ConsolidationVersion, version_id, populate_existing=True) assert reloaded is not None and reloaded.status is before_status [event] = ( await session.execute( select(AuditEvent).where(AuditEvent.action == AuditAction.FILE_DELETE, AuditEvent.object_id == file_id) ) ).scalars() assert event.user_id is None assert event.description == DESCRIPTION assert event.before_value is not None assert event.before_value["id"] == str(file_id) async def test_apply_mixed_keeps_osv_by_default(world: World) -> None: session = world.session objects = await _mixed_and_form_only(world) mixed_file: StoredFile = objects["mixed_file"] # type: ignore[assignment] mixed_balance: Document = objects["mixed_balance"] # type: ignore[assignment] osv: Document = objects["osv"] # type: ignore[assignment] await world.comment(mixed_balance) version = await world.version(ConsolidationStatus.DATA_CORRECTION) await world.bind(mixed_balance, version) file_id, osv_id, balance_id, job_id = mixed_file.id, osv.id, mixed_balance.id, objects["mixed_job"].id # type: ignore[attr-defined] code = await _run(session, "--apply") assert code == EXIT_OK assert await _count(session, StoredFile, StoredFile.id == file_id) == 1 assert await _count(session, Document, Document.id == osv_id) == 1 assert await _count(session, Document, Document.id == balance_id) == 0 assert await _count(session, RecognitionJob, RecognitionJob.id == job_id) == 1 assert await _count(session, Comment, Comment.object_id == balance_id) == 0 assert await _count(session, ConsolidationDocument, ConsolidationDocument.document_id == balance_id) == 0 events = ( ( await session.execute( select(AuditEvent).where(AuditEvent.action == AuditAction.FILE_DELETE, AuditEvent.object_id == file_id) ) ) .scalars() .all() ) [event] = events assert event.user_id is None and event.description == DESCRIPTION assert event.after_value is not None assert event.after_value["deleted_documents"] == [str(balance_id)] async def test_apply_include_mixed_deletes_whole_file(world: World) -> None: session = world.session objects = await _mixed_and_form_only(world) mixed_file: StoredFile = objects["mixed_file"] # type: ignore[assignment] file_id = mixed_file.id code = await _run(session, "--apply", "--include-mixed") assert code == EXIT_OK assert await _count(session, StoredFile, StoredFile.id == file_id) == 0 assert await _count(session, Document, Document.stored_file_id == file_id) == 0 assert await _count(session, StoredFile) == 1 # остался только файл с ОСВ async def test_osv_and_directories_untouched(world: World) -> None: session = world.session await _mixed_and_form_only(world) await make_osv_entry(session) models = (OsvEntry, Consolidation, ConsolidationVersion, LegalEntity, CompanyGroup) before = {model.__tablename__: await _count(session, model) for model in models} assert await _run(session, "--apply") == EXIT_OK assert {model.__tablename__: await _count(session, model) for model in models} == before assert await _count(session, AuditEvent, AuditEvent.action == AuditAction.FILE_DELETE) == FORM_AND_MIXED_FILES async def test_reupload_after_purge_not_duplicate(world: World, seeded: AsyncSession) -> None: session = world.session file, job = await make_stored_file(session, entity=world.entity, original_name="forms.pdf", content_hash="ab" * 32) await world.form_document(file, job, DocumentType.BALANCE) await session.execute(update(StoredFile).where(StoredFile.id == file.id).values(status=FileStatus.PROCESSED)) await session.flush() entity_id = world.entity.id assert await _run(session, "--apply") == EXIT_OK service = PrimaryFilesService.__new__(PrimaryFilesService) service._session = session # type: ignore[attr-defined] # find_duplicate читает только сессию assert await service.find_duplicate(entity_id, "ab" * 32) is None async def test_apply_uses_fresh_plan_for_file_lists(world: World) -> None: """T4.2: файл, появившийся после построения плана, удаляется по свежему плану под блокировками.""" session = world.session first, first_job = await make_stored_file(session, entity=world.entity, original_name="first.pdf") await world.form_document(first, first_job, DocumentType.BALANCE) stale = await purge_statements.plan_purge(session, include_mixed=False) late, late_job = await make_stored_file(session, entity=world.entity, original_name="late.pdf") await world.form_document(late, late_job, DocumentType.BALANCE) first_id, late_id = first.id, late.id await purge_statements.apply_purge(session, stale, now=datetime.datetime.now(datetime.UTC)) assert await _count(session, StoredFile, StoredFile.id.in_([first_id, late_id])) == 0 async def test_refusal_carries_fresh_plan_counters(world: World) -> None: """T4.4: отказ под блокировками несёт свежий план — счётчики отчёта не нулевые.""" objects = await _mixed_and_form_only(world) stale = await purge_statements.plan_purge(world.session, include_mixed=False) version = await world.version(ConsolidationStatus.VERIFIED) await world.bind(objects["income"], version) # type: ignore[arg-type] with pytest.raises(purge_statements.PurgeRefusedError) as raised: await purge_statements.apply_purge(world.session, stale, now=datetime.datetime.now(datetime.UTC)) refused = raised.value.plan assert refused.form_documents > 0 and refused.blocking assert "документов форм: 3" in purge_statements.render_report(refused)