782580beb4
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W
461 lines
19 KiB
Python
461 lines
19 KiB
Python
"""Очистка форм ББ/ОФР перед выкладкой v5.19 `scripts/purge_statements.py` (spec arm-core «Очистка форм перед
|
||
выкладкой»; P3-12, P3-13, Р-24, Р-27, Р-35).
|
||
|
||
Скрипт получает сессию из `transaction` — тесты подставляют `db_session` (транзакция теста), боевую БД не трогают.
|
||
"""
|
||
|
||
import datetime
|
||
import uuid
|
||
from collections.abc import AsyncIterator, Callable
|
||
from contextlib import AbstractAsyncContextManager, asynccontextmanager
|
||
|
||
import pytest
|
||
from sqlalchemy import func, select, update
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
|
||
from scripts import purge_statements
|
||
from src.models.audit_ex import AuditEvent, Comment, ExSnapshot, ExSnapshotItem
|
||
from src.models.consolidation import Consolidation, ConsolidationDocument, ConsolidationVersion
|
||
from src.models.directories import CompanyGroup, LegalEntity
|
||
from src.models.enums import (
|
||
AuditAction,
|
||
ConsolidationStatus,
|
||
DocumentStatus,
|
||
DocumentType,
|
||
ExSnapshotStatus,
|
||
FileStatus,
|
||
PeriodQuarter,
|
||
StatementType,
|
||
)
|
||
from src.models.files import Document, RecognitionJob, S3Object, StoredFile
|
||
from src.models.osv import OsvEntry
|
||
from src.models.statements import (
|
||
FinancialStatementColumn,
|
||
FinancialStatementLine,
|
||
FinancialStatementMeta,
|
||
FinancialStatementValue,
|
||
)
|
||
from src.services.primary_files import PrimaryFilesService
|
||
from tests.factories import (
|
||
make_company_group,
|
||
make_consolidation,
|
||
make_document,
|
||
make_legal_entity,
|
||
make_osv_entry,
|
||
make_regional_office,
|
||
make_related_group,
|
||
make_stored_file,
|
||
make_user,
|
||
make_version,
|
||
)
|
||
|
||
pytestmark = pytest.mark.integration
|
||
|
||
EXIT_OK, EXIT_REFUSED = 0, 1
|
||
DESCRIPTION = "очистка форм перед v5.19"
|
||
FORM_AND_MIXED_FILES = 2 # событий FILE_DELETE: файл только с формами и смешанный
|
||
|
||
|
||
def _transaction(session: AsyncSession) -> Callable[[], AbstractAsyncContextManager[AsyncSession]]:
|
||
@asynccontextmanager
|
||
async def factory() -> AsyncIterator[AsyncSession]:
|
||
yield session
|
||
|
||
return factory
|
||
|
||
|
||
async def _run(session: AsyncSession, *argv: str) -> int:
|
||
return await purge_statements.run(list(argv), transaction=_transaction(session))
|
||
|
||
|
||
async def _count(session: AsyncSession, model: type, *where: object) -> int:
|
||
return (await session.execute(select(func.count()).select_from(model).where(*where))).scalar_one()
|
||
|
||
|
||
class World:
|
||
"""ГК с ЮЛ, инициатор и счётчик номеров версий."""
|
||
|
||
def __init__(self, session: AsyncSession) -> None:
|
||
self.session = session
|
||
self.entity: LegalEntity
|
||
self.user_id: uuid.UUID
|
||
self._version_no = 1
|
||
|
||
async def build(self) -> "World":
|
||
session = self.session
|
||
office = await _office(session)
|
||
group = await make_company_group(session, name="ГК Очистка", office=office)
|
||
self.group = group
|
||
self.entity = await make_legal_entity(session, name="ООО Очистка", group=group)
|
||
self.user = await make_user(session, email="purge@armka.local", office=office)
|
||
self.consolidation, self.first_version = await make_consolidation(
|
||
session,
|
||
group=group,
|
||
year=2025,
|
||
quarter=PeriodQuarter.Q1,
|
||
members=[self.entity],
|
||
initiator=self.user,
|
||
status=ConsolidationStatus.INITIATED,
|
||
)
|
||
return self
|
||
|
||
async def version(self, status: ConsolidationStatus): # type: ignore[no-untyped-def]
|
||
"""Ещё одна версия консолидации в статусе `status`."""
|
||
self._version_no += 1
|
||
return await make_version(
|
||
self.session,
|
||
consolidation=self.consolidation,
|
||
version_no=self._version_no,
|
||
members=[self.entity],
|
||
initiator=self.user,
|
||
status=status,
|
||
)
|
||
|
||
async def form_document(
|
||
self,
|
||
file: StoredFile,
|
||
job: RecognitionJob,
|
||
document_type: DocumentType = DocumentType.BALANCE,
|
||
*,
|
||
status: DocumentStatus = DocumentStatus.RECOGNIZED,
|
||
) -> Document:
|
||
"""Документ формы с шапкой, колонкой, строкой и значением."""
|
||
session = self.session
|
||
ordinal = (
|
||
await session.execute(
|
||
select(func.coalesce(func.max(Document.file_ordinal) + 1, 0)).where(Document.stored_file_id == file.id)
|
||
)
|
||
).scalar_one()
|
||
now = datetime.datetime.now(datetime.UTC)
|
||
document = Document(
|
||
stored_file_id=file.id,
|
||
legal_entity_id=file.legal_entity_id,
|
||
document_type=document_type,
|
||
file_ordinal=ordinal,
|
||
period_year=2025,
|
||
period_quarter=PeriodQuarter.Q1,
|
||
status=status,
|
||
confirmed_by=self.user.id if status is DocumentStatus.CONFIRMED else None,
|
||
created_at=now,
|
||
updated_at=now,
|
||
)
|
||
session.add(document)
|
||
await session.flush()
|
||
if document_type in (DocumentType.BALANCE, DocumentType.INCOME):
|
||
meta = FinancialStatementMeta(
|
||
document_id=document.id,
|
||
recognition_job_id=job.id,
|
||
statement_type=StatementType.balance if document_type is DocumentType.BALANCE else StatementType.income,
|
||
)
|
||
session.add(meta)
|
||
await session.flush()
|
||
column = FinancialStatementColumn(meta_id=meta.id, column_index=0)
|
||
line = FinancialStatementLine(meta_id=meta.id, line_code="1600", row_index=1)
|
||
session.add_all([column, line])
|
||
await session.flush()
|
||
session.add(FinancialStatementValue(line_id=line.id, column_id=column.id, amount=1))
|
||
await session.flush()
|
||
return document
|
||
|
||
async def bind(self, document: Document, version) -> None: # type: ignore[no-untyped-def]
|
||
self.session.add(ConsolidationDocument(consolidation_version_id=version.id, document_id=document.id))
|
||
await self.session.flush()
|
||
|
||
async def comment(self, document: Document) -> None:
|
||
self.session.add(
|
||
Comment(
|
||
object_type="document",
|
||
object_id=document.id,
|
||
text="проверить",
|
||
author_id=self.user.id,
|
||
created_at=datetime.datetime.now(datetime.UTC),
|
||
)
|
||
)
|
||
await self.session.flush()
|
||
|
||
async def snapshot(self, version, *, document: Document | None = None) -> ExSnapshot: # type: ignore[no-untyped-def]
|
||
"""Срез ЕХ версии `version`; при `document` — с `ex_snapshot_item` на документ."""
|
||
related = await make_related_group(self.session, group=self.group)
|
||
now = datetime.datetime.now(datetime.UTC)
|
||
snapshot = ExSnapshot(
|
||
related_group_id=related.id,
|
||
company_group_id=self.group.id,
|
||
reporting_date=datetime.date(2025, 4, 1),
|
||
year=2025,
|
||
quarter=PeriodQuarter.Q1,
|
||
version_no=1,
|
||
status=ExSnapshotStatus.ACTUAL,
|
||
source_version_id=version.id,
|
||
source_version_no=version.version_no,
|
||
verified_at=now,
|
||
written_at=now,
|
||
)
|
||
self.session.add(snapshot)
|
||
await self.session.flush()
|
||
if document is not None:
|
||
self.session.add(
|
||
ExSnapshotItem(ex_snapshot_id=snapshot.id, item_type="primary_document", document_id=document.id)
|
||
)
|
||
await self.session.flush()
|
||
return snapshot
|
||
|
||
|
||
async def _office(session: AsyncSession): # type: ignore[no-untyped-def]
|
||
return await make_regional_office(session)
|
||
|
||
|
||
@pytest.fixture
|
||
async def world(seeded: AsyncSession) -> World:
|
||
return await World(seeded).build()
|
||
|
||
|
||
async def _mixed_and_form_only(world: World) -> dict[str, object]:
|
||
"""Файл только с ББ+ОФР (ББ CONFIRMED, привязан к INITIATED), смешанный файл (ОСВ + ББ), файл только с ОСВ."""
|
||
session = world.session
|
||
form_file, form_job = await make_stored_file(session, entity=world.entity, original_name="forms.pdf")
|
||
balance = await world.form_document(form_file, form_job, DocumentType.BALANCE, status=DocumentStatus.CONFIRMED)
|
||
income = await world.form_document(form_file, form_job, DocumentType.INCOME)
|
||
await world.bind(balance, world.first_version)
|
||
mixed_file, mixed_job = await make_stored_file(session, entity=world.entity, original_name="mixed.xls")
|
||
osv = await make_document(session, file=mixed_file, job=mixed_job)
|
||
mixed_balance = await world.form_document(mixed_file, mixed_job, DocumentType.BALANCE)
|
||
osv_file, osv_job = await make_stored_file(session, entity=world.entity, original_name="osv.xls")
|
||
await make_document(session, file=osv_file, job=osv_job)
|
||
return {
|
||
"form_file": form_file,
|
||
"form_job": form_job,
|
||
"balance": balance,
|
||
"income": income,
|
||
"mixed_file": mixed_file,
|
||
"mixed_job": mixed_job,
|
||
"osv": osv,
|
||
"mixed_balance": mixed_balance,
|
||
"osv_file": osv_file,
|
||
}
|
||
|
||
|
||
TABLES = (
|
||
StoredFile,
|
||
Document,
|
||
RecognitionJob,
|
||
FinancialStatementMeta,
|
||
FinancialStatementColumn,
|
||
FinancialStatementLine,
|
||
FinancialStatementValue,
|
||
Comment,
|
||
ConsolidationDocument,
|
||
OsvEntry,
|
||
AuditEvent,
|
||
)
|
||
|
||
|
||
async def _counts(session: AsyncSession) -> dict[str, int]:
|
||
return {model.__tablename__: await _count(session, model) for model in TABLES}
|
||
|
||
|
||
async def test_dry_run_reports_and_changes_nothing(world: World, capsys: pytest.CaptureFixture[str]) -> None:
|
||
await _mixed_and_form_only(world)
|
||
before = await _counts(world.session)
|
||
|
||
code = await _run(world.session)
|
||
|
||
out = capsys.readouterr().out
|
||
assert code == EXIT_OK
|
||
assert "файлов только с формами: 1" in out
|
||
assert "смешанных файлов: 1" in out
|
||
assert "документов форм: 3" in out
|
||
assert "подтверждённых документов: 1" in out
|
||
assert "привязок к неверифицированным версиям: 1" in out
|
||
assert await _counts(world.session) == before
|
||
|
||
|
||
async def test_usage_error(world: World) -> None:
|
||
assert await _run(world.session, "--unknown") == purge_statements.EXIT_USAGE
|
||
|
||
|
||
@pytest.mark.parametrize("status", [ConsolidationStatus.VERIFIED, ConsolidationStatus.ARCHIVED])
|
||
async def test_apply_refuses_verified_binding(
|
||
world: World, capsys: pytest.CaptureFixture[str], status: ConsolidationStatus
|
||
) -> None:
|
||
objects = await _mixed_and_form_only(world)
|
||
version = await world.version(status)
|
||
if status is ConsolidationStatus.ARCHIVED: # архивированная версия мешает, только если заморожена в ЕХ
|
||
await world.snapshot(version)
|
||
await world.bind(objects["income"], version) # type: ignore[arg-type]
|
||
before = await _counts(world.session)
|
||
|
||
code = await _run(world.session, "--apply")
|
||
|
||
out = capsys.readouterr()
|
||
assert code == EXIT_REFUSED
|
||
assert str(version.id) in out.out + out.err
|
||
assert status.value in out.out + out.err
|
||
assert await _counts(world.session) == before
|
||
|
||
|
||
async def test_apply_refuses_ex_snapshot_item(world: World, capsys: pytest.CaptureFixture[str]) -> None:
|
||
objects = await _mixed_and_form_only(world)
|
||
version = await world.version(ConsolidationStatus.CANCELLED)
|
||
snapshot = await world.snapshot(version, document=objects["balance"]) # type: ignore[arg-type]
|
||
before = await _counts(world.session)
|
||
|
||
code = await _run(world.session, "--apply")
|
||
|
||
out = capsys.readouterr()
|
||
assert code == EXIT_REFUSED
|
||
assert str(objects["balance"].id) in out.out + out.err # type: ignore[attr-defined]
|
||
assert str(snapshot.source_version_id) in out.out + out.err
|
||
assert await _counts(world.session) == before
|
||
|
||
|
||
async def test_apply_ignores_unverified_versions(world: World) -> None:
|
||
session = world.session
|
||
file, job = await make_stored_file(session, entity=world.entity, original_name="forms.pdf")
|
||
balance = await world.form_document(file, job, DocumentType.BALANCE)
|
||
income = await world.form_document(file, job, DocumentType.INCOME)
|
||
other = await world.form_document(file, job, DocumentType.BALANCE)
|
||
archived = await world.form_document(file, job, DocumentType.BALANCE)
|
||
versions = {
|
||
ConsolidationStatus.VERIFICATION_RF: balance,
|
||
ConsolidationStatus.REJECTED: income,
|
||
ConsolidationStatus.CANCELLED: other,
|
||
ConsolidationStatus.ARCHIVED: archived, # архивная без ex_snapshot (из CANCELLED)
|
||
}
|
||
statuses = {}
|
||
for status, document in versions.items():
|
||
version = await world.version(status)
|
||
await world.bind(document, version)
|
||
statuses[version.id] = status
|
||
|
||
code = await _run(session, "--apply")
|
||
|
||
assert code == EXIT_OK
|
||
assert await _count(session, ConsolidationDocument) == 0
|
||
assert await _count(session, Document, Document.stored_file_id == file.id) == 0
|
||
assert await _count(session, StoredFile, StoredFile.id == file.id) == 0
|
||
rows = (await session.execute(select(ConsolidationVersion.id, ConsolidationVersion.status))).all()
|
||
for version_id, status in rows:
|
||
if version_id in statuses:
|
||
assert status is statuses[version_id]
|
||
|
||
|
||
async def test_apply_deletes_form_only_file(world: World) -> None:
|
||
session = world.session
|
||
file, job = await make_stored_file(session, entity=world.entity, original_name="forms.pdf")
|
||
balance = await world.form_document(file, job, DocumentType.BALANCE, status=DocumentStatus.CONFIRMED)
|
||
await world.form_document(file, job, DocumentType.INCOME)
|
||
await world.form_document(file, job, DocumentType.DISCLOSURE)
|
||
await world.comment(balance)
|
||
version = await world.version(ConsolidationStatus.DATA_CORRECTION)
|
||
await world.bind(balance, version)
|
||
old, _ = await make_stored_file(session, entity=world.entity, original_name="old.pdf")
|
||
old.status = FileStatus.SUPERSEDED
|
||
old.superseded_by_file_id = file.id
|
||
await session.flush()
|
||
file_id, s3_id, old_id, version_id = file.id, file.s3_object_id, old.id, version.id
|
||
before_status = version.status
|
||
|
||
code = await _run(session, "--apply")
|
||
|
||
assert code == EXIT_OK
|
||
assert await _count(session, StoredFile, StoredFile.id == file_id) == 0
|
||
assert await _count(session, Document, Document.stored_file_id == file_id) == 0
|
||
for model in (FinancialStatementMeta, FinancialStatementColumn, FinancialStatementLine, FinancialStatementValue):
|
||
assert await _count(session, model) == 0, model.__tablename__
|
||
assert await _count(session, Comment) == 0
|
||
assert await _count(session, RecognitionJob, RecognitionJob.file_id == file_id) == 0
|
||
old_row = await session.get(StoredFile, old_id, populate_existing=True)
|
||
assert old_row is not None
|
||
assert old_row.superseded_by_file_id is None
|
||
assert old_row.status is FileStatus.SUPERSEDED
|
||
s3 = await session.get(S3Object, s3_id, populate_existing=True)
|
||
assert s3 is not None and s3.expires_at is not None
|
||
assert await _count(session, ConsolidationDocument) == 0
|
||
reloaded = await session.get(ConsolidationVersion, version_id, populate_existing=True)
|
||
assert reloaded is not None and reloaded.status is before_status
|
||
[event] = (
|
||
await session.execute(
|
||
select(AuditEvent).where(AuditEvent.action == AuditAction.FILE_DELETE, AuditEvent.object_id == file_id)
|
||
)
|
||
).scalars()
|
||
assert event.user_id is None
|
||
assert event.description == DESCRIPTION
|
||
assert event.before_value is not None
|
||
assert event.before_value["id"] == str(file_id)
|
||
|
||
|
||
async def test_apply_mixed_keeps_osv_by_default(world: World) -> None:
|
||
session = world.session
|
||
objects = await _mixed_and_form_only(world)
|
||
mixed_file: StoredFile = objects["mixed_file"] # type: ignore[assignment]
|
||
mixed_balance: Document = objects["mixed_balance"] # type: ignore[assignment]
|
||
osv: Document = objects["osv"] # type: ignore[assignment]
|
||
await world.comment(mixed_balance)
|
||
version = await world.version(ConsolidationStatus.DATA_CORRECTION)
|
||
await world.bind(mixed_balance, version)
|
||
file_id, osv_id, balance_id, job_id = mixed_file.id, osv.id, mixed_balance.id, objects["mixed_job"].id # type: ignore[attr-defined]
|
||
|
||
code = await _run(session, "--apply")
|
||
|
||
assert code == EXIT_OK
|
||
assert await _count(session, StoredFile, StoredFile.id == file_id) == 1
|
||
assert await _count(session, Document, Document.id == osv_id) == 1
|
||
assert await _count(session, Document, Document.id == balance_id) == 0
|
||
assert await _count(session, RecognitionJob, RecognitionJob.id == job_id) == 1
|
||
assert await _count(session, Comment, Comment.object_id == balance_id) == 0
|
||
assert await _count(session, ConsolidationDocument, ConsolidationDocument.document_id == balance_id) == 0
|
||
events = (
|
||
(
|
||
await session.execute(
|
||
select(AuditEvent).where(AuditEvent.action == AuditAction.FILE_DELETE, AuditEvent.object_id == file_id)
|
||
)
|
||
)
|
||
.scalars()
|
||
.all()
|
||
)
|
||
[event] = events
|
||
assert event.user_id is None and event.description == DESCRIPTION
|
||
assert event.after_value is not None
|
||
assert event.after_value["deleted_documents"] == [str(balance_id)]
|
||
|
||
|
||
async def test_apply_include_mixed_deletes_whole_file(world: World) -> None:
|
||
session = world.session
|
||
objects = await _mixed_and_form_only(world)
|
||
mixed_file: StoredFile = objects["mixed_file"] # type: ignore[assignment]
|
||
file_id = mixed_file.id
|
||
|
||
code = await _run(session, "--apply", "--include-mixed")
|
||
|
||
assert code == EXIT_OK
|
||
assert await _count(session, StoredFile, StoredFile.id == file_id) == 0
|
||
assert await _count(session, Document, Document.stored_file_id == file_id) == 0
|
||
assert await _count(session, StoredFile) == 1 # остался только файл с ОСВ
|
||
|
||
|
||
async def test_osv_and_directories_untouched(world: World) -> None:
|
||
session = world.session
|
||
await _mixed_and_form_only(world)
|
||
await make_osv_entry(session)
|
||
models = (OsvEntry, Consolidation, ConsolidationVersion, LegalEntity, CompanyGroup)
|
||
before = {model.__tablename__: await _count(session, model) for model in models}
|
||
|
||
assert await _run(session, "--apply") == EXIT_OK
|
||
|
||
assert {model.__tablename__: await _count(session, model) for model in models} == before
|
||
assert await _count(session, AuditEvent, AuditEvent.action == AuditAction.FILE_DELETE) == FORM_AND_MIXED_FILES
|
||
|
||
|
||
async def test_reupload_after_purge_not_duplicate(world: World, seeded: AsyncSession) -> None:
|
||
session = world.session
|
||
file, job = await make_stored_file(session, entity=world.entity, original_name="forms.pdf", content_hash="ab" * 32)
|
||
await world.form_document(file, job, DocumentType.BALANCE)
|
||
await session.execute(update(StoredFile).where(StoredFile.id == file.id).values(status=FileStatus.PROCESSED))
|
||
await session.flush()
|
||
entity_id = world.entity.id
|
||
assert await _run(session, "--apply") == EXIT_OK
|
||
|
||
service = PrimaryFilesService.__new__(PrimaryFilesService)
|
||
service._session = session # type: ignore[attr-defined] # find_duplicate читает только сессию
|
||
assert await service.find_duplicate(entity_id, "ab" * 32) is None
|