arm-core/api/tests/test_api_documents.py

"""Документ, строки ОСВ и подтверждение (spec documents-api «Документ и строки», «Подтверждение документа»;
audit-trail `DOCUMENT_CONFIRM`; design D-11; Review Focus 5 плана).

Основные сценарии — поверх фикстуры `app` (транзакция теста). Гонка «подтвердить» vs «автозамена» — на `committed_*`
(временная БД, сессия запроса как в проде): транзакция автозамены держит файл под `FOR UPDATE`, подтверждение ждёт её
и после commit видит `SUPERSEDED`.
"""

import asyncio
import datetime
import json
import uuid
from pathlib import Path
from typing import Any

import pytest
from sqlalchemy import func, select, text, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker

from src.models.access import AppUser
from src.models.audit_ex import AuditEvent
from src.models.directories import LegalEntity, RegionalOffice
from src.models.enums import (
    AuditAction,
    AuditSection,
    CounterpartyMatchKind,
    DocumentStatus,
    FileStatus,
    PeriodQuarter,
    RecognitionStatus,
)
from src.models.files import Document, RecognitionJob, StoredFile
from src.models.osv import OsvEntry
from tests.conftest import AuthClientFactory
from tests.factories import (
    make_company_group,
    make_document,
    make_entries_from_result,
    make_legal_entity,
    make_stored_file,
)

HTTP_OK = 200
HTTP_FORBIDDEN = 403
HTTP_NOT_FOUND = 404
HTTP_CONFLICT = 409
HTTP_UNPROCESSABLE = 422

KA_RF = "ka_rf@armka.local"  # Ростовский филиал, DOCUMENT_VIEW + DOCUMENT_EDIT
RKA = "rka@armka.local"  # Ростовский филиал, только DOCUMENT_VIEW
CLIENT_MANAGER = "client_manager@armka.local"  # Краснодарский филиал
CURATOR_GO = "curator_go@armka.local"  # головной офис, права DOCUMENT_VIEW нет
ROSTOV = "Ростовский региональный филиал"

GOLDEN_OSV_66 = Path("golden") / "expected_osv_parse" / "rb" / "2024-FY" / "ОСВ_66 за 2024.json"
GOLDEN_OSV_66_ENTRIES = 9  # 10 строк снимка минус TOTAL (R-13)
DEFAULT_ENTRIES_LIMIT = 500
MAX_ENTRIES_LIMIT = 2000
LOCK_WAIT_SECONDS = 10.0

pytestmark = pytest.mark.integration


async def _rostov_entity(session: AsyncSession, name: str = "ООО Документы") -> LegalEntity:
    office = (await session.execute(select(RegionalOffice).where(RegionalOffice.name == ROSTOV))).scalar_one()
    group = await make_company_group(session, name=f"ГК {name}", office=office)
    return await make_legal_entity(session, name=name, group=group)


async def _document(
    session: AsyncSession,
    *,
    status: DocumentStatus = DocumentStatus.NEEDS_CHECK,
    file_status: FileStatus = FileStatus.PROCESSED,
) -> tuple[StoredFile, RecognitionJob, Document]:
    entity = await _rostov_entity(session)
    file, job = await make_stored_file(session, entity=entity, status=file_status, job_status=RecognitionStatus.done)
    document = await make_document(session, file=file, job=job, status=status)
    return file, job, document


async def _user_id(session: AsyncSession, email: str) -> uuid.UUID:
    return (await session.execute(select(AppUser.id).where(AppUser.email == email))).scalar_one()


async def _confirm_events(session: AsyncSession, document_id: uuid.UUID) -> list[AuditEvent]:
    stmt = select(AuditEvent).where(
        AuditEvent.action == AuditAction.DOCUMENT_CONFIRM, AuditEvent.object_id == document_id
    )
    return list((await session.execute(stmt)).scalars().all())


def _golden(fixtures_dir: Path) -> dict[str, Any]:
    data: dict[str, Any] = json.loads((fixtures_dir / GOLDEN_OSV_66).read_text("utf-8"))
    return data


# --- карточка документа ----------------------------------------------------------------------------------------------


async def test_document_card(seeded: AsyncSession, auth_client: AuthClientFactory) -> None:
    file, job, document = await _document(seeded)
    document.locator = "sheet:Лист_2"
    job.group_schema = [
        {"locator": "sheet:Лист_1", "groups": [{"type": "ACCOUNT", "name": "Счет", "has_inn": None}]},
        {
            "locator": "sheet:Лист_2",
            "groups": [
                {"type": "ACCOUNT", "name": "Счет", "has_inn": None},
                {"type": "COUNTERPARTY", "name": "Контрагенты", "has_inn": True},
            ],
        },
    ]
    job.metrics = {"warnings": ["парсер"], "mapping_warnings": ["sheet=Лист_1: повтор", "sheet=Лист_2: сироты"]}
    await seeded.flush()
    client = await auth_client(KA_RF)

    response = await client.get(f"/documents/{document.id}")

    assert response.status_code == HTTP_OK, response.text
    assert response.json() == {
        "id": str(document.id),
        "file_id": str(file.id),
        "legal_entity_id": str(file.legal_entity_id),
        "document_type": "OSV",
        "osv_account_code": "60",
        "period_year": 2025,
        "period_quarter": "Q1",
        "is_cumulative": True,
        "locator": "sheet:Лист_2",
        "status": "NEEDS_CHECK",
        "period_check": None,
        "counterparty_check": None,
        "confirmed_by": None,
        "confirmed_at": None,
        "group_schema": [
            {"type": "ACCOUNT", "name": "Счет", "has_inn": None},
            {"type": "COUNTERPARTY", "name": "Контрагенты", "has_inn": True},
        ],
        # R-6: предупреждения маппинга файла целиком, без фильтра по листу
        "warnings": ["sheet=Лист_1: повтор", "sheet=Лист_2: сироты"],
    }


async def test_document_card_without_group_schema(seeded: AsyncSession, auth_client: AuthClientFactory) -> None:
    _, _, document = await _document(seeded)
    client = await auth_client(KA_RF)

    body = (await client.get(f"/documents/{document.id}")).json()

    assert body["group_schema"] is None
    assert body["warnings"] == []


# --- строки ----------------------------------------------------------------------------------------------------------


@pytest.mark.fixtures
async def test_entries_as_tree_on_golden_osv_66(
    seeded: AsyncSession, auth_client: AuthClientFactory, fixtures_dir: Path
) -> None:
    """«Строки деревом»: первая строка — счёт 66 уровня 0 без родителя, у строк уровня 2 родитель — субсчёт."""
    entity = await _rostov_entity(seeded)
    file, job = await make_stored_file(
        seeded, entity=entity, status=FileStatus.PROCESSED, job_status=RecognitionStatus.done
    )
    (document,) = await make_entries_from_result(seeded, file, job, _golden(fixtures_dir))
    client = await auth_client(KA_RF)

    response = await client.get(f"/documents/{document.id}/entries")

    assert response.status_code == HTTP_OK, response.text
    body = response.json()
    assert (body["total"], body["limit"], body["offset"]) == (GOLDEN_OSV_66_ENTRIES, DEFAULT_ENTRIES_LIMIT, 0)
    rows = body["items"]
    assert [row["row_number"] for row in rows] == sorted(row["row_number"] for row in rows)
    assert all(row["group_type"] != "TOTAL" for row in rows)
    first = rows[0]
    assert (first["row_value"], first["account_code"], first["level"], first["parent_entry_id"]) == (
        "66",
        "66",
        0,
        None,
    )
    assert first["closing_credit"] == "54.64"  # деньги — строкой, без float
    assert first["account_root"] == "66"
    by_id = {row["id"]: row for row in rows}
    level_2 = [row for row in rows if row["level"] == 2]  # noqa: PLR2004
    assert level_2
    for row in level_2:
        parent = by_id[row["parent_entry_id"]]
        assert (parent["level"], parent["group_type"]) == (1, "ACCOUNT")
        assert parent["account_code"] == row["account_code"]
        assert row["group_type"] == "COUNTERPARTY"
    yug = next(row for row in rows if row["row_number"] == 18)  # noqa: PLR2004
    assert yug["account_id"] is not None
    # значения снимка ОСВ_66 за 2024, строка 18: контрагент без ИНН в колонке
    assert (yug["counterparty_name"], yug["counterparty_inn"]) == ("Юг-Финанс / ПЕКАРНЯ СЧАСТЛИВОЧНАЯ", None)
    assert set(first) == {
        "id",
        "row_number",
        "level",
        "parent_entry_id",
        "row_value",
        "group_type",
        "group_name",
        "data_type",
        "account_code",
        "account_root",
        "account_id",
        "counterparty_name",
        "counterparty_inn",
        "counterparty_id",
        "match_kind",
        "opening_debit",
        "opening_credit",
        "turnover_debit",
        "turnover_credit",
        "closing_debit",
        "closing_credit",
    }


async def test_entries_expose_match_kind(seeded: AsyncSession, auth_client: AuthClientFactory) -> None:
    """v5.16: `match_kind` — способ сопоставления контрагента; у несопоставленной строки `null`."""
    entity = await _rostov_entity(seeded)
    counterparty = await make_legal_entity(seeded, name="ООО Контрагент Вид")
    file, job = await make_stored_file(
        seeded, entity=entity, status=FileStatus.PROCESSED, job_status=RecognitionStatus.done
    )
    result = json.loads(
        (Path(__file__).resolve().parents[1] / "contracts" / "examples" / "result-osv-minimal.json").read_text("utf-8")
    )
    (document,) = await make_entries_from_result(seeded, file, job, result)
    await seeded.execute(
        update(OsvEntry)
        .where(OsvEntry.document_id == document.id, OsvEntry.row_number == 2)  # noqa: PLR2004
        .values(counterparty_id=counterparty.id, counterparty_match_kind=CounterpartyMatchKind.MANUAL)
    )
    client = await auth_client(KA_RF)

    rows = (await client.get(f"/documents/{document.id}/entries")).json()["items"]

    assert [(row["row_number"], row["counterparty_id"], row["match_kind"]) for row in rows] == [
        (1, None, None),
        (2, str(counterparty.id), "MANUAL"),
    ]


async def test_entries_pagination_and_limits(seeded: AsyncSession, auth_client: AuthClientFactory) -> None:
    entity = await _rostov_entity(seeded)
    file, job = await make_stored_file(
        seeded, entity=entity, status=FileStatus.PROCESSED, job_status=RecognitionStatus.done
    )
    result = json.loads(
        (Path(__file__).resolve().parents[1] / "contracts" / "examples" / "result-osv-minimal.json").read_text("utf-8")
    )
    (document,) = await make_entries_from_result(seeded, file, job, result)
    client = await auth_client(KA_RF)
    url = f"/documents/{document.id}/entries"

    page = (await client.get(url, params={"limit": 1, "offset": 1})).json()
    widest = await client.get(url, params={"limit": MAX_ENTRIES_LIMIT})
    too_wide = await client.get(url, params={"limit": MAX_ENTRIES_LIMIT + 1})

    assert (page["total"], page["limit"], page["offset"]) == (2, 1, 1)
    assert [row["row_number"] for row in page["items"]] == [2]
    assert widest.status_code == HTTP_OK
    assert too_wide.status_code == HTTP_UNPROCESSABLE


# --- подтверждение ---------------------------------------------------------------------------------------------------


async def test_confirm_needs_check_document(seeded: AsyncSession, auth_client: AuthClientFactory) -> None:
    """«Подтверждение»: `ka_rf` подтверждает документ `NEEDS_CHECK` своего филиала."""
    _, _, document = await _document(seeded)
    user_id = await _user_id(seeded, KA_RF)
    client = await auth_client(KA_RF)

    response = await client.post(f"/documents/{document.id}/confirm")

    assert response.status_code == HTTP_OK, response.text
    body = response.json()
    assert body["status"] == "CONFIRMED"
    assert body["confirmed_by"] == str(user_id)
    assert body["confirmed_at"] is not None
    await seeded.refresh(document)
    assert document.status == DocumentStatus.CONFIRMED
    assert document.confirmed_by == user_id
    assert document.confirmed_at is not None
    (event,) = await _confirm_events(seeded, document.id)
    assert event.section == AuditSection.PRIMARY_DOCUMENTS
    assert event.object_type == "document"
    assert event.user_id == user_id
    assert event.before_value == {"status": "NEEDS_CHECK"}
    assert event.after_value == {"status": "CONFIRMED"}


async def test_confirm_recognized_document(seeded: AsyncSession, auth_client: AuthClientFactory) -> None:
    _, _, document = await _document(seeded, status=DocumentStatus.RECOGNIZED)
    client = await auth_client(KA_RF)

    response = await client.post(f"/documents/{document.id}/confirm")

    assert response.status_code == HTTP_OK, response.text
    (event,) = await _confirm_events(seeded, document.id)
    assert event.before_value == {"status": "RECOGNIZED"}


async def test_repeated_confirm_is_409(seeded: AsyncSession, auth_client: AuthClientFactory) -> None:
    _, _, document = await _document(seeded)
    client = await auth_client(KA_RF)

    first = await client.post(f"/documents/{document.id}/confirm")
    second = await client.post(f"/documents/{document.id}/confirm")

    assert first.status_code == HTTP_OK
    assert second.status_code == HTTP_CONFLICT
    assert second.json()["detail"]["code"] == "ALREADY_CONFIRMED"
    assert len(await _confirm_events(seeded, document.id)) == 1


async def test_error_document_is_not_confirmable(seeded: AsyncSession, auth_client: AuthClientFactory) -> None:
    _, _, document = await _document(seeded, status=DocumentStatus.ERROR)
    client = await auth_client(KA_RF)

    response = await client.post(f"/documents/{document.id}/confirm")

    assert response.status_code == HTTP_CONFLICT
    assert response.json()["detail"]["code"] == "DOCUMENT_NOT_CONFIRMABLE"
    assert await _confirm_events(seeded, document.id) == []


async def test_document_of_superseded_file_is_409(seeded: AsyncSession, auth_client: AuthClientFactory) -> None:
    _, _, document = await _document(seeded, file_status=FileStatus.SUPERSEDED)
    client = await auth_client(KA_RF)

    response = await client.post(f"/documents/{document.id}/confirm")

    assert response.status_code == HTTP_CONFLICT
    assert response.json()["detail"]["code"] == "FILE_SUPERSEDED"
    await seeded.refresh(document)
    assert document.status == DocumentStatus.NEEDS_CHECK
    assert await _confirm_events(seeded, document.id) == []


async def test_confirm_rereads_file_status(seeded: AsyncSession, auth_client: AuthClientFactory) -> None:
    """Файл заменён между чтением и подтверждением: объект файла в сессии устарел (`PROCESSED`), а в БД —
    `SUPERSEDED`; подтверждение перечитывает файл под блокировкой и отвечает 409."""
    file, _, document = await _document(seeded)
    await seeded.execute(
        text("UPDATE stored_file SET status = 'SUPERSEDED' WHERE id = :id"), {"id": file.id}
    )  # мимо identity map
    assert file.status == FileStatus.PROCESSED
    client = await auth_client(KA_RF)

    response = await client.post(f"/documents/{document.id}/confirm")

    assert response.status_code == HTTP_CONFLICT
    assert response.json()["detail"]["code"] == "FILE_SUPERSEDED"


async def _committed_document(maker: async_sessionmaker[AsyncSession]) -> tuple[uuid.UUID, uuid.UUID]:
    async with maker() as session, session.begin():
        entity = await _rostov_entity(session, "ООО Гонка")
        file, job = await make_stored_file(
            session, entity=entity, status=FileStatus.PROCESSED, job_status=RecognitionStatus.done
        )
        document = await make_document(session, file=file, job=job, status=DocumentStatus.NEEDS_CHECK)
        return file.id, document.id


async def _wait_for_lock_waiter(maker: async_sessionmaker[AsyncSession]) -> None:
    """Ждёт, пока какой-то сеанс БД встанет в ожидание блокировки строки (запрос подтверждения)."""
    deadline = asyncio.get_running_loop().time() + LOCK_WAIT_SECONDS
    while asyncio.get_running_loop().time() < deadline:
        async with maker() as session:
            waiting = (
                await session.execute(
                    text(
                        "SELECT count(*) FROM pg_stat_activity "
                        "WHERE datname = current_database() AND wait_event_type = 'Lock'"
                    )
                )
            ).scalar_one()
        if waiting:
            return
        await asyncio.sleep(0.05)
    raise AssertionError("подтверждение не встало в ожидание блокировки файла")


async def test_confirm_races_with_supersede(
    committed_sessionmaker: async_sessionmaker[AsyncSession], committed_client: AuthClientFactory
) -> None:
    """Review Focus 5: автозамена держит файл под `FOR UPDATE` и переводит его в `SUPERSEDED`; подтверждение,
    начатое в это время, ждёт блокировку и после commit автозамены отвечает 409 `FILE_SUPERSEDED`."""
    file_id, document_id = await _committed_document(committed_sessionmaker)
    client = await committed_client(KA_RF)

    async with committed_sessionmaker() as replacer, replacer.begin():
        await replacer.execute(select(StoredFile.id).where(StoredFile.id == file_id).with_for_update())
        await replacer.execute(
            update(StoredFile)
            .where(StoredFile.id == file_id)
            .values(status=FileStatus.SUPERSEDED, updated_at=datetime.datetime.now(datetime.UTC))
        )
        confirm = asyncio.create_task(client.post(f"/documents/{document_id}/confirm"))
        await _wait_for_lock_waiter(committed_sessionmaker)
        assert not confirm.done()
    response = await asyncio.wait_for(confirm, LOCK_WAIT_SECONDS)

    assert response.status_code == HTTP_CONFLICT, response.text
    assert response.json()["detail"]["code"] == "FILE_SUPERSEDED"
    async with committed_sessionmaker() as session:
        document = await session.get(Document, document_id)
        assert document is not None
        assert document.status == DocumentStatus.NEEDS_CHECK
        events = (
            await session.execute(
                select(func.count()).select_from(AuditEvent).where(AuditEvent.action == AuditAction.DOCUMENT_CONFIRM)
            )
        ).scalar_one()
        assert events == 0


# --- видимость и права -----------------------------------------------------------------------------------------------


async def test_confirm_requires_document_edit(seeded: AsyncSession, auth_client: AuthClientFactory) -> None:
    _, _, document = await _document(seeded)
    client = await auth_client(RKA)  # свой филиал, только DOCUMENT_VIEW

    assert (await client.get(f"/documents/{document.id}")).status_code == HTTP_OK
    response = await client.post(f"/documents/{document.id}/confirm")

    assert response.status_code == HTTP_FORBIDDEN
    await seeded.refresh(document)
    assert document.status == DocumentStatus.NEEDS_CHECK


async def test_reading_documents_requires_document_view(seeded: AsyncSession, auth_client: AuthClientFactory) -> None:
    _, _, document = await _document(seeded)
    client = await auth_client(CURATOR_GO)

    assert (await client.get(f"/documents/{document.id}")).status_code == HTTP_FORBIDDEN
    assert (await client.get(f"/documents/{document.id}/entries")).status_code == HTTP_FORBIDDEN


@pytest.mark.parametrize(
    ("method", "suffix"),
    [("get", ""), ("get", "/entries"), ("post", "/confirm")],
)
async def test_foreign_and_missing_document_are_same_404(
    seeded: AsyncSession, auth_client: AuthClientFactory, method: str, suffix: str
) -> None:
    _, _, document = await _document(seeded)
    client = await auth_client(CLIENT_MANAGER)  # Краснодарский филиал: ЮЛ Ростова не видит

    foreign = await client.request(method, f"/documents/{document.id}{suffix}")
    missing = await client.request(method, f"/documents/{uuid.uuid4()}{suffix}")

    assert foreign.status_code == missing.status_code == HTTP_NOT_FOUND
    assert foreign.json() == missing.json()
    assert foreign.json()["detail"]["code"] == "NOT_FOUND"
    await seeded.refresh(document)
    assert document.status == DocumentStatus.NEEDS_CHECK


async def test_period_quarter_enum_in_card(seeded: AsyncSession, auth_client: AuthClientFactory) -> None:
    entity = await _rostov_entity(seeded)
    file, job = await make_stored_file(seeded, entity=entity, status=FileStatus.PROCESSED)
    document = await make_document(seeded, file=file, job=job, quarter=PeriodQuarter.Q3, is_cumulative=False)
    client = await auth_client(KA_RF)

    body = (await client.get(f"/documents/{document.id}")).json()

    assert (body["period_quarter"], body["is_cumulative"]) == ("Q3", False)