"""Цикл маппинга результата распознавания в `document`/`osv_entry` (spec osv-result-mapping; design D-1, D-2, D-3, D-7, D-9; рулинги R-3, R-4, R-5; Review Focus 1, 2). Интеграционные тесты — на отдельной временной БД (`committed_sessionmaker`: цикл работает в собственных транзакциях), результат лежит в `FakeStorage` под `results/{file_id}/{job_id}/result.json`. Синтетические результаты строятся из примера контракта `contracts/examples/result-osv-minimal.json`; сценарий «Результат смапплен» и замер скорости — на golden-снимках парсера (маркер `fixtures`). """ import asyncio import copy import json import time import uuid from collections.abc import Callable from datetime import UTC, datetime from decimal import Decimal from pathlib import Path from typing import Any import pytest from loguru import logger from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from src.core.config import Settings, get_settings from src.models.accounting import ChartOfAccount from src.models.directories import LegalEntity from src.models.enums import ( DocumentCheckResult, DocumentStatus, DocumentType, FileStatus, OsvDataType, OsvGroupType, PeriodQuarter, RecognitionStatus, ) from src.models.files import Document, RecognitionJob, StoredFile from src.models.osv import OsvEntry from src.services import result_mapping as rm from src.services.result_mapping import ( MAPPING_ATTEMPTS_KEY, MAPPING_ERROR_KEY, MAPPING_WARNINGS_KEY, Candidate, MappingFatal, process_once, run_mapping_loop, validate_result_key, ) from tests.conftest import FakeStorage from tests.factories import make_document, make_legal_entity, make_result_object, make_stored_file API_DIR = Path(__file__).resolve().parents[1] MINIMAL_RESULT = API_DIR / "contracts" / "examples" / "result-osv-minimal.json" GOLDEN_OSV_66 = Path("golden") / "expected_osv_parse" / "rb" / "2024-FY" / "ОСВ_66 за 2024.json" GOLDEN_OSV_60 = Path("golden") / "expected_osv_parse" / "rb" / "2024-FY" / "ОСВ_60 за 2024.json" BUCKET = "arm-ka" OWNER_NAME = "АО «Русский бисквит»" PERF_BUDGET_SECONDS = 2.0 # цель плана: маппинг ~1000 строк ОСВ_60 за 2024 быстрее 2 с pytestmark = pytest.mark.integration # --- данные --------------------------------------------------------------------------------------------------------- def _minimal() -> dict[str, Any]: return json.loads(MINIMAL_RESULT.read_text("utf-8")) def _golden(fixtures_dir: Path, path: Path) -> dict[str, Any]: data = json.loads((fixtures_dir / path).read_text("utf-8")) # у снимка нет job_id/file_id (подставляются) и версии сервиса (в контракте она обязательна) data["service"] = {"version": "golden", **data["service"]} return data def _osv(result: dict[str, Any]) -> dict[str, Any]: return result["documents"][0] def _root(rows: list[dict[str, Any]]) -> dict[str, Any]: return next(row for row in rows if row["level"] == 0 and row["parent_row_number"] is None) def _settings() -> Settings: return get_settings() def _now() -> datetime: return datetime.now(UTC) async def _prepare( # noqa: PLR0913 - каждое поле подготовки задаётся тестом явно maker: async_sessionmaker[AsyncSession], storage: FakeStorage, result: dict[str, Any], *, entity: LegalEntity | None = None, owner_name: str = OWNER_NAME, owner_inn: str | None = None, job_status: RecognitionStatus = RecognitionStatus.done, file_status: FileStatus = FileStatus.PROCESSING, key: str | None = None, ) -> tuple[uuid.UUID, uuid.UUID, uuid.UUID]: """ЮЛ-владелец, файл `PROCESSING` с текущим заданием `done`, результат в `storage`; `(file_id, job_id, entity_id)`. `job_id`/`file_id` результата подставляются; `key` — подменить ключ `s3_object` результата (OQ-70).""" async with maker() as session, session.begin(): owner = entity or await make_legal_entity(session, name=owner_name, inn=owner_inn) file, job = await make_stored_file(session, entity=owner, status=file_status, job_status=job_status) job.finished_at = _now() job.metrics = copy.deepcopy(result.get("metrics")) # как inbox: метрики сервиса из конверта payload = copy.deepcopy(result) | {"job_id": str(job.id), "file_id": str(file.id)} s3 = await make_result_object(session, job=job, storage=storage, result=payload) if key is not None: storage.objects[key] = storage.objects[s3.key] s3.key = key return file.id, job.id, owner.id async def _file(maker: async_sessionmaker[AsyncSession], file_id: uuid.UUID) -> StoredFile: async with maker() as session: file = await session.get(StoredFile, file_id) assert file is not None return file async def _job(maker: async_sessionmaker[AsyncSession], job_id: uuid.UUID) -> RecognitionJob: async with maker() as session: job = await session.get(RecognitionJob, job_id) assert job is not None return job async def _documents(maker: async_sessionmaker[AsyncSession], file_id: uuid.UUID) -> list[Document]: async with maker() as session: rows = await session.execute(select(Document).where(Document.stored_file_id == file_id)) return list(rows.scalars()) async def _entries(maker: async_sessionmaker[AsyncSession], document_id: uuid.UUID) -> list[OsvEntry]: async with maker() as session: rows = await session.execute( select(OsvEntry).where(OsvEntry.document_id == document_id).order_by(OsvEntry.row_number) ) return list(rows.scalars()) # --- проверка ключа результата (без БД) ---------------------------------------------------------------------------- def _candidate(key: str, bucket: str = BUCKET) -> Candidate: return Candidate(job_id=JOB_ID, file_id=FILE_ID, bucket=bucket, key=key) JOB_ID = uuid.UUID("3f2a9c1e-7b4d-4e0a-9d51-2c8e6a1b0f01") FILE_ID = uuid.UUID("a1c4e7d2-5b36-4f89-8e12-9d0c3b7a4e02") @pytest.mark.parametrize( "key", [f"results/{FILE_ID}/{JOB_ID}/result.json", f"results/{FILE_ID}/{JOB_ID}/sub/result.json"], ) def test_validate_result_key_accepts_job_folder(key: str) -> None: validate_result_key(_candidate(key), BUCKET) @pytest.mark.parametrize( ("key", "bucket"), [ (f"results/{uuid.uuid4()}/{JOB_ID}/result.json", BUCKET), # чужой файл (f"results/{FILE_ID}/{uuid.uuid4()}/result.json", BUCKET), # чужое задание (f"uploads/{FILE_ID}/{JOB_ID}/result.json", BUCKET), # не папка результатов (f"results/{FILE_ID}/{JOB_ID}/", BUCKET), # нет имени объекта (f"results/{FILE_ID}/{JOB_ID}/../{uuid.uuid4()}/result.json", BUCKET), # выход из папки (f"results/{FILE_ID}/{JOB_ID}/result.json", "other-bucket"), # чужой бакет ], ) def test_validate_result_key_rejects_outside_job_folder(key: str, bucket: str) -> None: with pytest.raises(MappingFatal): validate_result_key(_candidate(key, bucket), BUCKET) # --- сценарии спеки ------------------------------------------------------------------------------------------------ @pytest.mark.fixtures async def test_golden_osv_66_mapped( # noqa: PLR0915 - один сценарий спеки целиком committed_sessionmaker: async_sessionmaker[AsyncSession], fixtures_dir: Path ): """«Результат смапплен», «Дерево и счета»: документ 66/2024/Q4/нарастающая, строки без TOTAL, `group_schema`.""" maker, storage = committed_sessionmaker, FakeStorage() golden = _golden(fixtures_dir, GOLDEN_OSV_66) file_id, job_id, entity_id = await _prepare(maker, storage, golden) assert await process_once(maker, storage, _settings()) is True file = await _file(maker, file_id) assert file.status is FileStatus.PROCESSED assert file.updated_at is not None [document] = await _documents(maker, file_id) assert document.document_type is DocumentType.OSV assert document.osv_account_code == "66" assert document.period_year == 2024 # noqa: PLR2004 - год из шапки снимка assert document.period_quarter is PeriodQuarter.Q4 assert document.is_cumulative is True assert document.locator == "sheet:Лист_1" assert document.legal_entity_id == entity_id assert document.status is DocumentStatus.RECOGNIZED # done, проверки чистые, шапка = владелец по имени assert document.counterparty_check is DocumentCheckResult.MATCH assert document.period_check is DocumentCheckResult.UNKNOWN rows = _osv(golden)["rows"] entries = await _entries(maker, document.id) assert len(entries) == len(rows) - 1 # всё, кроме TOTAL total_rows = {row["row_number"] for row in rows if row["group_type"] == "TOTAL"} assert total_rows and not total_rows & {entry.row_number for entry in entries} # в enum DBML TOTAL нет by_row = {entry.row_number: entry for entry in entries} for row in rows: if row["group_type"] == "TOTAL": continue entry = by_row[row["row_number"]] assert entry.recognition_job_id == job_id assert entry.owner_legal_entity_id == entity_id assert entry.is_modified is False assert entry.level == row["level"] assert entry.row_value == row["row_value"] for name in ("opening_debit", "opening_credit", "turnover_debit", "turnover_credit"): expected = row.get(name) assert getattr(entry, name) == (Decimal(expected) if expected is not None else None) if row["parent_row_number"] is None: assert entry.parent_entry_id is None else: assert entry.parent_entry_id == by_row[row["parent_row_number"]].id async with maker() as session: chart_6603 = (await session.execute(select(ChartOfAccount.id).where(ChartOfAccount.code == "66.03"))).scalar() [yug] = [e for e in entries if e.account_code == "66.03" and e.group_type is OsvGroupType.COUNTERPARTY] assert yug.counterparty_name == "Юг-Финанс / ПЕКАРНЯ СЧАСТЛИВОЧНАЯ" parent = next(e for e in entries if e.id == yug.parent_entry_id) assert parent.account_code == "66.03" assert parent.group_type is OsvGroupType.ACCOUNT assert yug.account_root == "66" assert yug.account_id == chart_6603 is not None job = await _job(maker, job_id) assert job.group_schema == [ { "locator": "sheet:Лист_1", "groups": [ {"type": "ACCOUNT", "name": "Счет", "has_inn": None}, {"type": "COUNTERPARTY", "name": "Контрагенты", "has_inn": None}, ], } ] assert job.metrics[MAPPING_WARNINGS_KEY] == [] assert job.metrics["rows"] == 9 # noqa: PLR2004 - метрики сервиса сохранены рядом с ключами маппинга @pytest.mark.fixtures async def test_mapping_performance_osv_60(committed_sessionmaker: async_sessionmaker[AsyncSession], fixtures_dir: Path): """Цель плана: маппинг самой длинной ОСВ (`ОСВ_60 за 2024`, ~1000 строк) — быстрее 2 с на один проход цикла.""" maker, storage = committed_sessionmaker, FakeStorage() golden = _golden(fixtures_dir, GOLDEN_OSV_60) file_id, _, _ = await _prepare(maker, storage, golden) started = time.perf_counter() assert await process_once(maker, storage, _settings()) is True elapsed = time.perf_counter() - started [document] = await _documents(maker, file_id) rows = _osv(golden)["rows"] entries = await _entries(maker, document.id) total_rows = sum(1 for row in rows if row["group_type"] == "TOTAL") assert len(entries) == len(rows) - total_rows logger.info("Маппинг ОСВ_60 за 2024: {rows} строк за {elapsed:.3f} с", rows=len(entries), elapsed=elapsed) print(f"\nPERF: ОСВ_60 за 2024 — {len(entries)} строк за {elapsed:.3f} с") # noqa: T201 - замер в отчёт assert elapsed < PERF_BUDGET_SECONDS async def test_result_key_outside_job_folder_is_error(committed_sessionmaker: async_sessionmaker[AsyncSession]): """«Ключ результата вне папки задания»: файл `ERROR` с ошибкой маппинга, документов нет, повтора нет.""" maker, storage = committed_sessionmaker, FakeStorage() file_id, job_id, _ = await _prepare( maker, storage, _minimal(), key=f"results/{uuid.uuid4()}/{uuid.uuid4()}/result.json" ) assert await process_once(maker, storage, _settings()) is True file = await _file(maker, file_id) assert file.status is FileStatus.ERROR job = await _job(maker, job_id) assert "results/" in job.metrics[MAPPING_ERROR_KEY] assert MAPPING_ATTEMPTS_KEY not in job.metrics # неповторяемая ошибка — без счётчика попыток assert await _documents(maker, file_id) == [] assert await process_once(maker, storage, _settings()) is False # повтора нет async def test_missing_result_object_is_error(committed_sessionmaker: async_sessionmaker[AsyncSession]): """Объекта результата нет в S3 — неповторяемая ошибка (D-2): файл `ERROR`.""" maker, storage = committed_sessionmaker, FakeStorage() file_id, job_id, _ = await _prepare(maker, storage, _minimal()) storage.objects.clear() assert await process_once(maker, storage, _settings()) is True assert (await _file(maker, file_id)).status is FileStatus.ERROR assert "не найден" in (await _job(maker, job_id)).metrics[MAPPING_ERROR_KEY] @pytest.mark.parametrize( ("documents", "status", "message"), [ ([], "needs_check", "схем"), # контракт требует documents ≥ 1 — результат не проходит схему ( [{"document_type": "BALANCE", "locator": "pages:1", "header": {}, "columns": [], "lines": []}], "needs_check", "нет документов ОСВ", ), ], ) async def test_result_without_osv_documents_is_error( committed_sessionmaker: async_sessionmaker[AsyncSession], documents: list[Any], status: str, message: str ): """Review Focus 2: результат без документов ОСВ → файл `ERROR` «нет документов ОСВ», не исключение цикла.""" maker, storage = committed_sessionmaker, FakeStorage() result = _minimal() | {"status": status, "documents": documents} file_id, job_id, _ = await _prepare(maker, storage, result, job_status=RecognitionStatus.needs_check) assert await process_once(maker, storage, _settings()) is True assert (await _file(maker, file_id)).status is FileStatus.ERROR assert message in (await _job(maker, job_id)).metrics[MAPPING_ERROR_KEY] assert await _documents(maker, file_id) == [] async def test_mapping_failure_retries_then_error( committed_sessionmaker: async_sessionmaker[AsyncSession], monkeypatch: pytest.MonkeyPatch ): """«Сбой маппинга с повтором»: откат (документов нет), `mapping_attempts` 1, 2 — файл `PROCESSING`; третья неудача — `ERROR` с `metrics.mapping_error`.""" maker, storage = committed_sessionmaker, FakeStorage() file_id, job_id, _ = await _prepare(maker, storage, _minimal()) real_map_file = rm.map_file calls: list[int] = [] async def failing_map_file(session: AsyncSession, **kwargs: Any) -> list[str]: await real_map_file(session, **kwargs) # вставка выполнена — и откатывается вместе с транзакцией calls.append(1) raise RuntimeError("подставная ошибка БД при вставке строк") monkeypatch.setattr(rm, "map_file", failing_map_file) for attempt in (1, 2): # R-15: повторяемая неудача — `False`, цикл выдерживает паузу MAPPING_POLL_SECONDS до следующей попытки assert await process_once(maker, storage, _settings()) is False job = await _job(maker, job_id) assert job.metrics[MAPPING_ATTEMPTS_KEY] == attempt assert MAPPING_ERROR_KEY not in job.metrics assert (await _file(maker, file_id)).status is FileStatus.PROCESSING assert await _documents(maker, file_id) == [] # третья неудача переводит файл в ERROR: кандидат покинул очередь, следующий можно брать без паузы — `True` assert await process_once(maker, storage, _settings()) is True job = await _job(maker, job_id) assert job.metrics[MAPPING_ATTEMPTS_KEY] == 3 # noqa: PLR2004 - MAPPING_MAX_ATTEMPTS по умолчанию assert "подставная ошибка БД" in job.metrics[MAPPING_ERROR_KEY] assert (await _file(maker, file_id)).status is FileStatus.ERROR assert len(calls) == 3 # noqa: PLR2004 - три попытки assert await process_once(maker, storage, _settings()) is False async def test_repeated_mapping_is_idempotent(committed_sessionmaker: async_sessionmaker[AsyncSession]): """«Повторный маппинг идемпотентен»: остатки прерванной вставки (документ и половина строк) удаляются, у файла ровно один набор документов и строк.""" maker, storage = committed_sessionmaker, FakeStorage() result = _minimal() rows = _osv(result)["rows"] rows.extend( rows[-1] | {"row_number": number, "counterparty_name": f"ООО Лютик-{number}", "counterparty_inn": None} for number in range(3, 7) ) file_id, job_id, entity_id = await _prepare(maker, storage, result) async with maker() as session, session.begin(): file = await session.get(StoredFile, file_id) job = await session.get(RecognitionJob, job_id) assert file is not None and job is not None stale = await make_document(session, file=file, job=job, account="60", status=DocumentStatus.NEEDS_CHECK) for row in rows[: len(rows) // 2]: session.add( OsvEntry( recognition_job_id=job_id, document_id=stale.id, owner_legal_entity_id=entity_id, row_number=row["row_number"], level=row["level"], row_value=row["row_value"], ) ) assert await process_once(maker, storage, _settings()) is True [document] = await _documents(maker, file_id) assert document.id != stale.id entries = await _entries(maker, document.id) assert [entry.row_number for entry in entries] == [row["row_number"] for row in rows] async with maker() as session: total = ( await session.execute( select(func.count()) .select_from(OsvEntry) .join(Document, Document.id == OsvEntry.document_id) .where(Document.stored_file_id == file_id) ) ).scalar_one() assert total == len(rows) async def test_two_sheets_of_same_account(committed_sessionmaker: async_sessionmaker[AsyncSession]): """«Два листа одного счёта»: один документ (первый лист), `NEEDS_CHECK`, предупреждение называет оба `locator`.""" maker, storage = committed_sessionmaker, FakeStorage() result = _minimal() first = _osv(result) second = copy.deepcopy(first) | {"locator": "sheet:Лист_2"} second["header"]["account_code_raw"] = "60" result["documents"] = [first, second] file_id, job_id, _ = await _prepare(maker, storage, result, owner_inn="7700000000") assert await process_once(maker, storage, _settings()) is True [document] = await _documents(maker, file_id) assert document.locator == "sheet:Лист_1" assert document.osv_account_code == "60" assert document.status is DocumentStatus.NEEDS_CHECK assert len(await _entries(maker, document.id)) == len(first["rows"]) job = await _job(maker, job_id) assert job.metrics[MAPPING_WARNINGS_KEY] == ["sheet=sheet:Лист_2: счёт 60 уже есть в листе sheet:Лист_1"] assert (await _file(maker, file_id)).status is FileStatus.PROCESSED async def test_header_of_another_organization(committed_sessionmaker: async_sessionmaker[AsyncSession]): """«Шапка другой организации»: файл ЮЛ «Пекарня Счастливочная», в шапке `АО "Русский бисквит"` → `MISMATCH`, `NEEDS_CHECK`.""" maker, storage = committed_sessionmaker, FakeStorage() result = _minimal() _osv(result)["header"] |= {"organization_name": 'АО "Русский бисквит"', "organization_inn": None} file_id, _, _ = await _prepare(maker, storage, result, owner_name="ООО «Пекарня Счастливочная»") assert await process_once(maker, storage, _settings()) is True [document] = await _documents(maker, file_id) assert document.counterparty_check is DocumentCheckResult.MISMATCH assert document.status is DocumentStatus.NEEDS_CHECK async def _q1_document( maker: async_sessionmaker[AsyncSession], entity_id: uuid.UUID, *, opening_credit: Decimal, file_status: FileStatus, ) -> None: """Документ ОСВ 60 за 2024 Q1 другого файла того же ЮЛ с корнем (level 0) и строкой-потомком.""" async with maker() as session, session.begin(): entity = await session.get(LegalEntity, entity_id) assert entity is not None file, job = await make_stored_file( session, entity=entity, status=file_status, job_status=RecognitionStatus.done ) document = await make_document(session, file=file, job=job, account="60", year=2024, quarter=PeriodQuarter.Q1) common = {"recognition_job_id": job.id, "document_id": document.id, "owner_legal_entity_id": entity_id} root = OsvEntry( **common, row_number=5, level=0, row_value="60", opening_debit=None, opening_credit=opening_credit ) session.add(root) await session.flush() # строка уровня 1 с меньшим номером не должна считаться корнем session.add(OsvEntry(**common, row_number=1, level=1, parent_entry_id=root.id, opening_credit=Decimal(1))) @pytest.mark.parametrize( ("q1_opening", "q1_file_status", "expected"), [ (Decimal("1000.00"), FileStatus.PROCESSED, True), # сальдо на начало совпало с Q1 (Decimal("1000.01"), FileStatus.PROCESSED, True), # в пределах ±0.01 (Decimal("999.00"), FileStatus.PROCESSED, False), # не совпало — по тексту (Decimal("1000.00"), FileStatus.SUPERSEDED, False), # Q1 из заменённого файла не учитывается ], ) async def test_cumulative_by_q1_data( committed_sessionmaker: async_sessionmaker[AsyncSession], q1_opening: Decimal, q1_file_status: FileStatus, expected: bool, ): """«Нарастающая по данным»: ОСВ 60 «за 3 квартал 2024» с `opening_*` корня = Q1 → `is_cumulative = true` и предупреждение «нарастающая по данным Q1» (ASSUMPTION(OQ-02)).""" maker, storage = committed_sessionmaker, FakeStorage() async with maker() as session, session.begin(): entity = await make_legal_entity(session, name=OWNER_NAME, inn="7700000000") await _q1_document(maker, entity.id, opening_credit=q1_opening, file_status=q1_file_status) result = _minimal() _osv(result)["header"] |= { "period_text": "3 квартал 2024 г.", "period_year": 2024, "period_quarter": "Q3", "is_cumulative": False, } assert _root(_osv(result)["rows"])["opening_credit"] == "1000.00" file_id, job_id, _ = await _prepare(maker, storage, result, entity=entity) assert await process_once(maker, storage, _settings()) is True [document] = await _documents(maker, file_id) assert document.period_quarter is PeriodQuarter.Q3 assert document.is_cumulative is expected warnings = (await _job(maker, job_id)).metrics[MAPPING_WARNINGS_KEY] if expected: assert len(warnings) == 1 assert "нарастающая по данным Q1" in warnings[0] assert "sheet:Лист_1" in warnings[0] assert document.status is DocumentStatus.NEEDS_CHECK else: assert warnings == [] assert document.status is DocumentStatus.RECOGNIZED @pytest.mark.parametrize( ("job_status", "expected"), [(RecognitionStatus.done, DocumentStatus.RECOGNIZED), (RecognitionStatus.needs_check, DocumentStatus.NEEDS_CHECK)], ) async def test_clean_result( committed_sessionmaker: async_sessionmaker[AsyncSession], job_status: RecognitionStatus, expected: DocumentStatus, ): """«Чистый результат»: задание `done`, шапка = владелец (ИНН), нарушений нет → `RECOGNIZED`, `MATCH`, `UNKNOWN`; задание `needs_check` → `NEEDS_CHECK`.""" maker, storage = committed_sessionmaker, FakeStorage() file_id, _, _ = await _prepare(maker, storage, _minimal(), owner_inn="7700000000", job_status=job_status) assert await process_once(maker, storage, _settings()) is True [document] = await _documents(maker, file_id) assert document.status is expected assert document.counterparty_check is DocumentCheckResult.MATCH assert document.period_check is DocumentCheckResult.UNKNOWN assert document.osv_account_code == "60" assert document.period_quarter is PeriodQuarter.Q1 assert document.is_cumulative is False assert (await _file(maker, file_id)).status is FileStatus.PROCESSED async def test_counterparty_resolved_in_entries(committed_sessionmaker: async_sessionmaker[AsyncSession]): """R-3: `counterparty_id` — только у `COUNTERPARTY` с `data_type ∈ {BU, SUM}`; `NU` хранится как пришёл, без резолва; строка-счёт без контрагента.""" maker, storage = committed_sessionmaker, FakeStorage() async with maker() as session, session.begin(): lyutik = await make_legal_entity(session, name="ООО «Лютик»", inn="7711111111") result = _minimal() rows = _osv(result)["rows"] rows.append(rows[1] | {"row_number": 3, "data_type": "NU"}) file_id, _, _ = await _prepare(maker, storage, result) assert await process_once(maker, storage, _settings()) is True [document] = await _documents(maker, file_id) account, bu, nu = await _entries(maker, document.id) assert account.group_type is OsvGroupType.ACCOUNT assert account.counterparty_id is None assert bu.counterparty_id == lyutik.id assert bu.counterparty_inn == "7711111111" assert nu.data_type is OsvDataType.NU assert nu.counterparty_id is None async def test_result_of_non_current_job_is_ignored(committed_sessionmaker: async_sessionmaker[AsyncSession]): """Review Focus 1: результат задания, которое уже не текущее для файла, не выбирается и файл не трогает.""" maker, storage = committed_sessionmaker, FakeStorage() file_id, old_job_id, _ = await _prepare(maker, storage, _minimal()) async with maker() as session, session.begin(): file = await session.get(StoredFile, file_id) assert file is not None new_job = RecognitionJob( file_id=file_id, pipeline=(await session.get(RecognitionJob, old_job_id)).pipeline, # type: ignore[union-attr] request_subject="recognition.tabular.request", schema_version="1.0", status=RecognitionStatus.processing, attempt=2, ) session.add(new_job) await session.flush() file.current_recognition_job_id = new_job.id before = await _file(maker, file_id) assert await process_once(maker, storage, _settings()) is False after = await _file(maker, file_id) assert (after.status, after.updated_at) == (FileStatus.PROCESSING, before.updated_at) assert await _documents(maker, file_id) == [] assert (await _job(maker, old_job_id)).group_schema is None async def test_job_replaced_while_reading_result( committed_sessionmaker: async_sessionmaker[AsyncSession], monkeypatch: pytest.MonkeyPatch ): """Review Focus 1, гонка: пока цикл читал S3, у файла сменилось текущее задание → транзакция маппинга ничего не меняет (задание и файл перечитываются под блокировкой).""" maker, storage = committed_sessionmaker, FakeStorage() file_id, job_id, _ = await _prepare(maker, storage, _minimal()) real_get_bytes = storage.get_bytes async def get_bytes_and_replace_job(key: str) -> bytes: async with maker() as session, session.begin(): file = await session.get(StoredFile, file_id) job = await session.get(RecognitionJob, job_id) assert file is not None and job is not None new_job = RecognitionJob( file_id=file_id, pipeline=job.pipeline, request_subject=job.request_subject, schema_version="1.0", status=RecognitionStatus.queued, attempt=2, ) session.add(new_job) await session.flush() file.current_recognition_job_id = new_job.id return await real_get_bytes(key) monkeypatch.setattr(storage, "get_bytes", get_bytes_and_replace_job) assert await process_once(maker, storage, _settings()) is True file = await _file(maker, file_id) assert file.status is FileStatus.PROCESSING assert await _documents(maker, file_id) == [] job = await _job(maker, job_id) assert job.group_schema is None assert MAPPING_ATTEMPTS_KEY not in (job.metrics or {}) async def test_run_mapping_loop_processes_two_candidates(committed_sessionmaker: async_sessionmaker[AsyncSession]): """`run_mapping_loop`: два кандидата подряд без паузы (`sleep(0)`), затем пауза `MAPPING_POLL_SECONDS`.""" maker, storage = committed_sessionmaker, FakeStorage() first, _, _ = await _prepare(maker, storage, _minimal()) second, _, _ = await _prepare(maker, storage, _minimal()) settings = _settings() sleeps: list[float] = [] async def fake_sleep(seconds: float) -> None: sleeps.append(seconds) if len(sleeps) == 3: # noqa: PLR2004 - два кандидата и один пустой проход raise asyncio.CancelledError with pytest.raises(asyncio.CancelledError): await run_mapping_loop(maker, storage, settings, sleep=fake_sleep) assert sleeps == [0, 0, settings.MAPPING_POLL_SECONDS] assert (await _file(maker, first)).status is FileStatus.PROCESSED assert (await _file(maker, second)).status is FileStatus.PROCESSED async def test_run_mapping_loop_pauses_after_retryable_failure( committed_sessionmaker: async_sessionmaker[AsyncSession], monkeypatch: pytest.MonkeyPatch ): """R-15: после повторяемой неудачи цикл спит `MAPPING_POLL_SECONDS` (попытки не сгорают за миллисекунды), после успешного маппинга — `sleep(0)`.""" maker, storage = committed_sessionmaker, FakeStorage() file_id, job_id, _ = await _prepare(maker, storage, _minimal()) real_map_file = rm.map_file failures = [RuntimeError("кратковременный сбой БД")] async def flaky_map_file(session: AsyncSession, **kwargs: Any) -> list[str]: if failures: raise failures.pop() return await real_map_file(session, **kwargs) monkeypatch.setattr(rm, "map_file", flaky_map_file) settings = _settings() sleeps: list[float] = [] async def fake_sleep(seconds: float) -> None: sleeps.append(seconds) if len(sleeps) == 3: # noqa: PLR2004 - неудача, успех, пустой проход raise asyncio.CancelledError with pytest.raises(asyncio.CancelledError): await run_mapping_loop(maker, storage, settings, sleep=fake_sleep) assert sleeps == [settings.MAPPING_POLL_SECONDS, 0, settings.MAPPING_POLL_SECONDS] assert (await _file(maker, file_id)).status is FileStatus.PROCESSED assert (await _job(maker, job_id)).metrics[MAPPING_ATTEMPTS_KEY] == 1 async def test_run_mapping_loop_survives_errors(monkeypatch: pytest.MonkeyPatch): """Ошибка прохода логируется, цикл не умирает и ждёт `MAPPING_POLL_SECONDS`.""" outcomes: list[Callable[[], bool]] = [lambda: (_ for _ in ()).throw(RuntimeError("сбой")), lambda: True] sleeps: list[float] = [] async def fake_process_once(*args: Any, **kwargs: Any) -> bool: return outcomes.pop(0)() async def fake_sleep(seconds: float) -> None: sleeps.append(seconds) if not outcomes: raise asyncio.CancelledError monkeypatch.setattr(rm, "process_once", fake_process_once) settings = _settings() records: list[Any] = [] handler_id = logger.add(lambda message: records.append(message.record), level="ERROR") try: with pytest.raises(asyncio.CancelledError): await run_mapping_loop(None, FakeStorage(), settings, sleep=fake_sleep) # type: ignore[arg-type] finally: logger.remove(handler_id) assert sleeps == [settings.MAPPING_POLL_SECONDS, 0] assert len(records) == 1