diff --git a/api/src/domain/statement_issues.py b/api/src/domain/statement_issues.py new file mode 100644 index 0000000..5c03500 --- /dev/null +++ b/api/src/domain/statement_issues.py @@ -0,0 +1,45 @@ +"""Черновик записи `document_issue` (DBML v5.19) — чистые данные без ORM. + +Маппер `1.x` пишет только `MAPPING_WARNING` (P3-5, P3-7): предупреждения черновика формы, листа ОСВ и повтор формы +в файле; проверки §7 и прочие коды ядра в `1.x` не пишутся. Запись в БД — `repositories/statements.add_issues`. +""" + +import uuid +from dataclasses import dataclass +from decimal import Decimal + +from src.models.enums import DocumentIssueKind, IssueSeverity, IssueSource + +__all__ = ["MAPPING_WARNING_CODE", "IssueDraft", "mapping_warning"] + +MAPPING_WARNING_CODE = "MAPPING_WARNING" + + +@dataclass(frozen=True) +class IssueDraft: + """Запись `document_issue` до вставки: `line_ids`/`column_id` — id уже созданных строк и колонок.""" + + kind: DocumentIssueKind + code: str + severity: IssueSeverity + message: str + source: IssueSource + line_ids: tuple[uuid.UUID, ...] = () + column_id: uuid.UUID | None = None + column_index: int | None = None + field: str | None = None + relation: str | None = None + codes: tuple[str, ...] = () + expected: Decimal | None = None + actual: Decimal | None = None + + +def mapping_warning(message: str) -> IssueDraft: + """Предупреждение маппинга ядра: `warning`, `MAPPING_WARNING`, `info`, источник `core`.""" + return IssueDraft( + kind=DocumentIssueKind.warning, + code=MAPPING_WARNING_CODE, + severity=IssueSeverity.info, + message=message, + source=IssueSource.core, + ) diff --git a/api/src/domain/statement_mapping.py b/api/src/domain/statement_mapping.py index 4c34b14..0101a49 100644 --- a/api/src/domain/statement_mapping.py +++ b/api/src/domain/statement_mapping.py @@ -11,6 +11,7 @@ from src.contract.result import StatementColumn, StatementDocument, StatementVal from src.models.enums import DocumentCheckResult, DocumentStatus, DocumentType, PeriodQuarter, StatementType __all__ = [ + "ColumnDraft", "LineDraft", "StatementDraft", "ValueDraft", @@ -19,6 +20,7 @@ __all__ = [ "quarter_of", "statement_period", "statement_status", + "value_clean", "values_ok", ] @@ -33,6 +35,18 @@ _QUARTER_ENDS = { _STATEMENT_TYPES = {DocumentType.BALANCE: StatementType.balance, DocumentType.INCOME: StatementType.income} +@dataclass +class ColumnDraft: + """Колонка формы; `period_kind` (v5.19): `absolute` при дате колонки, иначе `none` (в `1.x` относительных нет).""" + + column_index: int + label: str + period_date: datetime.date | None + is_current: bool | None + period_kind: str = "none" + label_confidence: Decimal | None = None + + @dataclass class ValueDraft: line_code: str @@ -41,6 +55,9 @@ class ValueDraft: string_value: str | None confidence: Decimal | None is_recognized: bool | None + row_index: int = 0 # индекс строки в `doc.lines` — ключ строки при вставке (код может повторяться) + source_text: str | None = None + needs_review: bool = False @dataclass @@ -49,6 +66,12 @@ class LineDraft: line_code_confidence: Decimal | None line_name: str line_name_confidence: Decimal | None + row_index: int = 0 # индекс строки в `doc.lines` результата; пропущенный повтор кода индекс не занимает + line_status: str = "ok" + line_code_original_value: str | None = None + source_ref: str | None = None + line_code_text: str | None = None + needs_review: bool = False @dataclass @@ -60,7 +83,7 @@ class StatementDraft: period_year: int | None period_quarter: PeriodQuarter | None header: dict[str, Any] - columns: list[tuple[int, str, datetime.date | None, bool | None]] + columns: list[ColumnDraft] lines: list[LineDraft] values: list[ValueDraft] warnings: list[str] = field(default_factory=list) @@ -116,7 +139,7 @@ def build_statement(doc: StatementDocument) -> StatementDraft: warnings.append(f"{doc.locator}: дата {period_date.isoformat()} не конец квартала — квартал не определён") latest = max((day for day in column_dates if day is not None), default=None) columns = [ - ( + ColumnDraft( column.column_index, column.label, column.period_date, @@ -124,6 +147,7 @@ def build_statement(doc: StatementDocument) -> StatementDraft: if latest is None else column.period_date == latest and column.column_index == min(c.column_index for c in source_columns if c.period_date == latest), + period_kind="absolute" if column.period_date is not None else "none", ) for column in source_columns ] @@ -131,7 +155,7 @@ def build_statement(doc: StatementDocument) -> StatementDraft: lines: list[LineDraft] = [] values: list[ValueDraft] = [] seen: set[str] = set() - for line in doc.lines: + for row_index, line in enumerate(doc.lines): if line.line_code in seen: warnings.append(f"{doc.locator}: код {line.line_code} повторяется — оставлена первая строка") continue @@ -142,6 +166,8 @@ def build_statement(doc: StatementDocument) -> StatementDraft: _decimal(line.line_code_confidence), line.line_name, _decimal(line.line_name_confidence), + row_index=row_index, + line_code_original_value=line.line_code, ) ) by_column: dict[int, StatementValue] = {} @@ -169,10 +195,25 @@ def build_statement(doc: StatementDocument) -> StatementDraft: value.string_value if value is not None else None, _decimal(value.confidence) if value is not None else None, value.is_recognized if value is not None else None, + row_index=row_index, + source_text=value.string_value if value is not None else None, + needs_review=value_needs_review(value), ) ) inn_confidence = _decimal(header.inn_confidence) - meta = { + header_source = { + "organization_name": {"text": header.organization_name, "confidence": header.organization_name_confidence}, + "inn": {"text": header.inn, "confidence": header.inn_confidence}, + "report_date": {"text": header.report_date_text, "confidence": None}, + "units": {"text": header.units, "confidence": None}, + "okud": {"text": header.okud, "confidence": None}, + "okpo": {"text": header.okpo, "confidence": None}, + "okopf": {"text": header.okopf, "confidence": None}, + "okfs": {"text": header.okfs, "confidence": None}, + "okei": {"text": None, "confidence": None}, + "address": {"text": header.address, "confidence": None}, + } + meta: dict[str, Any] = { "inn": header.inn, "inn_original_value": header.inn, "inn_confidence": inn_confidence, @@ -192,6 +233,11 @@ def build_statement(doc: StatementDocument) -> StatementDraft: "okopf": header.okopf, "okfs": header.okfs, "address": header.address, + "header_source": header_source, + "organization_name_original_value": header.organization_name, + "organization_name_modified": False, + "needs_review_fields": [], + "form_variant": None, } return StatementDraft( document_type=document_type, @@ -208,13 +254,22 @@ def build_statement(doc: StatementDocument) -> StatementDraft: ) +def value_clean(confidence: Decimal | None, is_recognized: bool | None) -> bool: + """Ячейка без сомнений: `is_recognized` не false, уверенность 100 или не задана (D-6, v5.19 `needs_review`).""" + return is_recognized is not False and (confidence is None or confidence == FULL_CONFIDENCE) + + +def value_needs_review(value: StatementValue | None) -> bool: + """`needs_review` ячейки при приёме `1.x`: значение есть и распознано с сомнением (пустая ячейка — нет).""" + if value is None: + return False + return not value_clean(_decimal(value.confidence), value.is_recognized) + + def values_ok(values: Iterable[tuple[bool, bool | None, Decimal | None]]) -> bool: """D-6: каждое значение правлено аналитиком или распознано без сомнений (`is_recognized` не false, уверенность 100/NULL).""" - return all( - modified or (recognized is not False and (confidence is None or confidence == FULL_CONFIDENCE)) - for modified, recognized, confidence in values - ) + return all(modified or value_clean(confidence, recognized) for modified, recognized, confidence in values) def statement_status( # noqa: PLR0913 - условия D-6 перечислены явно diff --git a/api/src/repositories/statements.py b/api/src/repositories/statements.py index 1f40bb4..1ad0218 100644 --- a/api/src/repositories/statements.py +++ b/api/src/repositories/statements.py @@ -12,6 +12,7 @@ from sqlalchemy import delete, func, insert, select from sqlalchemy.ext.asyncio import AsyncSession from src.domain.statement_aggregate import AggLine, CodeAggregate, aggregate_by_code +from src.domain.statement_issues import IssueDraft from src.domain.statement_mapping import StatementDraft from src.models.enums import DocumentStatus, DocumentType, FileStatus, IssueSeverity, StatementLineStatus from src.models.files import Document, StoredFile @@ -25,6 +26,7 @@ from src.models.statements import ( __all__ = [ "BATCH", + "add_issues", "add_line", "add_value", "code_amount", @@ -74,35 +76,39 @@ async def insert_statement( await session.flush() column_ids: dict[int, uuid.UUID] = {} columns: list[dict[str, Any]] = [] - for column_index, label, period_date, is_current in draft.columns: - column_ids[column_index] = uuid.uuid4() + for column in draft.columns: + column_ids[column.column_index] = uuid.uuid4() columns.append( { - "id": column_ids[column_index], + "id": column_ids[column.column_index], "meta_id": meta.id, - "column_index": column_index, - "label": label, - "period_date": period_date, - "is_current": is_current, - # до маппера A9 (план 3): как миграция v5.19 — `absolute` при дате, иначе `none` - "period_kind": "absolute" if period_date is not None else "none", + "column_index": column.column_index, + "label": column.label, + "period_date": column.period_date, + "is_current": column.is_current, + "period_kind": column.period_kind, + "label_confidence": column.label_confidence, } ) if columns: await session.execute(insert(FinancialStatementColumn).values(columns)) - line_ids: dict[str, uuid.UUID] = {} + line_ids: dict[int, uuid.UUID] = {} # по `row_index`: код строки может повторяться lines: list[dict[str, Any]] = [] - for row_index, line in enumerate(draft.lines): - line_ids[line.line_code] = uuid.uuid4() + for line in draft.lines: + line_ids[line.row_index] = uuid.uuid4() lines.append( { - "id": line_ids[line.line_code], + "id": line_ids[line.row_index], "meta_id": meta.id, - "row_index": row_index, # строка источника (до маппера `2.0`: порядок черновика) + "row_index": line.row_index, + "source_ref": line.source_ref, + "line_status": line.line_status, "line_code": line.line_code, - "line_code_original_value": line.line_code, + "line_code_text": line.line_code_text, + "line_code_original_value": line.line_code_original_value, "line_code_confidence": line.line_code_confidence, "line_code_modified": False, + "needs_review": line.needs_review, "line_name": line.line_name, "line_name_original_value": line.line_name, "line_name_confidence": line.line_name_confidence, @@ -114,13 +120,15 @@ async def insert_statement( values = [ { "id": uuid.uuid4(), - "line_id": line_ids[value.line_code], + "line_id": line_ids[value.row_index], "column_id": column_ids[value.column_index], "amount": value.amount, "string_value": value.string_value, "confidence": value.confidence, "is_recognized": value.is_recognized, "is_modified": False, + "source_text": value.source_text, + "needs_review": value.needs_review, } for value in draft.values ] @@ -129,6 +137,39 @@ async def insert_statement( return meta +async def add_issues( + session: AsyncSession, document_id: uuid.UUID, drafts: Sequence[IssueDraft], *, now: datetime.datetime +) -> None: + """Записи `document_issue` документа из черновиков (без commit).""" + if not drafts: + return + await session.execute( + insert(DocumentIssue).values( + [ + { + "id": uuid.uuid4(), + "document_id": document_id, + "kind": draft.kind, + "code": draft.code, + "severity": draft.severity, + "message": draft.message, + "line_ids": [str(line_id) for line_id in draft.line_ids] or None, + "column_id": draft.column_id, + "column_index": draft.column_index, + "codes": list(draft.codes) or None, + "field": draft.field, + "source": draft.source, + "relation": draft.relation, + "expected": draft.expected, + "actual": draft.actual, + "created_at": now, + } + for draft in drafts + ] + ) + ) + + async def delete_file_statements(session: AsyncSession, file_id: uuid.UUID) -> None: """`document_issue`, значения, строки, колонки и шапки форм всех документов файла (в порядке FK).""" documents = select(Document.id).where(Document.stored_file_id == file_id).scalar_subquery() diff --git a/api/src/services/result_mapping.py b/api/src/services/result_mapping.py index 74347e4..28e2543 100644 --- a/api/src/services/result_mapping.py +++ b/api/src/services/result_mapping.py @@ -52,6 +52,7 @@ from src.domain.osv_mapping import ( ) from src.domain.statement_aggregate import checks_input from src.domain.statement_checks import load_check_rules, run_checks +from src.domain.statement_issues import mapping_warning from src.domain.statement_lines import load_line_catalog from src.domain.statement_mapping import StatementDraft, build_statement, edition_for, statement_status, values_ok from src.models.accounting import ChartOfAccount @@ -344,30 +345,28 @@ def _split_duplicates(sheets: Sequence[OsvDocument]) -> tuple[list[OsvDocument], def _map_statement_docs( # noqa: PLR0913 - все входы именованные statement_docs: Sequence[StatementDocument], *, + ordinals: Mapping[int, int], file: StoredFile, owner: LegalEntity, owner_names: frozenset[str], now: datetime, -) -> tuple[list[tuple[Document, StatementDraft]], list[Document], list[str], set[DocumentType]]: +) -> tuple[list[tuple[Document, StatementDraft]], list[Document], list[str], dict[uuid.UUID, str]]: """Документы форм ББ/ОФР и расшифровок (spec «Документ формы ББ/ОФР», «Повтор формы в файле и прочие документы»; - D-2, D-3). Возвращает формы (документ + черновик; статус уточняется после вставки значений), документы - `DISCLOSURE`/`OTHER` (по одному на тип, `locator` через запятую, `RECOGNIZED`), предупреждения и типы форм, - у которых был повтор в файле (R-1: первый документ такого типа — `NEEDS_CHECK`).""" + D-2, D-3; P3-4, P3-6). Возвращает формы (документ + черновик; статус уточняется после вставки значений), + документы `DISCLOSURE`/`OTHER` (по одному на тип, `locator` через запятую, `RECOGNIZED`, `file_ordinal` — индекс + первого такого документа в `documents[]`), предупреждения маппинга и текст «другая форма» по `id` документов форм + одного типа и периода (Р-10, Р-18: хранятся все, каждый `NEEDS_CHECK`). `ordinals` — `id(документ результата)` → + индекс в `result.documents`.""" forms: list[tuple[Document, StatementDraft]] = [] - seen_forms: set[DocumentType] = set() - duplicated: set[DocumentType] = set() other_locators: dict[DocumentType, list[str]] = {} + other_ordinals: dict[DocumentType, int] = {} warnings: list[str] = [] for doc in statement_docs: doc_type = DocumentType(doc.document_type) if doc_type in (DocumentType.DISCLOSURE, DocumentType.OTHER): other_locators.setdefault(doc_type, []).append(doc.locator) + other_ordinals.setdefault(doc_type, ordinals[id(doc)]) continue - if doc_type in seen_forms: - warnings.append(f"{doc.locator}: вторая форма {doc_type.value} в файле пропущена (OQ-77)") - duplicated.add(doc_type) - continue - seen_forms.add(doc_type) draft = build_statement(doc) warnings.extend(draft.warnings) check = counterparty_check(doc.header.inn, doc.header.organization_name, owner.inn, owner_names) @@ -376,6 +375,7 @@ def _map_statement_docs( # noqa: PLR0913 - все входы именованн stored_file_id=file.id, legal_entity_id=file.legal_entity_id, document_type=doc_type, + file_ordinal=ordinals[id(doc)], osv_account_code=None, period_year=draft.period_year, period_quarter=draft.period_quarter, @@ -389,12 +389,26 @@ def _map_statement_docs( # noqa: PLR0913 - все входы именованн updated_at=now, ) forms.append((document, draft)) + same_period: dict[tuple[DocumentType, int, PeriodQuarter], int] = {} + for document, _ in forms: + if document.period_year is not None and document.period_quarter is not None: + key = (document.document_type, document.period_year, document.period_quarter) + same_period[key] = same_period.get(key, 0) + 1 + repeats: dict[uuid.UUID, str] = {} + for document, _ in forms: + if document.period_year is not None and document.period_quarter is not None: + key = (document.document_type, document.period_year, document.period_quarter) + if same_period[key] > 1: + text = f"в файле есть другая форма {document.document_type.value} за тот же период" + repeats[document.id] = text + warnings.append(f"{document.locator}: {text}") others = [ Document( id=uuid.uuid4(), stored_file_id=file.id, legal_entity_id=file.legal_entity_id, document_type=doc_type, + file_ordinal=other_ordinals[doc_type], locator=",".join(locators), status=DocumentStatus.RECOGNIZED, period_check=DocumentCheckResult.UNKNOWN, @@ -403,18 +417,20 @@ def _map_statement_docs( # noqa: PLR0913 - все входы именованн ) for doc_type, locators in other_locators.items() ] - return forms, others, warnings, duplicated + return forms, others, warnings, repeats -async def _insert_statements( +async def _insert_statements( # noqa: PLR0913 - все входы именованные session: AsyncSession, *, job: RecognitionJob, forms: Sequence[tuple[Document, StatementDraft]], - duplicated: set[DocumentType], + repeats: Mapping[uuid.UUID, str], job_done: bool, + now: datetime, ) -> None: - """Шапка, колонки, строки и значения каждой формы; статус документа по D-6 (акцепта при маппинге нет).""" + """Шапка, колонки, строки и значения каждой формы; предупреждения черновика и повтор формы — `MAPPING_WARNING` + документа; статус документа по D-6 (акцепта при маппинге нет).""" if not forms: return rules = load_check_rules() @@ -423,6 +439,8 @@ async def _insert_statements( edition = edition_for(document.reporting_date) form = document.document_type.value meta = await statements_repo.insert_statement(session, document_id=document.id, job_id=job.id, draft=draft) + warnings = [*draft.warnings, *([repeats[document.id]] if document.id in repeats else [])] + await statements_repo.add_issues(session, document.id, [mapping_warning(text) for text in warnings], now=now) columns, aggregates = await statements_repo.form_aggregate( session, meta.id, functools.partial(catalog.reference_name, edition, form) ) @@ -435,7 +453,7 @@ async def _insert_statements( period_known=document.period_year is not None and document.period_quarter is not None, values_ok=values_ok(await statements_repo.value_flags(session, meta.id)), has_lines=bool(draft.lines), - warnings=[*draft.warnings, *(["повтор формы"] if document.document_type in duplicated else [])], + warnings=warnings, ) @@ -467,9 +485,11 @@ async def map_file( # noqa: PLR0913 - все входы маппинга фай owner_names = index.owner_names(owner) job_done = job.status is RecognitionStatus.done + ordinals = {id(doc): index for index, doc in enumerate(result.documents)} # v5.19: номер документа в файле await _clear_file(session, file.id) kept, duplicate_warnings, warnings = _split_duplicates(sheets) documents: list[Document] = [] + sheet_issues: list[tuple[Document, list[str]]] = [] entries: list[dict[str, Any]] = [] for sheet in kept: fields = document_fields(sheet) @@ -506,6 +526,7 @@ async def map_file( # noqa: PLR0913 - все входы маппинга фай stored_file_id=file.id, legal_entity_id=file.legal_entity_id, document_type=DocumentType.OSV, + file_ordinal=ordinals[id(sheet)], osv_account_code=fields.osv_account_code, period_year=fields.period_year, period_quarter=fields.period_quarter, @@ -523,6 +544,7 @@ async def map_file( # noqa: PLR0913 - все входы маппинга фай updated_at=now, ) documents.append(document) + sheet_issues.append((document, [*duplicate_warnings.get(sheet.locator, ()), *sheet_warnings])) warnings.extend(sheet_warnings) entries.extend( _entry_values( @@ -534,25 +556,24 @@ async def map_file( # noqa: PLR0913 - все входы маппинга фай for entry in mapped.entries ) - forms, other_documents, form_warnings, duplicated = _map_statement_docs( - statement_docs, file=file, owner=owner, owner_names=owner_names, now=now + forms, other_documents, form_warnings, repeats = _map_statement_docs( + statement_docs, ordinals=ordinals, file=file, owner=owner, owner_names=owner_names, now=now ) warnings.extend(form_warnings) documents.extend(other_documents) documents.extend(document for document, _ in forms) - # v5.19: номер документа в файле уникален; до перехода на индекс в documents[] (задачи A5/A9) — порядок создания - for ordinal, document in enumerate(documents): - document.file_ordinal = ordinal session.add_all(documents) await session.flush() + for document, messages in sheet_issues: # P3-5: предупреждения листа ОСВ — и в `document_issue` + await statements_repo.add_issues(session, document.id, [mapping_warning(text) for text in messages], now=now) # T7.2: родитель раньше потомка на любой границе пачки (self-FK `parent_entry_id`) — не полагаемся на порядок строк # результата; сортировка устойчивая, внутри уровня порядок строк сохраняется entries.sort(key=lambda values: values["level"]) for start in range(0, len(entries), ENTRY_BATCH_SIZE): await session.execute(insert(OsvEntry).values(entries[start : start + ENTRY_BATCH_SIZE])) - await _insert_statements(session, job=job, forms=forms, duplicated=duplicated, job_done=job_done) + await _insert_statements(session, job=job, forms=forms, repeats=repeats, job_done=job_done, now=now) job.group_schema = [_group_schema(sheet) for sheet in sheets] job.metrics = {**_metrics(job), MAPPING_WARNINGS_KEY: warnings} diff --git a/api/tests/test_domain_statement_mapping.py b/api/tests/test_domain_statement_mapping.py index c3d7dcb..4bfdf61 100644 --- a/api/tests/test_domain_statement_mapping.py +++ b/api/tests/test_domain_statement_mapping.py @@ -12,6 +12,7 @@ from src.domain.statement_mapping import ( quarter_of, statement_period, statement_status, + value_clean, values_ok, ) from src.models.enums import DocumentCheckResult, DocumentStatus, DocumentType, PeriodQuarter, StatementType @@ -90,7 +91,10 @@ def test_build_statement_cells_and_header() -> None: assert draft.header["inn_verified"] is True assert draft.header["report_date_original_value"] == "на 30 сентября 2025 г." assert draft.header["units"] == draft.header["units_original_value"] == "тыс. руб." - assert draft.columns == [(0, "На 30.09.2025", d(2025, 9, 30), True), (1, "На 31.12.2024", d(2024, 12, 31), False)] + assert [(c.column_index, c.label, c.period_date, c.is_current) for c in draft.columns] == [ + (0, "На 30.09.2025", d(2025, 9, 30), True), + (1, "На 31.12.2024", d(2024, 12, 31), False), + ] assert [line.line_code for line in draft.lines] == ["1150", "1600"] # дубль 1150 отброшен cells = {(v.line_code, v.column_index): v for v in draft.values} assert len(cells) == expected_cells @@ -120,7 +124,7 @@ def test_period_from_columns_and_non_quarter() -> None: def test_no_dates_at_all() -> None: draft = build_statement(_doc(header={"form_name": "ББ"}, columns=[{"column_index": 0, "label": ""}])) assert (draft.period_year, draft.period_quarter, draft.period_date) == (None, None, None) - assert draft.columns == [(0, "", None, None)] + assert [(c.column_index, c.label, c.period_date, c.is_current) for c in draft.columns] == [(0, "", None, None)] def test_status_rules() -> None: @@ -162,7 +166,7 @@ def test_equal_column_dates_current_is_lowest_index_only() -> None: {"column_index": 3, "label": "в", "period_date": "2024-12-31"}, ] draft = build_statement(_doc(columns=columns, lines=[])) - assert {index: current for index, _, _, current in draft.columns} == {1: True, 2: False, 3: False} + assert {c.column_index: c.is_current for c in draft.columns} == {1: True, 2: False, 3: False} def test_column_without_date_next_to_dated_is_not_current() -> None: @@ -171,7 +175,7 @@ def test_column_without_date_next_to_dated_is_not_current() -> None: {"column_index": 1, "label": "На 30.09.2025", "period_date": "2025-09-30"}, ] draft = build_statement(_doc(columns=columns, lines=[])) - assert {index: current for index, _, _, current in draft.columns} == {0: False, 1: True} + assert {c.column_index: c.is_current for c in draft.columns} == {0: False, 1: True} def test_duplicate_code_values_belong_to_first_line() -> None: @@ -220,7 +224,10 @@ def test_duplicate_column_index_keeps_first_with_warning() -> None: {"column_index": 1, "label": "На 31.12.2024", "period_date": "2024-12-31"}, ] draft = build_statement(_doc(columns=columns)) - assert draft.columns == [(0, "На 30.09.2025", d(2025, 9, 30), True), (1, "На 31.12.2024", d(2024, 12, 31), False)] + assert [(c.column_index, c.label, c.period_date, c.is_current) for c in draft.columns] == [ + (0, "На 30.09.2025", d(2025, 9, 30), True), + (1, "На 31.12.2024", d(2024, 12, 31), False), + ] assert "sheet:ББ: колонка 0 повторяется — оставлена первая" in draft.warnings assert (draft.period_date, draft.period_quarter) == (d(2025, 9, 30), PeriodQuarter.Q3) assert [(v.line_code, v.column_index) for v in draft.values] == [("1150", 0), ("1150", 1), ("1600", 0), ("1600", 1)] @@ -241,3 +248,71 @@ def test_duplicate_cell_keeps_first_value_with_warning() -> None: draft = build_statement(_doc(lines=lines)) assert {(v.column_index): v.amount for v in draft.values} == {0: Decimal("10"), 1: None} assert "sheet:ББ: строка 1150: значение колонки 0 повторяется — оставлено первое" in draft.warnings + + +def test_1x_line_fields_domain() -> None: + """Строки 1.x несут `row_index` — индекс строки в `doc.lines` (пропущенный повтор индекс не занимает).""" + lines = [ + {"line_code": "1150", "line_name": "а", "values": []}, + {"line_code": "1150", "line_name": "повтор", "values": []}, + {"line_code": "1600", "line_name": "б", "values": []}, + ] + draft = build_statement(_doc(lines=lines)) + assert [(line.row_index, line.line_code) for line in draft.lines] == [(0, "1150"), (2, "1600")] + assert all( + line.line_status == "ok" and line.source_ref is None and line.line_code_text is None for line in draft.lines + ) + assert all(line.line_code_original_value == line.line_code and line.needs_review is False for line in draft.lines) + assert {v.row_index for v in draft.values} == {0, 2} + + +def test_value_clean_and_value_flags() -> None: + assert value_clean(Decimal("100"), True) is True + assert value_clean(None, None) is True + assert value_clean(Decimal("80"), True) is False + assert value_clean(Decimal("100"), False) is False + lines = [ + { + "line_code": "1150", + "line_name": "а", + "values": [ + {"column_index": 0, "amount": "10", "string_value": "10,0", "confidence": 100, "is_recognized": True}, + {"column_index": 1, "amount": "5", "string_value": "5", "confidence": 80, "is_recognized": True}, + ], + } + ] + draft = build_statement(_doc(lines=lines)) + by_column = {v.column_index: v for v in draft.values} + assert (by_column[0].source_text, by_column[0].needs_review) == ("10,0", False) + assert (by_column[1].source_text, by_column[1].needs_review) == ("5", True) + + +def test_1x_columns_period_kind() -> None: + draft = build_statement(_doc(columns=[{"column_index": 0, "label": "x"}])) + assert [(c.period_date, c.period_kind, c.label_confidence) for c in draft.columns] == [(None, "none", None)] + full = build_statement(_doc()) + assert [c.period_kind for c in full.columns] == ["absolute", "absolute"] + + +def test_1x_header_source() -> None: + draft = build_statement(_doc()) + source = draft.header["header_source"] + assert source["inn"] == {"text": "7700000000", "confidence": 100} + assert source["report_date"]["text"] == "на 30 сентября 2025 г." + assert source["organization_name"] == {"text": "ООО Пример", "confidence": None} + assert source["okei"] == {"text": None, "confidence": None} + assert set(source) == { + "organization_name", + "inn", + "report_date", + "units", + "okud", + "okpo", + "okopf", + "okfs", + "okei", + "address", + } + assert draft.header["organization_name_original_value"] == "ООО Пример" + assert draft.header["organization_name_modified"] is False + assert draft.header["needs_review_fields"] == [] and draft.header["form_variant"] is None diff --git a/api/tests/test_mapping_golden.py b/api/tests/test_mapping_golden.py index 996ba8f..c7f1ad1 100644 --- a/api/tests/test_mapping_golden.py +++ b/api/tests/test_mapping_golden.py @@ -110,7 +110,7 @@ def _check_sheet( # noqa: PLR0912 - все инварианты одного л async def _form_problems(session: AsyncSession, file_id: uuid.UUID, raw: dict[str, Any], name: str) -> list[str]: - """Формы смешанных снимков (s12 и др.): по документу на каждый тип ББ/ОФР снимка (повтор формы пропускается), + """Формы смешанных снимков (s12 и др.): по документу на каждую форму ББ/ОФР снимка (повтор формы хранится, P3-4), `DISCLOSURE`/`OTHER` — по одному документу на тип.""" types = Counter( document["document_type"] for document in raw["documents"] if document["document_type"] != DocumentType.OSV @@ -124,10 +124,15 @@ async def _form_problems(session: AsyncSession, file_id: uuid.UUID, raw: dict[st ) ).scalars() ) + merged = (DocumentType.DISCLOSURE, DocumentType.OTHER) + + def expected(kind: str) -> int: + return min(types[kind], 1) if kind in merged else types[kind] + return [ - f"{name}: документов {kind} {stored.get(kind, 0)}, ожидалось {1 if types[kind] else 0}" + f"{name}: документов {kind} {stored.get(kind, 0)}, ожидалось {expected(kind)}" for kind in sorted({*types, *stored}) - if stored.get(kind, 0) != (1 if types[kind] else 0) + if stored.get(kind, 0) != expected(kind) ] diff --git a/api/tests/test_mapping_statements.py b/api/tests/test_mapping_statements.py index cc1d426..99e6abe 100644 --- a/api/tests/test_mapping_statements.py +++ b/api/tests/test_mapping_statements.py @@ -221,8 +221,8 @@ async def test_all_statement_snapshots_map(seeded: AsyncSession, fixtures_dir: P assert await _count(seeded, FinancialStatementValue, meta_id) == lines * columns -async def test_duplicate_form_in_file_skipped(seeded: AsyncSession, fixtures_dir: Path, tmp_path: Path) -> None: - """R-1, spec «Повтор формы в файле»: вторая форма того же типа не создаётся, первая — `NEEDS_CHECK` даже чистая.""" +async def test_duplicate_form_in_file_stored(seeded: AsyncSession, fixtures_dir: Path, tmp_path: Path) -> None: + """P3-4 (Р-10, Р-18): вторая форма того же типа и периода хранится; обе `NEEDS_CHECK` (чистая — тоже).""" source = fixtures_dir / MEZH / "1_Форма2_бух_09.json" owner = await _owner(seeded, source) data = json.loads(source.read_text("utf-8")) @@ -230,11 +230,10 @@ async def test_duplicate_form_in_file_skipped(seeded: AsyncSession, fixtures_dir path = tmp_path / "duplicate.json" path.write_text(json.dumps(data, ensure_ascii=False), "utf-8") documents = await map_golden_snapshot(seeded, entity=owner, path=path, uploaded_by=None) - assert [d.document_type for d in documents] == [DocumentType.INCOME] - assert documents[0].status is DocumentStatus.NEEDS_CHECK # без повтора тот же снимок — RECOGNIZED - assert await _mapping_warnings(seeded, documents[0]) == [ - "sheet:повтор: вторая форма INCOME в файле пропущена (OQ-77)" - ] + assert [d.document_type for d in documents] == [DocumentType.INCOME, DocumentType.INCOME] + assert all(d.status is DocumentStatus.NEEDS_CHECK for d in documents) # без повтора тот же снимок — RECOGNIZED + assert sorted(d.file_ordinal for d in documents) == [0, 1] + assert any("другая форма INCOME за тот же период" in w for w in await _mapping_warnings(seeded, documents[0])) async def test_duplicate_column_index_does_not_fail_mapping( diff --git a/api/tests/test_result_mapping.py b/api/tests/test_result_mapping.py index 8c25454..871b466 100644 --- a/api/tests/test_result_mapping.py +++ b/api/tests/test_result_mapping.py @@ -54,6 +54,12 @@ from src.models.enums import ( ) from src.models.files import Document, RecognitionJob, StoredFile from src.models.osv import OsvEntry +from src.models.statements import ( + DocumentIssue, + FinancialStatementLine, + FinancialStatementMeta, + FinancialStatementValue, +) from src.repositories.counterparties import CounterpartyIndex from src.services import consolidation_documents from src.services import result_mapping as rm @@ -1781,3 +1787,197 @@ async def test_watchdog_timeout_writes_no_audit_event(committed_sessionmaker: as assert (await _file(maker, file_id)).status is FileStatus.ERROR assert (await _job(maker, job_id)).status is RecognitionStatus.timeout assert await _audit_count(maker) == before + + +# --- v5.19: поля 1.x, file_ordinal, вторая форма, MAPPING_WARNING (план 3, A9) --------------------------------------- + + +def _osv_sheet(account: str, locator: str) -> dict[str, Any]: + sheet = copy.deepcopy(_osv(_minimal())) + sheet["header"]["account_code_raw"] = account + sheet["locator"] = locator + return sheet + + +def _form(document_type: str, locator: str, *, report_date: str = "2025-09-30") -> dict[str, Any]: + form = copy.deepcopy(json.loads(STATEMENT_MINIMAL.read_text("utf-8"))["documents"][0]) + form["document_type"] = document_type + form["locator"] = locator + form["header"]["report_date"] = report_date + return form + + +async def _map_dict( + session: AsyncSession, documents: list[dict[str, Any]], *, owner_inn: str = "7700000000" +) -> tuple[StoredFile, list[Document]]: + """`map_file` на результате 1.x из словарей `documents`; документы файла по `file_ordinal`.""" + entity = await make_legal_entity(session, name=OWNER_NAME, inn=owner_inn) + file, job = await make_stored_file( + session, entity=entity, status=FileStatus.PROCESSING, job_status=RecognitionStatus.done + ) + payload = _minimal() | {"job_id": str(job.id), "file_id": str(file.id), "documents": documents} + result = RecognitionResult.model_validate(payload) + chart = {code: key for code, key in await session.execute(select(ChartOfAccount.code, ChartOfAccount.id))} + await rm.map_file( + session, job=job, file=file, result=result, chart=chart, index=await CounterpartyIndex.load(session), now=_now() + ) + stored = ( + ( + await session.execute( + select(Document).where(Document.stored_file_id == file.id).order_by(Document.file_ordinal) + ) + ) + .scalars() + .all() + ) + return file, list(stored) + + +async def _issues(session: AsyncSession, document: Document) -> list[DocumentIssue]: + stmt = select(DocumentIssue).where(DocumentIssue.document_id == document.id) + return list((await session.execute(stmt)).scalars()) + + +async def test_two_osv_sheets_file_ordinals(seeded: AsyncSession) -> None: + file, documents = await _map_dict(seeded, [_osv_sheet("60", "sheet:60"), _osv_sheet("62", "sheet:62")]) + assert [(d.osv_account_code, d.file_ordinal) for d in documents] == [("60", 0), ("62", 1)] + await seeded.refresh(file, ["status"]) + assert file.status is FileStatus.PROCESSED + + +async def test_pdf_balance_income_1x_ordinals(seeded: AsyncSession) -> None: + _, documents = await _map_dict(seeded, [_form("BALANCE", "pages:1"), _form("INCOME", "pages:2")]) + assert [(d.document_type, d.file_ordinal) for d in documents] == [ + (DocumentType.BALANCE, 0), + (DocumentType.INCOME, 1), + ] + + +async def test_duplicate_osv_sheet_takes_no_ordinal(seeded: AsyncSession) -> None: + _, documents = await _map_dict( + seeded, [_osv_sheet("60", "sheet:a"), _osv_sheet("60", "sheet:b"), _form("BALANCE", "sheet:ББ")] + ) + assert [(d.document_type, d.file_ordinal) for d in documents] == [(DocumentType.OSV, 0), (DocumentType.BALANCE, 2)] + + +async def test_disclosure_ordinal_is_first_in_documents(seeded: AsyncSession) -> None: + """P3-6: объединённый `DISCLOSURE` — индекс первого такого документа в `documents[]`.""" + bare = {"header": {}, "columns": [], "lines": []} + disclosure = {"document_type": "DISCLOSURE", "locator": "pages:3", **bare} + other = {"document_type": "OTHER", "locator": "pages:9", **bare} + _, documents = await _map_dict( + seeded, [_form("BALANCE", "pages:1"), disclosure, other, disclosure | {"locator": "pages:4"}] + ) + assert {(d.document_type, d.file_ordinal) for d in documents} == { + (DocumentType.BALANCE, 0), + (DocumentType.DISCLOSURE, 1), + (DocumentType.OTHER, 2), + } + + +async def test_1x_line_fields(seeded: AsyncSession) -> None: + form = _form("BALANCE", "sheet:ББ") + form["lines"] = [ + { + "line_code": code, + "line_name": f"строка {code}", + "values": [ + { + "column_index": 0, + "amount": "10", + "string_value": "10,0", + "confidence": confidence, + "is_recognized": True, + } + ], + } + for code, confidence in (("1100", 100), ("1200", 80), ("1600", 100)) + ] + _, [balance] = await _map_dict(seeded, [form]) + meta = ( + await seeded.execute(select(FinancialStatementMeta).where(FinancialStatementMeta.document_id == balance.id)) + ).scalar_one() + lines = ( + ( + await seeded.execute( + select(FinancialStatementLine) + .where(FinancialStatementLine.meta_id == meta.id) + .order_by(FinancialStatementLine.row_index) + ) + ) + .scalars() + .all() + ) + assert [line.row_index for line in lines] == [0, 1, 2] + assert all(line.line_status.value == "ok" and line.source_ref is None for line in lines) + assert all(line.line_code_original_value == line.line_code and line.needs_review is False for line in lines) + values = { + line.line_code: ( + await seeded.execute(select(FinancialStatementValue).where(FinancialStatementValue.line_id == line.id)) + ) + .scalars() + .all() + for line in lines + } + flags = {code: {v.needs_review for v in cells if v.confidence is not None} for code, cells in values.items()} + assert flags == {"1100": {False}, "1200": {True}, "1600": {False}} + assert {v.source_text for v in values["1100"] if v.string_value} == {"10,0"} + + +async def test_1x_duplicate_code_keeps_first(seeded: AsyncSession) -> None: + form = _form("INCOME", "sheet:ОФР") + form["lines"] = [ + {"line_code": "2110", "line_name": "а", "values": []}, + {"line_code": "2410", "line_name": "б", "values": []}, + {"line_code": "2410", "line_name": "повтор", "values": []}, + ] + _, [income] = await _map_dict(seeded, [form]) + meta = ( + await seeded.execute(select(FinancialStatementMeta).where(FinancialStatementMeta.document_id == income.id)) + ).scalar_one() + rows = ( + await seeded.execute( + select(FinancialStatementLine.row_index, FinancialStatementLine.line_code) + .where(FinancialStatementLine.meta_id == meta.id) + .order_by(FinancialStatementLine.row_index) + ) + ).all() + assert [tuple(row) for row in rows] == [(0, "2110"), (1, "2410")] + messages = [issue.message for issue in await _issues(seeded, income) if issue.code == "MAPPING_WARNING"] + assert any("2410" in message and "повторяется" in message for message in messages) + + +async def test_1x_second_form_same_period_stored(seeded: AsyncSession) -> None: + """Review Focus 1, P3-4 (Р-10/Р-18): две формы одного типа и периода хранятся, обе `NEEDS_CHECK`.""" + file, documents = await _map_dict(seeded, [_form("BALANCE", "sheet:а"), _form("BALANCE", "sheet:б")]) + assert [d.file_ordinal for d in documents] == [0, 1] + assert all(d.status is DocumentStatus.NEEDS_CHECK for d in documents) + for document in documents: + messages = [i.message for i in await _issues(seeded, document) if i.code == "MAPPING_WARNING"] + assert "в файле есть другая форма BALANCE за тот же период" in messages + issue = next(i for i in await _issues(seeded, document) if i.code == "MAPPING_WARNING") + assert (issue.kind.value, issue.severity.value, issue.source.value) == ("warning", "info", "core") + + +async def test_1x_second_form_other_period(seeded: AsyncSession) -> None: + _, documents = await _map_dict( + seeded, [_form("BALANCE", "sheet:а"), _form("BALANCE", "sheet:б", report_date="2024-12-31")] + ) + assert [(d.period_year, d.period_quarter) for d in documents] == [ + (2025, PeriodQuarter.Q3), + (2024, PeriodQuarter.Q4), + ] + for document in documents: + assert not any("другая форма" in i.message for i in await _issues(seeded, document)) + + +async def test_osv_orphans_mapping_warning_issue(seeded: AsyncSession) -> None: + sheet = _osv_sheet("60", "sheet:60") + sheet["rows"] = [*sheet["rows"], sheet["rows"][1] | {"row_number": 40, "parent_row_number": 99}] + file, [document] = await _map_dict(seeded, [sheet]) + issues = await _issues(seeded, document) + [orphans] = [i for i in issues if i.code == "MAPPING_WARNING"] + assert "без родителя" in orphans.message and "40" in orphans.message + assert (orphans.kind.value, orphans.severity.value, orphans.source.value) == ("warning", "info", "core") + job = await seeded.get(RecognitionJob, file.current_recognition_job_id) + assert job is not None and any("без родителя" in w for w in job.metrics[MAPPING_WARNINGS_KEY])