diff --git a/api/contracts/examples/result-osv-minimal.json b/api/contracts/examples/result-osv-minimal.json index f883a8d..b0a9e97 100644 --- a/api/contracts/examples/result-osv-minimal.json +++ b/api/contracts/examples/result-osv-minimal.json @@ -1,5 +1,5 @@ { - "schema_version": "1.1", + "schema_version": "1.2", "job_id": "3f2a9c1e-7b4d-4e0a-9d51-2c8e6a1b0f01", "file_id": "a1c4e7d2-5b36-4f89-8e12-9d0c3b7a4e02", "service": { "name": "tabular-parser", "version": "0.1.0" }, @@ -16,6 +16,9 @@ "period_year": 2025, "period_quarter": "Q1", "is_cumulative": false, + "period_start": "2025-01-01", + "period_end": "2025-03-31", + "is_from_year_start": true, "data_type": "BU", "currency": "RUB", "header_variant": 1 diff --git a/api/contracts/examples/result-statement-excel.json b/api/contracts/examples/result-statement-excel.json new file mode 100644 index 0000000..fcee529 --- /dev/null +++ b/api/contracts/examples/result-statement-excel.json @@ -0,0 +1,46 @@ +{ + "schema_version": "1.2", + "job_id": "3f1c7a2e-5b64-4d0a-9c1e-8a7b6d5e4f30", + "file_id": "a1b2c3d4-e5f6-4a7b-8c9d-0e1f2a3b4c5d", + "service": { "name": "tabular-parser", "version": "0.3.0" }, + "status": "needs_check", + "metrics": { "duration_ms": 412, "rows": 7, "warnings": ["sheet=Бухгалтерский баланс. Лист 1 значений с дробной частью при единице тыс. руб.: 3"] }, + "documents": [ + { + "document_type": "BALANCE", + "locator": "sheet:Бухгалтерский баланс. Лист 1,Бухгалтерский баланс. Лист 2", + "header": { "form_name": "Бухгалтерский баланс", "organization_name": "ООО \"Пример\"", "organization_name_confidence": 100, "inn": "7700000000", "inn_confidence": 100, "report_date_text": "на 30 сентября 2025 г.", "report_date": "2025-09-30", "units": "тыс. руб.", "okud": "0710001", "okpo": "12345678" }, + "columns": [ + { "column_index": 0, "label": "На 30 сентября 2025 г.", "period_date": "2025-09-30" }, + { "column_index": 1, "label": "На 31 декабря 2024 г.", "period_date": "2024-12-31" } + ], + "lines": [ + { "line_code": "1110", "line_code_confidence": 100, "line_name": "Нематериальные активы", "line_name_confidence": 100, "values": [ { "column_index": 0, "amount": "586260.26", "confidence": 100, "is_recognized": true }, { "column_index": 1, "amount": "905786.99", "confidence": 100, "is_recognized": true } ] }, + { "line_code": "11101", "line_code_confidence": 100, "line_name": "Нематериальные активы, готовые к использованию", "line_name_confidence": 100, "values": [ { "column_index": 0, "amount": "586260.26", "confidence": 100, "is_recognized": true }, { "column_index": 1, "amount": "900000.00", "confidence": 100, "is_recognized": true } ] }, + { "line_code": "1100", "line_code_confidence": 100, "line_name": "Итого по разделу I", "line_name_confidence": 100, "values": [ { "column_index": 0, "amount": "586260.26", "confidence": 100, "is_recognized": true }, { "column_index": 1, "amount": "905786.99", "confidence": 100, "is_recognized": true } ] }, + { "line_code": "1200", "line_code_confidence": 100, "line_name": "Итого по разделу II", "line_name_confidence": 100, "values": [ { "column_index": 0, "amount": null, "confidence": 100, "is_recognized": true }, { "column_index": 1, "amount": null, "confidence": 100, "is_recognized": true } ] }, + { "line_code": "1600", "line_code_confidence": 100, "line_name": "БАЛАНС", "line_name_confidence": 100, "values": [ { "column_index": 0, "amount": "586260.26", "confidence": 100, "is_recognized": true }, { "column_index": 1, "amount": "905786.99", "confidence": 100, "is_recognized": true } ] } + ], + "checks": { "relations_failed": [], "soft_relations_failed": [ { "column_index": 1, "relation": "1110 = 11101", "expected": "905786.99", "actual": "900000.00" } ], "duplicate_line_codes": [] } + }, + { + "document_type": "INCOME", + "locator": "sheet:Отчет о финансовых результатах", + "header": { "form_name": "Отчет о финансовых результатах", "organization_name": "ООО \"Пример\"", "organization_name_confidence": 100, "report_date_text": "за январь - сентябрь 2025 г.", "report_date": "2025-09-30", "units": "тыс. руб.", "okud": "0710002" }, + "columns": [ { "column_index": 0, "label": "За январь - сентябрь 2025 г.", "period_date": "2025-09-30" } ], + "lines": [ + { "line_code": "2110", "line_code_confidence": 100, "line_name": "Выручка", "line_name_confidence": 100, "values": [ { "column_index": 0, "amount": "1000.00", "confidence": 100, "is_recognized": true } ] }, + { "line_code": "2120", "line_code_confidence": 100, "line_name": "Себестоимость продаж", "line_name_confidence": 100, "values": [ { "column_index": 0, "amount": "-600.00", "confidence": 100, "is_recognized": true } ] }, + { "line_code": "2100", "line_code_confidence": 100, "line_name": "Валовая прибыль (убыток)", "line_name_confidence": 100, "values": [ { "column_index": 0, "amount": "400.00", "confidence": 100, "is_recognized": true } ] } + ], + "checks": { "relations_failed": [], "soft_relations_failed": [], "duplicate_line_codes": [] } + }, + { + "document_type": "DISCLOSURE", + "locator": "sheet:Пояснения", + "header": { "form_name": "Пояснения к бухгалтерскому балансу и отчету о финансовых результатах" }, + "columns": [], + "lines": [] + } + ] +} diff --git a/api/contracts/examples/result-statement-minimal.json b/api/contracts/examples/result-statement-minimal.json index 484f222..51e3b17 100644 --- a/api/contracts/examples/result-statement-minimal.json +++ b/api/contracts/examples/result-statement-minimal.json @@ -1,5 +1,5 @@ { - "schema_version": "1.0", + "schema_version": "1.2", "job_id": "8d6b1f30-2e94-4a7c-b5d8-0f1e9c3a6b03", "file_id": "c7e2a9b4-1d58-4630-a2f7-6b4d8e0c1f04", "service": { "name": "scan-gateway", "version": "0.1.0" }, @@ -44,7 +44,8 @@ { "column_index": 1, "amount": "-12.30", "confidence": 90, "is_recognized": true } ] } - ] + ], + "checks": { "relations_failed": [], "soft_relations_failed": [], "duplicate_line_codes": [] } } ] } diff --git a/api/contracts/recognition-result.schema.json b/api/contracts/recognition-result.schema.json index 5d329a8..9a168cf 100644 --- a/api/contracts/recognition-result.schema.json +++ b/api/contracts/recognition-result.schema.json @@ -1,13 +1,13 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "arm-ka/recognition-result/v1", - "title": "Результат распознавания файла (tabular-parser / scan-gateway), контракт v1.1", + "title": "Результат распознавания файла (tabular-parser / scan-gateway), контракт v1.2", "description": "Один файл (stored_file) → один job → N документов. Сервисы не знают БД: только структура файла. Нормализация счетов, резолв контрагентов, проверки — в arm-core.", "$comment": "Версионирование: schema_version = MAJOR.MINOR. MINOR (1.1) — необязательные поля и новые значения enum; MAJOR (2.0) — удаление/переименование полей, смена типов или семантики. Потребитель принимает тот же MAJOR и любой MINOR. Новый MINOR раскатывается сначала на потребителей (ядро и сервисы обновляют копии схем раньше, чем производитель начнёт слать 1.1), потому что закрытые enum отвергают незнакомые значения.", "type": "object", "required": ["schema_version", "job_id", "file_id", "service", "status", "documents"], "properties": { - "schema_version": { "type": "string", "pattern": "^1\\.\\d+$", "description": "MAJOR.MINOR; производитель контракта v1.1 пишет \"1.1\"" }, + "schema_version": { "type": "string", "pattern": "^1\\.\\d+$", "description": "MAJOR.MINOR; производитель контракта v1.2 пишет \"1.2\"" }, "job_id": { "type": "string", "format": "uuid" }, "file_id": { "type": "string", "format": "uuid" }, "service": { @@ -53,6 +53,16 @@ }, "$defs": { "money": { "type": ["string", "null"], "pattern": "^-?\\d+(\\.\\d{1,2})?$", "description": "Decimal как строка, чтобы не терять копейки в JSON" }, + "failedRelation": { + "type": "object", + "required": ["column_index", "relation", "expected", "actual"], + "properties": { + "column_index": { "type": "integer" }, + "relation": { "type": "string", "examples": ["1600 = 1100 + 1200"] }, + "expected": { "$ref": "#/$defs/money", "description": "левая часть соотношения как в файле" }, + "actual": { "$ref": "#/$defs/money", "description": "правая часть, вычисленная по строкам" } + } + }, "osvDocument": { "type": "object", "required": ["document_type", "locator", "header", "group_schema", "rows"], @@ -70,6 +80,9 @@ "period_year": { "type": ["integer", "null"] }, "period_quarter": { "enum": ["Q1", "Q2", "Q3", "Q4", null] }, "is_cumulative": { "type": ["boolean", "null"], "description": "по тексту; ядро может переопределить по данным (OQ-02)" }, + "period_start": { "type": ["string", "null"], "format": "date", "description": "начало периода по шапке: первое число квартала/года либо начало диапазона месяцев (OQ-131)" }, + "period_end": { "type": ["string", "null"], "format": "date", "description": "конец периода по шапке" }, + "is_from_year_start": { "type": ["boolean", "null"], "description": "period_start — 1 января; is_cumulative не переопределяет" }, "data_type": { "enum": ["BU", "NU", "SUM", "CURRENCY", "QUANTITY", null] }, "currency": { "type": ["string", "null"], "examples": ["RUB", "руб."] }, "header_variant": { "enum": [1, 2] } @@ -133,7 +146,7 @@ "type": "object", "required": ["document_type", "locator", "header", "columns", "lines"], "properties": { - "document_type": { "enum": ["BALANCE", "INCOME", "OTHER"] }, + "document_type": { "enum": ["BALANCE", "INCOME", "DISCLOSURE", "OTHER"] }, "locator": { "type": "string", "examples": ["pages:1-2"] }, "header": { "type": "object", @@ -191,6 +204,15 @@ } } } + }, + "checks": { + "type": "object", + "description": "Результаты самопроверки формы по соотношениям §7 (tabular-parser). Отсутствие объекта — «не проверялось» (OCR-путь); пустые массивы — «проверено, сошлось».", + "properties": { + "relations_failed": { "type": "array", "items": { "$ref": "#/$defs/failedRelation" }, "description": "нарушения жёстких соотношений: документ needs_check" }, + "soft_relations_failed": { "type": "array", "items": { "$ref": "#/$defs/failedRelation" }, "description": "нарушения мягких соотношений (OQ-12, подстроки «в том числе»): только предупреждение" }, + "duplicate_line_codes": { "type": "array", "items": { "type": "string" }, "description": "коды строк, встретившиеся в документе больше одного раза (осталась первая строка)" } + } } } } diff --git a/api/src/contract/common.py b/api/src/contract/common.py index 8fa3581..dd6b4a1 100644 --- a/api/src/contract/common.py +++ b/api/src/contract/common.py @@ -27,7 +27,7 @@ from pydantic import ( model_validator, ) -CONTRACT_VERSION = "1.1" +CONTRACT_VERSION = "1.2" # Шаблоны версий — буквально из схем SCHEMA_VERSION_PATTERN = r"^1\.\d+$" # свой MAJOR, любой MINOR diff --git a/api/src/contract/result.py b/api/src/contract/result.py index 4a89716..c7948c0 100644 --- a/api/src/contract/result.py +++ b/api/src/contract/result.py @@ -1,4 +1,4 @@ -"""Файл результата в S3 (`results/{file_id}/{job_id}/result.json`): `recognition-result.schema.json` v1.1. +"""Файл результата в S3 (`results/{file_id}/{job_id}/result.json`): `recognition-result.schema.json` v1.2. tabular-parser формирует этот файл по запросу; один файл → один job → N документов (ОСВ по счёту / ББ / ОФР / прочее). Модель намеренно совпадает с копией в arm-core (контракт — `../docs/contracts`, обзор — @@ -29,6 +29,7 @@ from src.contract.common import ( ) __all__ = [ + "FailedRelation", "OsvChecks", "OsvDocument", "OsvGroup", @@ -37,6 +38,7 @@ __all__ = [ "RecognitionResult", "ResultError", "ResultStatus", + "StatementChecks", "StatementColumn", "StatementDocument", "StatementHeader", @@ -72,6 +74,9 @@ class OsvHeader(ContractModel): period_year: IntLike | None = None period_quarter: Literal["Q1", "Q2", "Q3", "Q4"] | None = None is_cumulative: bool | None = None # по тексту; ядро может переопределить по данным (OQ-02) + period_start: ContractDate | None = None # начало периода по шапке (v1.2, OQ-131) + period_end: ContractDate | None = None + is_from_year_start: bool | None = None # period_start — 1 января; is_cumulative не переопределяет data_type: DataType | None = None currency: str | None = None header_variant: OptionalNotNull[HeaderVariant] = None @@ -162,12 +167,33 @@ class StatementLine(ContractModel): values: list[StatementValue] +class FailedRelation(ContractModel): + """Нарушенное соотношение §7 по одной колонке. + + `expected` — левая часть как в файле, `actual` — правая по строкам. + """ + + column_index: IntLike + relation: str # «1600 = 1100 + 1200» — текст из словаря statement_checks.yaml + expected: Money + actual: Money + + +class StatementChecks(ContractModel): + """Самопроверка формы (v1.2): отсутствие объекта — «не проверялось», пустые списки — «сошлось».""" + + relations_failed: list[FailedRelation] = Field(default_factory=list) + soft_relations_failed: list[FailedRelation] = Field(default_factory=list) + duplicate_line_codes: list[str] = Field(default_factory=list) + + class StatementDocument(ContractModel): - document_type: Literal["BALANCE", "INCOME", "OTHER"] + document_type: Literal["BALANCE", "INCOME", "DISCLOSURE", "OTHER"] locator: str # «pages:1-2» header: StatementHeader columns: list[StatementColumn] lines: list[StatementLine] + checks: OptionalNotNull[StatementChecks] = None ResultDocument = Annotated[OsvDocument | StatementDocument, Field(discriminator="document_type")] diff --git a/api/tests/integration/test_roundtrip.py b/api/tests/integration/test_roundtrip.py index 4d2a51b..cbba7ac 100644 --- a/api/tests/integration/test_roundtrip.py +++ b/api/tests/integration/test_roundtrip.py @@ -298,7 +298,7 @@ async def test_real_osv_through_worker_gives_done_and_valid_result_file( body = _envelope(await core.next_message(request), request, f"result:{request['job_id']}:{request['attempt']}") assert body["status"] == "done", body - assert body["result_schema_version"] == "1.1" + assert body["result_schema_version"] == "1.2" assert body["metrics"]["duration_ms"] >= 0 result_ref = body["result_s3"] assert result_ref["bucket"] == request["s3"]["bucket"] diff --git a/api/tests/unit/parser/test_cli.py b/api/tests/unit/parser/test_cli.py index 52d65b1..04e5fd2 100644 --- a/api/tests/unit/parser/test_cli.py +++ b/api/tests/unit/parser/test_cli.py @@ -45,7 +45,7 @@ def test_done_book_returns_0_and_prints_result_json(tmp_path: Path, capsys: pyte assert code == EXIT_DONE payload: dict[str, Any] = json.loads(out) assert payload["status"] == "done" - assert payload["schema_version"] == "1.1" + assert payload["schema_version"] == "1.2" assert schema_errors(payload) == [] assert RecognitionResult.model_validate(payload).to_wire() == payload assert payload["documents"][0]["header"]["account_code_raw"] == "60" @@ -58,7 +58,7 @@ def test_output_is_indented_and_keeps_model_field_order(tmp_path: Path, capsys: _code, out, _err = _run(capsys, str(path)) - assert out.startswith('{\n "schema_version": "1.1",\n') + assert out.startswith('{\n "schema_version": "1.2",\n') keys = list(json.loads(out)) assert keys != sorted(keys) # sort_keys не используется: порядок полей модели значим для golden diff --git a/api/tests/unit/parser/test_osv_parser.py b/api/tests/unit/parser/test_osv_parser.py index a69d163..1b59caa 100644 --- a/api/tests/unit/parser/test_osv_parser.py +++ b/api/tests/unit/parser/test_osv_parser.py @@ -202,7 +202,7 @@ def test_mini_osv_is_done_with_full_fields(rules: Rules) -> None: assert outcome.metrics.duration_ms >= 0 assert warnings_of(outcome) == ["sheet=ОСВ 60 hierarchy=outline"] payload = assert_valid_result(outcome) - assert payload["schema_version"] == "1.1" + assert payload["schema_version"] == "1.2" def test_spoiled_sum_needs_check_with_row_numbers(rules: Rules) -> None: diff --git a/api/tests/unit/test_contract.py b/api/tests/unit/test_contract.py index bce77dc..485ff26 100644 --- a/api/tests/unit/test_contract.py +++ b/api/tests/unit/test_contract.py @@ -58,8 +58,9 @@ EXAMPLE_KIND = { "result-osv-minimal.json": "result", "result-osv-needs-check.json": "result", "result-statement-minimal.json": "result", + "result-statement-excel.json": "result", } -EXAMPLES_COUNT = 9 +EXAMPLES_COUNT = 10 def _load(path: Path) -> Any: @@ -604,7 +605,7 @@ def test_message_status_enum_values() -> None: def test_contract_version_constant() -> None: - assert CONTRACT_VERSION == "1.1" + assert CONTRACT_VERSION == "1.2" def test_python_float_amount_is_rejected_with_clear_error() -> None: diff --git a/api/tests/unit/test_contract_v12.py b/api/tests/unit/test_contract_v12.py new file mode 100644 index 0000000..2480420 --- /dev/null +++ b/api/tests/unit/test_contract_v12.py @@ -0,0 +1,84 @@ +"""Контракт v1.2: новые необязательные поля и значение DISCLOSURE (docs change slice-2b design D-1).""" + +import datetime +from decimal import Decimal + +from src.contract.common import CONTRACT_VERSION +from src.contract.result import ( + FailedRelation, + OsvHeader, + StatementChecks, + StatementDocument, + StatementHeader, +) + + +def test_contract_version_is_1_2() -> None: + assert CONTRACT_VERSION == "1.2" + + +def test_statement_document_accepts_disclosure_and_checks() -> None: + document = StatementDocument( + document_type="DISCLOSURE", + locator="sheet:Пояснения", + header=StatementHeader(form_name="Пояснения"), + columns=[], + lines=[], + ) + assert document.checks is None + checked = StatementDocument.model_validate( + { + "document_type": "BALANCE", + "locator": "sheet:ББ", + "header": {}, + "columns": [], + "lines": [], + "checks": { + "relations_failed": [ + { + "column_index": 0, + "relation": "1600 = 1100 + 1200", + "expected": "10.00", + "actual": "9.00", + } + ], + "soft_relations_failed": [], + "duplicate_line_codes": ["1110"], + }, + } + ) + assert checked.checks == StatementChecks( + relations_failed=[ + FailedRelation( + column_index=0, + relation="1600 = 1100 + 1200", + expected=Decimal("10.00"), + actual=Decimal("9.00"), + ) + ], + soft_relations_failed=[], + duplicate_line_codes=["1110"], + ) + wire = checked.model_dump(mode="json", by_alias=True, exclude_none=True) + assert wire["checks"]["relations_failed"][0]["expected"] == "10.00" + + +def test_osv_header_period_bounds_are_optional_dates() -> None: + header = OsvHeader( + organization_name="ООО", + account_code_raw="60", + period_text="1 квартал 2025 г.", + ) + assert header.period_start is None and header.is_from_year_start is None + filled = OsvHeader.model_validate( + { + "organization_name": "ООО", + "account_code_raw": "60", + "period_text": "x", + "period_start": "2024-10-01", + "period_end": "2025-09-30", + "is_from_year_start": False, + } + ) + assert filled.period_start == datetime.date(2024, 10, 1) + assert filled.period_end == datetime.date(2025, 9, 30) diff --git a/api/tests/unit/test_golden_helpers.py b/api/tests/unit/test_golden_helpers.py index 9e50e57..fe0f47d 100644 --- a/api/tests/unit/test_golden_helpers.py +++ b/api/tests/unit/test_golden_helpers.py @@ -8,7 +8,7 @@ from golden_helpers import SNAPSHOT_DIR, golden_path, normalize_result, snapshot def _result() -> dict: return { - "schema_version": "1.1", + "schema_version": "1.2", "job_id": "00000000-0000-0000-0000-000000000000", "file_id": "00000000-0000-0000-0000-000000000000", "service": {"name": "tabular-parser", "version": "0.1.0"}, diff --git a/api/tests/unit/test_worker.py b/api/tests/unit/test_worker.py index 8d43762..7ee53f7 100644 --- a/api/tests/unit/test_worker.py +++ b/api/tests/unit/test_worker.py @@ -231,7 +231,7 @@ async def test_ack_is_published_before_s3_read_and_result(env: Env) -> None: assert subjects == {RESULT_SUBJECT} ack = env.bodies()[0] assert ack == { - "schema_version": "1.1", + "schema_version": "1.2", "job_id": JOB_ID, "file_id": FILE_ID, "status": "processing", @@ -368,7 +368,7 @@ async def test_unsupported_major_version_gives_schema_mismatch_without_ack(env: result = env.bodies()[0] _assert_message_valid(result) assert result["status"] == "error" - assert result["schema_version"] == "1.1" + assert result["schema_version"] == "1.2" assert result["job_id"] == JOB_ID assert result["file_id"] == FILE_ID assert result["error"]["code"] == "SCHEMA_MISMATCH" @@ -503,7 +503,7 @@ async def test_substituted_parser_gives_done_with_result_file() -> None: _assert_message_valid(envelope) assert envelope["status"] == "done" assert envelope["result_s3"] == {"bucket": BUCKET, "key": RESULT_KEY} - assert envelope["result_schema_version"] == "1.1" + assert envelope["result_schema_version"] == "1.2" assert "error" not in envelope assert envelope["metrics"]["rows"] == PARSER_ROWS assert envelope["metrics"]["duration_ms"] >= 0 @@ -847,7 +847,7 @@ async def test_worker_with_real_osv_parser_gives_done_and_valid_result_file(fixt envelope = environment.bodies()[1] _assert_message_valid(envelope) assert envelope["status"] == "done" - assert envelope["result_schema_version"] == "1.1" + assert envelope["result_schema_version"] == "1.2" assert envelope["result_s3"] == {"bucket": BUCKET, "key": RESULT_KEY} stored = environment.storage.json_objects[(BUCKET, RESULT_KEY)] assert [error.message for error in VALIDATORS["result"].iter_errors(stored)] == []