diff --git a/golden/expected_osv_parse/rb/2024-Q1/сч 60_ 1 кв 2024 г.json b/golden/expected_osv_parse/rb/2024-Q1/сч 60_ 1 кв 2024 г.json index 17f7610..63dfa1e 100644 --- a/golden/expected_osv_parse/rb/2024-Q1/сч 60_ 1 кв 2024 г.json +++ b/golden/expected_osv_parse/rb/2024-Q1/сч 60_ 1 кв 2024 г.json @@ -7,7 +7,8 @@ "metrics": { "rows": 612, "warnings": [ - "sheet=Лист_1 hierarchy=outline" + "sheet=Лист_1 hierarchy=outline", + "строки 625–626: ниже таблицы (после «Итого») есть числа — строки ниже «Итого» вне документа и не разобраны" ] }, "documents": [ diff --git a/golden/expected_osv_parse/synthetic/s29_xls_no_codepage.json b/golden/expected_osv_parse/synthetic/s29_xls_no_codepage.json new file mode 100644 index 0000000..074cdea --- /dev/null +++ b/golden/expected_osv_parse/synthetic/s29_xls_no_codepage.json @@ -0,0 +1,252 @@ +{ + "schema_version": "1.2", + "service": { + "name": "tabular-parser" + }, + "status": "done", + "metrics": { + "rows": 9, + "warnings": [ + "xls без кодовой страницы — строки прочитаны как cp1251", + "sheet=Лист_1 hierarchy=indent" + ] + }, + "documents": [ + { + "document_type": "OSV", + "locator": "sheet:Лист_1", + "header": { + "organization_name": "ООО \"Тестовая организация\"", + "organization_inn": null, + "account_code_raw": "60", + "period_text": "1 квартал 2025 г.", + "period_year": 2025, + "period_quarter": "Q1", + "is_cumulative": true, + "period_start": "2025-01-01", + "period_end": "2025-03-31", + "is_from_year_start": true, + "data_type": "BU", + "currency": "RUB", + "header_variant": 1 + }, + "group_schema": [ + { + "type": "ACCOUNT", + "name": "Счет" + }, + { + "type": "COUNTERPARTY", + "name": "Контрагенты", + "has_inn": true + } + ], + "columns": { + "opening_debit": "G", + "opening_credit": "H", + "turnover_debit": "I", + "turnover_credit": "J", + "closing_debit": "K", + "closing_credit": "L", + "inn": "E" + }, + "rows": [ + { + "row_number": 11, + "level": 0, + "parent_row_number": null, + "row_value": "60", + "group_type": "ACCOUNT", + "group_name": "Счет", + "account_code_raw": "60", + "counterparty_name": null, + "counterparty_inn": null, + "data_type": "BU", + "opening_debit": "135000.00", + "opening_credit": "262500.60", + "turnover_debit": "311000.35", + "turnover_credit": "572000.35", + "closing_debit": "115000.00", + "closing_credit": "503500.60", + "extra": null + }, + { + "row_number": 12, + "level": 1, + "parent_row_number": 11, + "row_value": "60.01", + "group_type": "ACCOUNT", + "group_name": "Счет", + "account_code_raw": "60.01", + "counterparty_name": null, + "counterparty_inn": null, + "data_type": "BU", + "opening_debit": "15000.00", + "opening_credit": "253000.50", + "turnover_debit": "273000.25", + "turnover_credit": "460000.00", + "closing_debit": "65000.00", + "closing_credit": "490000.25", + "extra": null + }, + { + "row_number": 13, + "level": 2, + "parent_row_number": 12, + "row_value": "ООО Альфа", + "group_type": "COUNTERPARTY", + "group_name": "Контрагенты", + "account_code_raw": "60.01", + "counterparty_name": "ООО Альфа", + "counterparty_inn": "7701234567", + "data_type": "BU", + "opening_debit": null, + "opening_credit": "250000.50", + "turnover_debit": "180000.25", + "turnover_credit": "420000.00", + "closing_debit": null, + "closing_credit": "490000.25", + "extra": null + }, + { + "row_number": 14, + "level": 2, + "parent_row_number": 12, + "row_value": "ООО Бета", + "group_type": "COUNTERPARTY", + "group_name": "Контрагенты", + "account_code_raw": "60.01", + "counterparty_name": "ООО Бета", + "counterparty_inn": "7707654321", + "data_type": "BU", + "opening_debit": "15000.00", + "opening_credit": null, + "turnover_debit": "90000.00", + "turnover_credit": "40000.00", + "closing_debit": "65000.00", + "closing_credit": null, + "extra": null + }, + { + "row_number": 15, + "level": 2, + "parent_row_number": 12, + "row_value": "Иванов Иван", + "group_type": "COUNTERPARTY", + "group_name": "Контрагенты", + "account_code_raw": "60.01", + "counterparty_name": "Иванов Иван", + "counterparty_inn": null, + "data_type": "BU", + "opening_debit": null, + "opening_credit": "3000.00", + "turnover_debit": "3000.00", + "turnover_credit": null, + "closing_debit": null, + "closing_credit": null, + "extra": null + }, + { + "row_number": 16, + "level": 1, + "parent_row_number": 11, + "row_value": "60.02", + "group_type": "ACCOUNT", + "group_name": "Счет", + "account_code_raw": "60.02", + "counterparty_name": null, + "counterparty_inn": null, + "data_type": "BU", + "opening_debit": "120000.00", + "opening_credit": "9500.10", + "turnover_debit": "38000.10", + "turnover_credit": "112000.35", + "closing_debit": "50000.00", + "closing_credit": "13500.35", + "extra": null + }, + { + "row_number": 17, + "level": 2, + "parent_row_number": 16, + "row_value": "ООО Гамма", + "group_type": "COUNTERPARTY", + "group_name": "Контрагенты", + "account_code_raw": "60.02", + "counterparty_name": "ООО Гамма", + "counterparty_inn": "5001234567", + "data_type": "BU", + "opening_debit": "120000.00", + "opening_credit": null, + "turnover_debit": "30000.00", + "turnover_credit": "100000.00", + "closing_debit": "50000.00", + "closing_credit": null, + "extra": null + }, + { + "row_number": 18, + "level": 2, + "parent_row_number": 16, + "row_value": "ООО Дельта", + "group_type": "COUNTERPARTY", + "group_name": "Контрагенты", + "account_code_raw": "60.02", + "counterparty_name": "ООО Дельта", + "counterparty_inn": "7812345678", + "data_type": "BU", + "opening_debit": null, + "opening_credit": "8000.10", + "turnover_debit": "8000.10", + "turnover_credit": "12000.35", + "closing_debit": null, + "closing_credit": "12000.35", + "extra": null + }, + { + "row_number": 19, + "level": 2, + "parent_row_number": 16, + "row_value": "ИП Сидоров", + "group_type": "COUNTERPARTY", + "group_name": "Контрагенты", + "account_code_raw": "60.02", + "counterparty_name": "ИП Сидоров", + "counterparty_inn": "500100732259", + "data_type": "BU", + "opening_debit": null, + "opening_credit": "1500.00", + "turnover_debit": null, + "turnover_credit": null, + "closing_debit": null, + "closing_credit": "1500.00", + "extra": null + }, + { + "row_number": 20, + "level": 0, + "parent_row_number": null, + "row_value": "Итого", + "group_type": "TOTAL", + "group_name": null, + "account_code_raw": null, + "counterparty_name": null, + "counterparty_inn": null, + "data_type": "BU", + "opening_debit": "135000.00", + "opening_credit": "262500.60", + "turnover_debit": "311000.35", + "turnover_credit": "572000.35", + "closing_debit": "115000.00", + "closing_credit": "503500.60", + "extra": null + } + ], + "checks": { + "total_row_matches_root": true, + "balance_identity_violations": [], + "parent_sum_violations": [] + } + } + ] +} diff --git a/golden/expected_osv_parse/synthetic/s30_val_kol_vo_materials.json b/golden/expected_osv_parse/synthetic/s30_val_kol_vo_materials.json new file mode 100644 index 0000000..c9f3731 --- /dev/null +++ b/golden/expected_osv_parse/synthetic/s30_val_kol_vo_materials.json @@ -0,0 +1,193 @@ +{ + "schema_version": "1.2", + "service": { + "name": "tabular-parser" + }, + "status": "done", + "metrics": { + "rows": 6, + "warnings": [ + "sheet=Лист_1 hierarchy=outline", + "валюта в шапке не найдена" + ] + }, + "documents": [ + { + "document_type": "OSV", + "locator": "sheet:Лист_1", + "header": { + "organization_name": "ООО \"Тестовая организация\"", + "organization_inn": null, + "account_code_raw": "10", + "period_text": "2 квартал 2025 г.", + "period_year": 2025, + "period_quarter": "Q2", + "is_cumulative": false, + "period_start": "2025-04-01", + "period_end": "2025-06-30", + "is_from_year_start": false, + "data_type": "BU", + "currency": null, + "header_variant": 2 + }, + "group_schema": [ + { + "type": "ACCOUNT", + "name": "Субконто" + }, + { + "type": "OTHER", + "name": "Номенклатура" + } + ], + "columns": { + "opening_debit": "C", + "opening_credit": "D", + "turnover_debit": "E", + "turnover_credit": "F", + "closing_debit": "G", + "closing_credit": "H" + }, + "rows": [ + { + "row_number": 9, + "level": 0, + "parent_row_number": null, + "row_value": "10", + "group_type": "ACCOUNT", + "group_name": "Субконто", + "account_code_raw": "10", + "counterparty_name": null, + "counterparty_inn": null, + "data_type": "BU", + "opening_debit": "1700.00", + "opening_credit": null, + "turnover_debit": "350.00", + "turnover_credit": "100.00", + "closing_debit": "1950.00", + "closing_credit": null, + "extra": null + }, + { + "row_number": 10, + "level": 1, + "parent_row_number": 9, + "row_value": "10.1", + "group_type": "ACCOUNT", + "group_name": "Субконто", + "account_code_raw": "10.1", + "counterparty_name": null, + "counterparty_inn": null, + "data_type": "BU", + "opening_debit": "1700.00", + "opening_credit": null, + "turnover_debit": "350.00", + "turnover_credit": "100.00", + "closing_debit": "1950.00", + "closing_credit": null, + "extra": null + }, + { + "row_number": 11, + "level": 2, + "parent_row_number": 10, + "row_value": "Вал", + "group_type": "OTHER", + "group_name": "Номенклатура", + "account_code_raw": "10.1", + "counterparty_name": null, + "counterparty_inn": null, + "data_type": "BU", + "opening_debit": "1500.00", + "opening_credit": null, + "turnover_debit": "300.00", + "turnover_credit": "100.00", + "closing_debit": "1700.00", + "closing_credit": null, + "extra": null + }, + { + "row_number": 12, + "level": 3, + "parent_row_number": 11, + "row_value": "Кол-во", + "group_type": "OTHER", + "group_name": null, + "account_code_raw": "10.1", + "counterparty_name": null, + "counterparty_inn": null, + "data_type": "QUANTITY", + "opening_debit": "10.00", + "opening_credit": null, + "turnover_debit": "2.00", + "turnover_credit": "1.00", + "closing_debit": "11.00", + "closing_credit": null, + "extra": null + }, + { + "row_number": 13, + "level": 2, + "parent_row_number": 10, + "row_value": "Болт М8", + "group_type": "OTHER", + "group_name": "Номенклатура", + "account_code_raw": "10.1", + "counterparty_name": null, + "counterparty_inn": null, + "data_type": "BU", + "opening_debit": "200.00", + "opening_credit": null, + "turnover_debit": "50.00", + "turnover_credit": null, + "closing_debit": "250.00", + "closing_credit": null, + "extra": null + }, + { + "row_number": 14, + "level": 3, + "parent_row_number": 13, + "row_value": "Кол-во", + "group_type": "OTHER", + "group_name": null, + "account_code_raw": "10.1", + "counterparty_name": null, + "counterparty_inn": null, + "data_type": "QUANTITY", + "opening_debit": "400.00", + "opening_credit": null, + "turnover_debit": "100.00", + "turnover_credit": null, + "closing_debit": "500.00", + "closing_credit": null, + "extra": null + }, + { + "row_number": 15, + "level": 0, + "parent_row_number": null, + "row_value": "Итого", + "group_type": "TOTAL", + "group_name": null, + "account_code_raw": null, + "counterparty_name": null, + "counterparty_inn": null, + "data_type": "BU", + "opening_debit": "1700.00", + "opening_credit": null, + "turnover_debit": "350.00", + "turnover_credit": "100.00", + "closing_debit": "1950.00", + "closing_credit": null, + "extra": null + } + ], + "checks": { + "total_row_matches_root": true, + "balance_identity_violations": [], + "parent_sum_violations": [] + } + } + ] +} diff --git a/golden/expected_osv_parse/synthetic/s31_rows_below_total.json b/golden/expected_osv_parse/synthetic/s31_rows_below_total.json new file mode 100644 index 0000000..8cfff18 --- /dev/null +++ b/golden/expected_osv_parse/synthetic/s31_rows_below_total.json @@ -0,0 +1,252 @@ +{ + "schema_version": "1.2", + "service": { + "name": "tabular-parser" + }, + "status": "done", + "metrics": { + "rows": 9, + "warnings": [ + "sheet=Лист_1 hierarchy=outline", + "строки 24–25: ниже таблицы (после «Итого») есть числа — строки ниже «Итого» вне документа и не разобраны" + ] + }, + "documents": [ + { + "document_type": "OSV", + "locator": "sheet:Лист_1", + "header": { + "organization_name": "ООО \"Тестовая организация\"", + "organization_inn": null, + "account_code_raw": "60", + "period_text": "1 квартал 2025 г.", + "period_year": 2025, + "period_quarter": "Q1", + "is_cumulative": true, + "period_start": "2025-01-01", + "period_end": "2025-03-31", + "is_from_year_start": true, + "data_type": "BU", + "currency": "RUB", + "header_variant": 1 + }, + "group_schema": [ + { + "type": "ACCOUNT", + "name": "Счет" + }, + { + "type": "COUNTERPARTY", + "name": "Контрагенты", + "has_inn": true + } + ], + "columns": { + "opening_debit": "G", + "opening_credit": "H", + "turnover_debit": "I", + "turnover_credit": "J", + "closing_debit": "K", + "closing_credit": "L", + "inn": "E" + }, + "rows": [ + { + "row_number": 11, + "level": 0, + "parent_row_number": null, + "row_value": "60", + "group_type": "ACCOUNT", + "group_name": "Счет", + "account_code_raw": "60", + "counterparty_name": null, + "counterparty_inn": null, + "data_type": "BU", + "opening_debit": "135000.00", + "opening_credit": "262500.60", + "turnover_debit": "311000.35", + "turnover_credit": "572000.35", + "closing_debit": "115000.00", + "closing_credit": "503500.60", + "extra": null + }, + { + "row_number": 12, + "level": 1, + "parent_row_number": 11, + "row_value": "60.01", + "group_type": "ACCOUNT", + "group_name": "Счет", + "account_code_raw": "60.01", + "counterparty_name": null, + "counterparty_inn": null, + "data_type": "BU", + "opening_debit": "15000.00", + "opening_credit": "253000.50", + "turnover_debit": "273000.25", + "turnover_credit": "460000.00", + "closing_debit": "65000.00", + "closing_credit": "490000.25", + "extra": null + }, + { + "row_number": 13, + "level": 2, + "parent_row_number": 12, + "row_value": "ООО Альфа", + "group_type": "COUNTERPARTY", + "group_name": "Контрагенты", + "account_code_raw": "60.01", + "counterparty_name": "ООО Альфа", + "counterparty_inn": "7701234567", + "data_type": "BU", + "opening_debit": null, + "opening_credit": "250000.50", + "turnover_debit": "180000.25", + "turnover_credit": "420000.00", + "closing_debit": null, + "closing_credit": "490000.25", + "extra": null + }, + { + "row_number": 14, + "level": 2, + "parent_row_number": 12, + "row_value": "ООО Бета", + "group_type": "COUNTERPARTY", + "group_name": "Контрагенты", + "account_code_raw": "60.01", + "counterparty_name": "ООО Бета", + "counterparty_inn": "7707654321", + "data_type": "BU", + "opening_debit": "15000.00", + "opening_credit": null, + "turnover_debit": "90000.00", + "turnover_credit": "40000.00", + "closing_debit": "65000.00", + "closing_credit": null, + "extra": null + }, + { + "row_number": 15, + "level": 2, + "parent_row_number": 12, + "row_value": "Иванов Иван", + "group_type": "COUNTERPARTY", + "group_name": "Контрагенты", + "account_code_raw": "60.01", + "counterparty_name": "Иванов Иван", + "counterparty_inn": null, + "data_type": "BU", + "opening_debit": null, + "opening_credit": "3000.00", + "turnover_debit": "3000.00", + "turnover_credit": null, + "closing_debit": null, + "closing_credit": null, + "extra": null + }, + { + "row_number": 16, + "level": 1, + "parent_row_number": 11, + "row_value": "60.02", + "group_type": "ACCOUNT", + "group_name": "Счет", + "account_code_raw": "60.02", + "counterparty_name": null, + "counterparty_inn": null, + "data_type": "BU", + "opening_debit": "120000.00", + "opening_credit": "9500.10", + "turnover_debit": "38000.10", + "turnover_credit": "112000.35", + "closing_debit": "50000.00", + "closing_credit": "13500.35", + "extra": null + }, + { + "row_number": 17, + "level": 2, + "parent_row_number": 16, + "row_value": "ООО Гамма", + "group_type": "COUNTERPARTY", + "group_name": "Контрагенты", + "account_code_raw": "60.02", + "counterparty_name": "ООО Гамма", + "counterparty_inn": "5001234567", + "data_type": "BU", + "opening_debit": "120000.00", + "opening_credit": null, + "turnover_debit": "30000.00", + "turnover_credit": "100000.00", + "closing_debit": "50000.00", + "closing_credit": null, + "extra": null + }, + { + "row_number": 18, + "level": 2, + "parent_row_number": 16, + "row_value": "ООО Дельта", + "group_type": "COUNTERPARTY", + "group_name": "Контрагенты", + "account_code_raw": "60.02", + "counterparty_name": "ООО Дельта", + "counterparty_inn": "7812345678", + "data_type": "BU", + "opening_debit": null, + "opening_credit": "8000.10", + "turnover_debit": "8000.10", + "turnover_credit": "12000.35", + "closing_debit": null, + "closing_credit": "12000.35", + "extra": null + }, + { + "row_number": 19, + "level": 2, + "parent_row_number": 16, + "row_value": "ИП Сидоров", + "group_type": "COUNTERPARTY", + "group_name": "Контрагенты", + "account_code_raw": "60.02", + "counterparty_name": "ИП Сидоров", + "counterparty_inn": "500100732259", + "data_type": "BU", + "opening_debit": null, + "opening_credit": "1500.00", + "turnover_debit": null, + "turnover_credit": null, + "closing_debit": null, + "closing_credit": "1500.00", + "extra": null + }, + { + "row_number": 20, + "level": 0, + "parent_row_number": null, + "row_value": "Итого", + "group_type": "TOTAL", + "group_name": null, + "account_code_raw": null, + "counterparty_name": null, + "counterparty_inn": null, + "data_type": "BU", + "opening_debit": "135000.00", + "opening_credit": "262500.60", + "turnover_debit": "311000.35", + "turnover_credit": "572000.35", + "closing_debit": "115000.00", + "closing_credit": "503500.60", + "extra": null + } + ], + "checks": { + "total_row_matches_root": true, + "balance_identity_violations": [], + "parent_sum_violations": [] + } + } + ] +} diff --git a/osv/synthetic/README.md b/osv/synthetic/README.md index 2520b2c..9ae48cc 100644 --- a/osv/synthetic/README.md +++ b/osv/synthetic/README.md @@ -53,10 +53,15 @@ cd tabular-parser/api | `s26_inn_below_sibling_sums.xlsx` | ИНН строкой **под** наименованием, сосед, с суммами | ИНН — своему контрагенту, `done` | | `s27_inn_below_sibling_blank.xlsx` | ИНН строкой под наименованием, сосед, без сумм (дефект main: ИНН уходил следующему контрагенту) | ИНН — своему контрагенту; направление — по блоку 60.02, кончающемуся строкой ИНН; `done` | | `s28_inn_bad_checksum.xlsx` | раскладка `s20` («над, не в дереве, без сумм»), но у ООО Альфа строка ИНН `7701234567` с неверной контрольной суммой (#35) | строка остаётся обычной, у Альфы ИНН нет, предупреждение «похожа на ИНН … контрольная сумма не сходится», `needs_check`; остальные контрагенты — со своими ИНН (направление «над» — по блоку 60.02) | +| `s29_xls_no_codepage.xls` | xls BIFF5 без записи CODEPAGE, строки в cp1251 (собран `tests/synthetic/xls_writer.py`); отступ — ведущими пробелами (AURORA-1662) | кириллица читается как cp1251, предупреждение «xls без кодовой страницы — строки прочитаны как cp1251», `done` | +| `s30_val_kol_vo_materials.xlsx` | счёт 10 (вариант 2, номенклатура): материал «Вал» без точки, под материалами строки «Кол-во» (AURORA-1663) | «Вал» — обычная строка (`data_type` шапки, `OTHER`), «Кол-во» — `QUANTITY`, `done` | +| `s31_rows_below_total.xlsx` | под «Итого» дописаны «Ответственный:», блок «ООО Альфа (ГК)» с числами и строка «Итого по ГК» (AURORA-1664) | таблица кончается на первой «Итого», хвост вне документа, предупреждение «ниже таблицы … есть числа», `done` | `s17`–`s19` — регрессия извлечения ИНН из текста (issue #9). `s14`–`s16` добавлены тестом покрытия словарей `tabular-parser` (`rules/*.yaml`, решение R-22): каждая запись словаря должна сработать на реальных или синтетических файлах; эти файлы закрывают записи, которых нет в реальных ОСВ. +`s29`–`s31` — tabular-parser#47 (кодировка xls без CODEPAGE, «Вал»/«Кол-во», строки ниже «Итого»). + `s20`–`s27` — восемь раскладок строки ИНН рядом с контрагентом (issue tabular-parser#35); ИНН с верной контрольной суммой — без подсказки в шапке другие не склеиваются. ## О файле xlsb diff --git a/osv/synthetic/s29_xls_no_codepage.xls b/osv/synthetic/s29_xls_no_codepage.xls new file mode 100644 index 0000000..c36c8ab Binary files /dev/null and b/osv/synthetic/s29_xls_no_codepage.xls differ diff --git a/osv/synthetic/s30_val_kol_vo_materials.xlsx b/osv/synthetic/s30_val_kol_vo_materials.xlsx new file mode 100644 index 0000000..e37b220 Binary files /dev/null and b/osv/synthetic/s30_val_kol_vo_materials.xlsx differ diff --git a/osv/synthetic/s31_rows_below_total.xlsx b/osv/synthetic/s31_rows_below_total.xlsx new file mode 100644 index 0000000..57ff817 Binary files /dev/null and b/osv/synthetic/s31_rows_below_total.xlsx differ