e1559c9770
api/tests/synthetic: детерминированный генератор s01…s13 (openpyxl, метки времени книги и zip зафиксированы), минимальный писатель BIFF12 для s13.xlsb (ruling R-9, читается pyxlsb), тесты сценариев спеки osv-parsing на файлах из fixtures/osv/synthetic/ (таблица «файл → ожидания», s13 равен s02 кроме предупреждения xlsb), тест побайтовой воспроизводимости генерации. synthetic_files() учитывает только файлы Excel (README.md каталога не файл набора). Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012vJkmBGKKqMoEMzvsXG3y2
415 lines
20 KiB
Python
415 lines
20 KiB
Python
"""Сценарии спеки `osv-parsing` на синтетических файлах `../fixtures/osv/synthetic/` (Срез 2a, Task 11).
|
||
|
||
Ожидания выведены из сценариев спеки и из содержимого, которое генератор записывает в файл (суммы и тексты ниже
|
||
перечислены в тесте явно), а не из текущего вывода парсера: файл — вход, оракулом он не служит. Без каталога fixtures
|
||
тесты пропускаются (маркер `fixtures`).
|
||
"""
|
||
|
||
from dataclasses import dataclass
|
||
from decimal import Decimal
|
||
from pathlib import Path
|
||
from typing import cast
|
||
|
||
import pytest
|
||
from result_contract import assert_valid_result
|
||
|
||
from src.contract import FileType, OsvDocument, ResultStatus
|
||
from src.contract.result import OsvRow
|
||
from src.parser import ParseOutcome, get_parser
|
||
|
||
pytestmark = pytest.mark.fixtures
|
||
|
||
SYNTHETIC_DIR = Path("osv") / "synthetic"
|
||
EXPECTED_NAMES = [
|
||
"s01_variant2_header.xlsx",
|
||
"s02_no_outline_indent.xlsx",
|
||
"s03_no_outline_no_indent.xlsx",
|
||
"s04_currency_quantity_rows.xlsx",
|
||
"s05_text_numbers.xlsx",
|
||
"s06_two_osv_and_other_sheet.xlsx",
|
||
"s07_no_opening_collapsed_closing.xlsx",
|
||
"s08_extra_kpp_column.xlsx",
|
||
"s09_subaccount_names.xlsx",
|
||
"s10_broken_sum.xlsx",
|
||
"s11_header_repeat.xlsx",
|
||
"s12_statement_and_osv.xlsx",
|
||
"s13_no_outline_indent.xlsb",
|
||
]
|
||
XLSB_MISSING_REASON = "нет файла s13_no_outline_indent.xlsb (xlsb не получен; см. README fixtures)"
|
||
VARIANT_2 = 2
|
||
XLSB_WARNING = "xlsb: outline level недоступен"
|
||
|
||
ALFA = "ООО Альфа"
|
||
BETA = "ООО Бета"
|
||
GAMMA = "ООО Гамма"
|
||
D = Decimal
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Expect:
|
||
"""Ожидания по файлу, общие для всех сценариев (шапка и итог разбора)."""
|
||
|
||
hierarchy: str
|
||
year: int
|
||
quarter: str
|
||
cumulative: bool
|
||
status: ResultStatus = ResultStatus.DONE
|
||
documents: int = 1
|
||
variant: int = 1
|
||
data_type: str = "BU"
|
||
|
||
|
||
EXPECTED: dict[str, Expect] = {
|
||
# «Период: 1 квартал 2026 г.» — вариант 2, данные «сумма»
|
||
"s01_variant2_header.xlsx": Expect("outline", 2026, "Q1", True, variant=2, data_type="SUM"),
|
||
"s02_no_outline_indent.xlsx": Expect("indent", 2025, "Q2", False),
|
||
# римский квартал «I кв.»
|
||
"s03_no_outline_no_indent.xlsx": Expect("account_pattern", 2025, "Q1", True),
|
||
"s04_currency_quantity_rows.xlsx": Expect("outline", 2025, "Q3", False),
|
||
# «полугодие»
|
||
"s05_text_numbers.xlsx": Expect("outline", 2025, "Q2", True),
|
||
"s06_two_osv_and_other_sheet.xlsx": Expect("outline", 2025, "Q1", True, documents=2),
|
||
# «9 месяцев»
|
||
"s07_no_opening_collapsed_closing.xlsx": Expect("outline", 2025, "Q3", True),
|
||
# диапазон дат «с 01.01.2025 по 31.03.2025»
|
||
"s08_extra_kpp_column.xlsx": Expect("outline", 2025, "Q1", True),
|
||
"s09_subaccount_names.xlsx": Expect("outline", 2025, "Q4", False),
|
||
"s10_broken_sum.xlsx": Expect("outline", 2025, "Q1", True, status=ResultStatus.NEEDS_CHECK),
|
||
# «12 месяцев»
|
||
"s11_header_repeat.xlsx": Expect("outline", 2025, "Q4", True),
|
||
# «январь - сентябрь», данные «НУ»
|
||
"s12_statement_and_osv.xlsx": Expect("outline", 2025, "Q3", True, data_type="NU"),
|
||
"s13_no_outline_indent.xlsb": Expect("indent", 2025, "Q2", False),
|
||
}
|
||
|
||
|
||
# --- Помощники ---
|
||
|
||
|
||
def synthetic_path(fixtures_dir: Path, name: str) -> Path:
|
||
return fixtures_dir / SYNTHETIC_DIR / name
|
||
|
||
|
||
def parse_path(path: Path) -> ParseOutcome:
|
||
return get_parser().parse(path.read_bytes(), cast(FileType, path.suffix.lstrip(".").upper()))
|
||
|
||
|
||
def parse_named(fixtures_dir: Path, name: str) -> ParseOutcome:
|
||
path = synthetic_path(fixtures_dir, name)
|
||
if not path.is_file():
|
||
if name.endswith(".xlsb"):
|
||
pytest.skip(XLSB_MISSING_REASON)
|
||
pytest.fail(f"нет файла {name}: запустите генератор (см. README fixtures)")
|
||
return parse_path(path)
|
||
|
||
|
||
def documents(outcome: ParseOutcome) -> list[OsvDocument]:
|
||
assert outcome.error is None, outcome.error
|
||
docs = list(outcome.documents)
|
||
assert all(isinstance(doc, OsvDocument) for doc in docs)
|
||
return cast(list[OsvDocument], docs)
|
||
|
||
|
||
def one_document(outcome: ParseOutcome) -> OsvDocument:
|
||
(doc,) = documents(outcome)
|
||
return doc
|
||
|
||
|
||
def warnings_of(outcome: ParseOutcome) -> list[str]:
|
||
assert outcome.metrics is not None
|
||
assert outcome.metrics.warnings is not None
|
||
return outcome.metrics.warnings
|
||
|
||
|
||
def by_text(doc: OsvDocument) -> dict[str, OsvRow]:
|
||
"""Строки по тексту (в синтетике тексты строк уникальны внутри листа)."""
|
||
rows = {row.row_value: row for row in doc.rows}
|
||
assert len(rows) == len(doc.rows), "тексты строк синтетики должны быть уникальны"
|
||
return rows
|
||
|
||
|
||
def rows_text(doc: OsvDocument) -> list[str]:
|
||
return [row.row_value for row in doc.rows]
|
||
|
||
|
||
def sums(row: OsvRow) -> tuple[Decimal | None, ...]:
|
||
return (
|
||
row.opening_debit,
|
||
row.opening_credit,
|
||
row.turnover_debit,
|
||
row.turnover_credit,
|
||
row.closing_debit,
|
||
row.closing_credit,
|
||
)
|
||
|
||
|
||
def tree(doc: OsvDocument) -> list[tuple[str, int, int | None, tuple[Decimal | None, ...]]]:
|
||
"""Дерево документа: текст, уровень, родитель, суммы — для сравнения двух файлов одной ведомости."""
|
||
return [(row.row_value, row.level, row.parent_row_number, sums(row)) for row in doc.rows]
|
||
|
||
|
||
# --- Состав набора и общие ожидания ---
|
||
|
||
|
||
def test_synthetic_set_is_complete(fixtures_dir: Path) -> None:
|
||
"""В `fixtures/osv/synthetic/` лежат файлы набора s01…s13 (s13 — xlsb; допустима деградация: нет только его)."""
|
||
present = sorted(path.name for path in (fixtures_dir / SYNTHETIC_DIR).glob("s*") if path.is_file())
|
||
assert present == [name for name in EXPECTED_NAMES if name in present]
|
||
missing = [name for name in EXPECTED_NAMES if name not in present]
|
||
assert missing in ([], [EXPECTED_NAMES[-1]]), f"нет файлов набора: {missing}"
|
||
|
||
|
||
def test_every_synthetic_file_is_listed(synthetic_file: Path) -> None:
|
||
"""Каждый файл каталога описан в таблице ожиданий (новый файл без ожиданий — ошибка набора)."""
|
||
assert synthetic_file.name in EXPECTED, f"добавьте ожидания для {synthetic_file.name}"
|
||
|
||
|
||
def test_header_status_and_hierarchy(synthetic_file: Path) -> None:
|
||
"""Статус, число документов, стратегия иерархии и шапка из таблицы `EXPECTED`; результат валиден по контракту."""
|
||
expect = EXPECTED[synthetic_file.name]
|
||
|
||
outcome = parse_path(synthetic_file)
|
||
|
||
assert outcome.status is expect.status, outcome.metrics
|
||
docs = documents(outcome)
|
||
assert len(docs) == expect.documents
|
||
hierarchy_lines = [w for w in warnings_of(outcome) if f"hierarchy={expect.hierarchy}" in w]
|
||
assert len(hierarchy_lines) == expect.documents, warnings_of(outcome)
|
||
for doc in docs:
|
||
assert doc.header.header_variant == expect.variant
|
||
assert doc.header.period_year == expect.year
|
||
assert doc.header.period_quarter == expect.quarter
|
||
assert doc.header.is_cumulative is expect.cumulative
|
||
assert doc.header.data_type == expect.data_type
|
||
assert doc.header.organization_name == 'ООО "Тестовая организация"'
|
||
assert doc.rows[-1].group_type == "TOTAL"
|
||
if expect.status is ResultStatus.DONE:
|
||
assert doc.checks is not None
|
||
assert doc.checks.total_row_matches_root is True
|
||
assert doc.checks.balance_identity_violations == []
|
||
assert doc.checks.parent_sum_violations == []
|
||
assert_valid_result(outcome)
|
||
|
||
|
||
# --- Сценарии спеки ---
|
||
|
||
|
||
def test_s01_variant2_header_and_subkonto_groups(fixtures_dir: Path) -> None:
|
||
"""«Вариант 2»: `header_variant = 2`, `group_schema = [ACCOUNT, COUNTERPARTY, CONTRACT]`, `period_quarter = Q1`."""
|
||
doc = one_document(parse_named(fixtures_dir, "s01_variant2_header.xlsx"))
|
||
|
||
assert doc.header.header_variant == VARIANT_2
|
||
assert [group.type for group in doc.group_schema] == ["ACCOUNT", "COUNTERPARTY", "CONTRACT"]
|
||
assert doc.header.period_quarter == "Q1"
|
||
assert doc.header.period_text == "1 квартал 2026 г."
|
||
rows = by_text(doc)
|
||
contract = rows["Договор поставки № 15 от 10.01.2026"]
|
||
assert contract.group_type == "CONTRACT"
|
||
assert contract.parent_row_number == rows[ALFA].row_number
|
||
assert rows[ALFA].group_type == "COUNTERPARTY"
|
||
assert rows[ALFA].counterparty_inn == "7701234567"
|
||
|
||
|
||
def test_s02_indent_levels_and_tree(fixtures_dir: Path) -> None:
|
||
"""«Без outline level»: `hierarchy=indent`, уровни 0/1/2, родитель контрагента — субсчёт."""
|
||
doc = one_document(parse_named(fixtures_dir, "s02_no_outline_indent.xlsx"))
|
||
|
||
rows = by_text(doc)
|
||
assert [(text, rows[text].level) for text in ("60", "60.01", ALFA, "60.02", GAMMA, "Итого")] == [
|
||
("60", 0),
|
||
("60.01", 1),
|
||
(ALFA, 2),
|
||
("60.02", 1),
|
||
(GAMMA, 2),
|
||
("Итого", 0),
|
||
]
|
||
assert rows[ALFA].parent_row_number == rows["60.01"].row_number
|
||
assert rows["60.01"].parent_row_number == rows["60"].row_number
|
||
assert rows["60"].parent_row_number is None
|
||
|
||
|
||
def test_s02_and_s03_describe_the_same_tree(fixtures_dir: Path) -> None:
|
||
"""Одна ведомость с отступами (s02) и с кодами счетов и жирностью (s03): дерево и суммы совпадают."""
|
||
indent = one_document(parse_named(fixtures_dir, "s02_no_outline_indent.xlsx"))
|
||
pattern = one_document(parse_named(fixtures_dir, "s03_no_outline_no_indent.xlsx"))
|
||
|
||
assert tree(indent) == tree(pattern)
|
||
|
||
|
||
def test_s03_account_pattern_levels(fixtures_dir: Path) -> None:
|
||
"""«Без outline и отступов»: уровни по кодам счетов и жирности, контрольные суммы сходятся (в общем тесте)."""
|
||
outcome = parse_named(fixtures_dir, "s03_no_outline_no_indent.xlsx")
|
||
doc = one_document(outcome)
|
||
|
||
rows = by_text(doc)
|
||
assert [rows[text].level for text in ("60", "60.01", ALFA, BETA, "Иванов Иван", "60.02", GAMMA)] == [
|
||
0,
|
||
1,
|
||
2,
|
||
2,
|
||
2,
|
||
1,
|
||
2,
|
||
]
|
||
assert rows[GAMMA].parent_row_number == rows["60.02"].row_number
|
||
assert outcome.status is ResultStatus.DONE
|
||
|
||
|
||
def test_s04_currency_and_quantity_rows(fixtures_dir: Path) -> None:
|
||
"""«Строки Вал./Кол.»: USD и Вал. — CURRENCY, Кол. — QUANTITY; в проверку суммы детей не входят, статус `done`."""
|
||
outcome = parse_named(fixtures_dir, "s04_currency_quantity_rows.xlsx")
|
||
doc = one_document(outcome)
|
||
|
||
rows = by_text(doc)
|
||
assert rows["USD"].data_type == "CURRENCY"
|
||
assert rows["Кол."].data_type == "QUANTITY"
|
||
assert rows["Вал."].data_type == "CURRENCY"
|
||
assert rows["USD"].parent_row_number == rows[ALFA].row_number
|
||
assert rows["Вал."].parent_row_number == rows[BETA].row_number
|
||
assert rows[ALFA].data_type == "BU"
|
||
assert rows[ALFA].group_type == "COUNTERPARTY"
|
||
# суммы показателей остаются в шести полях
|
||
assert rows["Кол."].opening_debit == D("10.00")
|
||
assert rows["USD"].opening_credit == D("2500.00")
|
||
assert outcome.status is ResultStatus.DONE
|
||
|
||
|
||
def test_s05_text_numbers(fixtures_dir: Path) -> None:
|
||
"""«Текстовые числа»: «1 234 567,89» → 1234567.89, «(1 234,00)» → -1234.00, «—»/«–»/«-» → null."""
|
||
doc = one_document(parse_named(fixtures_dir, "s05_text_numbers.xlsx"))
|
||
|
||
rows = by_text(doc)
|
||
assert sums(rows[ALFA]) == (D("1234567.89"), None, D("500.00"), D("-1234.00"), D("1236301.89"), None)
|
||
# неразрывные пробелы и «–»
|
||
assert sums(rows[BETA]) == (D("15000.00"), None, D("90000.00"), D("40000.00"), D("65000.00"), None)
|
||
# узкие неразрывные пробелы и «-»
|
||
assert sums(rows[GAMMA]) == (D("120000.00"), None, D("30000.00"), D("100000.00"), D("50000.00"), None)
|
||
|
||
|
||
def test_s06_two_osv_sheets_and_other_sheet(fixtures_dir: Path) -> None:
|
||
"""«Два листа ОСВ и посторонний лист»: два документа в порядке листов, в warnings — пропуск листа «Справка»."""
|
||
outcome = parse_named(fixtures_dir, "s06_two_osv_and_other_sheet.xlsx")
|
||
|
||
docs = documents(outcome)
|
||
assert [doc.locator for doc in docs] == ["sheet:ОСВ 60", "sheet:ОСВ 62"]
|
||
assert [doc.header.account_code_raw for doc in docs] == ["60", "62"]
|
||
assert any("Справка" in warning and "skipped" in warning for warning in warnings_of(outcome))
|
||
assert not any("ОСВ 60" in w and "skipped" in w or "ОСВ 62" in w and "skipped" in w for w in warnings_of(outcome))
|
||
assert outcome.metrics is not None
|
||
assert outcome.metrics.rows == sum(1 for doc in docs for row in doc.rows if row.group_type != "TOTAL")
|
||
|
||
|
||
def test_s07_no_opening_and_collapsed_closing(fixtures_dir: Path) -> None:
|
||
"""«Без начала и со свёрнутым сальдо»: `opening_* = null`, отрицательное свёрнутое — в `closing_credit`, warning."""
|
||
outcome = parse_named(fixtures_dir, "s07_no_opening_collapsed_closing.xlsx")
|
||
doc = one_document(outcome)
|
||
|
||
for row in doc.rows:
|
||
assert row.opening_debit is None
|
||
assert row.opening_credit is None
|
||
rows = by_text(doc)
|
||
assert (rows[ALFA].closing_debit, rows[ALFA].closing_credit) == (None, D("250000.50"))
|
||
assert (rows[BETA].closing_debit, rows[BETA].closing_credit) == (D("50000.00"), None)
|
||
assert (rows[GAMMA].closing_debit, rows[GAMMA].closing_credit) == (None, None)
|
||
assert rows[ALFA].turnover_credit == D("350000.50")
|
||
assert any("свёрнутое сальдо" in warning for warning in warnings_of(outcome))
|
||
assert outcome.status is ResultStatus.DONE
|
||
|
||
|
||
def test_s08_kpp_column_goes_to_extra(fixtures_dir: Path) -> None:
|
||
"""«Неизвестная колонка внутри таблицы»: `extra["КПП"]` у контрагентов, суммы не сдвинуты, ИНН в своей колонке."""
|
||
doc = one_document(parse_named(fixtures_dir, "s08_extra_kpp_column.xlsx"))
|
||
|
||
rows = by_text(doc)
|
||
assert rows[ALFA].extra == {"КПП": "770101001"}
|
||
assert rows[BETA].extra == {"КПП": "770701001"}
|
||
assert rows["Иванов Иван"].extra is None
|
||
assert rows["60"].extra is None
|
||
assert rows[ALFA].counterparty_inn == "7701234567"
|
||
assert sums(rows[ALFA]) == (None, D("250000.50"), D("180000.25"), D("420000.00"), None, D("490000.25"))
|
||
assert doc.columns is not None
|
||
assert doc.columns["inn"] == "E"
|
||
assert (doc.columns["opening_debit"], doc.columns["closing_credit"]) == ("G", "L")
|
||
|
||
|
||
def test_s09_subaccount_names_in_account_rows(fixtures_dir: Path) -> None:
|
||
"""«Название субсчёта в строке счёта»: ACCOUNT, `account_code_raw` без названия, `row_value` — исходный текст."""
|
||
doc = one_document(parse_named(fixtures_dir, "s09_subaccount_names.xlsx"))
|
||
|
||
rows = by_text(doc)
|
||
named = rows["60.01, Расчеты с поставщиками и подрядчиками"]
|
||
assert (named.group_type, named.account_code_raw) == ("ACCOUNT", "60.01")
|
||
assert rows["60.02, Расчеты по авансам выданным"].account_code_raw == "60.02"
|
||
# буквенный субсчёт (R-16): счёт, а не контрагент; дети наследуют его код
|
||
letter = rows["60.АВ, Расчеты по авансам (буквенный субсчёт)"]
|
||
assert (letter.group_type, letter.account_code_raw) == ("ACCOUNT", "60.АВ")
|
||
assert rows["ООО Дельта"].account_code_raw == "60.АВ"
|
||
assert rows["ООО Дельта"].group_type == "COUNTERPARTY"
|
||
assert rows[ALFA].account_code_raw == "60.01"
|
||
|
||
|
||
def test_s10_broken_sum_needs_check(fixtures_dir: Path) -> None:
|
||
"""«Испорченная сумма»: `needs_check`, нарушение тождества — у контрагента, суммы родителя — у его субсчёта."""
|
||
outcome = parse_named(fixtures_dir, "s10_broken_sum.xlsx")
|
||
doc = one_document(outcome)
|
||
|
||
rows = by_text(doc)
|
||
assert outcome.status is ResultStatus.NEEDS_CHECK
|
||
assert doc.checks is not None
|
||
assert doc.checks.balance_identity_violations == [rows[BETA].row_number]
|
||
assert doc.checks.parent_sum_violations == [rows["60.01"].row_number]
|
||
assert doc.checks.total_row_matches_root is True
|
||
assert rows[BETA].closing_debit == D("65001.00")
|
||
texts = [w for w in warnings_of(outcome) if "§1.4" in w]
|
||
assert any(str(rows[BETA].row_number) in w for w in texts)
|
||
assert any(str(rows["60.01"].row_number) in w for w in texts)
|
||
|
||
|
||
def test_s11_header_repeat_is_skipped(fixtures_dir: Path) -> None:
|
||
"""«Повтор заголовка внутри данных» (печать по страницам): строки заголовка не данные, дерево не нарушено."""
|
||
outcome = parse_named(fixtures_dir, "s11_header_repeat.xlsx")
|
||
doc = one_document(outcome)
|
||
|
||
assert rows_text(doc) == [
|
||
"60",
|
||
"60.01",
|
||
ALFA,
|
||
BETA,
|
||
"Иванов Иван",
|
||
"60.02",
|
||
GAMMA,
|
||
"ООО Дельта",
|
||
"ИП Сидоров",
|
||
"Итого",
|
||
]
|
||
rows = by_text(doc)
|
||
assert [rows[text].level for text in rows_text(doc)] == [0, 1, 2, 2, 2, 1, 2, 2, 2, 0]
|
||
assert rows["60.02"].parent_row_number == rows["60"].row_number
|
||
assert outcome.status is ResultStatus.DONE
|
||
|
||
|
||
def test_s12_statement_sheet_is_skipped_with_warning(fixtures_dir: Path) -> None:
|
||
"""«Лист ББ + лист ОСВ»: один документ, лист формы пропущен с предупреждением о подсрезе 2b."""
|
||
outcome = parse_named(fixtures_dir, "s12_statement_and_osv.xlsx")
|
||
|
||
(doc,) = documents(outcome)
|
||
assert doc.locator == "sheet:ОСВ 60"
|
||
assert any("Бухгалтерский баланс" in w and "skipped: statement" in w for w in warnings_of(outcome))
|
||
assert outcome.status is ResultStatus.DONE
|
||
|
||
|
||
def test_s13_xlsb_equals_s02_except_locator_and_warnings(fixtures_dir: Path) -> None:
|
||
"""«XLSB»: результат совпадает с xlsx-оригиналом (s02), кроме `locator` и предупреждения об outline."""
|
||
xlsb = parse_named(fixtures_dir, "s13_no_outline_indent.xlsb")
|
||
xlsx = parse_named(fixtures_dir, "s02_no_outline_indent.xlsx")
|
||
|
||
xlsb_doc, xlsx_doc = one_document(xlsb), one_document(xlsx)
|
||
assert xlsb_doc.model_copy(update={"locator": ""}) == xlsx_doc.model_copy(update={"locator": ""})
|
||
assert xlsb.status is xlsx.status is ResultStatus.DONE
|
||
assert XLSB_WARNING in warnings_of(xlsb)
|
||
assert [w for w in warnings_of(xlsb) if w != XLSB_WARNING] == warnings_of(xlsx)
|
||
assert any("hierarchy=indent" in w for w in warnings_of(xlsb))
|
||
assert xlsb.metrics is not None
|
||
assert xlsx.metrics is not None
|
||
assert xlsb.metrics.rows == xlsx.metrics.rows
|