arm-tabular-parser/api/tests/unit/parser/test_readers.py

"""Читатели книг: синтетическая книга openpyxl в памяти → `Sheet`; определение формата по сигнатуре; ошибки чтения."""

import datetime
import io
import zipfile
from collections.abc import Iterator
from decimal import Decimal
from types import TracebackType
from typing import NamedTuple

import openpyxl
import pytest
from openpyxl.styles import Alignment, Font

from src.contract import ErrorCode
from src.parser.errors import ParserError, UnreadableError, UnsupportedFormatError
from src.parser.grid import EMPTY_CELL, Cell, MergedRange
from src.parser.readers import read_book, read_book_detected
from src.parser.readers import xlsb as xlsb_reader
from src.parser.readers._common import make_cell
from src.parser.readers.xls import NO_CODEPAGE_WARNING
from tests.synthetic.xls_writer import write_xls


def _synthetic_xlsx() -> bytes:
    """Две страницы: шапка с merged-заголовками, outline 0/1/2, bold, отступ, число, дата, bool, пустые строки,
    ячейки из пробелов и NBSP, merged-диапазон с пробельным origin."""
    book = openpyxl.Workbook()
    osv = book.active
    assert osv is not None
    osv.title = "ОСВ"
    osv["A1"] = "ООО Ромашка"
    osv["A1"].font = Font(bold=True)
    osv["A3"] = "Счет"
    osv.merge_cells("A3:A4")
    osv["B3"] = "Сальдо на начало периода"
    osv.merge_cells("B3:C3")
    osv["B4"] = "Дебет"
    osv["C4"] = "Кредит"
    osv["A5"] = "62"
    osv["A5"].font = Font(bold=True)
    osv["B5"] = 1500
    osv["A6"] = "62.01"
    osv["A6"].alignment = Alignment(indent=2)
    osv["B6"] = 1500.25
    osv.row_dimensions[6].outline_level = 1
    osv["A7"] = "   Контрагент"
    osv["B7"] = datetime.datetime(2024, 9, 30)
    osv.row_dimensions[7].outline_level = 2
    osv["A9"] = " "
    osv["B9"] = "\xa0"
    osv["D3"] = " "
    osv.merge_cells("D3:E3")
    osv["A10"] = "Итого"
    osv["C10"] = True
    other = book.create_sheet("Прочее")
    other["A1"] = "x"
    buffer = io.BytesIO()
    book.save(buffer)
    return buffer.getvalue()


def test_xlsx_reader_builds_sheet_model() -> None:
    sheets, warnings = read_book(_synthetic_xlsx(), "XLSX")

    assert warnings == []
    assert [sheet.name for sheet in sheets] == ["ОСВ", "Прочее"]
    osv = sheets[0]
    assert osv.cell(1, 1) == Cell("ООО Ромашка", "text", bold=True)
    assert osv.cell(5, 1) == Cell("62", "text", bold=True)
    assert osv.cell(5, 2) == Cell(Decimal("1500"), "number")
    assert osv.cell(6, 1) == Cell("62.01", "text", indent=2)
    assert osv.cell(6, 2) == Cell(Decimal("1500.25"), "number")
    assert osv.cell(7, 1) == Cell("   Контрагент", "text", indent=3)
    assert osv.cell(7, 2) == Cell(datetime.datetime(2024, 9, 30), "date")
    assert osv.cell(10, 3) == Cell(True, "bool")
    assert osv.cell(9, 1) is EMPTY_CELL
    assert osv.cell(9, 2) is EMPTY_CELL
    assert (9, 1) not in osv.cells
    assert (9, 2) not in osv.cells
    assert osv.is_row_empty(8)
    assert osv.is_row_empty(9)
    assert osv.text_at(3, 5) == ""
    assert osv.merged_origin(3, 5) == (3, 4)
    assert set(osv.merged) == {MergedRange(3, 1, 4, 1), MergedRange(3, 2, 3, 3), MergedRange(3, 4, 3, 5)}
    assert osv.text_at(4, 1) == "Счет"
    assert osv.text_at(3, 3) == "Сальдо на начало периода"
    assert [osv.outline(row) for row in range(1, 11)] == [0, 0, 0, 0, 0, 1, 2, 0, 0, 0]
    assert (osv.n_rows, osv.n_cols) == (10, 5)
    assert all(cell.kind != "empty" for cell in osv.cells.values())
    assert sheets[1].row_cells(1) == [(1, Cell("x", "text"))]


def test_file_type_mismatch_reads_by_signature_and_reports_mismatch() -> None:
    read = read_book_detected(_synthetic_xlsx(), "XLS")

    assert [sheet.name for sheet in read.sheets] == ["ОСВ", "Прочее"]
    assert read.mismatch == "file_type=XLS, но содержимое OOXML: прочитано как XLSX"
    assert read.warnings == []  # несовпадение — не заметка читателя: книга делает из него предупреждение файла


def test_garbage_bytes_are_unsupported_format() -> None:
    with pytest.raises(UnsupportedFormatError) as caught:
        read_book(b"%PDF-1.7 not a workbook", "XLSX")

    assert caught.value.code is ErrorCode.UNSUPPORTED_FORMAT
    assert isinstance(caught.value, ParserError)


def test_truncated_zip_is_unreadable_with_one_line_reason() -> None:
    data = _synthetic_xlsx()

    with pytest.raises(UnreadableError) as caught:
        read_book(data[: len(data) // 2], "XLSX")

    assert caught.value.code is ErrorCode.UNREADABLE
    assert caught.value.message
    assert "\n" not in caught.value.message


def test_truncated_biff_is_unreadable() -> None:
    with pytest.raises(UnreadableError):
        read_book(bytes.fromhex("D0CF11E0A1B11AE1") + b"\x00" * 64, "XLS")


def _zip_with(name: str) -> bytes:
    buffer = io.BytesIO()
    with zipfile.ZipFile(buffer, "w") as archive:
        archive.writestr(name, b"\x00")
    return buffer.getvalue()


class _XlsbCell(NamedTuple):
    r: int
    c: int
    v: object


class _FakeXlsbSheet:
    def __init__(self, rows: list[list[_XlsbCell]]) -> None:
        self._rows = rows

    def __enter__(self) -> "_FakeXlsbSheet":
        return self

    def __exit__(self, *exc: object) -> None:
        return None

    def rows(self, sparse: bool = False) -> Iterator[list[_XlsbCell]]:
        assert sparse
        yield from self._rows


class _FakeXlsbBook:
    sheets = ("Лист1",)

    def __enter__(self) -> "_FakeXlsbBook":
        return self

    def __exit__(
        self, exc_type: type[BaseException] | None, exc: BaseException | None, tb: TracebackType | None
    ) -> None:
        return None

    def get_sheet(self, name: str) -> _FakeXlsbSheet:
        assert name == "Лист1"
        return _FakeXlsbSheet(
            [
                [_XlsbCell(0, 0, "Счет"), _XlsbCell(0, 1, None), _XlsbCell(0, 2, "  ")],
                [_XlsbCell(2, 0, "62.01"), _XlsbCell(2, 1, 0.1), _XlsbCell(2, 2, False)],
            ]
        )


def test_xlsb_signature_is_read_by_pyxlsb_without_outline(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setattr(xlsb_reader.pyxlsb, "open_workbook", lambda _stream: _FakeXlsbBook())

    sheets, warnings = read_book(_zip_with("xl/workbook.bin"), "XLSB")

    assert warnings == ["xlsb: outline level недоступен"]
    (sheet,) = sheets
    assert sheet.name == "Лист1"
    assert sheet.cell(1, 1) == Cell("Счет", "text")
    assert sheet.cell(1, 3) is EMPTY_CELL
    assert (1, 3) not in sheet.cells
    assert sheet.cell(3, 2) == Cell(Decimal("0.1"), "number")
    assert sheet.cell(3, 3) == Cell(False, "bool")
    assert sheet.merged == ()
    assert sheet.row_outline == {}
    assert (sheet.n_rows, sheet.n_cols) == (3, 3)


def test_zip_without_workbook_bin_goes_to_openpyxl_and_is_unreadable() -> None:
    with pytest.raises(UnreadableError):
        read_book(_zip_with("hello.txt"), "XLSX")


def _renamed_parts(data: bytes, renames: dict[str, str]) -> bytes:
    """Та же книга, у частей из `renames` — другое имя в архиве; ссылки `[Content_Types].xml` и `*.rels` прежние."""
    buffer = io.BytesIO()
    with zipfile.ZipFile(io.BytesIO(data)) as source, zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as target:
        for info in source.infolist():
            target.writestr(renames.get(info.filename, info.filename), source.read(info))
    return buffer.getvalue()


def _xlsx_with_shared_strings(part_name: str) -> bytes:
    """Минимальная книга с таблицей общих строк (openpyxl пишет строки inline) под именем `part_name` в архиве;
    `[Content_Types].xml` и `workbook.xml.rels` ссылаются на `xl/sharedStrings.xml`."""
    main = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"
    rel = "http://schemas.openxmlformats.org/officeDocument/2006/relationships"
    ct = "application/vnd.openxmlformats-officedocument.spreadsheetml"
    parts = {
        "[Content_Types].xml": (
            '<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">'
            '<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>'
            '<Default Extension="xml" ContentType="application/xml"/>'
            f'<Override PartName="/xl/workbook.xml" ContentType="{ct}.sheet.main+xml"/>'
            f'<Override PartName="/xl/worksheets/sheet1.xml" ContentType="{ct}.worksheet+xml"/>'
            f'<Override PartName="/xl/sharedStrings.xml" ContentType="{ct}.sharedStrings+xml"/>'
            "</Types>"
        ),
        "_rels/.rels": (
            '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'
            f'<Relationship Id="rId1" Type="{rel}/officeDocument" Target="xl/workbook.xml"/>'
            "</Relationships>"
        ),
        "xl/workbook.xml": (
            f'<workbook xmlns="{main}" xmlns:r="{rel}">'
            '<sheets><sheet name="ОСВ" sheetId="1" r:id="rId1"/></sheets></workbook>'
        ),
        "xl/_rels/workbook.xml.rels": (
            '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'
            f'<Relationship Id="rId1" Type="{rel}/worksheet" Target="worksheets/sheet1.xml"/>'
            f'<Relationship Id="rId2" Type="{rel}/sharedStrings" Target="sharedStrings.xml"/>'
            "</Relationships>"
        ),
        "xl/worksheets/sheet1.xml": (
            f'<worksheet xmlns="{main}"><sheetData>'
            '<row r="1"><c r="A1" t="s"><v>0</v></c><c r="B1"><v>1500</v></c></row>'
            "</sheetData></worksheet>"
        ),
        part_name: f'<sst xmlns="{main}" count="1" uniqueCount="1"><si><t>Счет 62</t></si></sst>',
    }
    buffer = io.BytesIO()
    with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive:
        for name, xml in parts.items():
            archive.writestr(name, '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>' + xml)
    return buffer.getvalue()


def test_shared_strings_part_name_is_case_insensitive_like_in_excel() -> None:
    """Некоторые выгрузки кладут таблицу строк как `xl/SharedStrings.xml` (AURORA-1707); Excel их открывает."""
    sheets, warnings = read_book(_xlsx_with_shared_strings("xl/SharedStrings.xml"), "XLSX")

    assert (sheets, warnings) == read_book(_xlsx_with_shared_strings("xl/sharedStrings.xml"), "XLSX")
    assert sheets[0].cell(1, 1) == Cell("Счет 62", "text")


def test_two_parts_matching_one_reference_are_not_guessed() -> None:
    data = _xlsx_with_shared_strings("xl/SharedStrings.xml")
    buffer = io.BytesIO(data)
    with zipfile.ZipFile(buffer, "a") as archive:
        archive.writestr("xl/SHAREDSTRINGS.xml", archive.read("xl/SharedStrings.xml"))

    with pytest.raises(UnreadableError, match="sharedStrings.xml"):
        read_book(buffer.getvalue(), "XLSX")


def test_part_found_by_relationship_is_case_insensitive_too() -> None:
    """Лист openpyxl находит по ссылке из `workbook.xml.rels`, стили — по `[Content_Types].xml`."""
    original = _synthetic_xlsx()
    renames = {"xl/worksheets/sheet1.xml": "xl/worksheets/Sheet1.xml", "xl/styles.xml": "XL/Styles.XML"}

    assert read_book(_renamed_parts(original, renames), "XLSX") == read_book(original, "XLSX")


@pytest.mark.parametrize(
    ("value", "expected_text"),
    [
        (2370012036.0, "2370012036"),  # xls хранит число как float: «.0» — артефакт чтения (OQ-162, #32)
        (501604329883.0, "501604329883"),
        (-15.0, "-15"),
        (1234.56, "1234.56"),
        (7706808138, "7706808138"),
    ],
)
def test_integral_float_is_read_without_fraction(value: float, expected_text: str) -> None:
    cell = make_cell(value)

    assert cell is not None
    assert (cell.kind, cell.text) == ("number", expected_text)
    assert cell.value == Decimal(str(value))


# --- xls без записи CODEPAGE (AURORA-1662) ---

_CYRILLIC_CELLS = {(1, 1): "Оборотно-сальдовая ведомость", (2, 2): Decimal("12.5")}


def test_xls_without_codepage_is_read_as_cp1251_with_flag() -> None:
    data = write_xls({"ОСВ 60": _CYRILLIC_CELLS}, codepage=None)

    read = read_book_detected(data, "XLS")

    assert [sheet.name for sheet in read.sheets] == ["ОСВ 60"]
    assert read.sheets[0].text_at(1, 1) == "Оборотно-сальдовая ведомость"
    assert read.no_codepage is True
    assert read.warnings == []
    assert NO_CODEPAGE_WARNING == "xls без кодовой страницы — строки прочитаны как cp1251"


def test_xls_with_codepage_is_read_as_before_without_warning() -> None:
    data = write_xls({"ОСВ 60": _CYRILLIC_CELLS}, codepage=1251)

    read = read_book_detected(data, "XLS")

    assert read.sheets[0].text_at(1, 1) == "Оборотно-сальдовая ведомость"
    assert read.no_codepage is False
    assert read.warnings == []