From 55dbf8d16d7ad4c5e00741e16e1c98b79d337b07 Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Thu, 22 Oct 2026 16:45:32 +0400 Subject: [PATCH] =?UTF-8?q?chore(arm=5Fstatements):=20=D0=BA=D0=BE=D0=BF?= =?UTF-8?q?=D0=B8=D1=8F=20arm-statements@25b2bb8=20(#50)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Копия api/arm_statements: 3390f92..25b2bb8 (1 коммитов), скрипт docs/scripts/sync-statements.sh. Коммиты arm-statements: - 72e2e72 fix(#8): шаг 2 — нумерация со сбоем OCR под заголовком, повтор шапки со склеенной нумерацией (AURORA-1517) statements-check scan-gateway: 0 — копия синхронна с arm-statements@origin/main (25b2bb8) Co-Authored-By: sync-statements.sh --- api/arm_statements/VERSION | 2 +- api/arm_statements/layout.py | 34 +++++++++++++----- api/arm_statements/segments.py | 6 ++-- api/arm_statements/tests/test_segments.py | 42 +++++++++++++++++++++++ 4 files changed, 72 insertions(+), 12 deletions(-) diff --git a/api/arm_statements/VERSION b/api/arm_statements/VERSION index efc6084..58f755c 100644 --- a/api/arm_statements/VERSION +++ b/api/arm_statements/VERSION @@ -1 +1 @@ -3390f92daddbfd4919d7f799ad2959cc35bfbd92 +25b2bb81195bb1e99f73b90e336cd175ae8b344c diff --git a/api/arm_statements/layout.py b/api/arm_statements/layout.py index 3f86a5b..9e9dc86 100644 --- a/api/arm_statements/layout.py +++ b/api/arm_statements/layout.py @@ -17,6 +17,7 @@ _MIN_NUMBERING_CELLS = 3 _SHORT_NOTE = 3 # «Пояснения» без подписи: значения не длиннее 3 символов _SKIP_STEP = 2 # скачок через одну пропущенную цифру нумерации _NUMBER = re.compile(r"[0-9]+") +_NUMBERING_TAIL = re.compile(r"\s+[0-9]{1,2}$") _FULL_CONFIDENCE = 100.0 @@ -24,6 +25,11 @@ def _text(cell: Cell | None) -> str: return "" if cell is None else cell.text.strip() +def caption_text(text: str) -> str: + """Подпись шапки для сравнения: `normalize_caption` без номера столбца, склеенного снизу («Коды\\n2» → «коды»).""" + return _NUMBERING_TAIL.sub("", normalize_caption(text)) + + def cell_at(rows: Grid, row: int, col: int) -> Cell | None: """Ячейка сетки; вне строки или сетки — `None`.""" if row < 0 or row >= len(rows) or col < 0: @@ -138,17 +144,27 @@ def first_code_row(rows: Grid, first: int, last: int, code_col: int, rules: Rule return None -def is_numbering_row(cells: Sequence[Cell | None], rules: Rules) -> bool: - """≥ 3 непустых ячеек-целых, слева направо +1, начало 1 или 2, не больше одного пропуска.""" +def is_numbering_row(cells: Sequence[Cell | None], rules: Rules, *, noisy: bool = False) -> bool: + """≥ 3 непустых ячеек-целых, слева направо +1, начало 1 или 2, не больше одного пропуска. + + `noisy` — строка сразу под шапкой (AURORA-1517): ячейка со сбоем OCR даёт своё единственное целое («• ■ 1» → 1, + «3гола» → 3), ячейка без цифр («/», «V») — пропуск, два целых в ячейке — не нумерация. + """ numbers: list[int] = [] for cell in cells: text = _text(cell) if not text: continue fixed = fix_lookalikes(text, rules.lookalikes, anywhere=True).strip() - if _NUMBER.fullmatch(fixed) is None: + if _NUMBER.fullmatch(fixed) is not None: + numbers.append(int(fixed)) + continue + if not noisy: return False - numbers.append(int(fixed)) + found = _NUMBER.findall(text) + if len(found) > 1: + return False + numbers.extend(int(number) for number in found) if len(numbers) < _MIN_NUMBERING_CELLS or numbers[0] not in (1, 2): return False skips = 0 @@ -212,7 +228,7 @@ def find_header_rows(rows: Grid, first: int, first_code: int, code_col: int, rul """Строка заголовка (одна или две, склеенные); не найдена — `()`.""" candidates = [r for r in range(first, first_code) if not is_numbering_row(rows[r], rules)] captioned = [ - r for r in candidates if rules.headers.code_caption.search(normalize_caption(_text(cell_at(rows, r, code_col)))) + r for r in candidates if rules.headers.code_caption.search(caption_text(_text(cell_at(rows, r, code_col)))) ] header: int | None = captioned[-1] if captioned else None if header is None: @@ -229,15 +245,17 @@ def find_header_rows(rows: Grid, first: int, first_code: int, code_col: int, rul def segment_roles(rows: Grid, first: int, last: int, header_rows: tuple[int, ...], rules: Rules) -> list[RowRole]: - """Роли строк `first..last`: выше заголовка — TITLE (только при найденном заголовке).""" + """Роли строк `first..last`: выше заголовка — TITLE (только при найденном заголовке); строка сразу под заголовком — + нумерация и при сбое OCR (`is_numbering_row(noisy=True)`).""" roles: list[RowRole] = [] top = min(header_rows) if header_rows else None + under = max(header_rows) + 1 if header_rows else None for row in range(first, last + 1): if row in header_rows: roles.append(RowRole.HEADER) elif top is not None and row < top: roles.append(RowRole.TITLE) - elif is_numbering_row(rows[row], rules): + elif is_numbering_row(rows[row], rules, noisy=row == under): roles.append(RowRole.NUMBERING) else: roles.append(RowRole.DATA) @@ -251,7 +269,7 @@ def _caption_column( for col, cell in enumerate(rows[row]): if col == code_col or (left_of_code and col > code_col): continue - if pattern.search(normalize_caption(_text(cell))): + if pattern.search(caption_text(_text(cell))): return col return None diff --git a/api/arm_statements/segments.py b/api/arm_statements/segments.py index 9bdce13..821925e 100644 --- a/api/arm_statements/segments.py +++ b/api/arm_statements/segments.py @@ -18,6 +18,7 @@ from arm_statements.columns import ( from arm_statements.issues import WarningCode, make_warning from arm_statements.layout import ( Grid, + caption_text, cell_at, find_code_column, find_header_rows, @@ -35,7 +36,6 @@ from arm_statements.model import ( StatementHeader, StatementWarning, ) -from arm_statements.text import normalize_caption @dataclass(frozen=True, slots=True) @@ -100,7 +100,7 @@ def _chunks(rows: Grid, breaks: Sequence[int]) -> list[tuple[int, int]]: def _caption_cells(rows: Grid, header_rows: Sequence[int]) -> frozenset[str]: - cells = (normalize_caption(_text(rows, r, c)) for r in header_rows for c in range(len(rows[r]))) + cells = (caption_text(_text(rows, r, c)) for r in header_rows for c in range(len(rows[r]))) return frozenset(c for c in cells if c) @@ -110,7 +110,7 @@ def _is_repeat_header(rows: Grid, row: int, code_col: int | None, reference: fro return False if code_col is not None and is_code_cell(_text(rows, row, code_col), rules): return False - normalized = [normalize_caption(c.text) for c in rows[row] if c is not None and c.text.strip()] + normalized = [caption_text(c.text) for c in rows[row] if c is not None and c.text.strip()] if any(rules.headers.code_caption.search(text) for text in normalized): return True return sum(1 for text in normalized if text and text in reference) >= _MIN_SAME_CELLS diff --git a/api/arm_statements/tests/test_segments.py b/api/arm_statements/tests/test_segments.py index 3f18448..c5c8fa2 100644 --- a/api/arm_statements/tests/test_segments.py +++ b/api/arm_statements/tests/test_segments.py @@ -215,3 +215,45 @@ def test_headerless_continuation_sheet_with_outside_numbers(rules: Rules) -> Non assert layout.warnings[0].locator == "Лист1!F2\nЛист2!F1" assert layout.segments[1].value_cols == ((3, 3), (4, 4)) assert layout.segments[1].extra_cols == () + + +BALANCE_CAPTIONS = ["Наименование показателя", "Коды", "На 30 июня\n2026 года", "На 31 декабря\n2025 года"] +ASSETS = [["Запасы", "1210", "5", "4"], ["Дебиторская задолженность", "1230", "3", "2"], ["Итого", "1200", "8", "6"]] +LIABILITIES = [["Заемные средства", "1510", "3", "2"], ["Кредиторская задолженность", "1520", "5", "4"]] + + +def test_numbering_with_ocr_noise_right_under_header(rules: Rules) -> None: + """AURORA-1517: нумерация под шапкой со сбоем OCR («/» вместо 1, мусор, протёкшее «года», «V» вместо 4).""" + for numbering in ( + ["/", "2", "3", "4"], + ["• ■ 1", "2", "3", "4"], + ["1", "2", "3гола", "гола\n4"], + ["1", "2", "3", "V"], + ): + layout = find_layout(ocr([BALANCE_CAPTIONS, numbering, *ASSETS]), parse_header(hdr(), rules), rules) + assert layout.roles[1] is RowRole.NUMBERING, numbering + + +def test_numbering_with_ocr_noise_under_repeat_header(rules: Rules) -> None: + """AURORA-1517, «Светлый путь»: «• ■ 1 2 3 4 5» под повторной шапкой второй страницы.""" + rows = [BALANCE_CAPTIONS, ["1", "2", "3", "4"], *ASSETS, BALANCE_CAPTIONS, ["• ■ 1", "2", "3", "4"], *LIABILITIES] + layout = find_layout(ocr(rows), parse_header(hdr(), rules), rules) + assert [s.first_row for s in layout.segments] == [0, 5] + assert layout.roles[6] is RowRole.NUMBERING and layout.roles[7] is RowRole.DATA + + +def test_noisy_numbering_elsewhere_stays_data(rules: Rules) -> None: + """Мягкая проверка — только сразу под шапкой; строка с кодом формы — не нумерация.""" + rows = [BALANCE_CAPTIONS, ["Запасы 1", "1210", "2", "3"], *ASSETS, ["/", "2", "3", "4"], *LIABILITIES] + layout = find_layout(ocr(rows), parse_header(hdr(), rules), rules) + assert layout.roles[1] is RowRole.DATA and layout.roles[5] is RowRole.DATA + + +def test_repeat_header_glued_with_numbering(rules: Rules) -> None: + """AURORA-1517, «Будамшинское»: повторная шапка, в каждой подписи которой снизу номер столбца.""" + glued = ["Наименование показателя\n1", "Коды\n2", "На 30 июня\n2026 года\n3", "На 31 декабря\n2025 года\n4"] + rows = [BALANCE_CAPTIONS, ["1", "2", "3", "4"], *ASSETS, glued, *LIABILITIES] + layout = find_layout(ocr(rows), parse_header(hdr(), rules), rules) + assert [s.first_row for s in layout.segments] == [0, 5] + assert layout.roles[5] is RowRole.HEADER and layout.roles[6] is RowRole.DATA + assert layout.segments[1].value_cols == ((2, 2), (3, 3))