chore(arm_statements): копия arm-statements@25b2bb8 (#50)
Копия api/arm_statements: 3390f92..25b2bb8 (1 коммитов), скрипт docs/scripts/sync-statements.sh. Коммиты arm-statements: - 72e2e72 fix(#8): шаг 2 — нумерация со сбоем OCR под заголовком, повтор шапки со склеенной нумерацией (AURORA-1517) statements-check scan-gateway: 0 — копия синхронна с arm-statements@origin/main (25b2bb8) Co-Authored-By: sync-statements.sh <noreply@aepif.ru>
This commit is contained in:
Andrey Epifancev
parent
b8447581ae
commit
55dbf8d16d
@ -1 +1 @@
|
||||
3390f92daddbfd4919d7f799ad2959cc35bfbd92
|
||||
25b2bb81195bb1e99f73b90e336cd175ae8b344c
|
||||
|
||||
@ -17,6 +17,7 @@ _MIN_NUMBERING_CELLS = 3
|
||||
_SHORT_NOTE = 3 # «Пояснения» без подписи: значения не длиннее 3 символов
|
||||
_SKIP_STEP = 2 # скачок через одну пропущенную цифру нумерации
|
||||
_NUMBER = re.compile(r"[0-9]+")
|
||||
_NUMBERING_TAIL = re.compile(r"\s+[0-9]{1,2}$")
|
||||
_FULL_CONFIDENCE = 100.0
|
||||
|
||||
|
||||
@ -24,6 +25,11 @@ def _text(cell: Cell | None) -> str:
|
||||
return "" if cell is None else cell.text.strip()
|
||||
|
||||
|
||||
def caption_text(text: str) -> str:
|
||||
"""Подпись шапки для сравнения: `normalize_caption` без номера столбца, склеенного снизу («Коды\\n2» → «коды»)."""
|
||||
return _NUMBERING_TAIL.sub("", normalize_caption(text))
|
||||
|
||||
|
||||
def cell_at(rows: Grid, row: int, col: int) -> Cell | None:
|
||||
"""Ячейка сетки; вне строки или сетки — `None`."""
|
||||
if row < 0 or row >= len(rows) or col < 0:
|
||||
@ -138,17 +144,27 @@ def first_code_row(rows: Grid, first: int, last: int, code_col: int, rules: Rule
|
||||
return None
|
||||
|
||||
|
||||
def is_numbering_row(cells: Sequence[Cell | None], rules: Rules) -> bool:
|
||||
"""≥ 3 непустых ячеек-целых, слева направо +1, начало 1 или 2, не больше одного пропуска."""
|
||||
def is_numbering_row(cells: Sequence[Cell | None], rules: Rules, *, noisy: bool = False) -> bool:
|
||||
"""≥ 3 непустых ячеек-целых, слева направо +1, начало 1 или 2, не больше одного пропуска.
|
||||
|
||||
`noisy` — строка сразу под шапкой (AURORA-1517): ячейка со сбоем OCR даёт своё единственное целое («• ■ 1» → 1,
|
||||
«3гола» → 3), ячейка без цифр («/», «V») — пропуск, два целых в ячейке — не нумерация.
|
||||
"""
|
||||
numbers: list[int] = []
|
||||
for cell in cells:
|
||||
text = _text(cell)
|
||||
if not text:
|
||||
continue
|
||||
fixed = fix_lookalikes(text, rules.lookalikes, anywhere=True).strip()
|
||||
if _NUMBER.fullmatch(fixed) is None:
|
||||
if _NUMBER.fullmatch(fixed) is not None:
|
||||
numbers.append(int(fixed))
|
||||
continue
|
||||
if not noisy:
|
||||
return False
|
||||
numbers.append(int(fixed))
|
||||
found = _NUMBER.findall(text)
|
||||
if len(found) > 1:
|
||||
return False
|
||||
numbers.extend(int(number) for number in found)
|
||||
if len(numbers) < _MIN_NUMBERING_CELLS or numbers[0] not in (1, 2):
|
||||
return False
|
||||
skips = 0
|
||||
@ -212,7 +228,7 @@ def find_header_rows(rows: Grid, first: int, first_code: int, code_col: int, rul
|
||||
"""Строка заголовка (одна или две, склеенные); не найдена — `()`."""
|
||||
candidates = [r for r in range(first, first_code) if not is_numbering_row(rows[r], rules)]
|
||||
captioned = [
|
||||
r for r in candidates if rules.headers.code_caption.search(normalize_caption(_text(cell_at(rows, r, code_col))))
|
||||
r for r in candidates if rules.headers.code_caption.search(caption_text(_text(cell_at(rows, r, code_col))))
|
||||
]
|
||||
header: int | None = captioned[-1] if captioned else None
|
||||
if header is None:
|
||||
@ -229,15 +245,17 @@ def find_header_rows(rows: Grid, first: int, first_code: int, code_col: int, rul
|
||||
|
||||
|
||||
def segment_roles(rows: Grid, first: int, last: int, header_rows: tuple[int, ...], rules: Rules) -> list[RowRole]:
|
||||
"""Роли строк `first..last`: выше заголовка — TITLE (только при найденном заголовке)."""
|
||||
"""Роли строк `first..last`: выше заголовка — TITLE (только при найденном заголовке); строка сразу под заголовком —
|
||||
нумерация и при сбое OCR (`is_numbering_row(noisy=True)`)."""
|
||||
roles: list[RowRole] = []
|
||||
top = min(header_rows) if header_rows else None
|
||||
under = max(header_rows) + 1 if header_rows else None
|
||||
for row in range(first, last + 1):
|
||||
if row in header_rows:
|
||||
roles.append(RowRole.HEADER)
|
||||
elif top is not None and row < top:
|
||||
roles.append(RowRole.TITLE)
|
||||
elif is_numbering_row(rows[row], rules):
|
||||
elif is_numbering_row(rows[row], rules, noisy=row == under):
|
||||
roles.append(RowRole.NUMBERING)
|
||||
else:
|
||||
roles.append(RowRole.DATA)
|
||||
@ -251,7 +269,7 @@ def _caption_column(
|
||||
for col, cell in enumerate(rows[row]):
|
||||
if col == code_col or (left_of_code and col > code_col):
|
||||
continue
|
||||
if pattern.search(normalize_caption(_text(cell))):
|
||||
if pattern.search(caption_text(_text(cell))):
|
||||
return col
|
||||
return None
|
||||
|
||||
|
||||
@ -18,6 +18,7 @@ from arm_statements.columns import (
|
||||
from arm_statements.issues import WarningCode, make_warning
|
||||
from arm_statements.layout import (
|
||||
Grid,
|
||||
caption_text,
|
||||
cell_at,
|
||||
find_code_column,
|
||||
find_header_rows,
|
||||
@ -35,7 +36,6 @@ from arm_statements.model import (
|
||||
StatementHeader,
|
||||
StatementWarning,
|
||||
)
|
||||
from arm_statements.text import normalize_caption
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@ -100,7 +100,7 @@ def _chunks(rows: Grid, breaks: Sequence[int]) -> list[tuple[int, int]]:
|
||||
|
||||
|
||||
def _caption_cells(rows: Grid, header_rows: Sequence[int]) -> frozenset[str]:
|
||||
cells = (normalize_caption(_text(rows, r, c)) for r in header_rows for c in range(len(rows[r])))
|
||||
cells = (caption_text(_text(rows, r, c)) for r in header_rows for c in range(len(rows[r])))
|
||||
return frozenset(c for c in cells if c)
|
||||
|
||||
|
||||
@ -110,7 +110,7 @@ def _is_repeat_header(rows: Grid, row: int, code_col: int | None, reference: fro
|
||||
return False
|
||||
if code_col is not None and is_code_cell(_text(rows, row, code_col), rules):
|
||||
return False
|
||||
normalized = [normalize_caption(c.text) for c in rows[row] if c is not None and c.text.strip()]
|
||||
normalized = [caption_text(c.text) for c in rows[row] if c is not None and c.text.strip()]
|
||||
if any(rules.headers.code_caption.search(text) for text in normalized):
|
||||
return True
|
||||
return sum(1 for text in normalized if text and text in reference) >= _MIN_SAME_CELLS
|
||||
|
||||
@ -215,3 +215,45 @@ def test_headerless_continuation_sheet_with_outside_numbers(rules: Rules) -> Non
|
||||
assert layout.warnings[0].locator == "Лист1!F2\nЛист2!F1"
|
||||
assert layout.segments[1].value_cols == ((3, 3), (4, 4))
|
||||
assert layout.segments[1].extra_cols == ()
|
||||
|
||||
|
||||
BALANCE_CAPTIONS = ["Наименование показателя", "Коды", "На 30 июня\n2026 года", "На 31 декабря\n2025 года"]
|
||||
ASSETS = [["Запасы", "1210", "5", "4"], ["Дебиторская задолженность", "1230", "3", "2"], ["Итого", "1200", "8", "6"]]
|
||||
LIABILITIES = [["Заемные средства", "1510", "3", "2"], ["Кредиторская задолженность", "1520", "5", "4"]]
|
||||
|
||||
|
||||
def test_numbering_with_ocr_noise_right_under_header(rules: Rules) -> None:
|
||||
"""AURORA-1517: нумерация под шапкой со сбоем OCR («/» вместо 1, мусор, протёкшее «года», «V» вместо 4)."""
|
||||
for numbering in (
|
||||
["/", "2", "3", "4"],
|
||||
["• ■ 1", "2", "3", "4"],
|
||||
["1", "2", "3гола", "гола\n4"],
|
||||
["1", "2", "3", "V"],
|
||||
):
|
||||
layout = find_layout(ocr([BALANCE_CAPTIONS, numbering, *ASSETS]), parse_header(hdr(), rules), rules)
|
||||
assert layout.roles[1] is RowRole.NUMBERING, numbering
|
||||
|
||||
|
||||
def test_numbering_with_ocr_noise_under_repeat_header(rules: Rules) -> None:
|
||||
"""AURORA-1517, «Светлый путь»: «• ■ 1 2 3 4 5» под повторной шапкой второй страницы."""
|
||||
rows = [BALANCE_CAPTIONS, ["1", "2", "3", "4"], *ASSETS, BALANCE_CAPTIONS, ["• ■ 1", "2", "3", "4"], *LIABILITIES]
|
||||
layout = find_layout(ocr(rows), parse_header(hdr(), rules), rules)
|
||||
assert [s.first_row for s in layout.segments] == [0, 5]
|
||||
assert layout.roles[6] is RowRole.NUMBERING and layout.roles[7] is RowRole.DATA
|
||||
|
||||
|
||||
def test_noisy_numbering_elsewhere_stays_data(rules: Rules) -> None:
|
||||
"""Мягкая проверка — только сразу под шапкой; строка с кодом формы — не нумерация."""
|
||||
rows = [BALANCE_CAPTIONS, ["Запасы 1", "1210", "2", "3"], *ASSETS, ["/", "2", "3", "4"], *LIABILITIES]
|
||||
layout = find_layout(ocr(rows), parse_header(hdr(), rules), rules)
|
||||
assert layout.roles[1] is RowRole.DATA and layout.roles[5] is RowRole.DATA
|
||||
|
||||
|
||||
def test_repeat_header_glued_with_numbering(rules: Rules) -> None:
|
||||
"""AURORA-1517, «Будамшинское»: повторная шапка, в каждой подписи которой снизу номер столбца."""
|
||||
glued = ["Наименование показателя\n1", "Коды\n2", "На 30 июня\n2026 года\n3", "На 31 декабря\n2025 года\n4"]
|
||||
rows = [BALANCE_CAPTIONS, ["1", "2", "3", "4"], *ASSETS, glued, *LIABILITIES]
|
||||
layout = find_layout(ocr(rows), parse_header(hdr(), rules), rules)
|
||||
assert [s.first_row for s in layout.segments] == [0, 5]
|
||||
assert layout.roles[5] is RowRole.HEADER and layout.roles[6] is RowRole.DATA
|
||||
assert layout.segments[1].value_cols == ((2, 2), (3, 3))
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user