update from current prod #8
195
api/src/core/PDFWorker/date_utils.py
Normal file
195
api/src/core/PDFWorker/date_utils.py
Normal file
@ -0,0 +1,195 @@
|
||||
import re
|
||||
from datetime import date, timedelta
|
||||
from typing import Optional
|
||||
|
||||
|
||||
# буквы, которые OCR ставит вместо цифр — заменяем только рядом с цифрами,
|
||||
# чтобы не портить нормальные слова ("Общество" остаётся словом)
|
||||
_LOOKALIKES = str.maketrans({
|
||||
"О": "0", "о": "0", "O": "0", "o": "0",
|
||||
"З": "3", "з": "3",
|
||||
"l": "1", "I": "1", "|": "1", "і": "1",
|
||||
"б": "6", "S": "5", "s": "5", "Ч": "4",
|
||||
})
|
||||
_LOOKALIKE_CHARS = "ОоOoЗзlI|ібSsЧ"
|
||||
_NEAR_DIGIT = re.compile(rf"(?<=\d)[{_LOOKALIKE_CHARS}]|[{_LOOKALIKE_CHARS}](?=\d)")
|
||||
|
||||
_ROMAN = {"I": 1, "II": 2, "III": 3, "IV": 4, "1V": 4, "11": 2, "111": 3}
|
||||
_ORDINAL = {
|
||||
"перв": 1, "втор": 2, "трет": 3, "четв": 4,
|
||||
}
|
||||
|
||||
|
||||
def _normalize(s: str) -> str:
|
||||
s = s.replace(" ", " ")
|
||||
# "|" как разделитель клеток формы: "2|0|2|4|" — это НЕ единица, а
|
||||
# разделитель; но "3l.12" — единица. Считаем | разделителем, если он
|
||||
# стоит между одиночными цифрами/пробелами по всей строке.
|
||||
if re.fullmatch(r"[\s\d|•·.:,]*", s.split("\n", 1)[0]):
|
||||
s = s.replace("|", " ")
|
||||
prev = None
|
||||
while prev != s: # "2ОО5" -> нужно два прохода
|
||||
prev = s
|
||||
s = _NEAR_DIGIT.sub(lambda m: m.group(0).translate(_LOOKALIKES), s)
|
||||
return s
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Месяцы
|
||||
# ----------------------------------------------------------------------------
|
||||
_MONTHS = {
|
||||
"январ": 1, "феврал": 2, "март": 3, "апрел": 4, "ма": 5, "июн": 6,
|
||||
"июл": 7, "август": 8, "сентябр": 9, "октябр": 10, "ноябр": 11, "декабр": 12,
|
||||
}
|
||||
|
||||
|
||||
def _month_num(word: str) -> Optional[int]:
|
||||
w = word.lower()
|
||||
for stem, n in _MONTHS.items():
|
||||
if w.startswith(stem):
|
||||
return n
|
||||
return None
|
||||
|
||||
|
||||
def _month_end(year: int, month: int) -> date:
|
||||
if month == 12:
|
||||
return date(year, 12, 31)
|
||||
return date(year, month + 1, 1).replace(day=1) - timedelta(days=1)
|
||||
|
||||
|
||||
def _year_full(y: str) -> int:
|
||||
y = int(y)
|
||||
return y if y > 100 else 2000 + y # "25" -> 2025
|
||||
|
||||
|
||||
_Y = r"(\d{4}|\d{2})\s*(?:г\.?|r\.?|год\w*)?" # год + необязательное "г." / "r." / "года"
|
||||
_MONTH_WORD = r"([а-яё]{3,8})"
|
||||
|
||||
|
||||
def _qh(num_str: str) -> Optional[int]:
|
||||
"""Номер квартала/полугодия из '1', 'I', 'IV', 'первый', '1-й'."""
|
||||
t = num_str.strip().upper().rstrip("-Й")
|
||||
if t in _ROMAN:
|
||||
return _ROMAN[t]
|
||||
if t.isdigit():
|
||||
return int(t)
|
||||
for stem, n in _ORDINAL.items():
|
||||
if t.lower().startswith(stem):
|
||||
return n
|
||||
return None
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Паттерны. Порядок = приоритет.
|
||||
# ----------------------------------------------------------------------------
|
||||
_NUM = r"(I{1,3}V?|IV|\d|перв\w*|втор\w*|трет\w*|четв\w*)"
|
||||
|
||||
_PATTERNS = [
|
||||
# --- диапазон "с 01.01.2025 по 30.09.2025": берём конец
|
||||
(re.compile(r"по\s*(\d{1,2})[.\-/ ](\d{1,2})[.\-/ ](\d{4}|\d{2})", re.I),
|
||||
lambda m: _safe(_year_full(m[3]), int(m[2]), int(m[1]))),
|
||||
# --- "31 декабря 2025 г." / "на 30 сентября 2025 года"
|
||||
(re.compile(rf"(\d{{1,2}})\s+{_MONTH_WORD}\s+{_Y}", re.I),
|
||||
lambda m: _safe(_year_full(m[3]), _month_num(m[2]), int(m[1]))),
|
||||
# --- "I квартал 2026", "1-й кв. 2026", "первый квартал"
|
||||
(re.compile(rf"{_NUM}\s*-?\s*(?:й|ый|ого|го)?\s*кв\w*\.?\s*{_Y}", re.I),
|
||||
lambda m: _period(m[2], (_qh(m[1]) or 0) * 3)),
|
||||
# --- "1 полугодие 2026", "первое полугодие", "1 п/г 2026"
|
||||
(re.compile(rf"{_NUM}\s*-?\s*(?:е|ое|го)?\s*(?:полугоди\w*|п/г)\.?\s*{_Y}", re.I),
|
||||
lambda m: _period(m[2], (_qh(m[1]) or 0) * 6)),
|
||||
# --- "9 месяцев 2025", "9 мес. 2025"
|
||||
(re.compile(rf"(\d{{1,2}})\s*мес\w*\.?\s*{_Y}", re.I),
|
||||
lambda m: _period(m[2], int(m[1]))),
|
||||
# --- "январь - сентябрь 2025": последний месяц
|
||||
(re.compile(rf"{_MONTH_WORD}\s*[-–—]\s*{_MONTH_WORD}\s*{_Y}", re.I),
|
||||
lambda m: _period(m[3], _month_num(m[2]))),
|
||||
# --- "за 2025 г." / "2025 год" / "2025г" / "2025 r."
|
||||
(re.compile(r"(?:за\s*)?(?:отч[её]тный\s*)?(\d{4})\s*(?:г\b|г\.|r\b|r\.|год)", re.I),
|
||||
lambda m: _period(m[1], 12)),
|
||||
]
|
||||
|
||||
|
||||
def _period(year_str: str, month: Optional[int]) -> Optional[date]:
|
||||
if not month or not 1 <= month <= 12:
|
||||
return None
|
||||
return _month_end(_year_full(year_str), month)
|
||||
|
||||
|
||||
def _safe(y: int, m: Optional[int], d: int) -> Optional[date]:
|
||||
try:
|
||||
return date(y, m, d) if m else None
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _text_date(s: str) -> Optional[date]:
|
||||
for rx, fn in _PATTERNS:
|
||||
m = rx.search(s)
|
||||
if m:
|
||||
d = fn(m)
|
||||
if d:
|
||||
return d
|
||||
return None
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Числовые формы
|
||||
# ----------------------------------------------------------------------------
|
||||
def _numeric_date(s: str) -> Optional[date]:
|
||||
groups = re.findall(r"\d+", s)
|
||||
if not groups:
|
||||
return None
|
||||
|
||||
# ровно три группы: "30.09.2025", "2025 03 31", "30/09/25"
|
||||
if len(groups) == 3:
|
||||
return _from_parts(*groups)
|
||||
|
||||
# иначе склеиваем цифры и смотрим на длину:
|
||||
# "2 0 2 4", "30 09 2 0 2 5", "3 0 0 9 2025", "| 3 | 0 | 0 | 9 | 2 | 0 | 2 | 5 |"
|
||||
digits = "".join(groups)
|
||||
if len(digits) == 8:
|
||||
return _from_digits(digits)
|
||||
if len(digits) == 4:
|
||||
return _year(int(digits))
|
||||
if len(digits) == 6: # "30 09 25"
|
||||
return _from_parts(digits[:2], digits[2:4], digits[4:])
|
||||
|
||||
# длинный хвост: "30.09.2025 0:00:00" — дата в первых трёх группах
|
||||
if len(groups) > 3:
|
||||
return _from_parts(*groups[:3])
|
||||
return None
|
||||
|
||||
|
||||
def _from_digits(g: str) -> Optional[date]:
|
||||
"""8 слитных цифр: ДДММГГГГ, ГГГГММДД, ГГГГДДММ — первый валидный вариант."""
|
||||
return (_from_parts(g[:2], g[2:4], g[4:]) # ДД ММ ГГГГ
|
||||
or _from_parts(g[6:], g[4:6], g[:4]) # ГГГГ ММ ДД
|
||||
or _from_parts(g[4:6], g[6:], g[:4])) # ГГГГ ДД ММ
|
||||
|
||||
|
||||
def _from_parts(a: str, b: str, c: str) -> Optional[date]:
|
||||
"""Пробуем DD MM YYYY и YYYY MM DD (год 2- или 4-значный только в позиции года)."""
|
||||
for d, m, y in ((a, b, c), (c, b, a)):
|
||||
if len(y) in (2, 4) and len(d) <= 2 and len(m) <= 2:
|
||||
try:
|
||||
return date(_year_full(y), int(m), int(d))
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _year(y: int) -> Optional[date]:
|
||||
"""Одинокий год -> 31.12 этого года (годовая отчётность)."""
|
||||
return date(y, 12, 31) if 1990 <= y <= 2100 else None
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
def _parse_date(value: str) -> Optional[date]:
|
||||
"""Вернуть дату или None, если это не дата (название компании и т.п.)."""
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
s = _normalize(value.strip())
|
||||
if not s:
|
||||
return None
|
||||
# текстовые формы ищем по всей строке, числовые — по первой (дальше мусор)
|
||||
return _text_date(s) or _numeric_date(s.split("\n", 1)[0]) or _numeric_date(s.replace("\n", " "))
|
||||
@ -1,13 +1,16 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
import re
|
||||
from dataclasses import dataclass, asdict
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Optional, Literal, TypedDict
|
||||
from loguru import logger
|
||||
|
||||
from src.schemas.enums import StatementType, FslRowKind
|
||||
from .consts import codes_mapping
|
||||
from .date_utils import _parse_date
|
||||
|
||||
@dataclass
|
||||
class Meta:
|
||||
@ -52,6 +55,32 @@ class ExtractResult(TypedDict):
|
||||
skipped_files: list[tuple[str, dict]]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ColumnLayout:
|
||||
"""Индексы значимых столбцов таблицы OCR: из строки заголовка или выведенные по строкам данных."""
|
||||
label_idx: Optional[int]
|
||||
code_idx: int
|
||||
value_idxs: tuple[int, ...]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ParsedRow:
|
||||
cells: dict[int, dict]
|
||||
layout: ColumnLayout
|
||||
kind: FslRowKind
|
||||
ordinal: int
|
||||
|
||||
|
||||
# column_index в БД: 0 - наименование, 1 - код, значения начинаются с 2
|
||||
_FIRST_VALUE_COLUMN = 2
|
||||
_YEAR_RE = re.compile(r"\d{4}")
|
||||
_BLANK_CHARS_RE = re.compile(r"[\s\-–—]")
|
||||
_HEADER_CODE = 'Код'
|
||||
NUM_RE = re.compile(r'[-+]?[\d.]+')
|
||||
ALLOWED_RE = re.compile(r'[^\d+\-.]')
|
||||
BRACKET_RE = re.compile(r'\(([^()]*)\)')
|
||||
|
||||
|
||||
_TABLE_TYPE_MAP: dict[str, StatementType] = {
|
||||
"Бухгалтерский баланс": StatementType.BALANCE,
|
||||
"Отчет о финансовых результатах": StatementType.INCOME,
|
||||
@ -82,40 +111,95 @@ def _is_recognized(confidence: float) -> bool:
|
||||
def _remove_spec_space(raw):
|
||||
return raw.replace('\n', ' ').replace('\t', ' ').strip()
|
||||
|
||||
def _parse_date(raw: str) -> date | None:
|
||||
# def _parse_date(raw: str) -> date | None:
|
||||
# try:
|
||||
# return datetime.strptime(raw, "%d %m %Y").date()
|
||||
# except:
|
||||
# return None
|
||||
|
||||
|
||||
def _parse_amount(raw: str):
|
||||
s = raw.strip().replace(',', '.').replace(' ', '')
|
||||
stripped_raw = s
|
||||
if not stripped_raw:
|
||||
return (None, stripped_raw == raw)
|
||||
m = BRACKET_RE.search(s)
|
||||
bracketed = bool(m)
|
||||
if bracketed:
|
||||
s = m.group(1)
|
||||
|
||||
s = ALLOWED_RE.sub('', s)
|
||||
|
||||
if s.count('.') > 1:
|
||||
*head, tail = s.split('.')
|
||||
s = ''.join(head) + tail
|
||||
|
||||
if bracketed:
|
||||
s = '-' + s.lstrip('+-')
|
||||
is_verified = (s == stripped_raw.replace(')', '').replace('(', ''))
|
||||
try:
|
||||
return datetime.strptime(raw, "%d %m %Y").date()
|
||||
return (Decimal(s), is_verified)
|
||||
except:
|
||||
return None
|
||||
|
||||
|
||||
def _parse_amount(raw: str) -> tuple[Optional[Decimal], bool]:
|
||||
stripped = raw.strip().replace(' ', '').replace('-', '')
|
||||
if not stripped:
|
||||
return (None, stripped == raw)
|
||||
if '(' in stripped and ')' in stripped:
|
||||
stripped = stripped.replace(')', '').replace('(', '-')
|
||||
to_parse = ''
|
||||
for c in stripped:
|
||||
if (c.isdigit() or c == '.' or c == '-'):
|
||||
to_parse += c
|
||||
is_verified = to_parse == stripped
|
||||
try:
|
||||
return (Decimal(stripped), is_verified)
|
||||
except (InvalidOperation, AttributeError, ValueError):
|
||||
return (None, is_verified)
|
||||
|
||||
|
||||
def _classify_row(cells: list[dict]) -> FslRowKind | None:
|
||||
if not (str(cells[1]['value']).strip()):
|
||||
|
||||
def _normalize_header(value) -> str:
|
||||
text = str(value or '').lower()
|
||||
text = ''.join(c if c.isalnum() else ' ' for c in text)
|
||||
return ' '.join(text.split())
|
||||
|
||||
|
||||
def _is_code_header(text: str) -> bool:
|
||||
normalized_text = text.strip().lower()
|
||||
return text.startswith('код')
|
||||
|
||||
|
||||
def _is_value_header(text: str) -> bool:
|
||||
return bool(_YEAR_RE.search(text)) or text.startswith(('на ', 'за '))
|
||||
|
||||
|
||||
def _cells_by_index(row: dict) -> dict[int, dict]:
|
||||
return {cell['column_index']: cell for cell in row['cells']}
|
||||
|
||||
|
||||
def _cell_value(cells: dict[int, dict], idx: Optional[int]) -> str:
|
||||
if idx is None or idx not in cells:
|
||||
return ''
|
||||
return str(cells[idx]['value'] or '')
|
||||
|
||||
|
||||
def _build_layout(cells: dict[int, dict]) -> Optional[ColumnLayout]:
|
||||
"""Возвращает раскладку столбцов, если строка является заголовком (есть ячейка «Код»)."""
|
||||
headers = {idx: _normalize_header(cells[idx]['value']) for idx in sorted(cells)}
|
||||
code_idx = next((idx for idx, text in headers.items() if _is_code_header(text)), None)
|
||||
if code_idx is None:
|
||||
return None
|
||||
|
||||
label_idx = next((idx for idx, text in headers.items() if 'наименование' in text), None)
|
||||
if label_idx is None:
|
||||
left_filled = [idx for idx, text in headers.items() if idx < code_idx and text]
|
||||
label_idx = left_filled[-1] if left_filled else None
|
||||
|
||||
value_idxs = tuple(
|
||||
idx for idx, text in headers.items()
|
||||
if idx > code_idx and _is_value_header(text)
|
||||
)
|
||||
return ColumnLayout(label_idx=label_idx, code_idx=code_idx, value_idxs=value_idxs)
|
||||
|
||||
|
||||
def _classify_row(cells: dict[int, dict], layout: ColumnLayout) -> FslRowKind | None:
|
||||
code = _cell_value(cells, layout.code_idx).strip()
|
||||
if not code:
|
||||
return None
|
||||
try:
|
||||
int(cells[1]["value"])
|
||||
int(code)
|
||||
except (TypeError, ValueError):
|
||||
return FslRowKind.HEADER
|
||||
if cells[0]["value"] == "Баланс":
|
||||
label = _cell_value(cells, layout.label_idx)
|
||||
if label == "Баланс":
|
||||
return FslRowKind.BALANCE
|
||||
elif cells[0]["value"].lower().startswith("итого"):
|
||||
elif label.lower().startswith("итого"):
|
||||
return FslRowKind.TOTAL
|
||||
return FslRowKind.ROW
|
||||
|
||||
@ -129,19 +213,127 @@ def _keep_digits_only(raw: str) -> str:
|
||||
return res.strip()
|
||||
|
||||
|
||||
def _resolve_table_type(rows) -> StatementType:
|
||||
def _is_blank(value: str) -> bool:
|
||||
# прочерк в отчётности означает ноль, считаем его пустым значением
|
||||
return not _BLANK_CHARS_RE.sub('', value)
|
||||
|
||||
|
||||
def _infer_layout(rows_cells: list[dict[int, dict]]) -> Optional[ColumnLayout]:
|
||||
"""Выводит раскладку столбцов по строкам данных, когда заголовка нет.
|
||||
|
||||
Столбец оценивается по всем строкам сразу, поэтому пустые ячейки в отдельных
|
||||
строках (например, в начале таблицы) не влияют на результат.
|
||||
"""
|
||||
columns: dict[int, list[str]] = {}
|
||||
for cells in rows_cells:
|
||||
for idx, cell in cells.items():
|
||||
columns.setdefault(idx, []).append(str(cell['value'] or '').strip())
|
||||
|
||||
# код — столбец с наибольшим числом известных кодов строк
|
||||
code_hits = {idx: sum(1 for v in values if v in codes_mapping) for idx, values in columns.items()}
|
||||
code_idx = min(code_hits, key=lambda idx: (-code_hits[idx], idx), default=None)
|
||||
if code_idx is None or code_hits[code_idx] == 0:
|
||||
return None
|
||||
|
||||
# наименование — столбец левее кода с наибольшим числом текстовых ячеек
|
||||
text_hits = {
|
||||
idx: sum(1 for v in values if any(c.isalpha() for c in v))
|
||||
for idx, values in columns.items() if idx < code_idx
|
||||
}
|
||||
label_idx = min(text_hits, key=lambda idx: (-text_hits[idx], -idx), default=None)
|
||||
if label_idx is not None and text_hits[label_idx] == 0:
|
||||
label_idx = None
|
||||
|
||||
# значения — столбцы правее кода, где суммы преобладают над мусором (полоса скана),
|
||||
# либо полностью пустые (период без данных)
|
||||
value_idxs = []
|
||||
for idx in sorted(columns):
|
||||
if idx <= code_idx:
|
||||
continue
|
||||
filled = [v for v in columns[idx] if not _is_blank(v)]
|
||||
amounts = sum(1 for v in filled if _parse_amount(v)[0] is not None)
|
||||
if amounts >= len(filled) - amounts:
|
||||
value_idxs.append(idx)
|
||||
|
||||
return ColumnLayout(label_idx=label_idx, code_idx=code_idx, value_idxs=tuple(value_idxs))
|
||||
|
||||
|
||||
def _parse_data_row(cells: dict[int, dict], layout: ColumnLayout, ordinal: int) -> Optional[ParsedRow]:
|
||||
kind = _classify_row(cells, layout)
|
||||
if kind is None or kind == FslRowKind.HEADER:
|
||||
return None
|
||||
return ParsedRow(cells, layout, kind, ordinal)
|
||||
|
||||
|
||||
def _synthetic_header_cells(layout: ColumnLayout) -> dict[int, dict]:
|
||||
"""Ячейки восстановленного заголовка: код и наименование, тексты периодов пустые."""
|
||||
cells = {layout.code_idx: {'value': _HEADER_CODE, 'confidence': _FULL_CONFIDENCE}}
|
||||
if layout.label_idx is not None:
|
||||
cells[layout.label_idx] = {'value': codes_mapping[_HEADER_CODE], 'confidence': _FULL_CONFIDENCE}
|
||||
return cells
|
||||
|
||||
|
||||
def _parse_table_rows(table_rows: list[dict]) -> Optional[list[ParsedRow]]:
|
||||
"""Разбирает строки таблицы; первым элементом результата всегда идёт заголовок.
|
||||
|
||||
- Раскладка берётся из строки с ячейкой «Код»; каждый повторный заголовок
|
||||
перезаписывает индексы столбцов.
|
||||
- Строки до первого заголовка разбираются по раскладке, выведенной из них самих
|
||||
(индексы заголовка могут не совпадать с ними). Заголовок пишется в начало таблицы.
|
||||
- Если заголовка нет, раскладка выводится по всем строкам, а заголовок создаётся
|
||||
с кодом и наименованием и пустыми текстами периодов.
|
||||
Если не удалось определить ни заголовок, ни столбец кода, возвращает None.
|
||||
"""
|
||||
rows_cells = [(row['row_index'], _cells_by_index(row)) for row in table_rows]
|
||||
first_header = next(
|
||||
(pos for pos, (_, cells) in enumerate(rows_cells) if _build_layout(cells) is not None),
|
||||
len(rows_cells),
|
||||
)
|
||||
|
||||
before_header = rows_cells[:first_header]
|
||||
inferred = _infer_layout([cells for _, cells in before_header]) if before_header else None
|
||||
leading_rows = []
|
||||
if inferred is not None:
|
||||
for ordinal, cells in before_header:
|
||||
parsed = _parse_data_row(cells, inferred, ordinal)
|
||||
if parsed is not None:
|
||||
leading_rows.append(parsed)
|
||||
|
||||
if first_header == len(rows_cells):
|
||||
if not leading_rows:
|
||||
return None
|
||||
header_ordinal = min(p.ordinal for p in leading_rows) - 1
|
||||
header = ParsedRow(_synthetic_header_cells(inferred), inferred, FslRowKind.HEADER, header_ordinal)
|
||||
return [header, *leading_rows]
|
||||
|
||||
header_row_index, header_cells = rows_cells[first_header]
|
||||
layout = _build_layout(header_cells)
|
||||
header_ordinal = min([header_row_index, *(p.ordinal - 1 for p in leading_rows)])
|
||||
parsed_rows = [ParsedRow(header_cells, layout, FslRowKind.HEADER, header_ordinal), *leading_rows]
|
||||
|
||||
for ordinal, cells in rows_cells[first_header + 1:]:
|
||||
header_layout = _build_layout(cells)
|
||||
if header_layout is not None:
|
||||
layout = header_layout
|
||||
continue
|
||||
parsed = _parse_data_row(cells, layout, ordinal)
|
||||
if parsed is not None:
|
||||
parsed_rows.append(parsed)
|
||||
|
||||
return parsed_rows
|
||||
|
||||
|
||||
def _resolve_table_type(parsed_rows: list[ParsedRow]) -> StatementType:
|
||||
cell_codes = {
|
||||
'1': 0,
|
||||
'2': 0
|
||||
}
|
||||
try:
|
||||
for r in rows:
|
||||
cells = list(sorted(r["cells"], key=lambda x: x['column_index']))
|
||||
kind = _classify_row(cells)
|
||||
if kind is None or kind == FslRowKind.HEADER:
|
||||
for parsed in parsed_rows:
|
||||
if parsed.kind == FslRowKind.HEADER:
|
||||
continue
|
||||
else:
|
||||
first_symbol = str(cells[1]['value'])[0]
|
||||
first_symbol = _cell_value(parsed.cells, parsed.layout.code_idx).strip()[0]
|
||||
cell_codes.setdefault(first_symbol, 0)
|
||||
cell_codes[first_symbol] += 1
|
||||
if cell_codes['1'] > cell_codes['2']:
|
||||
@ -177,9 +369,10 @@ def _process_file(
|
||||
units_verified = _is_recognized(units_item["confidence"]) and units_value.strip() == units_value_cleared
|
||||
|
||||
date_value = date_item["form_name"]
|
||||
date_string_value = _keep_digits_spaces_only(date_value)
|
||||
date_verified = date_value.strip() == date_string_value and _is_recognized(date_item['confidence'])
|
||||
date_parsed = _parse_date(date_string_value)
|
||||
# date_string_value = _keep_digits_spaces_only(date_value)
|
||||
# date_verified = date_value.strip() == date_string_value and _is_recognized(date_item['confidence'])
|
||||
date_verified = _is_recognized(date_item['confidence'])
|
||||
date_parsed = _parse_date(date_value)
|
||||
date_verified = date_verified and (date_parsed is not None)
|
||||
|
||||
shared = {
|
||||
@ -196,36 +389,38 @@ def _process_file(
|
||||
rows: list[DbRow] = []
|
||||
values: list[DbValue] = []
|
||||
|
||||
for table in data["table_structure"]:
|
||||
for table_number, table in enumerate(data["table_structure"]):
|
||||
parsed_rows = _parse_table_rows(table["rows"])
|
||||
if parsed_rows is None:
|
||||
logger.warning(
|
||||
"В таблице нет заголовка и по строкам не удалось определить столбец кода, таблица пропущена. "
|
||||
"file_uid={file_uid}, table_number={table_number}, table_label={table_label}",
|
||||
file_uid=file_id,
|
||||
table_number=table_number,
|
||||
table_label=(table.get("label") or {}).get("form_name"),
|
||||
)
|
||||
continue
|
||||
|
||||
meta = Meta(
|
||||
id=uuid.uuid4(),
|
||||
statement_type=_resolve_table_type(table["rows"]),
|
||||
statement_type=_resolve_table_type(parsed_rows),
|
||||
**shared,
|
||||
)
|
||||
metas.append(asdict(meta))
|
||||
header_found = False
|
||||
|
||||
for row in table["rows"]:
|
||||
cells = list(sorted(row["cells"], key=lambda x: x['column_index']))
|
||||
if len(cells) < 3:
|
||||
raise ValueError(
|
||||
f"Row {row['row_index']} in file {file['file_uid']} "
|
||||
f"has {len(cells)} cells (< 3)."
|
||||
)
|
||||
for parsed in parsed_rows:
|
||||
cells, layout, kind = parsed.cells, parsed.layout, parsed.kind
|
||||
code_cell = cells[layout.code_idx]
|
||||
|
||||
|
||||
kind = _classify_row(cells)
|
||||
if kind is None or (kind == FslRowKind.HEADER and header_found):
|
||||
continue
|
||||
elif (kind == FslRowKind.HEADER and not(header_found)):
|
||||
header_found = True
|
||||
label_cell = cells[0]
|
||||
code_cell = cells[1]
|
||||
is_line_code_valid = True
|
||||
line_code = _clear_value(code_cell["value"])
|
||||
if kind == FslRowKind.HEADER:
|
||||
line_code = 'Код'
|
||||
line_name = codes_mapping.get(line_code, '')
|
||||
if line_name == '':
|
||||
is_line_code_valid = False
|
||||
line_name = label_cell['value']
|
||||
line_name = _cell_value(cells, layout.label_idx)
|
||||
|
||||
line_code_recognized = _is_recognized(code_cell["confidence"]) and _clear_value(code_cell['value']) == line_code and is_line_code_valid
|
||||
|
||||
@ -240,20 +435,29 @@ def _process_file(
|
||||
line_name=line_name,
|
||||
line_name_recognized=line_name_recognized,
|
||||
row_kind=kind,
|
||||
ordinal=row["row_index"],
|
||||
ordinal=parsed.ordinal,
|
||||
)
|
||||
|
||||
rows.append(asdict(db_row))
|
||||
|
||||
for cell in cells:
|
||||
if cell["column_index"] < 2:
|
||||
continue
|
||||
raw = cell["value"]
|
||||
for value_number, source_idx in enumerate(layout.value_idxs):
|
||||
cell = cells.get(source_idx)
|
||||
parsed_amount = None
|
||||
if kind == FslRowKind.HEADER:
|
||||
if cell is None:
|
||||
# ячейки под столбцом заголовка нет — пишем пустое значение, чтобы не сдвигать столбцы
|
||||
raw = ''
|
||||
is_recognized = False
|
||||
elif kind == FslRowKind.HEADER:
|
||||
raw = cell["value"]
|
||||
is_recognized = (_clear_value(raw) == _remove_spec_space(raw) and _is_recognized(cell["confidence"]))
|
||||
raw = _clear_value(raw)
|
||||
parsed_date = _parse_date(raw)
|
||||
if parsed_date is None:
|
||||
raw = _clear_value(raw)
|
||||
else:
|
||||
raw = f"На {parsed_date.strftime('%d.%m.%Y')}"
|
||||
|
||||
else:
|
||||
raw = cell["value"]
|
||||
parsed_amount, is_num_recognized = _parse_amount(raw)
|
||||
is_recognized = _is_recognized(cell["confidence"]) and is_num_recognized
|
||||
raw = _clear_value(raw)
|
||||
@ -261,7 +465,7 @@ def _process_file(
|
||||
line_id=db_row.id,
|
||||
amount=parsed_amount,
|
||||
string_value=raw,
|
||||
column_index=cell["column_index"],
|
||||
column_index=_FIRST_VALUE_COLUMN + value_number,
|
||||
is_recognized=is_recognized,
|
||||
)
|
||||
values.append(
|
||||
|
||||
@ -4,10 +4,12 @@ from sqlalchemy.orm import selectinload
|
||||
|
||||
from src.models import FinancialStatementLine, FinancialStatementValue, FinancialStatementMeta
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy import select, update, delete, case
|
||||
|
||||
from src.schemas import FileFslPatchSchema
|
||||
|
||||
from src.core.PDFWorker.consts import codes_mapping
|
||||
|
||||
|
||||
class FinStatRepository:
|
||||
def __init__(self, session: AsyncSession):
|
||||
@ -27,7 +29,7 @@ class FinStatRepository:
|
||||
return list(result.scalars().all())
|
||||
|
||||
async def update_file_fsl(self, file_id, payload: FileFslPatchSchema):
|
||||
await self._mark_file_recognized(file_id)
|
||||
# await self._mark_file_recognized(file_id)
|
||||
for t in payload.tables:
|
||||
if t.meta:
|
||||
meta_values = {}
|
||||
@ -47,25 +49,25 @@ class FinStatRepository:
|
||||
if t.cells:
|
||||
for c in t.cells:
|
||||
line_id = c.line_id
|
||||
# разделение на код/наименование и цифровое значение
|
||||
if c.field is not None:
|
||||
values = {c.field: c.cell.value}
|
||||
stmt = update(FinancialStatementLine).where(FinancialStatementLine.id == line_id).values(**values)
|
||||
await self.session.execute(stmt)
|
||||
else:
|
||||
value_id = c.value_id
|
||||
raw = c.cell.value
|
||||
raw = str(c.cell.value)
|
||||
values = {}
|
||||
try:
|
||||
amount = Decimal(raw.strip().replace(',', '.'))
|
||||
values['amount'] = amount
|
||||
except:
|
||||
except Exception as exc:
|
||||
values['string_value'] = raw
|
||||
stmt = update(FinancialStatementValue).where(FinancialStatementValue.id == value_id).values(**values)
|
||||
await self.session.execute(stmt)
|
||||
|
||||
return file_id
|
||||
|
||||
async def _mark_file_recognized(self, file_id):
|
||||
async def mark_file_recognized(self, file_id):
|
||||
meta_values = {
|
||||
'inn_verified': True,
|
||||
'units_verified': True,
|
||||
@ -83,4 +85,21 @@ class FinStatRepository:
|
||||
|
||||
values = await self.session.execute(update(FinancialStatementValue)
|
||||
.where(FinancialStatementValue.line_id.in_(lines.scalars().all()))
|
||||
.values(is_recognized=True))
|
||||
.values(is_recognized=True))
|
||||
|
||||
async def remove_invalid_lines(self, file_id):
|
||||
meta_ids = select(FinancialStatementMeta.id).where(FinancialStatementMeta.stored_file_id == file_id)
|
||||
lines_stmt = select(FinancialStatementLine.id).where(FinancialStatementLine.meta_id.in_(meta_ids),
|
||||
FinancialStatementLine.line_code.notin_(codes_mapping.keys()))
|
||||
delete_values_stmt = delete(FinancialStatementValue).where(FinancialStatementValue.line_id.in_(lines_stmt))
|
||||
await self.session.execute(delete_values_stmt)
|
||||
delete_lines_stmt = delete(FinancialStatementLine).where(FinancialStatementLine.id.in_(lines_stmt))
|
||||
await self.session.execute(delete_lines_stmt)
|
||||
|
||||
|
||||
async def update_line_names(self, file_id):
|
||||
meta_ids = select(FinancialStatementMeta.id).where(FinancialStatementMeta.id == file_id)
|
||||
stmt = (update(FinancialStatementLine)
|
||||
.where(FinancialStatementLine.meta_id.in_(meta_ids), FinancialStatementLine.line_code.in_(codes_mapping.keys()))
|
||||
.values(line_name = case(codes_mapping, value=FinancialStatementLine.line_name)))
|
||||
await self.session.execute(stmt)
|
||||
@ -26,7 +26,10 @@ class FinStatService:
|
||||
|
||||
async def update_by_file_id(self, file_id: uuid.UUID, payload):
|
||||
try:
|
||||
await self.fin_stat_repo.mark_file_recognized(file_id)
|
||||
await self.fin_stat_repo.update_file_fsl(file_id, payload)
|
||||
await self.fin_stat_repo.remove_invalid_lines(file_id)
|
||||
await self.fin_stat_repo.update_line_names(file_id)
|
||||
await self.session.commit()
|
||||
|
||||
new_data = await self.get_by_file_id(file_id)
|
||||
|
||||
BIN
web/src/assets/fonts/Geist-Bold.woff2
Executable file
BIN
web/src/assets/fonts/Geist-Bold.woff2
Executable file
Binary file not shown.
BIN
web/src/assets/fonts/Geist-Light.woff2
Executable file
BIN
web/src/assets/fonts/Geist-Light.woff2
Executable file
Binary file not shown.
BIN
web/src/assets/fonts/Geist-Medium.woff2
Executable file
BIN
web/src/assets/fonts/Geist-Medium.woff2
Executable file
Binary file not shown.
BIN
web/src/assets/fonts/Geist-Regular.woff2
Executable file
BIN
web/src/assets/fonts/Geist-Regular.woff2
Executable file
Binary file not shown.
BIN
web/src/assets/fonts/Geist-SemiBold.woff2
Executable file
BIN
web/src/assets/fonts/Geist-SemiBold.woff2
Executable file
Binary file not shown.
BIN
web/src/assets/fonts/Geist-Thin.woff2
Executable file
BIN
web/src/assets/fonts/Geist-Thin.woff2
Executable file
Binary file not shown.
@ -109,8 +109,43 @@ code {
|
||||
padding: 4px 8px;
|
||||
background: var(--code-bg);
|
||||
} */
|
||||
@font-face {
|
||||
font-family: "Geist";
|
||||
src: url("./assets/fonts/Geist-SemiBold.woff2") format("woff2");
|
||||
font-style: normal;
|
||||
font-weight: 600;
|
||||
font-display: swap;
|
||||
}
|
||||
@font-face {
|
||||
font-family: "Geist";
|
||||
src: url("./assets/fonts/Geist-Light.woff2") format("woff2");
|
||||
font-style: normal;
|
||||
font-weight: 300;
|
||||
font-display: swap;
|
||||
}
|
||||
@font-face {
|
||||
font-family: "Geist";
|
||||
src: url("./assets/fonts/Geist-Bold.woff2") format("woff2");
|
||||
font-style: normal;
|
||||
font-weight: 700;
|
||||
font-display: swap;
|
||||
}
|
||||
@font-face {
|
||||
font-family: "Geist";
|
||||
src: url("./assets/fonts/Geist-Regular.woff2") format("woff2");
|
||||
font-style: normal;
|
||||
font-weight: 400;
|
||||
font-display: swap;
|
||||
}
|
||||
@font-face {
|
||||
font-family: "Geist";
|
||||
src: url("./assets/fonts/Geist-Thin.woff2") format("woff2");
|
||||
font-style: normal;
|
||||
font-weight: 100;
|
||||
font-display: swap;
|
||||
}
|
||||
|
||||
body {
|
||||
font-family: 'Gill Sans', 'Gill Sans MT', Calibri, 'Trebuchet MS', sans-serif;
|
||||
font-family: 'Geist', 'Gill Sans MT', Calibri, 'Trebuchet MS', sans-serif;
|
||||
margin: 0px;
|
||||
}
|
||||
|
||||
@ -10,6 +10,7 @@ createRoot(document.getElementById("root")!).render(
|
||||
theme={{
|
||||
token: {
|
||||
colorPrimary: "rgba(46, 166, 75, 0.8)",
|
||||
fontFamily: "Geist",
|
||||
colorPrimaryHover: "rgba(46, 166, 75, 0.9)",
|
||||
colorBgLayout: "rgb(247, 248, 250)",
|
||||
},
|
||||
|
||||
@ -11,6 +11,31 @@ const EMPTY_DRAFT: StatementDraft = {meta: {}, rows: {}}
|
||||
const META_KEYS: (keyof MetaSchema)[] = ["inn", "report_date", "units"]
|
||||
|
||||
function patchJson(json) {
|
||||
const initState = {
|
||||
'11': {
|
||||
header: 'ВНЕОБОРОТНЫЕ АКТИВЫ',
|
||||
is_found: false,
|
||||
},
|
||||
'12': {
|
||||
header: 'ОБОРОТНЫЕ АКТИВЫ',
|
||||
is_found: false,
|
||||
},
|
||||
'13': {
|
||||
header: 'КАПИТАЛ И РЕЗЕРВЫ',
|
||||
is_found: false,
|
||||
},
|
||||
'14': {
|
||||
header: 'ДОЛГОСРОЧНЫЕ ОБЯЗАТЕЛЬСТВА',
|
||||
is_found: false,
|
||||
},
|
||||
'15': {
|
||||
header: 'КРАТКОСРОЧНЫЕ ОБЯЗАТЕЛЬСТВА',
|
||||
is_found: false
|
||||
}
|
||||
}
|
||||
const ACTIVE_CODES = ['11', '12', '16']
|
||||
const PASSIVE_CODES = ['13', '14', '15', '17']
|
||||
|
||||
for (let j = 0; j < json.length; j++) {
|
||||
if (json[j].table.table_type === 'balance') {
|
||||
let source_rows = json[j].table.rows;
|
||||
@ -18,25 +43,38 @@ function patchJson(json) {
|
||||
let passive_added = false;
|
||||
let active_added = false;
|
||||
for (let i = 0; i < source_rows.length; i++) {
|
||||
if (source_rows[i].line_code?.value == 'Код' && !active_added) {
|
||||
rows.push(source_rows[i])
|
||||
let firstTwoSymbols = source_rows[i].line_code?.value || '';
|
||||
let firsts = firstTwoSymbols.slice(0,2)
|
||||
if (ACTIVE_CODES.includes(firsts) && !active_added) {
|
||||
|
||||
rows.push({line_id: 'active',
|
||||
cells: [],
|
||||
line_code: '',
|
||||
line_name: 'АКТИВ',
|
||||
row_type: 'section'})
|
||||
row_type: 'section'
|
||||
})
|
||||
active_added = true
|
||||
} else if (source_rows[i].line_code?.value && ['17', '13', '14', '15'].includes(source_rows[i].line_code.value.slice(0,2)) && !passive_added) {
|
||||
|
||||
|
||||
|
||||
} else if (PASSIVE_CODES.includes(firsts) && !passive_added) {
|
||||
rows.push({line_id: 'passive',
|
||||
cells: [],
|
||||
line_code: '',
|
||||
line_name: 'ПАССИВ',
|
||||
row_type: 'section'})
|
||||
passive_added = true;
|
||||
rows.push(source_rows[i])
|
||||
} else {
|
||||
rows.push(source_rows[i])
|
||||
}
|
||||
|
||||
if (initState[firsts] && !initState[firsts].is_found) {
|
||||
rows.push({line_id: `subsection_${firsts}`,
|
||||
cells: [],
|
||||
line_code: '',
|
||||
line_name: initState[firsts].header,
|
||||
row_type: 'subsection'})
|
||||
initState[firsts].is_found = true
|
||||
}
|
||||
rows.push(source_rows[i])
|
||||
}
|
||||
json[j].table.rows = rows;
|
||||
}
|
||||
@ -130,7 +168,26 @@ function applyDraftToData(data: FileStatementResponse[], draft: Draft): FileStat
|
||||
if (!rowDraft) return row
|
||||
return {
|
||||
...row,
|
||||
cells: row.cells.map(cell => rowDraft[cell.value_id] ?? cell)
|
||||
line_code: rowDraft['line_code'] ?? row['line_code'],
|
||||
line_name: rowDraft['line_name'] ?? row['line_name'],
|
||||
cells: row.cells.map(cell => {
|
||||
if (!rowDraft[cell.value_id]) {
|
||||
return cell
|
||||
}
|
||||
let newVal = rowDraft[cell.value_id].value;
|
||||
if (row.row_type === 'row' || row.row_type === 'balance' || row.row_type === 'total') {
|
||||
// try {
|
||||
// newVal = normalizeEditedValue(newVal);
|
||||
|
||||
// } catch (err) {
|
||||
// console.log("err", err.message)
|
||||
// alert(`${err.message}`)
|
||||
// newVal = cell.value;
|
||||
// }
|
||||
newVal = normalizeEditedValue(newVal);
|
||||
}
|
||||
return {...rowDraft[cell.value_id], value: newVal}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
@ -144,14 +201,14 @@ type EditableCellProps = {
|
||||
onChange: (value: string) => void;
|
||||
}
|
||||
|
||||
function normalizeEditedValue(originalValue, nextValue) {
|
||||
if (typeof originalValue === "number") {
|
||||
if (nextValue.trim() === "") return "";
|
||||
const parsed = Number(nextValue);
|
||||
return Number.isNaN(parsed) ? nextValue : parsed;
|
||||
}
|
||||
|
||||
return nextValue;
|
||||
function normalizeEditedValue(nextValue: string) {
|
||||
if (nextValue.trim() === "") return "";
|
||||
const parsed = Number(nextValue.replace(',', '.'));
|
||||
// if (Number.isNaN(parsed)) {
|
||||
// throw new Error('Необходимо ввести корректное число')
|
||||
// }
|
||||
// return parsed;
|
||||
return Number.isNaN(parsed) ? nextValue : parsed;
|
||||
}
|
||||
|
||||
const FIXED_CELL_KEYS = ["line_name", "line_code"];
|
||||
@ -163,6 +220,7 @@ const EditableCell = memo(function EditableCell({
|
||||
// minWidth = 160,
|
||||
}: EditableCellProps) {
|
||||
const verified = !!cell?.is_verified;
|
||||
const defValue = typeof (cell?.value) === 'number' ? cell?.value.toLocaleString() : cell.value
|
||||
|
||||
return (
|
||||
<td
|
||||
@ -176,7 +234,7 @@ const EditableCell = memo(function EditableCell({
|
||||
onChange={(e) => onChange(e.target.value)}
|
||||
/>
|
||||
) : (
|
||||
<span>{String(cell?.value ?? "")}</span>
|
||||
<span>{defValue ?? ""}</span>
|
||||
)}
|
||||
</td>
|
||||
);
|
||||
@ -232,10 +290,10 @@ const DataRow = memo(function DataRow({
|
||||
onChange={(nextValue) =>
|
||||
onCellChange(tableId, row.line_id, key, {
|
||||
...currentCell,
|
||||
...(key === "first_column" || key === "second_column"
|
||||
...(key === "line_code" || key === "line_name"
|
||||
? {}
|
||||
: { value_id: key }),
|
||||
value: normalizeEditedValue(currentCell.value, nextValue),
|
||||
value: nextValue,
|
||||
})
|
||||
}
|
||||
/>
|
||||
@ -285,10 +343,7 @@ const MetaTable = memo(function MetaTable({
|
||||
onChange={(nextValue) =>
|
||||
onMetaChange(metaKey, {
|
||||
...currentCell,
|
||||
value: normalizeEditedValue(
|
||||
currentCell.value,
|
||||
nextValue,
|
||||
),
|
||||
value: nextValue,
|
||||
})
|
||||
}
|
||||
/>
|
||||
@ -321,7 +376,7 @@ const Block = memo(function Block({item, draftBlock, isEditing, onMetaChange, on
|
||||
<tbody>
|
||||
{item.table.rows.map((row) => {
|
||||
|
||||
if (row.row_type !== 'section') {
|
||||
if (row.row_type !== 'section' && row.row_type !== 'subsection') {
|
||||
return <DataRow
|
||||
key={row.line_id}
|
||||
tableId={tableId}
|
||||
@ -330,8 +385,11 @@ const Block = memo(function Block({item, draftBlock, isEditing, onMetaChange, on
|
||||
isEditing={isEditing}
|
||||
onCellChange={onCellChange}
|
||||
|
||||
/> } else {
|
||||
return <tr key={row.line_id}><td>{row.line_name}</td></tr>}
|
||||
/> } else if (row.row_type === 'section') {
|
||||
return <tr key={row.line_id} style={{fontSize: 24}}><td>{row.line_name}</td></tr>
|
||||
} else {
|
||||
return <tr key={row.line_id}><td>{row.line_name}</td></tr>
|
||||
}
|
||||
|
||||
|
||||
})}
|
||||
@ -412,7 +470,6 @@ export default function PdfDataViewer({ fileId }) {
|
||||
}, []);
|
||||
|
||||
const handleSave = useCallback(async () => {
|
||||
const nextData = applyDraftToData(data, draft);
|
||||
|
||||
setData((prev) => applyDraftToData(prev, draft));
|
||||
setPending((prev) => mergeChanges(prev, draft));
|
||||
|
||||
@ -79,7 +79,6 @@
|
||||
padding: 22px;
|
||||
box-shadow: 0 16px 32px #0f172a14;
|
||||
color: #111827;
|
||||
font-family: Georgia, Times New Roman, serif;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user