feat: словарь периодов — даты, относительные подписи бланка 66н, исключения колонок
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W
This commit is contained in:
Andrey Epifancev
parent
aed37cc189
commit
19f70fea65
276
arm_statements/dates.py
Normal file
276
arm_statements/dates.py
Normal file
@ -0,0 +1,276 @@
|
|||||||
|
"""Даты и подписи колонок: разбор текстовых и числовых форм, относительные подписи бланка 66н (§11.3, §12 п. 4).
|
||||||
|
|
||||||
|
Вход `parse_date` — текст после `fix_lookalikes`. Регулярки и месяцы — данные (`rules/statement_periods.yaml`),
|
||||||
|
здесь только обработчики на каждый вид абсолютной формы."""
|
||||||
|
|
||||||
|
import calendar
|
||||||
|
import re
|
||||||
|
from collections.abc import Callable
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import date
|
||||||
|
|
||||||
|
from arm_statements.loader import PeriodRules, Rules
|
||||||
|
from arm_statements.model import Cell, PeriodKind
|
||||||
|
from arm_statements.text import fix_lookalikes, normalize_caption
|
||||||
|
|
||||||
|
__all__ = ["PeriodLabel", "expected_dates", "parse_date", "parse_period_label", "resolve_relative"]
|
||||||
|
|
||||||
|
_BARE_YEAR = re.compile(r"\d{4}")
|
||||||
|
_DIGITS = re.compile(r"\d+")
|
||||||
|
_ROMAN = {"I": 1, "II": 2, "III": 3, "IV": 4, "1V": 4, "11": 2, "111": 3}
|
||||||
|
_ORDINAL = {"перв": 1, "втор": 2, "трет": 3, "четв": 4}
|
||||||
|
_QUARTER_MONTHS = 3
|
||||||
|
_HALF_MONTHS = 6
|
||||||
|
_DECEMBER = 12
|
||||||
|
_SHORT_YEAR_LIMIT = 100 # меньше — двузначный год → 20YY
|
||||||
|
_CENTURY = 2000
|
||||||
|
_DIGITS_FULL = 8
|
||||||
|
_DIGITS_YEAR = 4
|
||||||
|
_DIGITS_SHORT = 6
|
||||||
|
_PARTS = 3
|
||||||
|
_SHORT_PART = 2
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PeriodLabel:
|
||||||
|
kind: PeriodKind
|
||||||
|
date: date | None
|
||||||
|
excluded: bool
|
||||||
|
bare_year: bool
|
||||||
|
|
||||||
|
|
||||||
|
def _year_full(year: str) -> int:
|
||||||
|
value = int(year)
|
||||||
|
return value if value >= _SHORT_YEAR_LIMIT else _CENTURY + value
|
||||||
|
|
||||||
|
|
||||||
|
def _month_num(word: str, rules: PeriodRules) -> int | None:
|
||||||
|
lowered = word.lower()
|
||||||
|
for stem, number in rules.months:
|
||||||
|
if lowered.startswith(stem):
|
||||||
|
return number
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _month_end(year: int, month: int) -> date | None:
|
||||||
|
if not 1 <= month <= _DECEMBER:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return date(year, month, calendar.monthrange(year, month)[1])
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _period_end(year: str, month: int | None) -> date | None:
|
||||||
|
return _month_end(_year_full(year), month) if month else None
|
||||||
|
|
||||||
|
|
||||||
|
def _safe(year: int, month: int | None, day: int) -> date | None:
|
||||||
|
try:
|
||||||
|
return date(year, month, day) if month else None
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _ordinal(word: str) -> int | None:
|
||||||
|
"""Номер квартала/полугодия из «1», «I», «IV», «первый», «1-й»."""
|
||||||
|
token = word.strip().upper().rstrip("-Й")
|
||||||
|
if token in _ROMAN:
|
||||||
|
return _ROMAN[token]
|
||||||
|
if token.isdigit():
|
||||||
|
return int(token)
|
||||||
|
lowered = token.lower()
|
||||||
|
for stem, number in _ORDINAL.items():
|
||||||
|
if lowered.startswith(stem):
|
||||||
|
return number
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _group(match: re.Match[str], name: str) -> str | None:
|
||||||
|
return match.group(name) if name in match.re.groupindex else None
|
||||||
|
|
||||||
|
|
||||||
|
def _handle_range_end(match: re.Match[str], rules: PeriodRules) -> date | None:
|
||||||
|
return _handle_day_month_year(match, rules)
|
||||||
|
|
||||||
|
|
||||||
|
def _handle_day_month_year(match: re.Match[str], rules: PeriodRules) -> date | None:
|
||||||
|
year, day = _group(match, "y"), _group(match, "d")
|
||||||
|
if year is None or day is None:
|
||||||
|
return None
|
||||||
|
month_word = _group(match, "mw")
|
||||||
|
month = _month_num(month_word, rules) if month_word is not None else int(_group(match, "m") or 0)
|
||||||
|
return _safe(_year_full(year), month, int(day))
|
||||||
|
|
||||||
|
|
||||||
|
def _handle_quarter(match: re.Match[str], rules: PeriodRules) -> date | None:
|
||||||
|
del rules
|
||||||
|
year, number = _group(match, "y"), _ordinal(_group(match, "q") or "")
|
||||||
|
return _period_end(year, (number or 0) * _QUARTER_MONTHS) if year else None
|
||||||
|
|
||||||
|
|
||||||
|
def _handle_half(match: re.Match[str], rules: PeriodRules) -> date | None:
|
||||||
|
del rules
|
||||||
|
year, number = _group(match, "y"), _ordinal(_group(match, "q") or "")
|
||||||
|
return _period_end(year, (number or 0) * _HALF_MONTHS) if year else None
|
||||||
|
|
||||||
|
|
||||||
|
def _handle_months(match: re.Match[str], rules: PeriodRules) -> date | None:
|
||||||
|
del rules
|
||||||
|
year, count = _group(match, "y"), _group(match, "n")
|
||||||
|
return _period_end(year, int(count)) if year and count else None
|
||||||
|
|
||||||
|
|
||||||
|
def _handle_month_range(match: re.Match[str], rules: PeriodRules) -> date | None:
|
||||||
|
year, word = _group(match, "y"), _group(match, "mw")
|
||||||
|
return _period_end(year, _month_num(word, rules)) if year and word else None
|
||||||
|
|
||||||
|
|
||||||
|
def _handle_year(match: re.Match[str], rules: PeriodRules) -> date | None:
|
||||||
|
year = _group(match, "y")
|
||||||
|
return _period_end(year, _DECEMBER) if year and _year_ok(int(year), rules) else None
|
||||||
|
|
||||||
|
|
||||||
|
_HANDLERS: dict[str, Callable[[re.Match[str], PeriodRules], date | None]] = {
|
||||||
|
"range_end": _handle_range_end,
|
||||||
|
"day_month_year": _handle_day_month_year,
|
||||||
|
"quarter": _handle_quarter,
|
||||||
|
"half": _handle_half,
|
||||||
|
"months": _handle_months,
|
||||||
|
"month_range": _handle_month_range,
|
||||||
|
"year": _handle_year,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _year_ok(year: int, rules: PeriodRules) -> bool:
|
||||||
|
return rules.min_year <= year <= rules.max_year
|
||||||
|
|
||||||
|
|
||||||
|
def _text_date(text: str, rules: PeriodRules) -> date | None:
|
||||||
|
for item in rules.absolute:
|
||||||
|
match = item.pattern.search(text)
|
||||||
|
if match:
|
||||||
|
found = _HANDLERS[item.kind](match, rules)
|
||||||
|
if found:
|
||||||
|
return found
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _from_parts(first: str, second: str, third: str) -> date | None:
|
||||||
|
"""ДД ММ ГГ[ГГ] и ГГ[ГГ] ММ ДД (год двух- или четырёхзначный только в позиции года)."""
|
||||||
|
for day, month, year in ((first, second, third), (third, second, first)):
|
||||||
|
if len(year) in (_SHORT_PART, _DIGITS_YEAR) and len(day) <= _SHORT_PART and len(month) <= _SHORT_PART:
|
||||||
|
try:
|
||||||
|
return date(_year_full(year), int(month), int(day))
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _from_digits(digits: str) -> date | None:
|
||||||
|
"""8 слитных цифр: ДДММГГГГ, ГГГГММДД, ГГГГДДММ — первый валидный вариант."""
|
||||||
|
return (
|
||||||
|
_from_parts(digits[:2], digits[2:4], digits[4:])
|
||||||
|
or _from_parts(digits[6:], digits[4:6], digits[:4])
|
||||||
|
or _from_parts(digits[4:6], digits[6:], digits[:4])
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _from_digits_run(digits: str, groups: list[str], rules: PeriodRules) -> date | None:
|
||||||
|
"""Склеенные цифры «2 0 2 4», «30 09 2 0 2 5»: решает их число."""
|
||||||
|
if len(digits) == _DIGITS_FULL:
|
||||||
|
return _from_digits(digits)
|
||||||
|
if len(digits) == _DIGITS_YEAR:
|
||||||
|
return date(int(digits), 12, 31) if _year_ok(int(digits), rules) else None
|
||||||
|
if len(digits) == _DIGITS_SHORT:
|
||||||
|
return _from_parts(digits[:2], digits[2:4], digits[4:])
|
||||||
|
if len(groups) > _PARTS: # «30.09.2025 0:00:00»
|
||||||
|
return _from_parts(*groups[:_PARTS])
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _numeric_date(text: str, rules: PeriodRules) -> date | None:
|
||||||
|
groups = _DIGITS.findall(text)
|
||||||
|
if not groups:
|
||||||
|
return None
|
||||||
|
if len(groups) == _PARTS:
|
||||||
|
return _from_parts(*groups)
|
||||||
|
return _from_digits_run("".join(groups), groups, rules)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_date(text: str | None, rules: Rules) -> date | None:
|
||||||
|
"""Дата из текста после очистки или None. Длиннее `max_text_length` не разбирается (проверка до регулярок)."""
|
||||||
|
if text is None:
|
||||||
|
return None
|
||||||
|
stripped = text.strip()
|
||||||
|
if not stripped or len(stripped) > rules.periods.max_text_length:
|
||||||
|
return None
|
||||||
|
periods = rules.periods
|
||||||
|
return (
|
||||||
|
_text_date(stripped, periods)
|
||||||
|
or _numeric_date(stripped.split("\n", 1)[0], periods)
|
||||||
|
or _numeric_date(stripped.replace("\n", " "), periods)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _is_excluded(caption: str, rules: Rules) -> bool:
|
||||||
|
return any(rx.search(caption) for rx in rules.headers.column_exclusions)
|
||||||
|
|
||||||
|
|
||||||
|
def _classify(text: str, rules: Rules) -> PeriodLabel:
|
||||||
|
caption = normalize_caption(text)
|
||||||
|
bare_year = _BARE_YEAR.fullmatch(text.strip()) is not None
|
||||||
|
if _is_excluded(caption, rules):
|
||||||
|
return PeriodLabel(PeriodKind.NONE, None, True, False)
|
||||||
|
for kind, pattern in rules.periods.relative:
|
||||||
|
if pattern.search(caption):
|
||||||
|
return PeriodLabel(kind, None, False, bare_year)
|
||||||
|
found = parse_date(text, rules)
|
||||||
|
return PeriodLabel(PeriodKind.ABSOLUTE if found else PeriodKind.NONE, found, False, bare_year)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_period_label(cell: Cell | None, rules: Rules) -> PeriodLabel:
|
||||||
|
"""Подпись колонки → вид периода и дата. Дата-значение ячейки Excel — абсолютная."""
|
||||||
|
if cell is not None and isinstance(cell.value, date):
|
||||||
|
return PeriodLabel(PeriodKind.ABSOLUTE, cell.value, False, False)
|
||||||
|
raw = cell.text.strip() if cell is not None else ""
|
||||||
|
if not raw:
|
||||||
|
return PeriodLabel(PeriodKind.NONE, None, False, False)
|
||||||
|
# длинный текст не чистим от двойников (backtracking); parse_date всё равно вернёт None
|
||||||
|
text = raw if len(raw) > rules.periods.max_text_length else fix_lookalikes(raw, rules.lookalikes)
|
||||||
|
return _classify(text, rules)
|
||||||
|
|
||||||
|
|
||||||
|
def _shift_years(day: date, years: int) -> date:
|
||||||
|
try:
|
||||||
|
return day.replace(year=day.year - years)
|
||||||
|
except ValueError: # 29.02 → 28.02
|
||||||
|
return day.replace(year=day.year - years, day=28)
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_relative(kind: PeriodKind, effective: date | None) -> date | None:
|
||||||
|
"""Дата относительной подписи; для ABSOLUTE и NONE — None (дата из подписи или от аналитика, P1-24)."""
|
||||||
|
if effective is None:
|
||||||
|
return None
|
||||||
|
if kind is PeriodKind.CURRENT:
|
||||||
|
return effective
|
||||||
|
if kind is PeriodKind.PREV_YEAR_END:
|
||||||
|
return date(effective.year - 1, 12, 31)
|
||||||
|
if kind is PeriodKind.PREV_PREV_YEAR_END:
|
||||||
|
return date(effective.year - 2, 12, 31)
|
||||||
|
if kind is PeriodKind.PREV_PERIOD:
|
||||||
|
return _shift_years(effective, 1)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def expected_dates(effective: date) -> frozenset[date]:
|
||||||
|
"""Даты колонок формы при отчётной дате: она сама, она − 1 и − 2 года, 31.12 года − 1 и года − 2."""
|
||||||
|
return frozenset(
|
||||||
|
{
|
||||||
|
effective,
|
||||||
|
_shift_years(effective, 1),
|
||||||
|
_shift_years(effective, 2),
|
||||||
|
date(effective.year - 1, 12, 31),
|
||||||
|
date(effective.year - 2, 12, 31),
|
||||||
|
}
|
||||||
|
)
|
||||||
94
arm_statements/tests/test_dates.py
Normal file
94
arm_statements/tests/test_dates.py
Normal file
@ -0,0 +1,94 @@
|
|||||||
|
from datetime import date
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from arm_statements.dates import expected_dates, parse_date, parse_period_label, resolve_relative
|
||||||
|
from arm_statements.loader import Rules
|
||||||
|
from arm_statements.model import Cell, PeriodKind
|
||||||
|
from arm_statements.text import fix_lookalikes
|
||||||
|
|
||||||
|
# Обязательные строки scan-gateway test_dates.py::test_parses_real_forms (спека «Шаг 2 — подпись и дата колонки»)
|
||||||
|
REAL_FORMS = [
|
||||||
|
("на 30 сентября 2025 г.", date(2025, 9, 30)),
|
||||||
|
("На 30 июня\n2025 г", date(2025, 6, 30)),
|
||||||
|
("31 декабря 2024 года", date(2024, 12, 31)),
|
||||||
|
("30 сентября 2025 r.", date(2025, 9, 30)),
|
||||||
|
("с 01.01.2025 по 30.09.2025", date(2025, 9, 30)),
|
||||||
|
("I квартал 2026", date(2026, 3, 31)),
|
||||||
|
("1-й кв. 2026", date(2026, 3, 31)),
|
||||||
|
("первый квартал 2025 г.", date(2025, 3, 31)),
|
||||||
|
("1 полугодие 2026", date(2026, 6, 30)),
|
||||||
|
("1 п/г 2026", date(2026, 6, 30)),
|
||||||
|
("9 месяцев 2025", date(2025, 9, 30)),
|
||||||
|
("9 мес. 2025", date(2025, 9, 30)),
|
||||||
|
("За январь - июнь\n2025 г.", date(2025, 6, 30)),
|
||||||
|
("январь – сентябрь 2025", date(2025, 9, 30)),
|
||||||
|
("за 2025 г.", date(2025, 12, 31)),
|
||||||
|
("2024 год", date(2024, 12, 31)),
|
||||||
|
("30.09.2025", date(2025, 9, 30)),
|
||||||
|
("2025 03 31", date(2025, 3, 31)),
|
||||||
|
("30/09/25", date(2025, 9, 30)),
|
||||||
|
("2 0 2 4", date(2024, 12, 31)),
|
||||||
|
("30 09 2 0 2 5", date(2025, 9, 30)),
|
||||||
|
("30092025", date(2025, 9, 30)),
|
||||||
|
("20250930", date(2025, 9, 30)),
|
||||||
|
("30 09 25", date(2025, 9, 30)),
|
||||||
|
("30.09.2025 0:00:00", date(2025, 9, 30)),
|
||||||
|
("2|0|2|4|", date(2024, 12, 31)),
|
||||||
|
("3l.12.2О24", date(2024, 12, 31)),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(("raw", "expected"), REAL_FORMS)
|
||||||
|
def test_parses_real_forms(rules: Rules, raw: str, expected: date) -> None:
|
||||||
|
assert parse_date(fix_lookalikes(raw, rules.lookalikes), rules) == expected
|
||||||
|
label = parse_period_label(Cell(raw, None, 100.0, "row:0"), rules)
|
||||||
|
assert (label.kind, label.date) == (PeriodKind.ABSOLUTE, expected)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("raw", ["Общество с ограниченной ответственностью", "", "+", "тыс. руб.", "1999999999999"])
|
||||||
|
def test_not_a_date(rules: Rules, raw: str) -> None:
|
||||||
|
assert parse_date(raw, rules) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_long_text_not_parsed(rules: Rules) -> None:
|
||||||
|
assert parse_date("на 30 сентября 2025 г." + "x" * 200, rules) is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("raw", "kind"),
|
||||||
|
[
|
||||||
|
("На отчетную дату", PeriodKind.CURRENT),
|
||||||
|
("За отчётный период", PeriodKind.CURRENT),
|
||||||
|
("На 31 декабря предыдущего года", PeriodKind.PREV_YEAR_END),
|
||||||
|
("На 31 декабря года, предшествующего предыдущему", PeriodKind.PREV_PREV_YEAR_END),
|
||||||
|
("За аналогичный период предыдущего года", PeriodKind.PREV_PERIOD),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_relative_labels(rules: Rules, raw: str, kind: PeriodKind) -> None:
|
||||||
|
assert parse_period_label(Cell(raw, None, 100.0, "row:0"), rules).kind is kind
|
||||||
|
|
||||||
|
|
||||||
|
def test_exclusion_and_excel_date_value(rules: Rules) -> None:
|
||||||
|
assert parse_period_label(Cell("Изменение, %", None, 100.0, "A!F1"), rules).excluded
|
||||||
|
label = parse_period_label(Cell("30.09.2025", date(2025, 9, 30), 100.0, "A!D1"), rules)
|
||||||
|
assert (label.kind, label.date) == (PeriodKind.ABSOLUTE, date(2025, 9, 30))
|
||||||
|
assert parse_period_label(Cell("2024", None, 100.0, "A!E1"), rules).bare_year
|
||||||
|
|
||||||
|
|
||||||
|
def test_empty_label(rules: Rules) -> None:
|
||||||
|
assert parse_period_label(None, rules).kind is PeriodKind.NONE
|
||||||
|
label = parse_period_label(Cell(" ", None, 100.0, "A!G1"), rules)
|
||||||
|
assert (label.kind, label.date, label.excluded, label.bare_year) == (PeriodKind.NONE, None, False, False)
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_relative_and_expected() -> None:
|
||||||
|
eff = date(2025, 9, 30)
|
||||||
|
assert resolve_relative(PeriodKind.PREV_YEAR_END, eff) == date(2024, 12, 31)
|
||||||
|
assert resolve_relative(PeriodKind.PREV_PREV_YEAR_END, eff) == date(2023, 12, 31)
|
||||||
|
assert resolve_relative(PeriodKind.PREV_PERIOD, date(2024, 2, 29)) == date(2023, 2, 28)
|
||||||
|
assert resolve_relative(PeriodKind.CURRENT, None) is None
|
||||||
|
assert resolve_relative(PeriodKind.ABSOLUTE, eff) is None and resolve_relative(PeriodKind.NONE, eff) is None
|
||||||
|
assert expected_dates(eff) == frozenset(
|
||||||
|
{eff, date(2024, 9, 30), date(2023, 9, 30), date(2024, 12, 31), date(2023, 12, 31)}
|
||||||
|
)
|
||||||
Reference in New Issue
Block a user