feat: словарь периодов — даты, относительные подписи бланка 66н, исключения колонок
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BJKMLHdxMwSUeYcdkwqX4W
This commit is contained in:
Andrey Epifancev
parent
aed37cc189
commit
19f70fea65
276
arm_statements/dates.py
Normal file
276
arm_statements/dates.py
Normal file
@ -0,0 +1,276 @@
|
||||
"""Даты и подписи колонок: разбор текстовых и числовых форм, относительные подписи бланка 66н (§11.3, §12 п. 4).
|
||||
|
||||
Вход `parse_date` — текст после `fix_lookalikes`. Регулярки и месяцы — данные (`rules/statement_periods.yaml`),
|
||||
здесь только обработчики на каждый вид абсолютной формы."""
|
||||
|
||||
import calendar
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
|
||||
from arm_statements.loader import PeriodRules, Rules
|
||||
from arm_statements.model import Cell, PeriodKind
|
||||
from arm_statements.text import fix_lookalikes, normalize_caption
|
||||
|
||||
__all__ = ["PeriodLabel", "expected_dates", "parse_date", "parse_period_label", "resolve_relative"]
|
||||
|
||||
_BARE_YEAR = re.compile(r"\d{4}")
|
||||
_DIGITS = re.compile(r"\d+")
|
||||
_ROMAN = {"I": 1, "II": 2, "III": 3, "IV": 4, "1V": 4, "11": 2, "111": 3}
|
||||
_ORDINAL = {"перв": 1, "втор": 2, "трет": 3, "четв": 4}
|
||||
_QUARTER_MONTHS = 3
|
||||
_HALF_MONTHS = 6
|
||||
_DECEMBER = 12
|
||||
_SHORT_YEAR_LIMIT = 100 # меньше — двузначный год → 20YY
|
||||
_CENTURY = 2000
|
||||
_DIGITS_FULL = 8
|
||||
_DIGITS_YEAR = 4
|
||||
_DIGITS_SHORT = 6
|
||||
_PARTS = 3
|
||||
_SHORT_PART = 2
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PeriodLabel:
|
||||
kind: PeriodKind
|
||||
date: date | None
|
||||
excluded: bool
|
||||
bare_year: bool
|
||||
|
||||
|
||||
def _year_full(year: str) -> int:
|
||||
value = int(year)
|
||||
return value if value >= _SHORT_YEAR_LIMIT else _CENTURY + value
|
||||
|
||||
|
||||
def _month_num(word: str, rules: PeriodRules) -> int | None:
|
||||
lowered = word.lower()
|
||||
for stem, number in rules.months:
|
||||
if lowered.startswith(stem):
|
||||
return number
|
||||
return None
|
||||
|
||||
|
||||
def _month_end(year: int, month: int) -> date | None:
|
||||
if not 1 <= month <= _DECEMBER:
|
||||
return None
|
||||
try:
|
||||
return date(year, month, calendar.monthrange(year, month)[1])
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _period_end(year: str, month: int | None) -> date | None:
|
||||
return _month_end(_year_full(year), month) if month else None
|
||||
|
||||
|
||||
def _safe(year: int, month: int | None, day: int) -> date | None:
|
||||
try:
|
||||
return date(year, month, day) if month else None
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _ordinal(word: str) -> int | None:
|
||||
"""Номер квартала/полугодия из «1», «I», «IV», «первый», «1-й»."""
|
||||
token = word.strip().upper().rstrip("-Й")
|
||||
if token in _ROMAN:
|
||||
return _ROMAN[token]
|
||||
if token.isdigit():
|
||||
return int(token)
|
||||
lowered = token.lower()
|
||||
for stem, number in _ORDINAL.items():
|
||||
if lowered.startswith(stem):
|
||||
return number
|
||||
return None
|
||||
|
||||
|
||||
def _group(match: re.Match[str], name: str) -> str | None:
|
||||
return match.group(name) if name in match.re.groupindex else None
|
||||
|
||||
|
||||
def _handle_range_end(match: re.Match[str], rules: PeriodRules) -> date | None:
|
||||
return _handle_day_month_year(match, rules)
|
||||
|
||||
|
||||
def _handle_day_month_year(match: re.Match[str], rules: PeriodRules) -> date | None:
|
||||
year, day = _group(match, "y"), _group(match, "d")
|
||||
if year is None or day is None:
|
||||
return None
|
||||
month_word = _group(match, "mw")
|
||||
month = _month_num(month_word, rules) if month_word is not None else int(_group(match, "m") or 0)
|
||||
return _safe(_year_full(year), month, int(day))
|
||||
|
||||
|
||||
def _handle_quarter(match: re.Match[str], rules: PeriodRules) -> date | None:
|
||||
del rules
|
||||
year, number = _group(match, "y"), _ordinal(_group(match, "q") or "")
|
||||
return _period_end(year, (number or 0) * _QUARTER_MONTHS) if year else None
|
||||
|
||||
|
||||
def _handle_half(match: re.Match[str], rules: PeriodRules) -> date | None:
|
||||
del rules
|
||||
year, number = _group(match, "y"), _ordinal(_group(match, "q") or "")
|
||||
return _period_end(year, (number or 0) * _HALF_MONTHS) if year else None
|
||||
|
||||
|
||||
def _handle_months(match: re.Match[str], rules: PeriodRules) -> date | None:
|
||||
del rules
|
||||
year, count = _group(match, "y"), _group(match, "n")
|
||||
return _period_end(year, int(count)) if year and count else None
|
||||
|
||||
|
||||
def _handle_month_range(match: re.Match[str], rules: PeriodRules) -> date | None:
|
||||
year, word = _group(match, "y"), _group(match, "mw")
|
||||
return _period_end(year, _month_num(word, rules)) if year and word else None
|
||||
|
||||
|
||||
def _handle_year(match: re.Match[str], rules: PeriodRules) -> date | None:
|
||||
year = _group(match, "y")
|
||||
return _period_end(year, _DECEMBER) if year and _year_ok(int(year), rules) else None
|
||||
|
||||
|
||||
_HANDLERS: dict[str, Callable[[re.Match[str], PeriodRules], date | None]] = {
|
||||
"range_end": _handle_range_end,
|
||||
"day_month_year": _handle_day_month_year,
|
||||
"quarter": _handle_quarter,
|
||||
"half": _handle_half,
|
||||
"months": _handle_months,
|
||||
"month_range": _handle_month_range,
|
||||
"year": _handle_year,
|
||||
}
|
||||
|
||||
|
||||
def _year_ok(year: int, rules: PeriodRules) -> bool:
|
||||
return rules.min_year <= year <= rules.max_year
|
||||
|
||||
|
||||
def _text_date(text: str, rules: PeriodRules) -> date | None:
|
||||
for item in rules.absolute:
|
||||
match = item.pattern.search(text)
|
||||
if match:
|
||||
found = _HANDLERS[item.kind](match, rules)
|
||||
if found:
|
||||
return found
|
||||
return None
|
||||
|
||||
|
||||
def _from_parts(first: str, second: str, third: str) -> date | None:
|
||||
"""ДД ММ ГГ[ГГ] и ГГ[ГГ] ММ ДД (год двух- или четырёхзначный только в позиции года)."""
|
||||
for day, month, year in ((first, second, third), (third, second, first)):
|
||||
if len(year) in (_SHORT_PART, _DIGITS_YEAR) and len(day) <= _SHORT_PART and len(month) <= _SHORT_PART:
|
||||
try:
|
||||
return date(_year_full(year), int(month), int(day))
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _from_digits(digits: str) -> date | None:
|
||||
"""8 слитных цифр: ДДММГГГГ, ГГГГММДД, ГГГГДДММ — первый валидный вариант."""
|
||||
return (
|
||||
_from_parts(digits[:2], digits[2:4], digits[4:])
|
||||
or _from_parts(digits[6:], digits[4:6], digits[:4])
|
||||
or _from_parts(digits[4:6], digits[6:], digits[:4])
|
||||
)
|
||||
|
||||
|
||||
def _from_digits_run(digits: str, groups: list[str], rules: PeriodRules) -> date | None:
|
||||
"""Склеенные цифры «2 0 2 4», «30 09 2 0 2 5»: решает их число."""
|
||||
if len(digits) == _DIGITS_FULL:
|
||||
return _from_digits(digits)
|
||||
if len(digits) == _DIGITS_YEAR:
|
||||
return date(int(digits), 12, 31) if _year_ok(int(digits), rules) else None
|
||||
if len(digits) == _DIGITS_SHORT:
|
||||
return _from_parts(digits[:2], digits[2:4], digits[4:])
|
||||
if len(groups) > _PARTS: # «30.09.2025 0:00:00»
|
||||
return _from_parts(*groups[:_PARTS])
|
||||
return None
|
||||
|
||||
|
||||
def _numeric_date(text: str, rules: PeriodRules) -> date | None:
|
||||
groups = _DIGITS.findall(text)
|
||||
if not groups:
|
||||
return None
|
||||
if len(groups) == _PARTS:
|
||||
return _from_parts(*groups)
|
||||
return _from_digits_run("".join(groups), groups, rules)
|
||||
|
||||
|
||||
def parse_date(text: str | None, rules: Rules) -> date | None:
|
||||
"""Дата из текста после очистки или None. Длиннее `max_text_length` не разбирается (проверка до регулярок)."""
|
||||
if text is None:
|
||||
return None
|
||||
stripped = text.strip()
|
||||
if not stripped or len(stripped) > rules.periods.max_text_length:
|
||||
return None
|
||||
periods = rules.periods
|
||||
return (
|
||||
_text_date(stripped, periods)
|
||||
or _numeric_date(stripped.split("\n", 1)[0], periods)
|
||||
or _numeric_date(stripped.replace("\n", " "), periods)
|
||||
)
|
||||
|
||||
|
||||
def _is_excluded(caption: str, rules: Rules) -> bool:
|
||||
return any(rx.search(caption) for rx in rules.headers.column_exclusions)
|
||||
|
||||
|
||||
def _classify(text: str, rules: Rules) -> PeriodLabel:
|
||||
caption = normalize_caption(text)
|
||||
bare_year = _BARE_YEAR.fullmatch(text.strip()) is not None
|
||||
if _is_excluded(caption, rules):
|
||||
return PeriodLabel(PeriodKind.NONE, None, True, False)
|
||||
for kind, pattern in rules.periods.relative:
|
||||
if pattern.search(caption):
|
||||
return PeriodLabel(kind, None, False, bare_year)
|
||||
found = parse_date(text, rules)
|
||||
return PeriodLabel(PeriodKind.ABSOLUTE if found else PeriodKind.NONE, found, False, bare_year)
|
||||
|
||||
|
||||
def parse_period_label(cell: Cell | None, rules: Rules) -> PeriodLabel:
|
||||
"""Подпись колонки → вид периода и дата. Дата-значение ячейки Excel — абсолютная."""
|
||||
if cell is not None and isinstance(cell.value, date):
|
||||
return PeriodLabel(PeriodKind.ABSOLUTE, cell.value, False, False)
|
||||
raw = cell.text.strip() if cell is not None else ""
|
||||
if not raw:
|
||||
return PeriodLabel(PeriodKind.NONE, None, False, False)
|
||||
# длинный текст не чистим от двойников (backtracking); parse_date всё равно вернёт None
|
||||
text = raw if len(raw) > rules.periods.max_text_length else fix_lookalikes(raw, rules.lookalikes)
|
||||
return _classify(text, rules)
|
||||
|
||||
|
||||
def _shift_years(day: date, years: int) -> date:
|
||||
try:
|
||||
return day.replace(year=day.year - years)
|
||||
except ValueError: # 29.02 → 28.02
|
||||
return day.replace(year=day.year - years, day=28)
|
||||
|
||||
|
||||
def resolve_relative(kind: PeriodKind, effective: date | None) -> date | None:
|
||||
"""Дата относительной подписи; для ABSOLUTE и NONE — None (дата из подписи или от аналитика, P1-24)."""
|
||||
if effective is None:
|
||||
return None
|
||||
if kind is PeriodKind.CURRENT:
|
||||
return effective
|
||||
if kind is PeriodKind.PREV_YEAR_END:
|
||||
return date(effective.year - 1, 12, 31)
|
||||
if kind is PeriodKind.PREV_PREV_YEAR_END:
|
||||
return date(effective.year - 2, 12, 31)
|
||||
if kind is PeriodKind.PREV_PERIOD:
|
||||
return _shift_years(effective, 1)
|
||||
return None
|
||||
|
||||
|
||||
def expected_dates(effective: date) -> frozenset[date]:
|
||||
"""Даты колонок формы при отчётной дате: она сама, она − 1 и − 2 года, 31.12 года − 1 и года − 2."""
|
||||
return frozenset(
|
||||
{
|
||||
effective,
|
||||
_shift_years(effective, 1),
|
||||
_shift_years(effective, 2),
|
||||
date(effective.year - 1, 12, 31),
|
||||
date(effective.year - 2, 12, 31),
|
||||
}
|
||||
)
|
||||
94
arm_statements/tests/test_dates.py
Normal file
94
arm_statements/tests/test_dates.py
Normal file
@ -0,0 +1,94 @@
|
||||
from datetime import date
|
||||
|
||||
import pytest
|
||||
|
||||
from arm_statements.dates import expected_dates, parse_date, parse_period_label, resolve_relative
|
||||
from arm_statements.loader import Rules
|
||||
from arm_statements.model import Cell, PeriodKind
|
||||
from arm_statements.text import fix_lookalikes
|
||||
|
||||
# Обязательные строки scan-gateway test_dates.py::test_parses_real_forms (спека «Шаг 2 — подпись и дата колонки»)
|
||||
REAL_FORMS = [
|
||||
("на 30 сентября 2025 г.", date(2025, 9, 30)),
|
||||
("На 30 июня\n2025 г", date(2025, 6, 30)),
|
||||
("31 декабря 2024 года", date(2024, 12, 31)),
|
||||
("30 сентября 2025 r.", date(2025, 9, 30)),
|
||||
("с 01.01.2025 по 30.09.2025", date(2025, 9, 30)),
|
||||
("I квартал 2026", date(2026, 3, 31)),
|
||||
("1-й кв. 2026", date(2026, 3, 31)),
|
||||
("первый квартал 2025 г.", date(2025, 3, 31)),
|
||||
("1 полугодие 2026", date(2026, 6, 30)),
|
||||
("1 п/г 2026", date(2026, 6, 30)),
|
||||
("9 месяцев 2025", date(2025, 9, 30)),
|
||||
("9 мес. 2025", date(2025, 9, 30)),
|
||||
("За январь - июнь\n2025 г.", date(2025, 6, 30)),
|
||||
("январь – сентябрь 2025", date(2025, 9, 30)),
|
||||
("за 2025 г.", date(2025, 12, 31)),
|
||||
("2024 год", date(2024, 12, 31)),
|
||||
("30.09.2025", date(2025, 9, 30)),
|
||||
("2025 03 31", date(2025, 3, 31)),
|
||||
("30/09/25", date(2025, 9, 30)),
|
||||
("2 0 2 4", date(2024, 12, 31)),
|
||||
("30 09 2 0 2 5", date(2025, 9, 30)),
|
||||
("30092025", date(2025, 9, 30)),
|
||||
("20250930", date(2025, 9, 30)),
|
||||
("30 09 25", date(2025, 9, 30)),
|
||||
("30.09.2025 0:00:00", date(2025, 9, 30)),
|
||||
("2|0|2|4|", date(2024, 12, 31)),
|
||||
("3l.12.2О24", date(2024, 12, 31)),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("raw", "expected"), REAL_FORMS)
|
||||
def test_parses_real_forms(rules: Rules, raw: str, expected: date) -> None:
|
||||
assert parse_date(fix_lookalikes(raw, rules.lookalikes), rules) == expected
|
||||
label = parse_period_label(Cell(raw, None, 100.0, "row:0"), rules)
|
||||
assert (label.kind, label.date) == (PeriodKind.ABSOLUTE, expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", ["Общество с ограниченной ответственностью", "", "+", "тыс. руб.", "1999999999999"])
|
||||
def test_not_a_date(rules: Rules, raw: str) -> None:
|
||||
assert parse_date(raw, rules) is None
|
||||
|
||||
|
||||
def test_long_text_not_parsed(rules: Rules) -> None:
|
||||
assert parse_date("на 30 сентября 2025 г." + "x" * 200, rules) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("raw", "kind"),
|
||||
[
|
||||
("На отчетную дату", PeriodKind.CURRENT),
|
||||
("За отчётный период", PeriodKind.CURRENT),
|
||||
("На 31 декабря предыдущего года", PeriodKind.PREV_YEAR_END),
|
||||
("На 31 декабря года, предшествующего предыдущему", PeriodKind.PREV_PREV_YEAR_END),
|
||||
("За аналогичный период предыдущего года", PeriodKind.PREV_PERIOD),
|
||||
],
|
||||
)
|
||||
def test_relative_labels(rules: Rules, raw: str, kind: PeriodKind) -> None:
|
||||
assert parse_period_label(Cell(raw, None, 100.0, "row:0"), rules).kind is kind
|
||||
|
||||
|
||||
def test_exclusion_and_excel_date_value(rules: Rules) -> None:
|
||||
assert parse_period_label(Cell("Изменение, %", None, 100.0, "A!F1"), rules).excluded
|
||||
label = parse_period_label(Cell("30.09.2025", date(2025, 9, 30), 100.0, "A!D1"), rules)
|
||||
assert (label.kind, label.date) == (PeriodKind.ABSOLUTE, date(2025, 9, 30))
|
||||
assert parse_period_label(Cell("2024", None, 100.0, "A!E1"), rules).bare_year
|
||||
|
||||
|
||||
def test_empty_label(rules: Rules) -> None:
|
||||
assert parse_period_label(None, rules).kind is PeriodKind.NONE
|
||||
label = parse_period_label(Cell(" ", None, 100.0, "A!G1"), rules)
|
||||
assert (label.kind, label.date, label.excluded, label.bare_year) == (PeriodKind.NONE, None, False, False)
|
||||
|
||||
|
||||
def test_resolve_relative_and_expected() -> None:
|
||||
eff = date(2025, 9, 30)
|
||||
assert resolve_relative(PeriodKind.PREV_YEAR_END, eff) == date(2024, 12, 31)
|
||||
assert resolve_relative(PeriodKind.PREV_PREV_YEAR_END, eff) == date(2023, 12, 31)
|
||||
assert resolve_relative(PeriodKind.PREV_PERIOD, date(2024, 2, 29)) == date(2023, 2, 28)
|
||||
assert resolve_relative(PeriodKind.CURRENT, None) is None
|
||||
assert resolve_relative(PeriodKind.ABSOLUTE, eff) is None and resolve_relative(PeriodKind.NONE, eff) is None
|
||||
assert expected_dates(eff) == frozenset(
|
||||
{eff, date(2024, 9, 30), date(2023, 9, 30), date(2024, 12, 31), date(2023, 12, 31)}
|
||||
)
|
||||
Reference in New Issue
Block a user