From df8a5285bb5ee6649b6bb1709497ecb46eee7e8a Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Sun, 4 Oct 2026 22:29:52 +0400 Subject: [PATCH] =?UTF-8?q?feat(domain):=20=D1=8F=D0=B4=D1=80=D0=BE=20?= =?UTF-8?q?=D0=B8=D0=BC=D0=BF=D0=BE=D1=80=D1=82=D0=B0=20=D1=81=D0=BF=D1=80?= =?UTF-8?q?=D0=B0=D0=B2=D0=BE=D1=87=D0=BD=D0=B8=D0=BA=D0=BE=D0=B2=20=D0=B8?= =?UTF-8?q?=D0=B7=20=D0=B2=D1=8B=D0=B3=D1=80=D1=83=D0=B7=D0=BA=D0=B8=20?= =?UTF-8?q?=D0=95=D0=A6=D0=9F=20(=C2=A713.1=E2=80=9313.2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - domain/directory_import.py: read_rows (xlsx первый лист / csv UTF-8 `;`/`,`, заголовки по нормализованному тексту, MISSING_COLUMNS / UNSUPPORTED_FILE), plan_import — чистая функция: операции плана и отчёт с кодами §13.2, ASSUMPTION(OQ-87..91); ИНН — через domain/inn.py. - openpyxl==3.1.5 (+ et_xmlfile==2.0.0) в requirements.txt, types-openpyxl в requirements-dev.txt. - Тесты без БД: нормализация, чтение csv/xlsx (Review Focus 1–3), каждый код §13.2, skip/update, фикстура «Русский Бисквит» (ФЛ в строке 12 → UNSUPPORTED_TYPE). - План фазы 2: Task 9 отмечена; tasks.md 3.1. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012vJkmBGKKqMoEMzvsXG3y2 --- api/requirements-dev.txt | 1 + api/requirements.txt | 2 + api/src/domain/directory_import.py | 552 +++++++++++++++ api/tests/test_domain_directory_import.py | 650 ++++++++++++++++++ .../2026-10-04-slice-1-directories-phase2.md | 6 +- openspec/changes/slice-1-directories/tasks.md | 2 +- 6 files changed, 1209 insertions(+), 4 deletions(-) create mode 100644 api/src/domain/directory_import.py create mode 100644 api/tests/test_domain_directory_import.py diff --git a/api/requirements-dev.txt b/api/requirements-dev.txt index f067282..d7aae10 100644 --- a/api/requirements-dev.txt +++ b/api/requirements-dev.txt @@ -11,3 +11,4 @@ PyJWT[crypto]>=2.9 pydbml>=1.1 jsonschema>=4.23 types-PyYAML>=6.0 +types-openpyxl>=3.1 diff --git a/api/requirements.txt b/api/requirements.txt index babec49..954ca54 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -16,6 +16,7 @@ click==8.4.2 cryptography==50.0.2 dnspython==2.8.0 email-validator==2.3.0 +et_xmlfile==2.0.0 fastapi==0.135.1 fastapi-cli==0.0.24 fastapi-cloud-cli==0.14.1 @@ -37,6 +38,7 @@ miniopy-async==1.23.4 multidict==6.7.1 nats-py==2.14.0 nkeys==0.2.1 +openpyxl==3.1.5 propcache==0.5.2 pycparser==3.0 pycryptodome==3.23.0 diff --git a/api/src/domain/directory_import.py b/api/src/domain/directory_import.py new file mode 100644 index 0000000..5141fde --- /dev/null +++ b/api/src/domain/directory_import.py @@ -0,0 +1,552 @@ +"""Импорт справочников из выгрузки ЕЦП CRM: чтение файла и план операций (`05-domain-rules.md` §13.1–13.2). + +Чистое ядро без БД (spec directory-import): `read_rows` превращает xlsx/csv в строки `ImportRow`, `plan_import` по +снимку справочника (`DirectorySnapshot`, строит сервис) решает судьбу каждой строки и возвращает операции (только +данные — применяет сервис) и отчёт с кодами §13.2. Идентификаторы новых объектов генерируются здесь (`uuid4`), чтобы +операции плана ссылались друг на друга до записи в БД. +""" + +import csv +import enum +import io +import re +import uuid +import zipfile +from collections.abc import Iterable, Sequence +from dataclasses import asdict, dataclass, field +from typing import Any + +from openpyxl import load_workbook +from openpyxl.utils.exceptions import InvalidFileException + +from src.common.exceptions import DomainValidationError +from src.domain.inn import normalize_inn, validate_inn +from src.models.enums import EntityType, RelatedGroupKind + + +class ImportCode(enum.StrEnum): + """Коды строк отчёта (§13.2). Итоговые: пять кодов пропуска и `LE_*`; остальные — сопутствующие события.""" + + UNSUPPORTED_TYPE = "UNSUPPORTED_TYPE" + INVALID_INN = "INVALID_INN" + DUPLICATE_IN_FILE = "DUPLICATE_IN_FILE" + LE_IN_OTHER_GROUP = "LE_IN_OTHER_GROUP" + GK_NAME_CONFLICT = "GK_NAME_CONFLICT" + LE_CREATED = "LE_CREATED" + LE_UPDATED = "LE_UPDATED" + LE_EXISTS = "LE_EXISTS" + RF_CREATED = "RF_CREATED" + GK_CREATED = "GK_CREATED" + GK_UPDATED = "GK_UPDATED" + GK_EXISTS = "GK_EXISTS" + GSK_CREATED = "GSK_CREATED" + GSZ_CREATED = "GSZ_CREATED" + MEMBER_ADDED = "MEMBER_ADDED" + UNKNOWN_FLAG = "UNKNOWN_FLAG" + + +class OnExisting(enum.StrEnum): + """Стратегия для существующих ГК/ЮЛ (§13.3): оставить или обновить имя/РФ/тип.""" + + skip = "skip" + update = "update" + + +# --------------------------------------------------------------------------- формат файла (§13.1) + +REQUIRED_COLUMNS = ("инн", "idгк", "группакомпаний", "организация", "тип") +OPTIONAL_COLUMNS = ("базовыйрфгквcrm", "гск", "гсз") +# Исходные имена колонок — для перечня в `MISSING_COLUMNS` +COLUMN_TITLES: dict[str, str] = { + "инн": "ИНН", + "idгк": "iD ГК", + "базовыйрфгквcrm": "Базовый РФ ГК в CRM", + "группакомпаний": "Группа компаний", + "организация": "Организация", + "гск": "ГСК", + "гсз": "ГСЗ", + "тип": "Тип", +} +FLAG_GSK = "входитвгск" +FLAG_GSZ = "входитвгсз" + +MISSING_COLUMNS = "MISSING_COLUMNS" +UNSUPPORTED_FILE = "UNSUPPORTED_FILE" + +_WHITESPACE_RE = re.compile(r"\s+") + + +def normalize_header(raw: object) -> str: + """Заголовок/флаг для сопоставления: нижний регистр, без пробельных символов, `ё` = `е` (§13.1).""" + return _WHITESPACE_RE.sub("", normalize_cell(raw).lower()).replace("ё", "е") + + +def normalize_cell(raw: object) -> str: + """Значение ячейки строкой: `None` → `""`; целое число (в т.ч. `3528052718.0` из xlsx) — без дробной части; trim.""" + if raw is None: + return "" + if isinstance(raw, bool): + return str(raw) + if isinstance(raw, int): + return str(raw) + if isinstance(raw, float) and raw.is_integer(): + return str(int(raw)) + return str(raw).strip() + + +class FileFormatError(Exception): + """Ошибка файла целиком (§13.3: прерывает импорт): `MISSING_COLUMNS` (с перечнем) или `UNSUPPORTED_FILE`.""" + + def __init__(self, code: str, message: str, missing: list[str] | None = None) -> None: + super().__init__(message) + self.code = code + self.message = message + self.missing = missing if missing is not None else [] + + +@dataclass(frozen=True) +class ImportRow: + """Строка выгрузки после нормализации ячеек; `row` — номер строки в файле (1-based, как в Excel).""" + + row: int + inn: str + crm_id: str + office_name: str + group_name: str + entity_name: str + gsk_flag: str + gsz_flag: str + type_raw: str + + +def read_rows(filename: str, content: bytes) -> list[ImportRow]: + """Строки данных из xlsx (первый лист) или csv (UTF-8, `;`/`,`); формат выбирается по расширению имени файла. + + Заголовки — первая непустая строка, сопоставляются по `normalize_header`; пустые строки пропускаются. + """ + extension = filename.rsplit(".", 1)[-1].lower() if "." in filename else "" + if extension == "xlsx": + raw_rows = _xlsx_rows(content) + elif extension == "csv": + raw_rows = _csv_rows(content) + else: + raise FileFormatError(UNSUPPORTED_FILE, "поддерживаются только файлы xlsx и csv") + return _rows_from_table(raw_rows) + + +def _xlsx_rows(content: bytes) -> list[tuple[int, list[str]]]: + try: + workbook = load_workbook(io.BytesIO(content), read_only=True, data_only=True) + except (zipfile.BadZipFile, InvalidFileException, KeyError, ValueError, OSError, EOFError) as error: + raise FileFormatError(UNSUPPORTED_FILE, "файл xlsx не читается") from error + try: + sheet = workbook.worksheets[0] + # Размер листа из файла бывает неверным (экспорт сторонних систем) — читаем все строки фактически + sheet.reset_dimensions() + return [ + (number, [normalize_cell(value) for value in values]) + for number, values in enumerate(sheet.iter_rows(values_only=True), start=1) + ] + finally: + workbook.close() + + +def _csv_rows(content: bytes) -> list[tuple[int, list[str]]]: + try: + text = content.decode("utf-8-sig") + except UnicodeDecodeError as error: + raise FileFormatError(UNSUPPORTED_FILE, "кодировка csv не распознана: ожидается UTF-8") from error + header_line = next((line for line in text.splitlines() if line.strip()), "") + delimiter = ";" if header_line.count(";") > header_line.count(",") else "," + try: + return [ + (number, [normalize_cell(value) for value in values]) + for number, values in enumerate(csv.reader(io.StringIO(text, newline=""), delimiter=delimiter), start=1) + ] + except csv.Error as error: + raise FileFormatError(UNSUPPORTED_FILE, "файл csv не читается") from error + + +def _rows_from_table(raw_rows: Iterable[tuple[int, list[str]]]) -> list[ImportRow]: + columns: dict[str, int] | None = None + result: list[ImportRow] = [] + for number, cells in raw_rows: + if not any(cells): + continue + if columns is None: + columns = _header_columns(cells) + else: + result.append(_import_row(number, cells, columns)) + if columns is None: + _header_columns([]) + return result + + +def _import_row(number: int, cells: list[str], columns: dict[str, int]) -> ImportRow: + def cell(name: str) -> str: + index = columns.get(name) + return cells[index] if index is not None and index < len(cells) else "" + + return ImportRow( + row=number, + inn=cell("инн"), + crm_id=cell("idгк"), + office_name=cell("базовыйрфгквcrm"), + group_name=cell("группакомпаний"), + entity_name=cell("организация"), + gsk_flag=cell("гск"), + gsz_flag=cell("гсз"), + type_raw=cell("тип"), + ) + + +def _header_columns(cells: list[str]) -> dict[str, int]: + """Индексы известных колонок (первое вхождение); нет обязательной → `MISSING_COLUMNS` с исходными именами.""" + columns: dict[str, int] = {} + for index, value in enumerate(cells): + name = normalize_header(value) + if name in COLUMN_TITLES and name not in columns: + columns[name] = index + missing = [COLUMN_TITLES[name] for name in REQUIRED_COLUMNS if name not in columns] + if missing: + raise FileFormatError(MISSING_COLUMNS, "нет обязательных колонок: " + ", ".join(missing), missing) + return columns + + +# --------------------------------------------------------------------------- снимок справочника + + +@dataclass +class OfficeSnap: + id: uuid.UUID + name: str + + +@dataclass +class GroupSnap: + id: uuid.UUID + crm_id: str + name: str + office_id: uuid.UUID | None + gsk_id: uuid.UUID | None + gsz_id: uuid.UUID | None + + +@dataclass +class EntitySnap: + """ЮЛ/ИП; `gsk_group_id`/`gsz_group_id` — текущие членства (`related_group_member.is_current`).""" + + id: uuid.UUID + inn: str + name: str + entity_type: EntityType + group_id: uuid.UUID | None + gsk_group_id: uuid.UUID | None + gsz_group_id: uuid.UUID | None + + +@dataclass +class DirectorySnapshot: + offices_by_name: dict[str, OfficeSnap] + groups_by_crm: dict[str, GroupSnap] + entities_by_inn: dict[str, EntitySnap] + + +# --------------------------------------------------------------------------- операции плана + + +@dataclass(frozen=True) +class CreateOffice: + id: uuid.UUID + name: str + + +@dataclass(frozen=True) +class CreateGroup: + id: uuid.UUID + crm_id: str + name: str + office_id: uuid.UUID | None + + +@dataclass(frozen=True) +class UpdateGroup: + """`None` в поле — без изменений; офис меняется только непустым значением из файла.""" + + id: uuid.UUID + name: str | None + office_id: uuid.UUID | None + + +@dataclass(frozen=True) +class CreateEntity: + id: uuid.UUID + inn: str + name: str + entity_type: EntityType + group_id: uuid.UUID + office_id: uuid.UUID | None + + +@dataclass(frozen=True) +class UpdateEntity: + """`None` в поле — без изменений.""" + + id: uuid.UUID + name: str | None + entity_type: EntityType | None + + +@dataclass(frozen=True) +class CreateRelatedGroup: + id: uuid.UUID + group_id: uuid.UUID + kind: RelatedGroupKind + name: str + + +@dataclass(frozen=True) +class AddMember: + related_group_id: uuid.UUID + entity_id: uuid.UUID + kind: RelatedGroupKind + + +Operation = CreateOffice | CreateGroup | UpdateGroup | CreateEntity | UpdateEntity | CreateRelatedGroup | AddMember + + +# --------------------------------------------------------------------------- отчёт + + +@dataclass +class RowResult: + row: int + inn: str + code: ImportCode + message: str + + +@dataclass +class ImportReport: + """Счётчики и построчный список отчёта (§13.3); `errors` — число пропущенных строк.""" + + rows: int = 0 + groups_created: int = 0 + groups_updated: int = 0 + groups_skipped: int = 0 + entities_created: int = 0 + entities_updated: int = 0 + entities_skipped: int = 0 + memberships_created: int = 0 + offices_created: int = 0 + errors: int = 0 + lines: list[RowResult] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + """JSON-совместимый словарь (коды — строки) для ответа и `audit_event.after_value`.""" + data = asdict(self) + data["lines"] = [{**line, "code": str(line["code"])} for line in data["lines"]] + return data + + +@dataclass +class ImportPlan: + operations: list[Operation] + report: ImportReport + + +# --------------------------------------------------------------------------- план (§13.2) + +_ENTITY_TYPES: dict[str, EntityType] = {"юл": EntityType.ЮЛ, "ип": EntityType.ИП} +_FLAGS: tuple[tuple[RelatedGroupKind, str, str], ...] = ( + (RelatedGroupKind.gsk, FLAG_GSK, "ГСК"), + (RelatedGroupKind.gsz, FLAG_GSZ, "ГСЗ"), +) +_RELATED_CREATED = {RelatedGroupKind.gsk: ImportCode.GSK_CREATED, RelatedGroupKind.gsz: ImportCode.GSZ_CREATED} + + +@dataclass +class _GroupState: + """ГК, уже встреченная в файле: имя из первой строки, действующий РФ и ГСК/ГСЗ (из снимка или созданные).""" + + id: uuid.UUID + file_name: str + first_row: int + office_id: uuid.UUID | None + related: dict[RelatedGroupKind, uuid.UUID | None] + + +class _RowSkipped(Exception): + """Строка пропущена с итоговым кодом (код уже записан в отчёт).""" + + +class _Planner: + def __init__(self, snapshot: DirectorySnapshot, on_existing: OnExisting) -> None: + self.snapshot = snapshot + self.on_existing = on_existing + self.operations: list[Operation] = [] + self.report = ImportReport() + self.offices: dict[str, uuid.UUID] = {name: office.id for name, office in snapshot.offices_by_name.items()} + self.groups: dict[str, _GroupState] = {} + self.seen_inns: dict[str, int] = {} + + def line(self, row: ImportRow, inn: str, code: ImportCode, message: str) -> None: + self.report.lines.append(RowResult(row=row.row, inn=inn, code=code, message=message)) + + def skip(self, row: ImportRow, inn: str, code: ImportCode, message: str) -> _RowSkipped: + self.line(row, inn, code, message) + self.report.errors += 1 + return _RowSkipped() + + def plan_row(self, row: ImportRow) -> None: + inn = normalize_inn(row.inn) + # ASSUMPTION(OQ-87): типы вне {ЮЛ, ИП} (`нерез.`, `ФЛ`, `нет инн`, пусто) пропускаются + entity_type = _ENTITY_TYPES.get(row.type_raw.lower()) + if entity_type is None: + raise self.skip(row, inn, ImportCode.UNSUPPORTED_TYPE, f"тип «{row.type_raw}» не поддерживается: ЮЛ или ИП") + try: + # ASSUMPTION(OQ-88): только длина (ЮЛ — 10, ИП — 12) и цифры, контрольные разряды не проверяются + inn = validate_inn(inn, entity_type) + except DomainValidationError as error: + raise self.skip(row, inn, ImportCode.INVALID_INN, error.message) from error + if inn in self.seen_inns: + raise self.skip( + row, inn, ImportCode.DUPLICATE_IN_FILE, f"ИНН уже встречался в строке {self.seen_inns[inn]}" + ) + self.seen_inns[inn] = row.row + group = self.plan_group(row, inn) + entity_id, current = self.plan_entity(row, inn, entity_type, group) + self.plan_flags(row, inn, group, entity_id, current) + + def plan_office(self, row: ImportRow, inn: str) -> uuid.UUID | None: + name = row.office_name + if not name: + return None + office_id = self.offices.get(name) + if office_id is None: + # ASSUMPTION(OQ-89): РФ, которого нет в справочнике (в т.ч. «ЦКБ», «Головная организация»), создаётся + office_id = uuid.uuid4() + self.offices[name] = office_id + self.operations.append(CreateOffice(id=office_id, name=name)) + self.report.offices_created += 1 + self.line(row, inn, ImportCode.RF_CREATED, f"создан филиал «{name}»") + return office_id + + def plan_group(self, row: ImportRow, inn: str) -> _GroupState: + crm_id = row.crm_id + if not crm_id: + raise self.skip(row, inn, ImportCode.GK_NAME_CONFLICT, "пустой iD ГК") + state = self.groups.get(crm_id) + if state is not None: + if row.group_name != state.file_name: + message = ( + f"ГК {crm_id}: имя «{row.group_name}» расходится с «{state.file_name}» (строка {state.first_row})" + ) + raise self.skip(row, inn, ImportCode.GK_NAME_CONFLICT, message) + return state + office_id = self.plan_office(row, inn) + snap = self.snapshot.groups_by_crm.get(crm_id) + if snap is None: + group_id = uuid.uuid4() + self.operations.append(CreateGroup(id=group_id, crm_id=crm_id, name=row.group_name, office_id=office_id)) + self.report.groups_created += 1 + self.line(row, inn, ImportCode.GK_CREATED, f"создана ГК {crm_id} «{row.group_name}»") + state = _GroupState(group_id, row.group_name, row.row, office_id, {kind: None for kind in RelatedGroupKind}) + else: + state = self.existing_group(row, inn, snap, office_id) + self.groups[crm_id] = state + return state + + def existing_group(self, row: ImportRow, inn: str, snap: GroupSnap, office_id: uuid.UUID | None) -> _GroupState: + new_name = row.group_name if row.group_name != snap.name else None + new_office = office_id if office_id is not None and office_id != snap.office_id else None + effective_office = snap.office_id + if self.on_existing is OnExisting.update and (new_name is not None or new_office is not None): + self.operations.append(UpdateGroup(id=snap.id, name=new_name, office_id=new_office)) + self.report.groups_updated += 1 + self.line(row, inn, ImportCode.GK_UPDATED, f"обновлена ГК {snap.crm_id} «{row.group_name}»") + effective_office = new_office if new_office is not None else snap.office_id + else: + self.report.groups_skipped += 1 + self.line(row, inn, ImportCode.GK_EXISTS, f"ГК {snap.crm_id} «{snap.name}» уже есть") + related = {RelatedGroupKind.gsk: snap.gsk_id, RelatedGroupKind.gsz: snap.gsz_id} + return _GroupState(snap.id, row.group_name, row.row, effective_office, related) + + def plan_entity( + self, row: ImportRow, inn: str, entity_type: EntityType, group: _GroupState + ) -> tuple[uuid.UUID, EntitySnap | None]: + snap = self.snapshot.entities_by_inn.get(inn) + if snap is None: + entity_id = uuid.uuid4() + self.operations.append( + CreateEntity( + id=entity_id, + inn=inn, + name=row.entity_name, + entity_type=entity_type, + group_id=group.id, + office_id=group.office_id, + ) + ) + self.report.entities_created += 1 + self.line(row, inn, ImportCode.LE_CREATED, f"создано {entity_type.value} «{row.entity_name}»") + return entity_id, None + if snap.group_id != group.id: + # ASSUMPTION(OQ-90): ЮЛ из другой ГК (или без ГК) импорт не переводит — только явная операция §13.4 + raise self.skip(row, inn, ImportCode.LE_IN_OTHER_GROUP, f"«{snap.name}» уже состоит в другой ГК") + new_name = row.entity_name if row.entity_name != snap.name else None + new_type = entity_type if entity_type != snap.entity_type else None + if self.on_existing is OnExisting.update and (new_name is not None or new_type is not None): + self.operations.append(UpdateEntity(id=snap.id, name=new_name, entity_type=new_type)) + self.report.entities_updated += 1 + self.line(row, inn, ImportCode.LE_UPDATED, f"обновлено {entity_type.value} «{row.entity_name}»") + else: + self.report.entities_skipped += 1 + self.line(row, inn, ImportCode.LE_EXISTS, f"«{snap.name}» уже есть") + return snap.id, snap + + def plan_flags( + self, row: ImportRow, inn: str, group: _GroupState, entity_id: uuid.UUID, current: EntitySnap | None + ) -> None: + for kind, expected, title in _FLAGS: + raw = row.gsk_flag if kind is RelatedGroupKind.gsk else row.gsz_flag + if not raw: + continue + if normalize_header(raw) != expected: + self.line( + row, inn, ImportCode.UNKNOWN_FLAG, f"значение «{raw}» в колонке {title} не распознано: не входит" + ) + continue + related_id = group.related[kind] + if related_id is None: + # ASSUMPTION(OQ-91): одна группа вида на ГК, имя — имя ГК + related_id = uuid.uuid4() + group.related[kind] = related_id + self.operations.append( + CreateRelatedGroup(id=related_id, group_id=group.id, kind=kind, name=group.file_name) + ) + self.line(row, inn, _RELATED_CREATED[kind], f"создана {title} «{group.file_name}»") + membership = None + if current is not None: + membership = current.gsk_group_id if kind is RelatedGroupKind.gsk else current.gsz_group_id + # Текущее членство (в этой или другой группе того же вида) импорт не меняет и не закрывает + if membership is None: + self.operations.append(AddMember(related_group_id=related_id, entity_id=entity_id, kind=kind)) + self.report.memberships_created += 1 + self.line(row, inn, ImportCode.MEMBER_ADDED, f"добавлено в {title} «{group.file_name}»") + + +def plan_import(rows: Sequence[ImportRow], snapshot: DirectorySnapshot, on_existing: OnExisting) -> ImportPlan: + """План импорта по §13.2: строки в порядке файла, у каждой ровно один итоговый код и события того же `row`. + + Накопители «создано в этом файле» (РФ по имени, ГК по `iD ГК`, ГСК/ГСЗ по ГК, ИНН) гарантируют, что одна ГК из + многих строк создаётся один раз. Снимок не изменяется. + """ + planner = _Planner(snapshot, on_existing) + for row in rows: + planner.report.rows += 1 + try: + planner.plan_row(row) + except _RowSkipped: + continue + return ImportPlan(operations=planner.operations, report=planner.report) diff --git a/api/tests/test_domain_directory_import.py b/api/tests/test_domain_directory_import.py new file mode 100644 index 0000000..0494d57 --- /dev/null +++ b/api/tests/test_domain_directory_import.py @@ -0,0 +1,650 @@ +"""Ядро импорта справочников из выгрузки ЕЦП CRM (§13.1–13.2, spec directory-import): чтение файла и план без БД.""" + +import csv +import io +import uuid +from pathlib import Path +from typing import TypeVar + +import openpyxl +import pytest + +from src.domain.directory_import import ( + AddMember, + CreateEntity, + CreateGroup, + CreateOffice, + CreateRelatedGroup, + DirectorySnapshot, + EntitySnap, + FileFormatError, + GroupSnap, + ImportCode, + ImportPlan, + ImportRow, + OfficeSnap, + OnExisting, + UpdateEntity, + UpdateGroup, + normalize_cell, + normalize_header, + plan_import, + read_rows, +) +from src.models.enums import EntityType, RelatedGroupKind + +T = TypeVar("T") + +HEADERS = ["ИНН", "iD ГК", "Базовый РФ ГК в CRM", "Группа компаний", "Организация", "ГСК", "ГСЗ", "Тип"] +FINAL_CODES = { + ImportCode.UNSUPPORTED_TYPE, + ImportCode.INVALID_INN, + ImportCode.DUPLICATE_IN_FILE, + ImportCode.LE_IN_OTHER_GROUP, + ImportCode.GK_NAME_CONFLICT, + ImportCode.LE_CREATED, + ImportCode.LE_UPDATED, + ImportCode.LE_EXISTS, +} + + +# ---------------------------------------------------------------- нормализация + + +def test_normalize_header_lowercases_and_drops_spaces() -> None: + assert normalize_header("Базовый РФ ГК в CRM") == "базовыйрфгквcrm" + assert normalize_header(" iD ГК ") == "idгк" + assert normalize_header("Входит в ГСК") == "входитвгск" + + +def test_normalize_header_treats_yo_as_ye() -> None: + assert normalize_header("Объём Ёлки") == "объемелки" + + +def test_normalize_header_of_none_is_empty() -> None: + assert normalize_header(None) == "" + + +def test_normalize_cell_numeric_inn_has_no_fraction() -> None: + """Review Focus 2: ИНН из xlsx числом (float) → строка без `.0`.""" + assert normalize_cell(3528052718.0) == "3528052718" + assert normalize_cell(352819549855.0) == "352819549855" + assert normalize_cell(3528052718) == "3528052718" + + +def test_normalize_cell_none_and_strings() -> None: + assert normalize_cell(None) == "" + assert normalize_cell(" ЮЛ ") == "ЮЛ" + + +# ---------------------------------------------------------------- чтение файла + + +def _csv_bytes(rows: list[list[str]], delimiter: str = ";", bom: bool = True) -> bytes: + buffer = io.StringIO() + writer = csv.writer(buffer, delimiter=delimiter, lineterminator="\n") + writer.writerows(rows) + return buffer.getvalue().encode("utf-8-sig" if bom else "utf-8") + + +def _data_row(inn: str = "3528052718", type_: str = "ЮЛ") -> list[str]: + return [ + inn, + "1-AZFL6BQ", + "Санкт-Петербургский РФ", + "Русский Бисквит", + "РУССКИЙ БИСКВИТ, АО", + "входит в ГСК", + "", + type_, + ] + + +@pytest.mark.parametrize("delimiter", [";", ","]) +def test_read_csv_with_either_delimiter(delimiter: str) -> None: + content = _csv_bytes([HEADERS, _data_row()], delimiter=delimiter) + rows = read_rows("export.csv", content) + assert rows == [ + ImportRow( + row=2, + inn="3528052718", + crm_id="1-AZFL6BQ", + office_name="Санкт-Петербургский РФ", + group_name="Русский Бисквит", + entity_name="РУССКИЙ БИСКВИТ, АО", + gsk_flag="входит в ГСК", + gsz_flag="", + type_raw="ЮЛ", + ) + ] + + +def test_read_csv_without_bom() -> None: + rows = read_rows("export.CSV", _csv_bytes([HEADERS, _data_row()], bom=False)) + assert [r.inn for r in rows] == ["3528052718"] + + +def test_read_csv_columns_reordered_and_extra_column_ignored() -> None: + headers = ["Лишняя", "Тип", "Организация", "Группа компаний", "iD ГК", "ИНН"] + content = _csv_bytes([headers, ["x", "ИП", "ИП Юганов", "Русский Бисквит", "1-AZFL6BQ", "352819549855"]]) + (row,) = read_rows("export.csv", content) + assert (row.inn, row.type_raw, row.entity_name, row.group_name, row.crm_id) == ( + "352819549855", + "ИП", + "ИП Юганов", + "Русский Бисквит", + "1-AZFL6BQ", + ) + assert (row.office_name, row.gsk_flag, row.gsz_flag) == ("", "", "") + + +def test_read_csv_skips_empty_lines_and_keeps_file_row_numbers() -> None: + content = _csv_bytes([HEADERS, _data_row("3528052718"), ["", "", "", "", "", "", "", ""], _data_row("3528104476")]) + rows = read_rows("export.csv", content) + assert [(r.row, r.inn) for r in rows] == [(2, "3528052718"), (4, "3528104476")] + + +def test_read_csv_missing_type_column() -> None: + content = _csv_bytes([HEADERS[:-1], _data_row()[:-1]]) + with pytest.raises(FileFormatError) as error: + read_rows("export.csv", content) + assert error.value.code == "MISSING_COLUMNS" + assert error.value.missing == ["Тип"] + + +def test_read_csv_not_utf8_is_unsupported() -> None: + """Review Focus 3: csv в cp1251 → UNSUPPORTED_FILE «ожидается UTF-8», не 500.""" + content = ";".join(HEADERS).encode("cp1251") + with pytest.raises(FileFormatError) as error: + read_rows("export.csv", content) + assert error.value.code == "UNSUPPORTED_FILE" + assert "UTF-8" in error.value.message + + +def test_read_txt_is_unsupported() -> None: + with pytest.raises(FileFormatError) as error: + read_rows("export.txt", _csv_bytes([HEADERS, _data_row()])) + assert error.value.code == "UNSUPPORTED_FILE" + + +def test_read_broken_xlsx_is_unsupported() -> None: + with pytest.raises(FileFormatError) as error: + read_rows("export.xlsx", b"not a zip at all") + assert error.value.code == "UNSUPPORTED_FILE" + + +def test_read_xlsx_header_in_second_row_numeric_inn_and_trailing_empty_rows() -> None: + """Review Focus 1–2: заголовок в первой непустой строке, ИНН числом, 3 пустые строки в хвосте пропускаются.""" + workbook = openpyxl.Workbook() + sheet = workbook.active + assert sheet is not None + for column, header in enumerate(HEADERS, start=1): + sheet.cell(row=2, column=column, value=header) + first: list[float | str | None] = [ + 3528052718.0, + "1-AZFL6BQ", + "Санкт-Петербургский РФ", + "Русский Бисквит", + "АО", + "входит в ГСК", + None, + "ЮЛ", + ] + second: list[float | str | None] = [ + 352819549855, + "1-AZFL6BQ", + "Санкт-Петербургский РФ", + "Русский Бисквит", + "ИП", + None, + None, + "ИП", + ] + for column, value in enumerate(first, start=1): + sheet.cell(row=3, column=column, value=value) + for column, value in enumerate(second, start=1): + sheet.cell(row=4, column=column, value=value) + for row in (5, 6, 7): + sheet.cell(row=row, column=1, value="") + sheet.cell(row=row, column=8, value=None) + buffer = io.BytesIO() + workbook.save(buffer) + + rows = read_rows("export.xlsx", buffer.getvalue()) + + assert [(r.row, r.inn, r.type_raw) for r in rows] == [(3, "3528052718", "ЮЛ"), (4, "352819549855", "ИП")] + assert rows[0].gsk_flag == "входит в ГСК" + + +# ---------------------------------------------------------------- план импорта: помощники + + +def _row( # noqa: PLR0913 - строка выгрузки: все поля значимы для теста + row: int, + *, + inn: str = "3528052718", + type_raw: str = "ЮЛ", + crm_id: str = "1-AZFL6BQ", + group_name: str = "Русский Бисквит", + office_name: str = "Санкт-Петербургский РФ", + entity_name: str = "РУССКИЙ БИСКВИТ, АО", + gsk_flag: str = "", + gsz_flag: str = "", +) -> ImportRow: + return ImportRow( + row=row, + inn=inn, + crm_id=crm_id, + office_name=office_name, + group_name=group_name, + entity_name=entity_name, + gsk_flag=gsk_flag, + gsz_flag=gsz_flag, + type_raw=type_raw, + ) + + +def _empty() -> DirectorySnapshot: + return DirectorySnapshot(offices_by_name={}, groups_by_crm={}, entities_by_inn={}) + + +def _final_codes(plan: ImportPlan) -> dict[int, ImportCode]: + finals: dict[int, ImportCode] = {} + for line in plan.report.lines: + if line.code in FINAL_CODES: + assert line.row not in finals, f"строка {line.row}: два итоговых кода" + finals[line.row] = line.code + return finals + + +def _events(plan: ImportPlan, row: int) -> list[ImportCode]: + return [line.code for line in plan.report.lines if line.row == row and line.code not in FINAL_CODES] + + +def _ops(plan: ImportPlan, kind: type[T]) -> list[T]: + return [op for op in plan.operations if isinstance(op, kind)] + + +OFFICE_ID = uuid.uuid4() +GROUP_ID = uuid.uuid4() +OTHER_GROUP_ID = uuid.uuid4() +ENTITY_ID = uuid.uuid4() +GSK_ID = uuid.uuid4() + + +def _existing( + entity_group: uuid.UUID | None = GROUP_ID, + group_name: str = "Русский Бисквит", + gsk_member: uuid.UUID | None = None, +) -> DirectorySnapshot: + return DirectorySnapshot( + offices_by_name={"Санкт-Петербургский РФ": OfficeSnap(id=OFFICE_ID, name="Санкт-Петербургский РФ")}, + groups_by_crm={ + "1-AZFL6BQ": GroupSnap( + id=GROUP_ID, crm_id="1-AZFL6BQ", name=group_name, office_id=OFFICE_ID, gsk_id=None, gsz_id=None + ), + "1-OTHER": GroupSnap( + id=OTHER_GROUP_ID, crm_id="1-OTHER", name="Другая", office_id=None, gsk_id=None, gsz_id=None + ), + }, + entities_by_inn={ + "3528052718": EntitySnap( + id=ENTITY_ID, + inn="3528052718", + name="РУССКИЙ БИСКВИТ, АО", + entity_type=EntityType.ЮЛ, + group_id=entity_group, + gsk_group_id=gsk_member, + gsz_group_id=None, + ) + }, + ) + + +# ---------------------------------------------------------------- план импорта: по коду + + +@pytest.mark.parametrize( + ("rows", "snapshot", "expected_final", "expected_events"), + [ + pytest.param([_row(2, type_raw="ФЛ")], _empty(), ImportCode.UNSUPPORTED_TYPE, [], id="UNSUPPORTED_TYPE-ФЛ"), + pytest.param([_row(2, type_raw="нерез.")], _empty(), ImportCode.UNSUPPORTED_TYPE, [], id="UNSUPPORTED_TYPE"), + pytest.param([_row(2, type_raw="")], _empty(), ImportCode.UNSUPPORTED_TYPE, [], id="UNSUPPORTED_TYPE-пусто"), + pytest.param([_row(2, inn="35280527")], _empty(), ImportCode.INVALID_INN, [], id="INVALID_INN-короткий"), + pytest.param( + [_row(2, inn="3528052718", type_raw="ИП")], _empty(), ImportCode.INVALID_INN, [], id="INVALID_INN-ИП-10" + ), + pytest.param([_row(2, inn="LV12345678")], _empty(), ImportCode.INVALID_INN, [], id="INVALID_INN-буквы"), + pytest.param([_row(2, crm_id="")], _empty(), ImportCode.GK_NAME_CONFLICT, [], id="GK_NAME_CONFLICT-пустой"), + pytest.param( + [_row(2)], + _empty(), + ImportCode.LE_CREATED, + [ImportCode.RF_CREATED, ImportCode.GK_CREATED], + id="LE_CREATED", + ), + pytest.param([_row(2)], _existing(), ImportCode.LE_EXISTS, [ImportCode.GK_EXISTS], id="LE_EXISTS"), + pytest.param( + [_row(2)], + _existing(entity_group=OTHER_GROUP_ID), + ImportCode.LE_IN_OTHER_GROUP, + [ImportCode.GK_EXISTS], + id="LE_IN_OTHER_GROUP", + ), + pytest.param( + [_row(2)], + _existing(entity_group=None), + ImportCode.LE_IN_OTHER_GROUP, + [ImportCode.GK_EXISTS], + id="LE_IN_OTHER_GROUP-без-ГК", + ), + pytest.param( + [_row(2, gsk_flag="входит в ГСК")], + _empty(), + ImportCode.LE_CREATED, + [ImportCode.RF_CREATED, ImportCode.GK_CREATED, ImportCode.GSK_CREATED, ImportCode.MEMBER_ADDED], + id="GSK_CREATED+MEMBER_ADDED", + ), + pytest.param( + [_row(2, gsz_flag="Входит в ГСЗ")], + _empty(), + ImportCode.LE_CREATED, + [ImportCode.RF_CREATED, ImportCode.GK_CREATED, ImportCode.GSZ_CREATED, ImportCode.MEMBER_ADDED], + id="GSZ_CREATED+MEMBER_ADDED", + ), + pytest.param( + [_row(2, gsk_flag="не входит в ГСК")], + _empty(), + ImportCode.LE_CREATED, + [ImportCode.RF_CREATED, ImportCode.GK_CREATED, ImportCode.UNKNOWN_FLAG], + id="UNKNOWN_FLAG", + ), + ], +) +def test_plan_row_code( + rows: list[ImportRow], + snapshot: DirectorySnapshot, + expected_final: ImportCode, + expected_events: list[ImportCode], +) -> None: + plan = plan_import(rows, snapshot, OnExisting.skip) + assert _final_codes(plan) == {2: expected_final} + assert _events(plan, 2) == expected_events + assert plan.report.rows == 1 + + +def test_skipped_rows_count_as_errors_and_produce_no_operations() -> None: + rows = [_row(2, type_raw="ФЛ"), _row(3, inn="12"), _row(4, crm_id="")] + plan = plan_import(rows, _empty(), OnExisting.skip) + assert plan.operations == [] + assert plan.report.errors == len(rows) + assert plan.report.rows == len(rows) + + +def test_duplicate_in_file_after_first_occurrence() -> None: + rows = [_row(2, inn="352819549855", type_raw="ИП"), _row(3, inn="352819549855", type_raw="ип")] + plan = plan_import(rows, _empty(), OnExisting.skip) + assert _final_codes(plan) == {2: ImportCode.LE_CREATED, 3: ImportCode.DUPLICATE_IN_FILE} + assert plan.report.errors == 1 + assert len(_ops(plan, CreateEntity)) == 1 + + +def test_inn_with_spaces_is_normalized() -> None: + plan = plan_import([_row(2, inn=" 3528 052 718 ")], _empty(), OnExisting.skip) + (entity,) = _ops(plan, CreateEntity) + assert entity.inn == "3528052718" + assert plan.report.lines[-1].inn == "3528052718" + + +def test_gk_name_conflict_inside_file() -> None: + rows = [_row(2), _row(3, inn="3528104476", group_name="Русский Бисквит 2")] + plan = plan_import(rows, _empty(), OnExisting.skip) + assert _final_codes(plan) == {2: ImportCode.LE_CREATED, 3: ImportCode.GK_NAME_CONFLICT} + assert len(_ops(plan, CreateGroup)) == 1 + assert len(_ops(plan, CreateEntity)) == 1 + assert plan.report.errors == 1 + + +def test_empty_snapshot_creates_office_group_entity_with_links() -> None: + plan = plan_import([_row(2, type_raw="юл")], _empty(), OnExisting.skip) + (office,) = _ops(plan, CreateOffice) + (group,) = _ops(plan, CreateGroup) + (entity,) = _ops(plan, CreateEntity) + assert office.name == "Санкт-Петербургский РФ" + assert (group.crm_id, group.name, group.office_id) == ("1-AZFL6BQ", "Русский Бисквит", office.id) + assert (entity.inn, entity.name, entity.entity_type) == ("3528052718", "РУССКИЙ БИСКВИТ, АО", EntityType.ЮЛ) + assert (entity.group_id, entity.office_id) == (group.id, office.id) + report = plan.report + assert (report.offices_created, report.groups_created, report.entities_created, report.errors) == (1, 1, 1, 0) + rf_line = next(line for line in report.lines if line.code == ImportCode.RF_CREATED) + assert (rf_line.row, rf_line.message) == (2, "создан филиал «Санкт-Петербургский РФ»") + + +def test_empty_office_name_gives_group_without_office() -> None: + plan = plan_import([_row(2, office_name="")], _empty(), OnExisting.skip) + assert _ops(plan, CreateOffice) == [] + (group,) = _ops(plan, CreateGroup) + (entity,) = _ops(plan, CreateEntity) + assert group.office_id is None + assert entity.office_id is None + + +def test_existing_office_is_reused() -> None: + snapshot = _empty() + snapshot.offices_by_name["Санкт-Петербургский РФ"] = OfficeSnap(id=OFFICE_ID, name="Санкт-Петербургский РФ") + plan = plan_import([_row(2)], snapshot, OnExisting.skip) + assert _ops(plan, CreateOffice) == [] + (group,) = _ops(plan, CreateGroup) + assert group.office_id == OFFICE_ID + assert plan.report.offices_created == 0 + + +def test_two_rows_of_one_group_create_it_once() -> None: + rows = [_row(2), _row(3, inn="3528104476", entity_name="ПЕКАРНЯ СЧАСТЛИВОЧНАЯ, ООО")] + plan = plan_import(rows, _empty(), OnExisting.skip) + (group,) = _ops(plan, CreateGroup) + assert len(_ops(plan, CreateOffice)) == 1 + assert [e.group_id for e in _ops(plan, CreateEntity)] == [group.id, group.id] + assert plan.report.groups_created == 1 + assert _events(plan, 3) == [] + + +def test_flag_on_two_entities_creates_one_related_group_and_two_members() -> None: + rows = [_row(2, gsk_flag="входит в ГСК"), _row(3, inn="3528104476", gsk_flag="входит в ГСК")] + plan = plan_import(rows, _empty(), OnExisting.skip) + (group,) = _ops(plan, CreateGroup) + (related,) = _ops(plan, CreateRelatedGroup) + assert (related.group_id, related.kind, related.name) == (group.id, RelatedGroupKind.gsk, "Русский Бисквит") + members = _ops(plan, AddMember) + assert [m.related_group_id for m in members] == [related.id, related.id] + assert [m.kind for m in members] == [RelatedGroupKind.gsk, RelatedGroupKind.gsk] + assert [m.entity_id for m in members] == [e.id for e in _ops(plan, CreateEntity)] + assert plan.report.memberships_created == len(rows) + + +def test_existing_related_group_of_group_is_reused() -> None: + snapshot = _existing() + snapshot.groups_by_crm["1-AZFL6BQ"].gsk_id = GSK_ID + plan = plan_import([_row(2, inn="3528104476", gsk_flag="входит в ГСК")], snapshot, OnExisting.skip) + assert _ops(plan, CreateRelatedGroup) == [] + (member,) = _ops(plan, AddMember) + assert member.related_group_id == GSK_ID + + +def test_existing_membership_is_kept() -> None: + snapshot = _existing(gsk_member=GSK_ID) + snapshot.groups_by_crm["1-AZFL6BQ"].gsk_id = GSK_ID + plan = plan_import([_row(2, gsk_flag="входит в ГСК")], snapshot, OnExisting.skip) + assert _ops(plan, AddMember) == [] + assert _events(plan, 2) == [ImportCode.GK_EXISTS] + + +def test_membership_in_other_related_group_is_not_moved() -> None: + other_gsk = uuid.uuid4() + plan = plan_import([_row(2, gsk_flag="входит в ГСК")], _existing(gsk_member=other_gsk), OnExisting.skip) + assert _ops(plan, AddMember) == [] + assert ImportCode.MEMBER_ADDED not in _events(plan, 2) + + +def test_entity_in_other_group_gets_no_membership() -> None: + plan = plan_import([_row(2, gsk_flag="входит в ГСК")], _existing(entity_group=OTHER_GROUP_ID), OnExisting.skip) + assert _ops(plan, AddMember) == [] + assert _ops(plan, CreateRelatedGroup) == [] + assert plan.report.errors == 1 + + +def test_changed_names_with_skip_are_left_as_is() -> None: + rows = [_row(2, entity_name="РУССКИЙ БИСКВИТ, ПАО")] + plan = plan_import(rows, _existing(group_name="Старое имя"), OnExisting.skip) + assert plan.operations == [] + assert _final_codes(plan) == {2: ImportCode.LE_EXISTS} + assert _events(plan, 2) == [ImportCode.GK_EXISTS] + report = plan.report + assert (report.groups_skipped, report.entities_skipped, report.groups_updated, report.entities_updated) == ( + 1, + 1, + 0, + 0, + ) + + +def test_changed_names_with_update_produce_updates() -> None: + rows = [_row(2, entity_name="РУССКИЙ БИСКВИТ, ПАО")] + plan = plan_import(rows, _existing(group_name="Старое имя"), OnExisting.update) + assert plan.operations == [ + UpdateGroup(id=GROUP_ID, name="Русский Бисквит", office_id=None), + UpdateEntity(id=ENTITY_ID, name="РУССКИЙ БИСКВИТ, ПАО", entity_type=None), + ] + assert _final_codes(plan) == {2: ImportCode.LE_UPDATED} + assert _events(plan, 2) == [ImportCode.GK_UPDATED] + report = plan.report + assert (report.groups_updated, report.entities_updated, report.groups_skipped, report.entities_skipped) == ( + 1, + 1, + 0, + 0, + ) + + +def test_update_without_differences_is_exists() -> None: + plan = plan_import([_row(2)], _existing(), OnExisting.update) + assert plan.operations == [] + assert _final_codes(plan) == {2: ImportCode.LE_EXISTS} + assert _events(plan, 2) == [ImportCode.GK_EXISTS] + + +def test_update_moves_group_to_new_office_and_new_entity_follows() -> None: + rows = [_row(2, inn="3528104476", office_name="Вологодский РФ")] + plan = plan_import(rows, _existing(), OnExisting.update) + (office,) = _ops(plan, CreateOffice) + (update,) = _ops(plan, UpdateGroup) + (entity,) = _ops(plan, CreateEntity) + assert update == UpdateGroup(id=GROUP_ID, name=None, office_id=office.id) + assert (entity.group_id, entity.office_id) == (GROUP_ID, office.id) + + +def test_empty_office_in_file_keeps_group_office_for_new_entity() -> None: + rows = [_row(2, inn="3528104476", office_name="")] + plan = plan_import(rows, _existing(), OnExisting.update) + assert _ops(plan, UpdateGroup) == [] + (entity,) = _ops(plan, CreateEntity) + assert entity.office_id == OFFICE_ID + + +def test_update_changes_entity_type() -> None: + snapshot = _existing() + snapshot.entities_by_inn["352819549855"] = EntitySnap( + id=ENTITY_ID, + inn="352819549855", + name="ИП Юганов", + entity_type=EntityType.ЮЛ, + group_id=GROUP_ID, + gsk_group_id=None, + gsz_group_id=None, + ) + plan = plan_import( + [_row(2, inn="352819549855", type_raw="ИП", entity_name="ИП Юганов")], snapshot, OnExisting.update + ) + assert plan.operations == [UpdateEntity(id=ENTITY_ID, name=None, entity_type=EntityType.ИП)] + + +def test_group_events_once_per_group() -> None: + rows = [_row(2), _row(3, inn="3528104476")] + plan = plan_import(rows, _existing(group_name="Старое имя"), OnExisting.update) + assert len(_ops(plan, UpdateGroup)) == 1 + assert _events(plan, 2) == [ImportCode.GK_UPDATED] + assert _events(plan, 3) == [] + assert plan.report.groups_updated == 1 + + +def test_report_to_dict() -> None: + plan = plan_import([_row(2, type_raw="ФЛ")], _empty(), OnExisting.skip) + data = plan.report.to_dict() + assert data["rows"] == 1 + assert data["errors"] == 1 + assert data["groups_created"] == 0 + assert data["lines"] == [ + {"row": 2, "inn": "3528052718", "code": "UNSUPPORTED_TYPE", "message": data["lines"][0]["message"]} + ] + assert isinstance(data["lines"][0]["code"], str) + assert set(data) == { + "rows", + "groups_created", + "groups_updated", + "groups_skipped", + "entities_created", + "entities_updated", + "entities_skipped", + "memberships_created", + "offices_created", + "errors", + "lines", + } + + +# ---------------------------------------------------------------- реальная выгрузка + + +# Ожидания по фактическому содержимому `ecp-export-russkiy-biskvit.xlsx` (лист «Выписки_01.04.26», строки 2–22): +# строка 11 — «ИП Юганов Максим Анатольевич» (ИП, ИНН 352819549855), строка 12 — «Юганов Максим Анатольевич» (ФЛ, тот же +# ИНН). Тип проверяется раньше дубля (§13.2), поэтому строка 12 → UNSUPPORTED_TYPE, DUPLICATE_IN_FILE в срезе нет. +# Остальные 20 строк — 18 ЮЛ и 2 ИП (строки 11, 14) → LE_CREATED. +RB_ROWS = 21 +RB_FL_ROW = 12 +RB_LEGAL_ENTITIES = 18 +RB_SOLE_PROPRIETORS = 2 +RB_EXPECTED_FINALS = { + number: ImportCode.UNSUPPORTED_TYPE if number == RB_FL_ROW else ImportCode.LE_CREATED for number in range(2, 23) +} +RB_GSK_MEMBERS = ["ПЕКАРНЯ СЧАСТЛИВОЧНАЯ, ООО", "РУССКИЙ БИСКВИТ, АО"] + + +@pytest.mark.fixtures +def test_russkiy_biskvit_export_on_empty_directory(fixtures_dir: Path) -> None: + """Срез «Русский Бисквит» на пустом справочнике: 1 ГК, 1 РФ, 20 лиц (18 ЮЛ + 2 ИП), 1 ГСК с двумя членами.""" + path = fixtures_dir / "directories" / "ecp-export-russkiy-biskvit.xlsx" + rows = read_rows(path.name, path.read_bytes()) + by_row = {r.row: r for r in rows} + assert (by_row[11].inn, by_row[11].type_raw) == ("352819549855", "ИП") + assert (by_row[RB_FL_ROW].inn, by_row[RB_FL_ROW].type_raw) == ("352819549855", "ФЛ") + assert [r.type_raw for r in rows].count("ЮЛ") == RB_LEGAL_ENTITIES + assert [r.type_raw for r in rows].count("ИП") == RB_SOLE_PROPRIETORS + + plan = plan_import(rows, _empty(), OnExisting.skip) + report = plan.report + + assert report.rows == RB_ROWS + assert _final_codes(plan) == RB_EXPECTED_FINALS + assert (report.groups_created, report.offices_created, report.errors) == (1, 1, 1) + assert report.entities_created == RB_LEGAL_ENTITIES + RB_SOLE_PROPRIETORS + (group,) = _ops(plan, CreateGroup) + assert (group.crm_id, group.name) == ("1-AZFL6BQ", "Русский Бисквит") + (office,) = _ops(plan, CreateOffice) + assert office.name == "Санкт-Петербургский РФ" + (related,) = _ops(plan, CreateRelatedGroup) + assert (related.kind, related.name) == (RelatedGroupKind.gsk, "Русский Бисквит") + entities = {e.id: e.name for e in _ops(plan, CreateEntity)} + members = _ops(plan, AddMember) + assert sorted(entities[m.entity_id] for m in members) == RB_GSK_MEMBERS + assert report.memberships_created == len(RB_GSK_MEMBERS) + assert all(m.kind is RelatedGroupKind.gsk for m in members) diff --git a/docs/superpowers/plans/2026-10-04-slice-1-directories-phase2.md b/docs/superpowers/plans/2026-10-04-slice-1-directories-phase2.md index 44d66e7..f0a58a4 100644 --- a/docs/superpowers/plans/2026-10-04-slice-1-directories-phase2.md +++ b/docs/superpowers/plans/2026-10-04-slice-1-directories-phase2.md @@ -121,13 +121,13 @@ def plan_import(rows: Sequence[ImportRow], snapshot: DirectorySnapshot, on_exist 7. Флаги: `normalize_header(flag)` == `FLAG_GSK`/`FLAG_GSZ` → членство; непустое иное → `UNKNOWN_FLAG` (событие, не ошибка). Для флага: ГСК/ГСЗ ГК из снимка/накопителя или `CreateRelatedGroup(kind, name=имя ГК)` (`GSK_CREATED`/`GSZ_CREATED`, `# ASSUMPTION(OQ-91)`); у ЮЛ нет текущего членства этого вида → `AddMember` (`MEMBER_ADDED`, `memberships_created += 1`); есть в этой группе → ничего; есть в другой группе того же вида → ничего (импорт не переводит; событие не пишется). `rows` = число непустых строк данных. Итоговый код строки — ровно один из восьми; события — дополнительные `RowResult` с тем же `row`. -- [ ] **Step 1: Тесты** (`tests/test_domain_directory_import.py`, без БД): +- [x] **Step 1: Тесты** (`tests/test_domain_directory_import.py`, без БД): - `normalize_header("Базовый РФ ГК в CRM") == "базовыйрфгквcrm"`, `normalize_cell(3528052718.0) == "3528052718"`, `normalize_cell(None) == ""`. - `read_rows` csv `;` и `,`, utf-8-sig, перестановка колонок, лишняя колонка; без «Тип» → `FileFormatError.code == "MISSING_COLUMNS"`, `missing == ["Тип"]`; `.txt` → `UNSUPPORTED_FILE`; xlsx, собранный `openpyxl.Workbook()` с заголовком во 2-й строке, ИНН числом, 3 пустыми строками в хвосте (Review Focus 1–2) → 2 строки данных. - `plan_import` на синтетических строках, по одному тесту на каждый код (параметризация: строка → ожидаемый итоговый код и события), пустой снимок и снимок с существующими; `update` vs `skip`; две строки одной ГК → один `CreateGroup`; флаг у двух ЮЛ одной ГК → один `CreateRelatedGroup` и два `AddMember`; ЮЛ в другой ГК → нет `AddMember`. - `@pytest.mark.fixtures`: `read_rows` + `plan_import` на `fixtures_dir / "directories/ecp-export-russkiy-biskvit.xlsx"` с пустым снимком: `report.rows == 21`, `groups_created == 1`, `offices_created == 1`, `entities_created == 18`, ровно одна `CreateRelatedGroup(kind=gsk)`, два `AddMember`, нет `gsz`; коды строк ФЛ/дубля — зафиксировать по фактическому порядку в файле (посмотреть файл в тесте, не угадывать) и записать ожидание явно. -- [ ] **Step 2:** падает → **Step 3:** реализация (+ `openpyxl` в requirements, `make venv`) → **Step 4:** `pytest tests/test_domain_directory_import.py -v`, `mypy src` (strict для `src.domain.*`), `ruff` → PASS. -- [ ] **Step 5: Commit** `feat(domain): ядро импорта справочников из выгрузки ЕЦП (§13.1–13.2)`. +- [x] **Step 2:** падает → **Step 3:** реализация (+ `openpyxl` в requirements, `make venv`) → **Step 4:** `pytest tests/test_domain_directory_import.py -v`, `mypy src` (strict для `src.domain.*`), `ruff` → PASS. +- [x] **Step 5: Commit** `feat(domain): ядро импорта справочников из выгрузки ЕЦП (§13.1–13.2)`. --- diff --git a/openspec/changes/slice-1-directories/tasks.md b/openspec/changes/slice-1-directories/tasks.md index 99a4ffc..3ab58cf 100644 --- a/openspec/changes/slice-1-directories/tasks.md +++ b/openspec/changes/slice-1-directories/tasks.md @@ -46,7 +46,7 @@ test-integration`). Пути — от корня репозитория. Доп ## 3. Импорт из ЕЦП CRM -- [ ] 3.1 **Чтение файла и ядро решений.** `openpyxl` в `api/requirements.txt` (пин); `domain/directory_import.py`: +- [x] 3.1 **Чтение файла и ядро решений.** `openpyxl` в `api/requirements.txt` (пин); `domain/directory_import.py`: `read_rows(file) -> (headers, rows)` для xlsx/csv, нормализация заголовков (`MISSING_COLUMNS`), `ImportCode` (StrEnum = §13.2), `plan_import(rows, snapshot, on_existing) -> ImportPlan(operations, report)` — чистая функция; `ASSUMPTION(OQ-87..91)` в коде. Unit-тесты без БД: синтетический csv, покрывающий каждый код §13.2 ровно один раз; xlsx-фикстура