feat(domain): ядро импорта справочников из выгрузки ЕЦП (§13.1–13.2)
- domain/directory_import.py: read_rows (xlsx первый лист / csv UTF-8 `;`/`,`, заголовки по нормализованному тексту, MISSING_COLUMNS / UNSUPPORTED_FILE), plan_import — чистая функция: операции плана и отчёт с кодами §13.2, ASSUMPTION(OQ-87..91); ИНН — через domain/inn.py. - openpyxl==3.1.5 (+ et_xmlfile==2.0.0) в requirements.txt, types-openpyxl в requirements-dev.txt. - Тесты без БД: нормализация, чтение csv/xlsx (Review Focus 1–3), каждый код §13.2, skip/update, фикстура «Русский Бисквит» (ФЛ в строке 12 → UNSUPPORTED_TYPE). - План фазы 2: Task 9 отмечена; tasks.md 3.1. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012vJkmBGKKqMoEMzvsXG3y2
This commit is contained in:
Andrey Epifancev
parent
dd176a3c69
commit
df8a5285bb
@ -11,3 +11,4 @@ PyJWT[crypto]>=2.9
|
||||
pydbml>=1.1
|
||||
jsonschema>=4.23
|
||||
types-PyYAML>=6.0
|
||||
types-openpyxl>=3.1
|
||||
|
||||
@ -16,6 +16,7 @@ click==8.4.2
|
||||
cryptography==50.0.2
|
||||
dnspython==2.8.0
|
||||
email-validator==2.3.0
|
||||
et_xmlfile==2.0.0
|
||||
fastapi==0.135.1
|
||||
fastapi-cli==0.0.24
|
||||
fastapi-cloud-cli==0.14.1
|
||||
@ -37,6 +38,7 @@ miniopy-async==1.23.4
|
||||
multidict==6.7.1
|
||||
nats-py==2.14.0
|
||||
nkeys==0.2.1
|
||||
openpyxl==3.1.5
|
||||
propcache==0.5.2
|
||||
pycparser==3.0
|
||||
pycryptodome==3.23.0
|
||||
|
||||
552
api/src/domain/directory_import.py
Normal file
552
api/src/domain/directory_import.py
Normal file
@ -0,0 +1,552 @@
|
||||
"""Импорт справочников из выгрузки ЕЦП CRM: чтение файла и план операций (`05-domain-rules.md` §13.1–13.2).
|
||||
|
||||
Чистое ядро без БД (spec directory-import): `read_rows` превращает xlsx/csv в строки `ImportRow`, `plan_import` по
|
||||
снимку справочника (`DirectorySnapshot`, строит сервис) решает судьбу каждой строки и возвращает операции (только
|
||||
данные — применяет сервис) и отчёт с кодами §13.2. Идентификаторы новых объектов генерируются здесь (`uuid4`), чтобы
|
||||
операции плана ссылались друг на друга до записи в БД.
|
||||
"""
|
||||
|
||||
import csv
|
||||
import enum
|
||||
import io
|
||||
import re
|
||||
import uuid
|
||||
import zipfile
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from openpyxl import load_workbook
|
||||
from openpyxl.utils.exceptions import InvalidFileException
|
||||
|
||||
from src.common.exceptions import DomainValidationError
|
||||
from src.domain.inn import normalize_inn, validate_inn
|
||||
from src.models.enums import EntityType, RelatedGroupKind
|
||||
|
||||
|
||||
class ImportCode(enum.StrEnum):
|
||||
"""Коды строк отчёта (§13.2). Итоговые: пять кодов пропуска и `LE_*`; остальные — сопутствующие события."""
|
||||
|
||||
UNSUPPORTED_TYPE = "UNSUPPORTED_TYPE"
|
||||
INVALID_INN = "INVALID_INN"
|
||||
DUPLICATE_IN_FILE = "DUPLICATE_IN_FILE"
|
||||
LE_IN_OTHER_GROUP = "LE_IN_OTHER_GROUP"
|
||||
GK_NAME_CONFLICT = "GK_NAME_CONFLICT"
|
||||
LE_CREATED = "LE_CREATED"
|
||||
LE_UPDATED = "LE_UPDATED"
|
||||
LE_EXISTS = "LE_EXISTS"
|
||||
RF_CREATED = "RF_CREATED"
|
||||
GK_CREATED = "GK_CREATED"
|
||||
GK_UPDATED = "GK_UPDATED"
|
||||
GK_EXISTS = "GK_EXISTS"
|
||||
GSK_CREATED = "GSK_CREATED"
|
||||
GSZ_CREATED = "GSZ_CREATED"
|
||||
MEMBER_ADDED = "MEMBER_ADDED"
|
||||
UNKNOWN_FLAG = "UNKNOWN_FLAG"
|
||||
|
||||
|
||||
class OnExisting(enum.StrEnum):
|
||||
"""Стратегия для существующих ГК/ЮЛ (§13.3): оставить или обновить имя/РФ/тип."""
|
||||
|
||||
skip = "skip"
|
||||
update = "update"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- формат файла (§13.1)
|
||||
|
||||
REQUIRED_COLUMNS = ("инн", "idгк", "группакомпаний", "организация", "тип")
|
||||
OPTIONAL_COLUMNS = ("базовыйрфгквcrm", "гск", "гсз")
|
||||
# Исходные имена колонок — для перечня в `MISSING_COLUMNS`
|
||||
COLUMN_TITLES: dict[str, str] = {
|
||||
"инн": "ИНН",
|
||||
"idгк": "iD ГК",
|
||||
"базовыйрфгквcrm": "Базовый РФ ГК в CRM",
|
||||
"группакомпаний": "Группа компаний",
|
||||
"организация": "Организация",
|
||||
"гск": "ГСК",
|
||||
"гсз": "ГСЗ",
|
||||
"тип": "Тип",
|
||||
}
|
||||
FLAG_GSK = "входитвгск"
|
||||
FLAG_GSZ = "входитвгсз"
|
||||
|
||||
MISSING_COLUMNS = "MISSING_COLUMNS"
|
||||
UNSUPPORTED_FILE = "UNSUPPORTED_FILE"
|
||||
|
||||
_WHITESPACE_RE = re.compile(r"\s+")
|
||||
|
||||
|
||||
def normalize_header(raw: object) -> str:
|
||||
"""Заголовок/флаг для сопоставления: нижний регистр, без пробельных символов, `ё` = `е` (§13.1)."""
|
||||
return _WHITESPACE_RE.sub("", normalize_cell(raw).lower()).replace("ё", "е")
|
||||
|
||||
|
||||
def normalize_cell(raw: object) -> str:
|
||||
"""Значение ячейки строкой: `None` → `""`; целое число (в т.ч. `3528052718.0` из xlsx) — без дробной части; trim."""
|
||||
if raw is None:
|
||||
return ""
|
||||
if isinstance(raw, bool):
|
||||
return str(raw)
|
||||
if isinstance(raw, int):
|
||||
return str(raw)
|
||||
if isinstance(raw, float) and raw.is_integer():
|
||||
return str(int(raw))
|
||||
return str(raw).strip()
|
||||
|
||||
|
||||
class FileFormatError(Exception):
|
||||
"""Ошибка файла целиком (§13.3: прерывает импорт): `MISSING_COLUMNS` (с перечнем) или `UNSUPPORTED_FILE`."""
|
||||
|
||||
def __init__(self, code: str, message: str, missing: list[str] | None = None) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.missing = missing if missing is not None else []
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImportRow:
|
||||
"""Строка выгрузки после нормализации ячеек; `row` — номер строки в файле (1-based, как в Excel)."""
|
||||
|
||||
row: int
|
||||
inn: str
|
||||
crm_id: str
|
||||
office_name: str
|
||||
group_name: str
|
||||
entity_name: str
|
||||
gsk_flag: str
|
||||
gsz_flag: str
|
||||
type_raw: str
|
||||
|
||||
|
||||
def read_rows(filename: str, content: bytes) -> list[ImportRow]:
|
||||
"""Строки данных из xlsx (первый лист) или csv (UTF-8, `;`/`,`); формат выбирается по расширению имени файла.
|
||||
|
||||
Заголовки — первая непустая строка, сопоставляются по `normalize_header`; пустые строки пропускаются.
|
||||
"""
|
||||
extension = filename.rsplit(".", 1)[-1].lower() if "." in filename else ""
|
||||
if extension == "xlsx":
|
||||
raw_rows = _xlsx_rows(content)
|
||||
elif extension == "csv":
|
||||
raw_rows = _csv_rows(content)
|
||||
else:
|
||||
raise FileFormatError(UNSUPPORTED_FILE, "поддерживаются только файлы xlsx и csv")
|
||||
return _rows_from_table(raw_rows)
|
||||
|
||||
|
||||
def _xlsx_rows(content: bytes) -> list[tuple[int, list[str]]]:
|
||||
try:
|
||||
workbook = load_workbook(io.BytesIO(content), read_only=True, data_only=True)
|
||||
except (zipfile.BadZipFile, InvalidFileException, KeyError, ValueError, OSError, EOFError) as error:
|
||||
raise FileFormatError(UNSUPPORTED_FILE, "файл xlsx не читается") from error
|
||||
try:
|
||||
sheet = workbook.worksheets[0]
|
||||
# Размер листа из файла бывает неверным (экспорт сторонних систем) — читаем все строки фактически
|
||||
sheet.reset_dimensions()
|
||||
return [
|
||||
(number, [normalize_cell(value) for value in values])
|
||||
for number, values in enumerate(sheet.iter_rows(values_only=True), start=1)
|
||||
]
|
||||
finally:
|
||||
workbook.close()
|
||||
|
||||
|
||||
def _csv_rows(content: bytes) -> list[tuple[int, list[str]]]:
|
||||
try:
|
||||
text = content.decode("utf-8-sig")
|
||||
except UnicodeDecodeError as error:
|
||||
raise FileFormatError(UNSUPPORTED_FILE, "кодировка csv не распознана: ожидается UTF-8") from error
|
||||
header_line = next((line for line in text.splitlines() if line.strip()), "")
|
||||
delimiter = ";" if header_line.count(";") > header_line.count(",") else ","
|
||||
try:
|
||||
return [
|
||||
(number, [normalize_cell(value) for value in values])
|
||||
for number, values in enumerate(csv.reader(io.StringIO(text, newline=""), delimiter=delimiter), start=1)
|
||||
]
|
||||
except csv.Error as error:
|
||||
raise FileFormatError(UNSUPPORTED_FILE, "файл csv не читается") from error
|
||||
|
||||
|
||||
def _rows_from_table(raw_rows: Iterable[tuple[int, list[str]]]) -> list[ImportRow]:
|
||||
columns: dict[str, int] | None = None
|
||||
result: list[ImportRow] = []
|
||||
for number, cells in raw_rows:
|
||||
if not any(cells):
|
||||
continue
|
||||
if columns is None:
|
||||
columns = _header_columns(cells)
|
||||
else:
|
||||
result.append(_import_row(number, cells, columns))
|
||||
if columns is None:
|
||||
_header_columns([])
|
||||
return result
|
||||
|
||||
|
||||
def _import_row(number: int, cells: list[str], columns: dict[str, int]) -> ImportRow:
|
||||
def cell(name: str) -> str:
|
||||
index = columns.get(name)
|
||||
return cells[index] if index is not None and index < len(cells) else ""
|
||||
|
||||
return ImportRow(
|
||||
row=number,
|
||||
inn=cell("инн"),
|
||||
crm_id=cell("idгк"),
|
||||
office_name=cell("базовыйрфгквcrm"),
|
||||
group_name=cell("группакомпаний"),
|
||||
entity_name=cell("организация"),
|
||||
gsk_flag=cell("гск"),
|
||||
gsz_flag=cell("гсз"),
|
||||
type_raw=cell("тип"),
|
||||
)
|
||||
|
||||
|
||||
def _header_columns(cells: list[str]) -> dict[str, int]:
|
||||
"""Индексы известных колонок (первое вхождение); нет обязательной → `MISSING_COLUMNS` с исходными именами."""
|
||||
columns: dict[str, int] = {}
|
||||
for index, value in enumerate(cells):
|
||||
name = normalize_header(value)
|
||||
if name in COLUMN_TITLES and name not in columns:
|
||||
columns[name] = index
|
||||
missing = [COLUMN_TITLES[name] for name in REQUIRED_COLUMNS if name not in columns]
|
||||
if missing:
|
||||
raise FileFormatError(MISSING_COLUMNS, "нет обязательных колонок: " + ", ".join(missing), missing)
|
||||
return columns
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- снимок справочника
|
||||
|
||||
|
||||
@dataclass
|
||||
class OfficeSnap:
|
||||
id: uuid.UUID
|
||||
name: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class GroupSnap:
|
||||
id: uuid.UUID
|
||||
crm_id: str
|
||||
name: str
|
||||
office_id: uuid.UUID | None
|
||||
gsk_id: uuid.UUID | None
|
||||
gsz_id: uuid.UUID | None
|
||||
|
||||
|
||||
@dataclass
|
||||
class EntitySnap:
|
||||
"""ЮЛ/ИП; `gsk_group_id`/`gsz_group_id` — текущие членства (`related_group_member.is_current`)."""
|
||||
|
||||
id: uuid.UUID
|
||||
inn: str
|
||||
name: str
|
||||
entity_type: EntityType
|
||||
group_id: uuid.UUID | None
|
||||
gsk_group_id: uuid.UUID | None
|
||||
gsz_group_id: uuid.UUID | None
|
||||
|
||||
|
||||
@dataclass
|
||||
class DirectorySnapshot:
|
||||
offices_by_name: dict[str, OfficeSnap]
|
||||
groups_by_crm: dict[str, GroupSnap]
|
||||
entities_by_inn: dict[str, EntitySnap]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- операции плана
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CreateOffice:
|
||||
id: uuid.UUID
|
||||
name: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CreateGroup:
|
||||
id: uuid.UUID
|
||||
crm_id: str
|
||||
name: str
|
||||
office_id: uuid.UUID | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class UpdateGroup:
|
||||
"""`None` в поле — без изменений; офис меняется только непустым значением из файла."""
|
||||
|
||||
id: uuid.UUID
|
||||
name: str | None
|
||||
office_id: uuid.UUID | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CreateEntity:
|
||||
id: uuid.UUID
|
||||
inn: str
|
||||
name: str
|
||||
entity_type: EntityType
|
||||
group_id: uuid.UUID
|
||||
office_id: uuid.UUID | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class UpdateEntity:
|
||||
"""`None` в поле — без изменений."""
|
||||
|
||||
id: uuid.UUID
|
||||
name: str | None
|
||||
entity_type: EntityType | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CreateRelatedGroup:
|
||||
id: uuid.UUID
|
||||
group_id: uuid.UUID
|
||||
kind: RelatedGroupKind
|
||||
name: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AddMember:
|
||||
related_group_id: uuid.UUID
|
||||
entity_id: uuid.UUID
|
||||
kind: RelatedGroupKind
|
||||
|
||||
|
||||
Operation = CreateOffice | CreateGroup | UpdateGroup | CreateEntity | UpdateEntity | CreateRelatedGroup | AddMember
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- отчёт
|
||||
|
||||
|
||||
@dataclass
|
||||
class RowResult:
|
||||
row: int
|
||||
inn: str
|
||||
code: ImportCode
|
||||
message: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImportReport:
|
||||
"""Счётчики и построчный список отчёта (§13.3); `errors` — число пропущенных строк."""
|
||||
|
||||
rows: int = 0
|
||||
groups_created: int = 0
|
||||
groups_updated: int = 0
|
||||
groups_skipped: int = 0
|
||||
entities_created: int = 0
|
||||
entities_updated: int = 0
|
||||
entities_skipped: int = 0
|
||||
memberships_created: int = 0
|
||||
offices_created: int = 0
|
||||
errors: int = 0
|
||||
lines: list[RowResult] = field(default_factory=list)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""JSON-совместимый словарь (коды — строки) для ответа и `audit_event.after_value`."""
|
||||
data = asdict(self)
|
||||
data["lines"] = [{**line, "code": str(line["code"])} for line in data["lines"]]
|
||||
return data
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImportPlan:
|
||||
operations: list[Operation]
|
||||
report: ImportReport
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- план (§13.2)
|
||||
|
||||
_ENTITY_TYPES: dict[str, EntityType] = {"юл": EntityType.ЮЛ, "ип": EntityType.ИП}
|
||||
_FLAGS: tuple[tuple[RelatedGroupKind, str, str], ...] = (
|
||||
(RelatedGroupKind.gsk, FLAG_GSK, "ГСК"),
|
||||
(RelatedGroupKind.gsz, FLAG_GSZ, "ГСЗ"),
|
||||
)
|
||||
_RELATED_CREATED = {RelatedGroupKind.gsk: ImportCode.GSK_CREATED, RelatedGroupKind.gsz: ImportCode.GSZ_CREATED}
|
||||
|
||||
|
||||
@dataclass
|
||||
class _GroupState:
|
||||
"""ГК, уже встреченная в файле: имя из первой строки, действующий РФ и ГСК/ГСЗ (из снимка или созданные)."""
|
||||
|
||||
id: uuid.UUID
|
||||
file_name: str
|
||||
first_row: int
|
||||
office_id: uuid.UUID | None
|
||||
related: dict[RelatedGroupKind, uuid.UUID | None]
|
||||
|
||||
|
||||
class _RowSkipped(Exception):
|
||||
"""Строка пропущена с итоговым кодом (код уже записан в отчёт)."""
|
||||
|
||||
|
||||
class _Planner:
|
||||
def __init__(self, snapshot: DirectorySnapshot, on_existing: OnExisting) -> None:
|
||||
self.snapshot = snapshot
|
||||
self.on_existing = on_existing
|
||||
self.operations: list[Operation] = []
|
||||
self.report = ImportReport()
|
||||
self.offices: dict[str, uuid.UUID] = {name: office.id for name, office in snapshot.offices_by_name.items()}
|
||||
self.groups: dict[str, _GroupState] = {}
|
||||
self.seen_inns: dict[str, int] = {}
|
||||
|
||||
def line(self, row: ImportRow, inn: str, code: ImportCode, message: str) -> None:
|
||||
self.report.lines.append(RowResult(row=row.row, inn=inn, code=code, message=message))
|
||||
|
||||
def skip(self, row: ImportRow, inn: str, code: ImportCode, message: str) -> _RowSkipped:
|
||||
self.line(row, inn, code, message)
|
||||
self.report.errors += 1
|
||||
return _RowSkipped()
|
||||
|
||||
def plan_row(self, row: ImportRow) -> None:
|
||||
inn = normalize_inn(row.inn)
|
||||
# ASSUMPTION(OQ-87): типы вне {ЮЛ, ИП} (`нерез.`, `ФЛ`, `нет инн`, пусто) пропускаются
|
||||
entity_type = _ENTITY_TYPES.get(row.type_raw.lower())
|
||||
if entity_type is None:
|
||||
raise self.skip(row, inn, ImportCode.UNSUPPORTED_TYPE, f"тип «{row.type_raw}» не поддерживается: ЮЛ или ИП")
|
||||
try:
|
||||
# ASSUMPTION(OQ-88): только длина (ЮЛ — 10, ИП — 12) и цифры, контрольные разряды не проверяются
|
||||
inn = validate_inn(inn, entity_type)
|
||||
except DomainValidationError as error:
|
||||
raise self.skip(row, inn, ImportCode.INVALID_INN, error.message) from error
|
||||
if inn in self.seen_inns:
|
||||
raise self.skip(
|
||||
row, inn, ImportCode.DUPLICATE_IN_FILE, f"ИНН уже встречался в строке {self.seen_inns[inn]}"
|
||||
)
|
||||
self.seen_inns[inn] = row.row
|
||||
group = self.plan_group(row, inn)
|
||||
entity_id, current = self.plan_entity(row, inn, entity_type, group)
|
||||
self.plan_flags(row, inn, group, entity_id, current)
|
||||
|
||||
def plan_office(self, row: ImportRow, inn: str) -> uuid.UUID | None:
|
||||
name = row.office_name
|
||||
if not name:
|
||||
return None
|
||||
office_id = self.offices.get(name)
|
||||
if office_id is None:
|
||||
# ASSUMPTION(OQ-89): РФ, которого нет в справочнике (в т.ч. «ЦКБ», «Головная организация»), создаётся
|
||||
office_id = uuid.uuid4()
|
||||
self.offices[name] = office_id
|
||||
self.operations.append(CreateOffice(id=office_id, name=name))
|
||||
self.report.offices_created += 1
|
||||
self.line(row, inn, ImportCode.RF_CREATED, f"создан филиал «{name}»")
|
||||
return office_id
|
||||
|
||||
def plan_group(self, row: ImportRow, inn: str) -> _GroupState:
|
||||
crm_id = row.crm_id
|
||||
if not crm_id:
|
||||
raise self.skip(row, inn, ImportCode.GK_NAME_CONFLICT, "пустой iD ГК")
|
||||
state = self.groups.get(crm_id)
|
||||
if state is not None:
|
||||
if row.group_name != state.file_name:
|
||||
message = (
|
||||
f"ГК {crm_id}: имя «{row.group_name}» расходится с «{state.file_name}» (строка {state.first_row})"
|
||||
)
|
||||
raise self.skip(row, inn, ImportCode.GK_NAME_CONFLICT, message)
|
||||
return state
|
||||
office_id = self.plan_office(row, inn)
|
||||
snap = self.snapshot.groups_by_crm.get(crm_id)
|
||||
if snap is None:
|
||||
group_id = uuid.uuid4()
|
||||
self.operations.append(CreateGroup(id=group_id, crm_id=crm_id, name=row.group_name, office_id=office_id))
|
||||
self.report.groups_created += 1
|
||||
self.line(row, inn, ImportCode.GK_CREATED, f"создана ГК {crm_id} «{row.group_name}»")
|
||||
state = _GroupState(group_id, row.group_name, row.row, office_id, {kind: None for kind in RelatedGroupKind})
|
||||
else:
|
||||
state = self.existing_group(row, inn, snap, office_id)
|
||||
self.groups[crm_id] = state
|
||||
return state
|
||||
|
||||
def existing_group(self, row: ImportRow, inn: str, snap: GroupSnap, office_id: uuid.UUID | None) -> _GroupState:
|
||||
new_name = row.group_name if row.group_name != snap.name else None
|
||||
new_office = office_id if office_id is not None and office_id != snap.office_id else None
|
||||
effective_office = snap.office_id
|
||||
if self.on_existing is OnExisting.update and (new_name is not None or new_office is not None):
|
||||
self.operations.append(UpdateGroup(id=snap.id, name=new_name, office_id=new_office))
|
||||
self.report.groups_updated += 1
|
||||
self.line(row, inn, ImportCode.GK_UPDATED, f"обновлена ГК {snap.crm_id} «{row.group_name}»")
|
||||
effective_office = new_office if new_office is not None else snap.office_id
|
||||
else:
|
||||
self.report.groups_skipped += 1
|
||||
self.line(row, inn, ImportCode.GK_EXISTS, f"ГК {snap.crm_id} «{snap.name}» уже есть")
|
||||
related = {RelatedGroupKind.gsk: snap.gsk_id, RelatedGroupKind.gsz: snap.gsz_id}
|
||||
return _GroupState(snap.id, row.group_name, row.row, effective_office, related)
|
||||
|
||||
def plan_entity(
|
||||
self, row: ImportRow, inn: str, entity_type: EntityType, group: _GroupState
|
||||
) -> tuple[uuid.UUID, EntitySnap | None]:
|
||||
snap = self.snapshot.entities_by_inn.get(inn)
|
||||
if snap is None:
|
||||
entity_id = uuid.uuid4()
|
||||
self.operations.append(
|
||||
CreateEntity(
|
||||
id=entity_id,
|
||||
inn=inn,
|
||||
name=row.entity_name,
|
||||
entity_type=entity_type,
|
||||
group_id=group.id,
|
||||
office_id=group.office_id,
|
||||
)
|
||||
)
|
||||
self.report.entities_created += 1
|
||||
self.line(row, inn, ImportCode.LE_CREATED, f"создано {entity_type.value} «{row.entity_name}»")
|
||||
return entity_id, None
|
||||
if snap.group_id != group.id:
|
||||
# ASSUMPTION(OQ-90): ЮЛ из другой ГК (или без ГК) импорт не переводит — только явная операция §13.4
|
||||
raise self.skip(row, inn, ImportCode.LE_IN_OTHER_GROUP, f"«{snap.name}» уже состоит в другой ГК")
|
||||
new_name = row.entity_name if row.entity_name != snap.name else None
|
||||
new_type = entity_type if entity_type != snap.entity_type else None
|
||||
if self.on_existing is OnExisting.update and (new_name is not None or new_type is not None):
|
||||
self.operations.append(UpdateEntity(id=snap.id, name=new_name, entity_type=new_type))
|
||||
self.report.entities_updated += 1
|
||||
self.line(row, inn, ImportCode.LE_UPDATED, f"обновлено {entity_type.value} «{row.entity_name}»")
|
||||
else:
|
||||
self.report.entities_skipped += 1
|
||||
self.line(row, inn, ImportCode.LE_EXISTS, f"«{snap.name}» уже есть")
|
||||
return snap.id, snap
|
||||
|
||||
def plan_flags(
|
||||
self, row: ImportRow, inn: str, group: _GroupState, entity_id: uuid.UUID, current: EntitySnap | None
|
||||
) -> None:
|
||||
for kind, expected, title in _FLAGS:
|
||||
raw = row.gsk_flag if kind is RelatedGroupKind.gsk else row.gsz_flag
|
||||
if not raw:
|
||||
continue
|
||||
if normalize_header(raw) != expected:
|
||||
self.line(
|
||||
row, inn, ImportCode.UNKNOWN_FLAG, f"значение «{raw}» в колонке {title} не распознано: не входит"
|
||||
)
|
||||
continue
|
||||
related_id = group.related[kind]
|
||||
if related_id is None:
|
||||
# ASSUMPTION(OQ-91): одна группа вида на ГК, имя — имя ГК
|
||||
related_id = uuid.uuid4()
|
||||
group.related[kind] = related_id
|
||||
self.operations.append(
|
||||
CreateRelatedGroup(id=related_id, group_id=group.id, kind=kind, name=group.file_name)
|
||||
)
|
||||
self.line(row, inn, _RELATED_CREATED[kind], f"создана {title} «{group.file_name}»")
|
||||
membership = None
|
||||
if current is not None:
|
||||
membership = current.gsk_group_id if kind is RelatedGroupKind.gsk else current.gsz_group_id
|
||||
# Текущее членство (в этой или другой группе того же вида) импорт не меняет и не закрывает
|
||||
if membership is None:
|
||||
self.operations.append(AddMember(related_group_id=related_id, entity_id=entity_id, kind=kind))
|
||||
self.report.memberships_created += 1
|
||||
self.line(row, inn, ImportCode.MEMBER_ADDED, f"добавлено в {title} «{group.file_name}»")
|
||||
|
||||
|
||||
def plan_import(rows: Sequence[ImportRow], snapshot: DirectorySnapshot, on_existing: OnExisting) -> ImportPlan:
|
||||
"""План импорта по §13.2: строки в порядке файла, у каждой ровно один итоговый код и события того же `row`.
|
||||
|
||||
Накопители «создано в этом файле» (РФ по имени, ГК по `iD ГК`, ГСК/ГСЗ по ГК, ИНН) гарантируют, что одна ГК из
|
||||
многих строк создаётся один раз. Снимок не изменяется.
|
||||
"""
|
||||
planner = _Planner(snapshot, on_existing)
|
||||
for row in rows:
|
||||
planner.report.rows += 1
|
||||
try:
|
||||
planner.plan_row(row)
|
||||
except _RowSkipped:
|
||||
continue
|
||||
return ImportPlan(operations=planner.operations, report=planner.report)
|
||||
650
api/tests/test_domain_directory_import.py
Normal file
650
api/tests/test_domain_directory_import.py
Normal file
@ -0,0 +1,650 @@
|
||||
"""Ядро импорта справочников из выгрузки ЕЦП CRM (§13.1–13.2, spec directory-import): чтение файла и план без БД."""
|
||||
|
||||
import csv
|
||||
import io
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import TypeVar
|
||||
|
||||
import openpyxl
|
||||
import pytest
|
||||
|
||||
from src.domain.directory_import import (
|
||||
AddMember,
|
||||
CreateEntity,
|
||||
CreateGroup,
|
||||
CreateOffice,
|
||||
CreateRelatedGroup,
|
||||
DirectorySnapshot,
|
||||
EntitySnap,
|
||||
FileFormatError,
|
||||
GroupSnap,
|
||||
ImportCode,
|
||||
ImportPlan,
|
||||
ImportRow,
|
||||
OfficeSnap,
|
||||
OnExisting,
|
||||
UpdateEntity,
|
||||
UpdateGroup,
|
||||
normalize_cell,
|
||||
normalize_header,
|
||||
plan_import,
|
||||
read_rows,
|
||||
)
|
||||
from src.models.enums import EntityType, RelatedGroupKind
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
HEADERS = ["ИНН", "iD ГК", "Базовый РФ ГК в CRM", "Группа компаний", "Организация", "ГСК", "ГСЗ", "Тип"]
|
||||
FINAL_CODES = {
|
||||
ImportCode.UNSUPPORTED_TYPE,
|
||||
ImportCode.INVALID_INN,
|
||||
ImportCode.DUPLICATE_IN_FILE,
|
||||
ImportCode.LE_IN_OTHER_GROUP,
|
||||
ImportCode.GK_NAME_CONFLICT,
|
||||
ImportCode.LE_CREATED,
|
||||
ImportCode.LE_UPDATED,
|
||||
ImportCode.LE_EXISTS,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- нормализация
|
||||
|
||||
|
||||
def test_normalize_header_lowercases_and_drops_spaces() -> None:
|
||||
assert normalize_header("Базовый РФ ГК в CRM") == "базовыйрфгквcrm"
|
||||
assert normalize_header(" iD ГК ") == "idгк"
|
||||
assert normalize_header("Входит в ГСК") == "входитвгск"
|
||||
|
||||
|
||||
def test_normalize_header_treats_yo_as_ye() -> None:
|
||||
assert normalize_header("Объём Ёлки") == "объемелки"
|
||||
|
||||
|
||||
def test_normalize_header_of_none_is_empty() -> None:
|
||||
assert normalize_header(None) == ""
|
||||
|
||||
|
||||
def test_normalize_cell_numeric_inn_has_no_fraction() -> None:
|
||||
"""Review Focus 2: ИНН из xlsx числом (float) → строка без `.0`."""
|
||||
assert normalize_cell(3528052718.0) == "3528052718"
|
||||
assert normalize_cell(352819549855.0) == "352819549855"
|
||||
assert normalize_cell(3528052718) == "3528052718"
|
||||
|
||||
|
||||
def test_normalize_cell_none_and_strings() -> None:
|
||||
assert normalize_cell(None) == ""
|
||||
assert normalize_cell(" ЮЛ ") == "ЮЛ"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- чтение файла
|
||||
|
||||
|
||||
def _csv_bytes(rows: list[list[str]], delimiter: str = ";", bom: bool = True) -> bytes:
|
||||
buffer = io.StringIO()
|
||||
writer = csv.writer(buffer, delimiter=delimiter, lineterminator="\n")
|
||||
writer.writerows(rows)
|
||||
return buffer.getvalue().encode("utf-8-sig" if bom else "utf-8")
|
||||
|
||||
|
||||
def _data_row(inn: str = "3528052718", type_: str = "ЮЛ") -> list[str]:
|
||||
return [
|
||||
inn,
|
||||
"1-AZFL6BQ",
|
||||
"Санкт-Петербургский РФ",
|
||||
"Русский Бисквит",
|
||||
"РУССКИЙ БИСКВИТ, АО",
|
||||
"входит в ГСК",
|
||||
"",
|
||||
type_,
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("delimiter", [";", ","])
|
||||
def test_read_csv_with_either_delimiter(delimiter: str) -> None:
|
||||
content = _csv_bytes([HEADERS, _data_row()], delimiter=delimiter)
|
||||
rows = read_rows("export.csv", content)
|
||||
assert rows == [
|
||||
ImportRow(
|
||||
row=2,
|
||||
inn="3528052718",
|
||||
crm_id="1-AZFL6BQ",
|
||||
office_name="Санкт-Петербургский РФ",
|
||||
group_name="Русский Бисквит",
|
||||
entity_name="РУССКИЙ БИСКВИТ, АО",
|
||||
gsk_flag="входит в ГСК",
|
||||
gsz_flag="",
|
||||
type_raw="ЮЛ",
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_read_csv_without_bom() -> None:
|
||||
rows = read_rows("export.CSV", _csv_bytes([HEADERS, _data_row()], bom=False))
|
||||
assert [r.inn for r in rows] == ["3528052718"]
|
||||
|
||||
|
||||
def test_read_csv_columns_reordered_and_extra_column_ignored() -> None:
|
||||
headers = ["Лишняя", "Тип", "Организация", "Группа компаний", "iD ГК", "ИНН"]
|
||||
content = _csv_bytes([headers, ["x", "ИП", "ИП Юганов", "Русский Бисквит", "1-AZFL6BQ", "352819549855"]])
|
||||
(row,) = read_rows("export.csv", content)
|
||||
assert (row.inn, row.type_raw, row.entity_name, row.group_name, row.crm_id) == (
|
||||
"352819549855",
|
||||
"ИП",
|
||||
"ИП Юганов",
|
||||
"Русский Бисквит",
|
||||
"1-AZFL6BQ",
|
||||
)
|
||||
assert (row.office_name, row.gsk_flag, row.gsz_flag) == ("", "", "")
|
||||
|
||||
|
||||
def test_read_csv_skips_empty_lines_and_keeps_file_row_numbers() -> None:
|
||||
content = _csv_bytes([HEADERS, _data_row("3528052718"), ["", "", "", "", "", "", "", ""], _data_row("3528104476")])
|
||||
rows = read_rows("export.csv", content)
|
||||
assert [(r.row, r.inn) for r in rows] == [(2, "3528052718"), (4, "3528104476")]
|
||||
|
||||
|
||||
def test_read_csv_missing_type_column() -> None:
|
||||
content = _csv_bytes([HEADERS[:-1], _data_row()[:-1]])
|
||||
with pytest.raises(FileFormatError) as error:
|
||||
read_rows("export.csv", content)
|
||||
assert error.value.code == "MISSING_COLUMNS"
|
||||
assert error.value.missing == ["Тип"]
|
||||
|
||||
|
||||
def test_read_csv_not_utf8_is_unsupported() -> None:
|
||||
"""Review Focus 3: csv в cp1251 → UNSUPPORTED_FILE «ожидается UTF-8», не 500."""
|
||||
content = ";".join(HEADERS).encode("cp1251")
|
||||
with pytest.raises(FileFormatError) as error:
|
||||
read_rows("export.csv", content)
|
||||
assert error.value.code == "UNSUPPORTED_FILE"
|
||||
assert "UTF-8" in error.value.message
|
||||
|
||||
|
||||
def test_read_txt_is_unsupported() -> None:
|
||||
with pytest.raises(FileFormatError) as error:
|
||||
read_rows("export.txt", _csv_bytes([HEADERS, _data_row()]))
|
||||
assert error.value.code == "UNSUPPORTED_FILE"
|
||||
|
||||
|
||||
def test_read_broken_xlsx_is_unsupported() -> None:
|
||||
with pytest.raises(FileFormatError) as error:
|
||||
read_rows("export.xlsx", b"not a zip at all")
|
||||
assert error.value.code == "UNSUPPORTED_FILE"
|
||||
|
||||
|
||||
def test_read_xlsx_header_in_second_row_numeric_inn_and_trailing_empty_rows() -> None:
|
||||
"""Review Focus 1–2: заголовок в первой непустой строке, ИНН числом, 3 пустые строки в хвосте пропускаются."""
|
||||
workbook = openpyxl.Workbook()
|
||||
sheet = workbook.active
|
||||
assert sheet is not None
|
||||
for column, header in enumerate(HEADERS, start=1):
|
||||
sheet.cell(row=2, column=column, value=header)
|
||||
first: list[float | str | None] = [
|
||||
3528052718.0,
|
||||
"1-AZFL6BQ",
|
||||
"Санкт-Петербургский РФ",
|
||||
"Русский Бисквит",
|
||||
"АО",
|
||||
"входит в ГСК",
|
||||
None,
|
||||
"ЮЛ",
|
||||
]
|
||||
second: list[float | str | None] = [
|
||||
352819549855,
|
||||
"1-AZFL6BQ",
|
||||
"Санкт-Петербургский РФ",
|
||||
"Русский Бисквит",
|
||||
"ИП",
|
||||
None,
|
||||
None,
|
||||
"ИП",
|
||||
]
|
||||
for column, value in enumerate(first, start=1):
|
||||
sheet.cell(row=3, column=column, value=value)
|
||||
for column, value in enumerate(second, start=1):
|
||||
sheet.cell(row=4, column=column, value=value)
|
||||
for row in (5, 6, 7):
|
||||
sheet.cell(row=row, column=1, value="")
|
||||
sheet.cell(row=row, column=8, value=None)
|
||||
buffer = io.BytesIO()
|
||||
workbook.save(buffer)
|
||||
|
||||
rows = read_rows("export.xlsx", buffer.getvalue())
|
||||
|
||||
assert [(r.row, r.inn, r.type_raw) for r in rows] == [(3, "3528052718", "ЮЛ"), (4, "352819549855", "ИП")]
|
||||
assert rows[0].gsk_flag == "входит в ГСК"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- план импорта: помощники
|
||||
|
||||
|
||||
def _row( # noqa: PLR0913 - строка выгрузки: все поля значимы для теста
|
||||
row: int,
|
||||
*,
|
||||
inn: str = "3528052718",
|
||||
type_raw: str = "ЮЛ",
|
||||
crm_id: str = "1-AZFL6BQ",
|
||||
group_name: str = "Русский Бисквит",
|
||||
office_name: str = "Санкт-Петербургский РФ",
|
||||
entity_name: str = "РУССКИЙ БИСКВИТ, АО",
|
||||
gsk_flag: str = "",
|
||||
gsz_flag: str = "",
|
||||
) -> ImportRow:
|
||||
return ImportRow(
|
||||
row=row,
|
||||
inn=inn,
|
||||
crm_id=crm_id,
|
||||
office_name=office_name,
|
||||
group_name=group_name,
|
||||
entity_name=entity_name,
|
||||
gsk_flag=gsk_flag,
|
||||
gsz_flag=gsz_flag,
|
||||
type_raw=type_raw,
|
||||
)
|
||||
|
||||
|
||||
def _empty() -> DirectorySnapshot:
|
||||
return DirectorySnapshot(offices_by_name={}, groups_by_crm={}, entities_by_inn={})
|
||||
|
||||
|
||||
def _final_codes(plan: ImportPlan) -> dict[int, ImportCode]:
|
||||
finals: dict[int, ImportCode] = {}
|
||||
for line in plan.report.lines:
|
||||
if line.code in FINAL_CODES:
|
||||
assert line.row not in finals, f"строка {line.row}: два итоговых кода"
|
||||
finals[line.row] = line.code
|
||||
return finals
|
||||
|
||||
|
||||
def _events(plan: ImportPlan, row: int) -> list[ImportCode]:
|
||||
return [line.code for line in plan.report.lines if line.row == row and line.code not in FINAL_CODES]
|
||||
|
||||
|
||||
def _ops(plan: ImportPlan, kind: type[T]) -> list[T]:
|
||||
return [op for op in plan.operations if isinstance(op, kind)]
|
||||
|
||||
|
||||
OFFICE_ID = uuid.uuid4()
|
||||
GROUP_ID = uuid.uuid4()
|
||||
OTHER_GROUP_ID = uuid.uuid4()
|
||||
ENTITY_ID = uuid.uuid4()
|
||||
GSK_ID = uuid.uuid4()
|
||||
|
||||
|
||||
def _existing(
|
||||
entity_group: uuid.UUID | None = GROUP_ID,
|
||||
group_name: str = "Русский Бисквит",
|
||||
gsk_member: uuid.UUID | None = None,
|
||||
) -> DirectorySnapshot:
|
||||
return DirectorySnapshot(
|
||||
offices_by_name={"Санкт-Петербургский РФ": OfficeSnap(id=OFFICE_ID, name="Санкт-Петербургский РФ")},
|
||||
groups_by_crm={
|
||||
"1-AZFL6BQ": GroupSnap(
|
||||
id=GROUP_ID, crm_id="1-AZFL6BQ", name=group_name, office_id=OFFICE_ID, gsk_id=None, gsz_id=None
|
||||
),
|
||||
"1-OTHER": GroupSnap(
|
||||
id=OTHER_GROUP_ID, crm_id="1-OTHER", name="Другая", office_id=None, gsk_id=None, gsz_id=None
|
||||
),
|
||||
},
|
||||
entities_by_inn={
|
||||
"3528052718": EntitySnap(
|
||||
id=ENTITY_ID,
|
||||
inn="3528052718",
|
||||
name="РУССКИЙ БИСКВИТ, АО",
|
||||
entity_type=EntityType.ЮЛ,
|
||||
group_id=entity_group,
|
||||
gsk_group_id=gsk_member,
|
||||
gsz_group_id=None,
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- план импорта: по коду
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("rows", "snapshot", "expected_final", "expected_events"),
|
||||
[
|
||||
pytest.param([_row(2, type_raw="ФЛ")], _empty(), ImportCode.UNSUPPORTED_TYPE, [], id="UNSUPPORTED_TYPE-ФЛ"),
|
||||
pytest.param([_row(2, type_raw="нерез.")], _empty(), ImportCode.UNSUPPORTED_TYPE, [], id="UNSUPPORTED_TYPE"),
|
||||
pytest.param([_row(2, type_raw="")], _empty(), ImportCode.UNSUPPORTED_TYPE, [], id="UNSUPPORTED_TYPE-пусто"),
|
||||
pytest.param([_row(2, inn="35280527")], _empty(), ImportCode.INVALID_INN, [], id="INVALID_INN-короткий"),
|
||||
pytest.param(
|
||||
[_row(2, inn="3528052718", type_raw="ИП")], _empty(), ImportCode.INVALID_INN, [], id="INVALID_INN-ИП-10"
|
||||
),
|
||||
pytest.param([_row(2, inn="LV12345678")], _empty(), ImportCode.INVALID_INN, [], id="INVALID_INN-буквы"),
|
||||
pytest.param([_row(2, crm_id="")], _empty(), ImportCode.GK_NAME_CONFLICT, [], id="GK_NAME_CONFLICT-пустой"),
|
||||
pytest.param(
|
||||
[_row(2)],
|
||||
_empty(),
|
||||
ImportCode.LE_CREATED,
|
||||
[ImportCode.RF_CREATED, ImportCode.GK_CREATED],
|
||||
id="LE_CREATED",
|
||||
),
|
||||
pytest.param([_row(2)], _existing(), ImportCode.LE_EXISTS, [ImportCode.GK_EXISTS], id="LE_EXISTS"),
|
||||
pytest.param(
|
||||
[_row(2)],
|
||||
_existing(entity_group=OTHER_GROUP_ID),
|
||||
ImportCode.LE_IN_OTHER_GROUP,
|
||||
[ImportCode.GK_EXISTS],
|
||||
id="LE_IN_OTHER_GROUP",
|
||||
),
|
||||
pytest.param(
|
||||
[_row(2)],
|
||||
_existing(entity_group=None),
|
||||
ImportCode.LE_IN_OTHER_GROUP,
|
||||
[ImportCode.GK_EXISTS],
|
||||
id="LE_IN_OTHER_GROUP-без-ГК",
|
||||
),
|
||||
pytest.param(
|
||||
[_row(2, gsk_flag="входит в ГСК")],
|
||||
_empty(),
|
||||
ImportCode.LE_CREATED,
|
||||
[ImportCode.RF_CREATED, ImportCode.GK_CREATED, ImportCode.GSK_CREATED, ImportCode.MEMBER_ADDED],
|
||||
id="GSK_CREATED+MEMBER_ADDED",
|
||||
),
|
||||
pytest.param(
|
||||
[_row(2, gsz_flag="Входит в ГСЗ")],
|
||||
_empty(),
|
||||
ImportCode.LE_CREATED,
|
||||
[ImportCode.RF_CREATED, ImportCode.GK_CREATED, ImportCode.GSZ_CREATED, ImportCode.MEMBER_ADDED],
|
||||
id="GSZ_CREATED+MEMBER_ADDED",
|
||||
),
|
||||
pytest.param(
|
||||
[_row(2, gsk_flag="не входит в ГСК")],
|
||||
_empty(),
|
||||
ImportCode.LE_CREATED,
|
||||
[ImportCode.RF_CREATED, ImportCode.GK_CREATED, ImportCode.UNKNOWN_FLAG],
|
||||
id="UNKNOWN_FLAG",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_plan_row_code(
|
||||
rows: list[ImportRow],
|
||||
snapshot: DirectorySnapshot,
|
||||
expected_final: ImportCode,
|
||||
expected_events: list[ImportCode],
|
||||
) -> None:
|
||||
plan = plan_import(rows, snapshot, OnExisting.skip)
|
||||
assert _final_codes(plan) == {2: expected_final}
|
||||
assert _events(plan, 2) == expected_events
|
||||
assert plan.report.rows == 1
|
||||
|
||||
|
||||
def test_skipped_rows_count_as_errors_and_produce_no_operations() -> None:
|
||||
rows = [_row(2, type_raw="ФЛ"), _row(3, inn="12"), _row(4, crm_id="")]
|
||||
plan = plan_import(rows, _empty(), OnExisting.skip)
|
||||
assert plan.operations == []
|
||||
assert plan.report.errors == len(rows)
|
||||
assert plan.report.rows == len(rows)
|
||||
|
||||
|
||||
def test_duplicate_in_file_after_first_occurrence() -> None:
|
||||
rows = [_row(2, inn="352819549855", type_raw="ИП"), _row(3, inn="352819549855", type_raw="ип")]
|
||||
plan = plan_import(rows, _empty(), OnExisting.skip)
|
||||
assert _final_codes(plan) == {2: ImportCode.LE_CREATED, 3: ImportCode.DUPLICATE_IN_FILE}
|
||||
assert plan.report.errors == 1
|
||||
assert len(_ops(plan, CreateEntity)) == 1
|
||||
|
||||
|
||||
def test_inn_with_spaces_is_normalized() -> None:
|
||||
plan = plan_import([_row(2, inn=" 3528 052 718 ")], _empty(), OnExisting.skip)
|
||||
(entity,) = _ops(plan, CreateEntity)
|
||||
assert entity.inn == "3528052718"
|
||||
assert plan.report.lines[-1].inn == "3528052718"
|
||||
|
||||
|
||||
def test_gk_name_conflict_inside_file() -> None:
|
||||
rows = [_row(2), _row(3, inn="3528104476", group_name="Русский Бисквит 2")]
|
||||
plan = plan_import(rows, _empty(), OnExisting.skip)
|
||||
assert _final_codes(plan) == {2: ImportCode.LE_CREATED, 3: ImportCode.GK_NAME_CONFLICT}
|
||||
assert len(_ops(plan, CreateGroup)) == 1
|
||||
assert len(_ops(plan, CreateEntity)) == 1
|
||||
assert plan.report.errors == 1
|
||||
|
||||
|
||||
def test_empty_snapshot_creates_office_group_entity_with_links() -> None:
|
||||
plan = plan_import([_row(2, type_raw="юл")], _empty(), OnExisting.skip)
|
||||
(office,) = _ops(plan, CreateOffice)
|
||||
(group,) = _ops(plan, CreateGroup)
|
||||
(entity,) = _ops(plan, CreateEntity)
|
||||
assert office.name == "Санкт-Петербургский РФ"
|
||||
assert (group.crm_id, group.name, group.office_id) == ("1-AZFL6BQ", "Русский Бисквит", office.id)
|
||||
assert (entity.inn, entity.name, entity.entity_type) == ("3528052718", "РУССКИЙ БИСКВИТ, АО", EntityType.ЮЛ)
|
||||
assert (entity.group_id, entity.office_id) == (group.id, office.id)
|
||||
report = plan.report
|
||||
assert (report.offices_created, report.groups_created, report.entities_created, report.errors) == (1, 1, 1, 0)
|
||||
rf_line = next(line for line in report.lines if line.code == ImportCode.RF_CREATED)
|
||||
assert (rf_line.row, rf_line.message) == (2, "создан филиал «Санкт-Петербургский РФ»")
|
||||
|
||||
|
||||
def test_empty_office_name_gives_group_without_office() -> None:
|
||||
plan = plan_import([_row(2, office_name="")], _empty(), OnExisting.skip)
|
||||
assert _ops(plan, CreateOffice) == []
|
||||
(group,) = _ops(plan, CreateGroup)
|
||||
(entity,) = _ops(plan, CreateEntity)
|
||||
assert group.office_id is None
|
||||
assert entity.office_id is None
|
||||
|
||||
|
||||
def test_existing_office_is_reused() -> None:
|
||||
snapshot = _empty()
|
||||
snapshot.offices_by_name["Санкт-Петербургский РФ"] = OfficeSnap(id=OFFICE_ID, name="Санкт-Петербургский РФ")
|
||||
plan = plan_import([_row(2)], snapshot, OnExisting.skip)
|
||||
assert _ops(plan, CreateOffice) == []
|
||||
(group,) = _ops(plan, CreateGroup)
|
||||
assert group.office_id == OFFICE_ID
|
||||
assert plan.report.offices_created == 0
|
||||
|
||||
|
||||
def test_two_rows_of_one_group_create_it_once() -> None:
|
||||
rows = [_row(2), _row(3, inn="3528104476", entity_name="ПЕКАРНЯ СЧАСТЛИВОЧНАЯ, ООО")]
|
||||
plan = plan_import(rows, _empty(), OnExisting.skip)
|
||||
(group,) = _ops(plan, CreateGroup)
|
||||
assert len(_ops(plan, CreateOffice)) == 1
|
||||
assert [e.group_id for e in _ops(plan, CreateEntity)] == [group.id, group.id]
|
||||
assert plan.report.groups_created == 1
|
||||
assert _events(plan, 3) == []
|
||||
|
||||
|
||||
def test_flag_on_two_entities_creates_one_related_group_and_two_members() -> None:
|
||||
rows = [_row(2, gsk_flag="входит в ГСК"), _row(3, inn="3528104476", gsk_flag="входит в ГСК")]
|
||||
plan = plan_import(rows, _empty(), OnExisting.skip)
|
||||
(group,) = _ops(plan, CreateGroup)
|
||||
(related,) = _ops(plan, CreateRelatedGroup)
|
||||
assert (related.group_id, related.kind, related.name) == (group.id, RelatedGroupKind.gsk, "Русский Бисквит")
|
||||
members = _ops(plan, AddMember)
|
||||
assert [m.related_group_id for m in members] == [related.id, related.id]
|
||||
assert [m.kind for m in members] == [RelatedGroupKind.gsk, RelatedGroupKind.gsk]
|
||||
assert [m.entity_id for m in members] == [e.id for e in _ops(plan, CreateEntity)]
|
||||
assert plan.report.memberships_created == len(rows)
|
||||
|
||||
|
||||
def test_existing_related_group_of_group_is_reused() -> None:
|
||||
snapshot = _existing()
|
||||
snapshot.groups_by_crm["1-AZFL6BQ"].gsk_id = GSK_ID
|
||||
plan = plan_import([_row(2, inn="3528104476", gsk_flag="входит в ГСК")], snapshot, OnExisting.skip)
|
||||
assert _ops(plan, CreateRelatedGroup) == []
|
||||
(member,) = _ops(plan, AddMember)
|
||||
assert member.related_group_id == GSK_ID
|
||||
|
||||
|
||||
def test_existing_membership_is_kept() -> None:
|
||||
snapshot = _existing(gsk_member=GSK_ID)
|
||||
snapshot.groups_by_crm["1-AZFL6BQ"].gsk_id = GSK_ID
|
||||
plan = plan_import([_row(2, gsk_flag="входит в ГСК")], snapshot, OnExisting.skip)
|
||||
assert _ops(plan, AddMember) == []
|
||||
assert _events(plan, 2) == [ImportCode.GK_EXISTS]
|
||||
|
||||
|
||||
def test_membership_in_other_related_group_is_not_moved() -> None:
|
||||
other_gsk = uuid.uuid4()
|
||||
plan = plan_import([_row(2, gsk_flag="входит в ГСК")], _existing(gsk_member=other_gsk), OnExisting.skip)
|
||||
assert _ops(plan, AddMember) == []
|
||||
assert ImportCode.MEMBER_ADDED not in _events(plan, 2)
|
||||
|
||||
|
||||
def test_entity_in_other_group_gets_no_membership() -> None:
|
||||
plan = plan_import([_row(2, gsk_flag="входит в ГСК")], _existing(entity_group=OTHER_GROUP_ID), OnExisting.skip)
|
||||
assert _ops(plan, AddMember) == []
|
||||
assert _ops(plan, CreateRelatedGroup) == []
|
||||
assert plan.report.errors == 1
|
||||
|
||||
|
||||
def test_changed_names_with_skip_are_left_as_is() -> None:
|
||||
rows = [_row(2, entity_name="РУССКИЙ БИСКВИТ, ПАО")]
|
||||
plan = plan_import(rows, _existing(group_name="Старое имя"), OnExisting.skip)
|
||||
assert plan.operations == []
|
||||
assert _final_codes(plan) == {2: ImportCode.LE_EXISTS}
|
||||
assert _events(plan, 2) == [ImportCode.GK_EXISTS]
|
||||
report = plan.report
|
||||
assert (report.groups_skipped, report.entities_skipped, report.groups_updated, report.entities_updated) == (
|
||||
1,
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
)
|
||||
|
||||
|
||||
def test_changed_names_with_update_produce_updates() -> None:
|
||||
rows = [_row(2, entity_name="РУССКИЙ БИСКВИТ, ПАО")]
|
||||
plan = plan_import(rows, _existing(group_name="Старое имя"), OnExisting.update)
|
||||
assert plan.operations == [
|
||||
UpdateGroup(id=GROUP_ID, name="Русский Бисквит", office_id=None),
|
||||
UpdateEntity(id=ENTITY_ID, name="РУССКИЙ БИСКВИТ, ПАО", entity_type=None),
|
||||
]
|
||||
assert _final_codes(plan) == {2: ImportCode.LE_UPDATED}
|
||||
assert _events(plan, 2) == [ImportCode.GK_UPDATED]
|
||||
report = plan.report
|
||||
assert (report.groups_updated, report.entities_updated, report.groups_skipped, report.entities_skipped) == (
|
||||
1,
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
)
|
||||
|
||||
|
||||
def test_update_without_differences_is_exists() -> None:
|
||||
plan = plan_import([_row(2)], _existing(), OnExisting.update)
|
||||
assert plan.operations == []
|
||||
assert _final_codes(plan) == {2: ImportCode.LE_EXISTS}
|
||||
assert _events(plan, 2) == [ImportCode.GK_EXISTS]
|
||||
|
||||
|
||||
def test_update_moves_group_to_new_office_and_new_entity_follows() -> None:
|
||||
rows = [_row(2, inn="3528104476", office_name="Вологодский РФ")]
|
||||
plan = plan_import(rows, _existing(), OnExisting.update)
|
||||
(office,) = _ops(plan, CreateOffice)
|
||||
(update,) = _ops(plan, UpdateGroup)
|
||||
(entity,) = _ops(plan, CreateEntity)
|
||||
assert update == UpdateGroup(id=GROUP_ID, name=None, office_id=office.id)
|
||||
assert (entity.group_id, entity.office_id) == (GROUP_ID, office.id)
|
||||
|
||||
|
||||
def test_empty_office_in_file_keeps_group_office_for_new_entity() -> None:
|
||||
rows = [_row(2, inn="3528104476", office_name="")]
|
||||
plan = plan_import(rows, _existing(), OnExisting.update)
|
||||
assert _ops(plan, UpdateGroup) == []
|
||||
(entity,) = _ops(plan, CreateEntity)
|
||||
assert entity.office_id == OFFICE_ID
|
||||
|
||||
|
||||
def test_update_changes_entity_type() -> None:
|
||||
snapshot = _existing()
|
||||
snapshot.entities_by_inn["352819549855"] = EntitySnap(
|
||||
id=ENTITY_ID,
|
||||
inn="352819549855",
|
||||
name="ИП Юганов",
|
||||
entity_type=EntityType.ЮЛ,
|
||||
group_id=GROUP_ID,
|
||||
gsk_group_id=None,
|
||||
gsz_group_id=None,
|
||||
)
|
||||
plan = plan_import(
|
||||
[_row(2, inn="352819549855", type_raw="ИП", entity_name="ИП Юганов")], snapshot, OnExisting.update
|
||||
)
|
||||
assert plan.operations == [UpdateEntity(id=ENTITY_ID, name=None, entity_type=EntityType.ИП)]
|
||||
|
||||
|
||||
def test_group_events_once_per_group() -> None:
|
||||
rows = [_row(2), _row(3, inn="3528104476")]
|
||||
plan = plan_import(rows, _existing(group_name="Старое имя"), OnExisting.update)
|
||||
assert len(_ops(plan, UpdateGroup)) == 1
|
||||
assert _events(plan, 2) == [ImportCode.GK_UPDATED]
|
||||
assert _events(plan, 3) == []
|
||||
assert plan.report.groups_updated == 1
|
||||
|
||||
|
||||
def test_report_to_dict() -> None:
|
||||
plan = plan_import([_row(2, type_raw="ФЛ")], _empty(), OnExisting.skip)
|
||||
data = plan.report.to_dict()
|
||||
assert data["rows"] == 1
|
||||
assert data["errors"] == 1
|
||||
assert data["groups_created"] == 0
|
||||
assert data["lines"] == [
|
||||
{"row": 2, "inn": "3528052718", "code": "UNSUPPORTED_TYPE", "message": data["lines"][0]["message"]}
|
||||
]
|
||||
assert isinstance(data["lines"][0]["code"], str)
|
||||
assert set(data) == {
|
||||
"rows",
|
||||
"groups_created",
|
||||
"groups_updated",
|
||||
"groups_skipped",
|
||||
"entities_created",
|
||||
"entities_updated",
|
||||
"entities_skipped",
|
||||
"memberships_created",
|
||||
"offices_created",
|
||||
"errors",
|
||||
"lines",
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- реальная выгрузка
|
||||
|
||||
|
||||
# Ожидания по фактическому содержимому `ecp-export-russkiy-biskvit.xlsx` (лист «Выписки_01.04.26», строки 2–22):
|
||||
# строка 11 — «ИП Юганов Максим Анатольевич» (ИП, ИНН 352819549855), строка 12 — «Юганов Максим Анатольевич» (ФЛ, тот же
|
||||
# ИНН). Тип проверяется раньше дубля (§13.2), поэтому строка 12 → UNSUPPORTED_TYPE, DUPLICATE_IN_FILE в срезе нет.
|
||||
# Остальные 20 строк — 18 ЮЛ и 2 ИП (строки 11, 14) → LE_CREATED.
|
||||
RB_ROWS = 21
|
||||
RB_FL_ROW = 12
|
||||
RB_LEGAL_ENTITIES = 18
|
||||
RB_SOLE_PROPRIETORS = 2
|
||||
RB_EXPECTED_FINALS = {
|
||||
number: ImportCode.UNSUPPORTED_TYPE if number == RB_FL_ROW else ImportCode.LE_CREATED for number in range(2, 23)
|
||||
}
|
||||
RB_GSK_MEMBERS = ["ПЕКАРНЯ СЧАСТЛИВОЧНАЯ, ООО", "РУССКИЙ БИСКВИТ, АО"]
|
||||
|
||||
|
||||
@pytest.mark.fixtures
|
||||
def test_russkiy_biskvit_export_on_empty_directory(fixtures_dir: Path) -> None:
|
||||
"""Срез «Русский Бисквит» на пустом справочнике: 1 ГК, 1 РФ, 20 лиц (18 ЮЛ + 2 ИП), 1 ГСК с двумя членами."""
|
||||
path = fixtures_dir / "directories" / "ecp-export-russkiy-biskvit.xlsx"
|
||||
rows = read_rows(path.name, path.read_bytes())
|
||||
by_row = {r.row: r for r in rows}
|
||||
assert (by_row[11].inn, by_row[11].type_raw) == ("352819549855", "ИП")
|
||||
assert (by_row[RB_FL_ROW].inn, by_row[RB_FL_ROW].type_raw) == ("352819549855", "ФЛ")
|
||||
assert [r.type_raw for r in rows].count("ЮЛ") == RB_LEGAL_ENTITIES
|
||||
assert [r.type_raw for r in rows].count("ИП") == RB_SOLE_PROPRIETORS
|
||||
|
||||
plan = plan_import(rows, _empty(), OnExisting.skip)
|
||||
report = plan.report
|
||||
|
||||
assert report.rows == RB_ROWS
|
||||
assert _final_codes(plan) == RB_EXPECTED_FINALS
|
||||
assert (report.groups_created, report.offices_created, report.errors) == (1, 1, 1)
|
||||
assert report.entities_created == RB_LEGAL_ENTITIES + RB_SOLE_PROPRIETORS
|
||||
(group,) = _ops(plan, CreateGroup)
|
||||
assert (group.crm_id, group.name) == ("1-AZFL6BQ", "Русский Бисквит")
|
||||
(office,) = _ops(plan, CreateOffice)
|
||||
assert office.name == "Санкт-Петербургский РФ"
|
||||
(related,) = _ops(plan, CreateRelatedGroup)
|
||||
assert (related.kind, related.name) == (RelatedGroupKind.gsk, "Русский Бисквит")
|
||||
entities = {e.id: e.name for e in _ops(plan, CreateEntity)}
|
||||
members = _ops(plan, AddMember)
|
||||
assert sorted(entities[m.entity_id] for m in members) == RB_GSK_MEMBERS
|
||||
assert report.memberships_created == len(RB_GSK_MEMBERS)
|
||||
assert all(m.kind is RelatedGroupKind.gsk for m in members)
|
||||
@ -121,13 +121,13 @@ def plan_import(rows: Sequence[ImportRow], snapshot: DirectorySnapshot, on_exist
|
||||
7. Флаги: `normalize_header(flag)` == `FLAG_GSK`/`FLAG_GSZ` → членство; непустое иное → `UNKNOWN_FLAG` (событие, не ошибка). Для флага: ГСК/ГСЗ ГК из снимка/накопителя или `CreateRelatedGroup(kind, name=имя ГК)` (`GSK_CREATED`/`GSZ_CREATED`, `# ASSUMPTION(OQ-91)`); у ЮЛ нет текущего членства этого вида → `AddMember` (`MEMBER_ADDED`, `memberships_created += 1`); есть в этой группе → ничего; есть в другой группе того же вида → ничего (импорт не переводит; событие не пишется).
|
||||
`rows` = число непустых строк данных. Итоговый код строки — ровно один из восьми; события — дополнительные `RowResult` с тем же `row`.
|
||||
|
||||
- [ ] **Step 1: Тесты** (`tests/test_domain_directory_import.py`, без БД):
|
||||
- [x] **Step 1: Тесты** (`tests/test_domain_directory_import.py`, без БД):
|
||||
- `normalize_header("Базовый РФ ГК в CRM") == "базовыйрфгквcrm"`, `normalize_cell(3528052718.0) == "3528052718"`, `normalize_cell(None) == ""`.
|
||||
- `read_rows` csv `;` и `,`, utf-8-sig, перестановка колонок, лишняя колонка; без «Тип» → `FileFormatError.code == "MISSING_COLUMNS"`, `missing == ["Тип"]`; `.txt` → `UNSUPPORTED_FILE`; xlsx, собранный `openpyxl.Workbook()` с заголовком во 2-й строке, ИНН числом, 3 пустыми строками в хвосте (Review Focus 1–2) → 2 строки данных.
|
||||
- `plan_import` на синтетических строках, по одному тесту на каждый код (параметризация: строка → ожидаемый итоговый код и события), пустой снимок и снимок с существующими; `update` vs `skip`; две строки одной ГК → один `CreateGroup`; флаг у двух ЮЛ одной ГК → один `CreateRelatedGroup` и два `AddMember`; ЮЛ в другой ГК → нет `AddMember`.
|
||||
- `@pytest.mark.fixtures`: `read_rows` + `plan_import` на `fixtures_dir / "directories/ecp-export-russkiy-biskvit.xlsx"` с пустым снимком: `report.rows == 21`, `groups_created == 1`, `offices_created == 1`, `entities_created == 18`, ровно одна `CreateRelatedGroup(kind=gsk)`, два `AddMember`, нет `gsz`; коды строк ФЛ/дубля — зафиксировать по фактическому порядку в файле (посмотреть файл в тесте, не угадывать) и записать ожидание явно.
|
||||
- [ ] **Step 2:** падает → **Step 3:** реализация (+ `openpyxl` в requirements, `make venv`) → **Step 4:** `pytest tests/test_domain_directory_import.py -v`, `mypy src` (strict для `src.domain.*`), `ruff` → PASS.
|
||||
- [ ] **Step 5: Commit** `feat(domain): ядро импорта справочников из выгрузки ЕЦП (§13.1–13.2)`.
|
||||
- [x] **Step 2:** падает → **Step 3:** реализация (+ `openpyxl` в requirements, `make venv`) → **Step 4:** `pytest tests/test_domain_directory_import.py -v`, `mypy src` (strict для `src.domain.*`), `ruff` → PASS.
|
||||
- [x] **Step 5: Commit** `feat(domain): ядро импорта справочников из выгрузки ЕЦП (§13.1–13.2)`.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@ -46,7 +46,7 @@ test-integration`). Пути — от корня репозитория. Доп
|
||||
|
||||
## 3. Импорт из ЕЦП CRM
|
||||
|
||||
- [ ] 3.1 **Чтение файла и ядро решений.** `openpyxl` в `api/requirements.txt` (пин); `domain/directory_import.py`:
|
||||
- [x] 3.1 **Чтение файла и ядро решений.** `openpyxl` в `api/requirements.txt` (пин); `domain/directory_import.py`:
|
||||
`read_rows(file) -> (headers, rows)` для xlsx/csv, нормализация заголовков (`MISSING_COLUMNS`), `ImportCode` (StrEnum = §13.2),
|
||||
`plan_import(rows, snapshot, on_existing) -> ImportPlan(operations, report)` — чистая функция; `ASSUMPTION(OQ-87..91)` в коде.
|
||||
Unit-тесты без БД: синтетический csv, покрывающий каждый код §13.2 ровно один раз; xlsx-фикстура
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user