Raw version migrate smb to openclaude
This commit is contained in:
parent
44f23947a3
commit
79a35edcf4
@ -1,5 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from tempfile import gettempdir
|
from tempfile import gettempdir
|
||||||
|
|
||||||
@ -9,6 +10,18 @@ SFM_SMB_BASE_PATH = "//sgo-fc01-r06/sfm/ОТЧЕТ_7081У"
|
|||||||
SFM_DEFAULT_INPUT_DIR = r"//sgo-fc01-r06/sfm/ОТЧЕТ_7081У"
|
SFM_DEFAULT_INPUT_DIR = r"//sgo-fc01-r06/sfm/ОТЧЕТ_7081У"
|
||||||
ACCESS_SUPPORT_CONTACT = "администратору СФМ"
|
ACCESS_SUPPORT_CONTACT = "администратору СФМ"
|
||||||
|
|
||||||
|
# TODO(BIQ-36593): подтвердить SPACE_ID для прода (сейчас — значение,
|
||||||
|
# проверенное вручную на тестовом пространстве "AURORA"). ID составной
|
||||||
|
# ("storageId$spaceId") — храним как есть, без разбора, через
|
||||||
|
# env-переменную, чтобы не хардкодить прод-значение в коде. В проде
|
||||||
|
# SPACE_ID личного пространства пользователя определяется динамически
|
||||||
|
# через opencloud_access.resolve_personal_drive(access_token), это
|
||||||
|
# значение используется только как тестовый дефолт (AURORA).
|
||||||
|
OPENCLOUD_SPACE_ID = os.getenv(
|
||||||
|
"SFM_OPENCLOUD_SPACE_ID",
|
||||||
|
"28c0cecf-1e20-4664-b165-5c9d5515e70f$95e71bcd-3c89-4b77-b271-63c6d553f4fd",
|
||||||
|
)
|
||||||
|
|
||||||
EDIT_PATH_OPEN_KEY = "edit_path_open"
|
EDIT_PATH_OPEN_KEY = "edit_path_open"
|
||||||
EDIT_PATH_VALUE_KEY = "edit_path_value"
|
EDIT_PATH_VALUE_KEY = "edit_path_value"
|
||||||
# Runtime-only user paths storage. In bank contour we use tmpfs.
|
# Runtime-only user paths storage. In bank contour we use tmpfs.
|
||||||
@ -25,3 +38,23 @@ def resolve_user_id() -> str:
|
|||||||
)
|
)
|
||||||
full_name = str(profile.get("name") or profile.get("full_name") or "anonymous")
|
full_name = str(profile.get("name") or profile.get("full_name") or "anonymous")
|
||||||
return f"{email}|{full_name}"
|
return f"{email}|{full_name}"
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_opencloud_access_token() -> str | None:
|
||||||
|
"""Достаёт access_token для OpenCloud из token_data в сессии.
|
||||||
|
|
||||||
|
НЕ ПОДТВЕРЖДЕНО: в проде (raisa_streamlit_oauth) неизвестно, содержит ли
|
||||||
|
token_data ключ "access_token" — в локальном mock_provider.py его нет
|
||||||
|
(только email/preferred_username/name). Нужно проверить на проде через
|
||||||
|
st.write(token_data.keys()).
|
||||||
|
|
||||||
|
Для локального теста (LOCATION=local, mock-провайдер) — fallback на
|
||||||
|
переменную окружения SFM_OPENCLOUD_ACCESS_TOKEN из .env. В проде эта
|
||||||
|
переменная не задана, так что fallback не сработает и не помешает.
|
||||||
|
"""
|
||||||
|
token_data = st.session_state.get("token_data", {})
|
||||||
|
if isinstance(token_data, dict):
|
||||||
|
value = token_data.get("access_token")
|
||||||
|
if value:
|
||||||
|
return str(value)
|
||||||
|
return os.getenv("SFM_OPENCLOUD_ACCESS_TOKEN") or None
|
||||||
|
|||||||
173
app/opencloud_access.py
Normal file
173
app/opencloud_access.py
Normal file
@ -0,0 +1,173 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
from urllib.parse import quote
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
OPENCLOUD_DOMAIN = "docs.raisa.go.rshbank.ru"
|
||||||
|
OPENCLOUD_UPLOAD_TIMEOUT_SECONDS = 30
|
||||||
|
OPENCLOUD_GRAPH_TIMEOUT_SECONDS = 30
|
||||||
|
|
||||||
|
# Content-Type подобран по расширению — сервер OpenCloud (как и обычный WebDAV)
|
||||||
|
# не требует конкретного значения для PUT, но так удобнее просматривать файлы
|
||||||
|
# в интерфейсе.
|
||||||
|
_CONTENT_TYPES = {
|
||||||
|
".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||||
|
".xlsm": "application/vnd.ms-excel.sheet.macroEnabled.12",
|
||||||
|
".log": "text/plain; charset=utf-8",
|
||||||
|
}
|
||||||
|
_DEFAULT_CONTENT_TYPE = "application/octet-stream"
|
||||||
|
|
||||||
|
|
||||||
|
class OpenCloudUploadError(RuntimeError):
|
||||||
|
"""Ошибка загрузки файла в OpenCloud по WebDAV."""
|
||||||
|
|
||||||
|
|
||||||
|
class OpenCloudDriveResolutionError(RuntimeError):
|
||||||
|
"""Ошибка получения личного пространства (drive) пользователя в OpenCloud."""
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_personal_drive(access_token: str) -> dict:
|
||||||
|
"""Находит личное пространство пользователя через Graph API OpenCloud.
|
||||||
|
|
||||||
|
Личное пространство в OpenCloud — не подпапка внутри общего пространства,
|
||||||
|
а отдельный drive (driveType == "personal") со своим составным id вида
|
||||||
|
"{storageId}${spaceId}". Эта функция делает GET /graph/v1.0/me/drives,
|
||||||
|
выбирает запись с driveType == "personal" и возвращает словарь с ключами
|
||||||
|
"id" (SPACE_ID для загрузки) и "web_dav_url" (готовый базовый WebDAV URL
|
||||||
|
из root.webDavUrl, на случай если он понадобится вместо ручной сборки).
|
||||||
|
"""
|
||||||
|
if not access_token:
|
||||||
|
raise OpenCloudDriveResolutionError(
|
||||||
|
"Отсутствует access_token для запроса личного пространства OpenCloud."
|
||||||
|
)
|
||||||
|
|
||||||
|
url = f"https://{OPENCLOUD_DOMAIN}/graph/v1.0/me/drives"
|
||||||
|
headers = {"Authorization": f"Bearer {access_token}"}
|
||||||
|
|
||||||
|
try:
|
||||||
|
response = requests.get(
|
||||||
|
url, headers=headers, timeout=OPENCLOUD_GRAPH_TIMEOUT_SECONDS
|
||||||
|
)
|
||||||
|
except requests.RequestException as exc:
|
||||||
|
raise OpenCloudDriveResolutionError(
|
||||||
|
f"Ошибка сети при запросе списка пространств OpenCloud: {exc}"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
if response.status_code == 401:
|
||||||
|
raise OpenCloudDriveResolutionError(
|
||||||
|
"OpenCloud отклонил access_token (HTTP 401) при запросе "
|
||||||
|
"списка пространств."
|
||||||
|
)
|
||||||
|
if response.status_code != 200:
|
||||||
|
raise OpenCloudDriveResolutionError(
|
||||||
|
"Не удалось получить список пространств OpenCloud: "
|
||||||
|
f"HTTP {response.status_code}, body={response.text[:500]!r}"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
drives = response.json().get("value", [])
|
||||||
|
except ValueError as exc:
|
||||||
|
raise OpenCloudDriveResolutionError(
|
||||||
|
f"Некорректный JSON-ответ при запросе списка пространств OpenCloud: {exc}"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
for drive in drives:
|
||||||
|
if not isinstance(drive, dict) or drive.get("driveType") != "personal":
|
||||||
|
continue
|
||||||
|
drive_id = drive.get("id")
|
||||||
|
if not drive_id:
|
||||||
|
continue
|
||||||
|
web_dav_url = (drive.get("root") or {}).get("webDavUrl")
|
||||||
|
return {"id": drive_id, "web_dav_url": web_dav_url}
|
||||||
|
|
||||||
|
raise OpenCloudDriveResolutionError(
|
||||||
|
'Личное пространство (driveType="personal") не найдено среди '
|
||||||
|
"пространств пользователя в OpenCloud."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def build_opencloud_url(
|
||||||
|
space_id: str, folder_segments: tuple[str, ...] | list[str], file_name: str
|
||||||
|
) -> str:
|
||||||
|
"""Собирает URL вида /dav/spaces/{SPACE_ID}/{FOLDER}/{FILENAME}.
|
||||||
|
|
||||||
|
Каждый сегмент папки и имя файла кодируются ОТДЕЛЬНО через urllib.parse.quote —
|
||||||
|
requests не кодирует кириллицу/пробелы в уже собранном URL так, как нужно
|
||||||
|
серверу (проверено вручную, см. open_cloud_upload_fixed.py).
|
||||||
|
|
||||||
|
space_id передаётся как есть — по факту наблюдений это составной идентификатор
|
||||||
|
"storageId$spaceId" (обе части через "$"), но эта функция его не разбирает
|
||||||
|
и не проверяет формат, чтобы не завязываться на недокументированное поведение.
|
||||||
|
"""
|
||||||
|
if not space_id:
|
||||||
|
raise OpenCloudUploadError("Не задан SPACE_ID для OpenCloud.")
|
||||||
|
if not file_name:
|
||||||
|
raise OpenCloudUploadError("Не задано имя файла для загрузки в OpenCloud.")
|
||||||
|
|
||||||
|
encoded_folder = "/".join(
|
||||||
|
quote(segment, safe="") for segment in folder_segments if segment
|
||||||
|
)
|
||||||
|
encoded_file_name = quote(file_name, safe="")
|
||||||
|
|
||||||
|
path = f"/dav/spaces/{space_id}"
|
||||||
|
if encoded_folder:
|
||||||
|
path += f"/{encoded_folder}"
|
||||||
|
path += f"/{encoded_file_name}"
|
||||||
|
return f"https://{OPENCLOUD_DOMAIN}{path}"
|
||||||
|
|
||||||
|
|
||||||
|
def upload_local_file_to_opencloud(
|
||||||
|
local_path: Path,
|
||||||
|
space_id: str,
|
||||||
|
folder_segments: tuple[str, ...] | list[str],
|
||||||
|
file_name: str,
|
||||||
|
access_token: str,
|
||||||
|
) -> str:
|
||||||
|
"""Загружает локальный файл в OpenCloud через WebDAV PUT (просто attach).
|
||||||
|
|
||||||
|
Возвращает URL, по которому файл загружен (для отображения пользователю /
|
||||||
|
записи в протокол).
|
||||||
|
|
||||||
|
ВРЕМЕННО (тестовый прогон): папку НЕ создаём (без MKCOL) — просто
|
||||||
|
кладём файл прямо в личную папку участника (folder_segments — как
|
||||||
|
правило один сегмент, например "Райков М.С."). Структуру Out/год/
|
||||||
|
месяц/день/номер_запуска вернём отдельным шагом после того, как
|
||||||
|
подтвердим на тесте базовый сценарий "файл долетает в нужную папку".
|
||||||
|
"""
|
||||||
|
if not access_token:
|
||||||
|
raise OpenCloudUploadError(
|
||||||
|
"Отсутствует access_token для загрузки в OpenCloud. "
|
||||||
|
"Проверьте наличие ключа 'access_token' в st.session_state['token_data']."
|
||||||
|
)
|
||||||
|
if not local_path.exists():
|
||||||
|
raise OpenCloudUploadError(f"Локальный файл не найден: {local_path}")
|
||||||
|
|
||||||
|
url = build_opencloud_url(space_id, folder_segments, file_name)
|
||||||
|
content_type = _CONTENT_TYPES.get(local_path.suffix.lower(), _DEFAULT_CONTENT_TYPE)
|
||||||
|
headers = {
|
||||||
|
"Authorization": f"Bearer {access_token}",
|
||||||
|
"Content-Type": content_type,
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
with local_path.open("rb") as file_data:
|
||||||
|
response = requests.put(
|
||||||
|
url,
|
||||||
|
data=file_data,
|
||||||
|
headers=headers,
|
||||||
|
timeout=OPENCLOUD_UPLOAD_TIMEOUT_SECONDS,
|
||||||
|
)
|
||||||
|
except requests.RequestException as exc:
|
||||||
|
raise OpenCloudUploadError(
|
||||||
|
f"Ошибка сети при загрузке в OpenCloud: url='{url}', {exc}"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
if response.status_code not in (200, 201, 204):
|
||||||
|
raise OpenCloudUploadError(
|
||||||
|
"Загрузка в OpenCloud не удалась: "
|
||||||
|
f"HTTP {response.status_code}, url='{url}', "
|
||||||
|
f"body={response.text[:500]!r}"
|
||||||
|
)
|
||||||
|
return url
|
||||||
@ -2,15 +2,17 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import tempfile
|
import tempfile
|
||||||
import time
|
import time
|
||||||
|
import zipfile
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
from ..opencloud_access import OpenCloudUploadError, upload_local_file_to_opencloud
|
||||||
from .config import (
|
from .config import (
|
||||||
|
build_date_parts,
|
||||||
build_process_log_file_name,
|
build_process_log_file_name,
|
||||||
build_protocol_file_name,
|
build_protocol_file_name,
|
||||||
build_report_file_name,
|
build_report_file_name,
|
||||||
join_path,
|
|
||||||
)
|
)
|
||||||
from .file_manager import (
|
from .file_manager import (
|
||||||
ProcessedCopySucceededDeleteFailedError,
|
ProcessedCopySucceededDeleteFailedError,
|
||||||
@ -23,7 +25,6 @@ from .file_manager import (
|
|||||||
move_file_to_processed,
|
move_file_to_processed,
|
||||||
read_remote_file_bytes,
|
read_remote_file_bytes,
|
||||||
scan_input_files,
|
scan_input_files,
|
||||||
upload_local_file,
|
|
||||||
)
|
)
|
||||||
from .parser import FileParseResult, parse_xml_content
|
from .parser import FileParseResult, parse_xml_content
|
||||||
from .protocol import ProtocolRow, write_protocol
|
from .protocol import ProtocolRow, write_protocol
|
||||||
@ -51,6 +52,8 @@ def run_batch(
|
|||||||
samba_user: str,
|
samba_user: str,
|
||||||
samba_password: str,
|
samba_password: str,
|
||||||
launcher: str,
|
launcher: str,
|
||||||
|
opencloud_space_id: str | None,
|
||||||
|
opencloud_access_token: str | None,
|
||||||
artifacts_dir: str | None = None,
|
artifacts_dir: str | None = None,
|
||||||
) -> BatchResult:
|
) -> BatchResult:
|
||||||
started_at = time.perf_counter()
|
started_at = time.perf_counter()
|
||||||
@ -66,6 +69,33 @@ def run_batch(
|
|||||||
log_path="",
|
log_path="",
|
||||||
messages=["Введите логин SAMBA."],
|
messages=["Введите логин SAMBA."],
|
||||||
)
|
)
|
||||||
|
if not opencloud_access_token:
|
||||||
|
return BatchResult(
|
||||||
|
status="error",
|
||||||
|
processed=0,
|
||||||
|
partial=0,
|
||||||
|
errors=1,
|
||||||
|
report_path="",
|
||||||
|
protocol_path="",
|
||||||
|
log_path="",
|
||||||
|
messages=[
|
||||||
|
"Не удалось получить access_token для OpenCloud "
|
||||||
|
"(проверьте авторизацию)."
|
||||||
|
],
|
||||||
|
)
|
||||||
|
if not opencloud_space_id:
|
||||||
|
return BatchResult(
|
||||||
|
status="error",
|
||||||
|
processed=0,
|
||||||
|
partial=0,
|
||||||
|
errors=1,
|
||||||
|
report_path="",
|
||||||
|
protocol_path="",
|
||||||
|
log_path="",
|
||||||
|
messages=[
|
||||||
|
"Не удалось определить личное пространство пользователя в OpenCloud."
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
samba_conn = connect_samba(
|
samba_conn = connect_samba(
|
||||||
smb_base_path=smb_base_path,
|
smb_base_path=smb_base_path,
|
||||||
@ -157,9 +187,6 @@ def run_batch(
|
|||||||
run_context.date_key, run_context.run_number
|
run_context.date_key, run_context.run_number
|
||||||
)
|
)
|
||||||
|
|
||||||
report_path = join_path(run_context.run_dir, report_file_name)
|
|
||||||
protocol_path = join_path(run_context.run_dir, protocol_file_name)
|
|
||||||
log_path = join_path(run_context.run_dir, log_file_name)
|
|
||||||
with tempfile.TemporaryDirectory() as temp_dir:
|
with tempfile.TemporaryDirectory() as temp_dir:
|
||||||
temp_dir_path = Path(temp_dir)
|
temp_dir_path = Path(temp_dir)
|
||||||
local_report = temp_dir_path / "report.xlsx"
|
local_report = temp_dir_path / "report.xlsx"
|
||||||
@ -210,14 +237,31 @@ def run_batch(
|
|||||||
# Use CRLF to keep one-record-per-line in Windows viewers.
|
# Use CRLF to keep one-record-per-line in Windows viewers.
|
||||||
local_log.write_text("\r\n".join(log_lines) + "\r\n", encoding="utf-8")
|
local_log.write_text("\r\n".join(log_lines) + "\r\n", encoding="utf-8")
|
||||||
|
|
||||||
_upload_artifacts(
|
try:
|
||||||
|
report_path, protocol_path, log_path = _upload_artifacts(
|
||||||
local_report=local_report,
|
local_report=local_report,
|
||||||
local_protocol=local_protocol,
|
local_protocol=local_protocol,
|
||||||
local_log=local_log,
|
local_log=local_log,
|
||||||
report_path=report_path,
|
report_file_name=report_file_name,
|
||||||
protocol_path=protocol_path,
|
protocol_file_name=protocol_file_name,
|
||||||
log_path=log_path,
|
log_file_name=log_file_name,
|
||||||
samba_conn=samba_conn,
|
opencloud_space_id=opencloud_space_id,
|
||||||
|
opencloud_access_token=opencloud_access_token,
|
||||||
|
)
|
||||||
|
except OpenCloudUploadError as exc:
|
||||||
|
return BatchResult(
|
||||||
|
status="error",
|
||||||
|
processed=0,
|
||||||
|
partial=0,
|
||||||
|
errors=1,
|
||||||
|
report_path="",
|
||||||
|
protocol_path="",
|
||||||
|
log_path="",
|
||||||
|
messages=[
|
||||||
|
*messages,
|
||||||
|
f"Файлы обработаны, но не удалось загрузить результат в "
|
||||||
|
f"OpenCloud: {exc}",
|
||||||
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
processed = sum(1 for row in protocol_rows if row.status == "success")
|
processed = sum(1 for row in protocol_rows if row.status == "success")
|
||||||
@ -252,6 +296,210 @@ def run_batch(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def run_local_zip_batch(
|
||||||
|
zip_path: Path,
|
||||||
|
launcher: str,
|
||||||
|
opencloud_space_id: str | None,
|
||||||
|
opencloud_access_token: str | None,
|
||||||
|
) -> BatchResult:
|
||||||
|
"""Локальный тестовый прогон: zip с XML -> отчёт/протокол/лог -> OpenCloud.
|
||||||
|
|
||||||
|
В отличие от run_batch, НЕ использует SMB вообще — ни для чтения входных
|
||||||
|
файлов, ни для перемещения в Processed/Err. Предназначена для ручного
|
||||||
|
теста всей цепочки (парсинг + загрузка в облако) без сетевой шары.
|
||||||
|
Файлы с некорректным именем/битым XML просто попадают в протокол со
|
||||||
|
статусом "error" — никуда не перемещаются (перемещать их и некуда,
|
||||||
|
это временная распакованная копия).
|
||||||
|
"""
|
||||||
|
started_at = time.perf_counter()
|
||||||
|
if not opencloud_access_token:
|
||||||
|
return BatchResult(
|
||||||
|
status="error",
|
||||||
|
processed=0,
|
||||||
|
partial=0,
|
||||||
|
errors=1,
|
||||||
|
report_path="",
|
||||||
|
protocol_path="",
|
||||||
|
log_path="",
|
||||||
|
messages=[
|
||||||
|
"Не удалось получить access_token для OpenCloud "
|
||||||
|
"(проверьте авторизацию)."
|
||||||
|
],
|
||||||
|
)
|
||||||
|
if not opencloud_space_id:
|
||||||
|
return BatchResult(
|
||||||
|
status="error",
|
||||||
|
processed=0,
|
||||||
|
partial=0,
|
||||||
|
errors=1,
|
||||||
|
report_path="",
|
||||||
|
protocol_path="",
|
||||||
|
log_path="",
|
||||||
|
messages=[
|
||||||
|
"Не удалось определить личное пространство пользователя в OpenCloud."
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
protocol_rows: list[ProtocolRow] = []
|
||||||
|
log_lines: list[str] = [f"{_now()} | local zip batch start: {zip_path.name}"]
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as extract_dir:
|
||||||
|
extract_dir_path = Path(extract_dir)
|
||||||
|
try:
|
||||||
|
with zipfile.ZipFile(zip_path) as archive:
|
||||||
|
archive.extractall(extract_dir_path)
|
||||||
|
except zipfile.BadZipFile as exc:
|
||||||
|
return BatchResult(
|
||||||
|
status="error",
|
||||||
|
processed=0,
|
||||||
|
partial=0,
|
||||||
|
errors=1,
|
||||||
|
report_path="",
|
||||||
|
protocol_path="",
|
||||||
|
log_path="",
|
||||||
|
messages=[f"Не удалось распаковать архив: {exc}"],
|
||||||
|
)
|
||||||
|
|
||||||
|
xml_paths = sorted(extract_dir_path.rglob("*.xml"))
|
||||||
|
log_lines.append(
|
||||||
|
f"{_now()} | распаковано, найдено XML-файлов: {len(xml_paths)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
report_writer = StreamingReportWriter()
|
||||||
|
for xml_path in xml_paths:
|
||||||
|
file_name = xml_path.name
|
||||||
|
try:
|
||||||
|
content = xml_path.read_bytes()
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
protocol_rows.append(
|
||||||
|
ProtocolRow(
|
||||||
|
file_name=file_name,
|
||||||
|
status="error",
|
||||||
|
message=f"Ошибка чтения файла из архива: {exc}",
|
||||||
|
operations_total=0,
|
||||||
|
rows_written=0,
|
||||||
|
launcher=launcher,
|
||||||
|
processed_at=datetime.now(),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
log_lines.append(f"{_now()} | read failed: {file_name}. {exc}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
# parse_xml_content сам проверяет имя файла и проставляет
|
||||||
|
# fatal_error, если оно не соответствует шаблону — отдельная
|
||||||
|
# валидация здесь не нужна.
|
||||||
|
parse_result = parse_xml_content(file_name=file_name, xml_content=content)
|
||||||
|
for row in parse_result.rows:
|
||||||
|
report_writer.append_row(
|
||||||
|
ReportRow(
|
||||||
|
file_name=row.file_name,
|
||||||
|
record_id=row.record_id,
|
||||||
|
operation_index=row.operation_index,
|
||||||
|
operation_fields=row.operation_fields,
|
||||||
|
participant_fields=row.participant_fields,
|
||||||
|
is_continuation=row.is_continuation,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
status, message = _resolve_parse_status(parse_result)
|
||||||
|
protocol_rows.append(
|
||||||
|
ProtocolRow(
|
||||||
|
file_name=file_name,
|
||||||
|
status=status,
|
||||||
|
message=message,
|
||||||
|
operations_total=parse_result.total_operations,
|
||||||
|
rows_written=len(parse_result.rows),
|
||||||
|
launcher=launcher,
|
||||||
|
processed_at=datetime.now(),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
log_lines.append(f"{_now()} | parsed {file_name}: {status}")
|
||||||
|
|
||||||
|
if not protocol_rows:
|
||||||
|
protocol_rows.append(
|
||||||
|
ProtocolRow(
|
||||||
|
file_name="-",
|
||||||
|
status=EMPTY_BATCH_STATUS,
|
||||||
|
message="В архиве не найдено XML-файлов.",
|
||||||
|
operations_total=0,
|
||||||
|
rows_written=0,
|
||||||
|
launcher=launcher,
|
||||||
|
processed_at=datetime.now(),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
log_lines.append(f"{_now()} | empty batch")
|
||||||
|
|
||||||
|
_, _, _, date_key = build_date_parts()
|
||||||
|
run_number = 1
|
||||||
|
report_file_name = build_report_file_name(date_key, run_number)
|
||||||
|
protocol_file_name = build_protocol_file_name(date_key, run_number)
|
||||||
|
log_file_name = build_process_log_file_name(date_key, run_number)
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as artifacts_dir:
|
||||||
|
artifacts_dir_path = Path(artifacts_dir)
|
||||||
|
local_report = artifacts_dir_path / "report.xlsx"
|
||||||
|
local_protocol = artifacts_dir_path / "protocol.xlsx"
|
||||||
|
local_log = artifacts_dir_path / "process.log"
|
||||||
|
|
||||||
|
report_writer.save(local_report)
|
||||||
|
write_protocol(protocol_rows, local_protocol)
|
||||||
|
local_log.write_text("\r\n".join(log_lines) + "\r\n", encoding="utf-8")
|
||||||
|
|
||||||
|
try:
|
||||||
|
report_path, protocol_path, log_path = _upload_artifacts(
|
||||||
|
local_report=local_report,
|
||||||
|
local_protocol=local_protocol,
|
||||||
|
local_log=local_log,
|
||||||
|
report_file_name=report_file_name,
|
||||||
|
protocol_file_name=protocol_file_name,
|
||||||
|
log_file_name=log_file_name,
|
||||||
|
opencloud_space_id=opencloud_space_id,
|
||||||
|
opencloud_access_token=opencloud_access_token,
|
||||||
|
)
|
||||||
|
except OpenCloudUploadError as exc:
|
||||||
|
return BatchResult(
|
||||||
|
status="error",
|
||||||
|
processed=0,
|
||||||
|
partial=0,
|
||||||
|
errors=1,
|
||||||
|
report_path="",
|
||||||
|
protocol_path="",
|
||||||
|
log_path="",
|
||||||
|
messages=[
|
||||||
|
*log_lines,
|
||||||
|
f"Файлы обработаны, но не удалось загрузить результат "
|
||||||
|
f"в OpenCloud: {exc}",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
processed = sum(1 for row in protocol_rows if row.status == "success")
|
||||||
|
partial = sum(1 for row in protocol_rows if row.status == "partial")
|
||||||
|
errors = sum(1 for row in protocol_rows if row.status == "error")
|
||||||
|
status = (
|
||||||
|
"success"
|
||||||
|
if errors == 0 and partial == 0
|
||||||
|
else ("partial" if partial > 0 else "error")
|
||||||
|
)
|
||||||
|
messages = [
|
||||||
|
_build_result_message(
|
||||||
|
status=status, processed=processed, partial=partial, errors=errors
|
||||||
|
)
|
||||||
|
]
|
||||||
|
duration_seconds = round(time.perf_counter() - started_at, 2)
|
||||||
|
messages.append(f"Время выполнения: {_format_duration(duration_seconds)}.")
|
||||||
|
messages.extend(log_lines[-5:])
|
||||||
|
return BatchResult(
|
||||||
|
status=status,
|
||||||
|
processed=processed,
|
||||||
|
partial=partial,
|
||||||
|
errors=errors,
|
||||||
|
report_path=report_path,
|
||||||
|
protocol_path=protocol_path,
|
||||||
|
log_path=log_path,
|
||||||
|
messages=messages,
|
||||||
|
duration_seconds=duration_seconds,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _parse_remote_xml(
|
def _parse_remote_xml(
|
||||||
file_name: str, remote_path: str, samba_conn, stage_timings: dict[str, float]
|
file_name: str, remote_path: str, samba_conn, stage_timings: dict[str, float]
|
||||||
) -> FileParseResult:
|
) -> FileParseResult:
|
||||||
@ -414,14 +662,40 @@ def _upload_artifacts(
|
|||||||
local_report: Path,
|
local_report: Path,
|
||||||
local_protocol: Path,
|
local_protocol: Path,
|
||||||
local_log: Path,
|
local_log: Path,
|
||||||
report_path: str,
|
report_file_name: str,
|
||||||
protocol_path: str,
|
protocol_file_name: str,
|
||||||
log_path: str,
|
log_file_name: str,
|
||||||
samba_conn,
|
opencloud_space_id: str,
|
||||||
) -> None:
|
opencloud_access_token: str,
|
||||||
upload_local_file(local_report, report_path, samba_conn=samba_conn)
|
) -> tuple[str, str, str]:
|
||||||
upload_local_file(local_protocol, protocol_path, samba_conn=samba_conn)
|
# ВРЕМЕННО (тестовый прогон): грузим плоско, прямо в корень личного
|
||||||
upload_local_file(local_log, log_path, samba_conn=samba_conn)
|
# пространства пользователя (найденного через resolve_personal_drive),
|
||||||
|
# без MKCOL и без структуры Out/год/месяц/день/номер запуска. Сначала
|
||||||
|
# проверяем на тесте, что файл вообще долетает — структуру подпапок
|
||||||
|
# вернём отдельным шагом.
|
||||||
|
folder_segments: tuple[str, ...] = ()
|
||||||
|
report_url = upload_local_file_to_opencloud(
|
||||||
|
local_report,
|
||||||
|
space_id=opencloud_space_id,
|
||||||
|
folder_segments=folder_segments,
|
||||||
|
file_name=report_file_name,
|
||||||
|
access_token=opencloud_access_token,
|
||||||
|
)
|
||||||
|
protocol_url = upload_local_file_to_opencloud(
|
||||||
|
local_protocol,
|
||||||
|
space_id=opencloud_space_id,
|
||||||
|
folder_segments=folder_segments,
|
||||||
|
file_name=protocol_file_name,
|
||||||
|
access_token=opencloud_access_token,
|
||||||
|
)
|
||||||
|
log_url = upload_local_file_to_opencloud(
|
||||||
|
local_log,
|
||||||
|
space_id=opencloud_space_id,
|
||||||
|
folder_segments=folder_segments,
|
||||||
|
file_name=log_file_name,
|
||||||
|
access_token=opencloud_access_token,
|
||||||
|
)
|
||||||
|
return report_url, protocol_url, log_url
|
||||||
|
|
||||||
|
|
||||||
def _now() -> str:
|
def _now() -> str:
|
||||||
|
|||||||
32
main.py
32
main.py
@ -1,8 +1,14 @@
|
|||||||
import streamlit as st
|
import streamlit as st
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
|
|
||||||
from app.app_config import ACCESS_SUPPORT_CONTACT, SFM_SMB_BASE_PATH, resolve_user_id
|
from app.app_config import (
|
||||||
|
ACCESS_SUPPORT_CONTACT,
|
||||||
|
SFM_SMB_BASE_PATH,
|
||||||
|
resolve_opencloud_access_token,
|
||||||
|
resolve_user_id,
|
||||||
|
)
|
||||||
from app.auth.service import AuthService, clear_session
|
from app.auth.service import AuthService, clear_session
|
||||||
|
from app.opencloud_access import OpenCloudDriveResolutionError, resolve_personal_drive
|
||||||
from app.path_settings import (
|
from app.path_settings import (
|
||||||
close_path_editor,
|
close_path_editor,
|
||||||
ensure_user_path_state,
|
ensure_user_path_state,
|
||||||
@ -12,7 +18,7 @@ from app.path_settings import (
|
|||||||
)
|
)
|
||||||
from app.path_store import set_user_path
|
from app.path_store import set_user_path
|
||||||
from app.pipeline.config import join_path
|
from app.pipeline.config import join_path
|
||||||
from app.pipeline.service import run_batch
|
from app.pipeline.service import BatchResult, run_batch
|
||||||
from app.samba_access import check_access_and_list_files, diagnose_smb_access
|
from app.samba_access import check_access_and_list_files, diagnose_smb_access
|
||||||
|
|
||||||
|
|
||||||
@ -73,12 +79,34 @@ def main() -> None:
|
|||||||
):
|
):
|
||||||
with st.spinner("Выполняется обработка..."):
|
with st.spinner("Выполняется обработка..."):
|
||||||
launcher_name = samba_user.strip() or user_id
|
launcher_name = samba_user.strip() or user_id
|
||||||
|
access_token = resolve_opencloud_access_token()
|
||||||
|
opencloud_space_id = None
|
||||||
|
opencloud_error = None
|
||||||
|
if access_token:
|
||||||
|
try:
|
||||||
|
opencloud_space_id = resolve_personal_drive(access_token)["id"]
|
||||||
|
except OpenCloudDriveResolutionError as exc:
|
||||||
|
opencloud_error = str(exc)
|
||||||
|
if opencloud_error:
|
||||||
|
batch_result = BatchResult(
|
||||||
|
status="error",
|
||||||
|
processed=0,
|
||||||
|
partial=0,
|
||||||
|
errors=1,
|
||||||
|
report_path="",
|
||||||
|
protocol_path="",
|
||||||
|
log_path="",
|
||||||
|
messages=[opencloud_error],
|
||||||
|
)
|
||||||
|
else:
|
||||||
batch_result = run_batch(
|
batch_result = run_batch(
|
||||||
smb_base_path=SFM_SMB_BASE_PATH,
|
smb_base_path=SFM_SMB_BASE_PATH,
|
||||||
input_dir=processing_input_dir,
|
input_dir=processing_input_dir,
|
||||||
samba_user=samba_user,
|
samba_user=samba_user,
|
||||||
samba_password=samba_password,
|
samba_password=samba_password,
|
||||||
launcher=launcher_name,
|
launcher=launcher_name,
|
||||||
|
opencloud_space_id=opencloud_space_id,
|
||||||
|
opencloud_access_token=access_token,
|
||||||
artifacts_dir=resolved_input_dir,
|
artifacts_dir=resolved_input_dir,
|
||||||
)
|
)
|
||||||
if batch_result.status == "success":
|
if batch_result.status == "success":
|
||||||
|
|||||||
@ -9,3 +9,4 @@ openpyxl
|
|||||||
xlsxwriter==3.2.9
|
xlsxwriter==3.2.9
|
||||||
pytest
|
pytest
|
||||||
smbprotocol==1.15.0
|
smbprotocol==1.15.0
|
||||||
|
requests==2.32.3
|
||||||
|
|||||||
53
scripts/local_opencloud_smoke.py
Normal file
53
scripts/local_opencloud_smoke.py
Normal file
@ -0,0 +1,53 @@
|
|||||||
|
"""Локальный ручной тест: чисто attach в OpenCloud, без Streamlit.
|
||||||
|
|
||||||
|
Использование:
|
||||||
|
1. Впишите ниже ACCESS_TOKEN, SPACE_ID (если отличается от дефолта),
|
||||||
|
FOLDER и путь к тестовому файлу.
|
||||||
|
2. Запустите: python scripts/local_opencloud_smoke.py
|
||||||
|
|
||||||
|
Использует тот же app/opencloud_access.py, что и продакшн-код в
|
||||||
|
app/pipeline/service.py — так что если тут сработает, значит и в
|
||||||
|
приложении сработает (сама логика загрузки не отличается).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||||
|
|
||||||
|
from app.opencloud_access import OpenCloudUploadError, upload_local_file_to_opencloud
|
||||||
|
|
||||||
|
# --- Заполните перед запуском ---
|
||||||
|
ACCESS_TOKEN = "PASTE_TOKEN_HERE"
|
||||||
|
SPACE_ID = "28c0cecf-1e20-4664-b165-5c9d5515e70f" # тестовое пространство AURORA
|
||||||
|
FOLDER = "Райков М.С." # ваша личная папка в OpenCloud
|
||||||
|
LOCAL_FILE = Path(__file__).resolve().parent / "test_upload.xlsx"
|
||||||
|
REMOTE_FILE_NAME = "test_upload.xlsx"
|
||||||
|
# ---------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
if not LOCAL_FILE.exists():
|
||||||
|
LOCAL_FILE.write_bytes(b"local opencloud smoke test")
|
||||||
|
print(f"Тестовый файл создан: {LOCAL_FILE}")
|
||||||
|
|
||||||
|
try:
|
||||||
|
url = upload_local_file_to_opencloud(
|
||||||
|
LOCAL_FILE,
|
||||||
|
space_id=SPACE_ID,
|
||||||
|
folder_segments=(FOLDER,),
|
||||||
|
file_name=REMOTE_FILE_NAME,
|
||||||
|
access_token=ACCESS_TOKEN,
|
||||||
|
)
|
||||||
|
except OpenCloudUploadError as exc:
|
||||||
|
print(f"ОШИБКА: {exc}")
|
||||||
|
raise SystemExit(1) from exc
|
||||||
|
|
||||||
|
print(f"OK, загружено: {url}")
|
||||||
|
print("Проверьте файл в веб-интерфейсе OpenCloud, в папке", FOLDER)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
112
scripts/local_zip_test_app.py
Normal file
112
scripts/local_zip_test_app.py
Normal file
@ -0,0 +1,112 @@
|
|||||||
|
"""Тестовая страница: прикрепляете .zip с XML, он распаковывается,
|
||||||
|
обрабатывается тем же парсером, что и прод, а готовый отчёт/протокол/лог
|
||||||
|
грузятся в OpenCloud. Без SMB на входе — специально для локального теста
|
||||||
|
всей цепочки целиком.
|
||||||
|
|
||||||
|
Не трогает main.py и основной SMB-пайплайн.
|
||||||
|
|
||||||
|
Запуск:
|
||||||
|
streamlit run scripts/local_zip_test_app.py
|
||||||
|
|
||||||
|
Токен/папку можно один раз прописать в .env (см. .env.example) — поля
|
||||||
|
на странице предзаполнятся, но их всегда можно поменять руками.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||||
|
|
||||||
|
import streamlit as st
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
|
||||||
|
from app.opencloud_access import OpenCloudDriveResolutionError, resolve_personal_drive
|
||||||
|
from app.pipeline.service import run_local_zip_batch
|
||||||
|
|
||||||
|
load_dotenv()
|
||||||
|
|
||||||
|
st.set_page_config(page_title="СФМ — локальный тест", page_icon="📦")
|
||||||
|
st.title("Локальный тест: zip → обработка → OpenCloud")
|
||||||
|
st.caption(
|
||||||
|
"Прикрепляете .zip с XML-файлами, он распаковывается и обрабатывается "
|
||||||
|
"тем же кодом парсинга, что и прод (app/pipeline/parser.py), а отчёт, "
|
||||||
|
"протокол и лог грузятся в OpenCloud (app/opencloud_access.py). "
|
||||||
|
"SMB тут не используется вообще."
|
||||||
|
)
|
||||||
|
|
||||||
|
access_token = st.text_input(
|
||||||
|
"Access token",
|
||||||
|
value=os.getenv("SFM_OPENCLOUD_ACCESS_TOKEN", ""),
|
||||||
|
type="password",
|
||||||
|
help="Bearer-токен из Keycloak. Можно прописать в .env как SFM_OPENCLOUD_ACCESS_TOKEN.",
|
||||||
|
)
|
||||||
|
space_id_override = st.text_input(
|
||||||
|
"SPACE_ID (переопределить вручную, необязательно)",
|
||||||
|
value=os.getenv("SFM_OPENCLOUD_SPACE_ID", ""),
|
||||||
|
help=(
|
||||||
|
"Если оставить пустым — личное пространство определится "
|
||||||
|
"автоматически по токену через resolve_personal_drive()."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
launcher = st.text_input("Запустил (для протокола)", value="local-test")
|
||||||
|
|
||||||
|
uploaded_zip = st.file_uploader("Архив с XML-файлами (.zip)", type=["zip"])
|
||||||
|
|
||||||
|
if st.button("Начать обработку", type="primary", use_container_width=True):
|
||||||
|
if not access_token:
|
||||||
|
st.error("Укажите access token.")
|
||||||
|
elif uploaded_zip is None:
|
||||||
|
st.error("Прикрепите .zip-архив.")
|
||||||
|
else:
|
||||||
|
space_id = space_id_override
|
||||||
|
if not space_id:
|
||||||
|
try:
|
||||||
|
drive = resolve_personal_drive(access_token)
|
||||||
|
except OpenCloudDriveResolutionError as exc:
|
||||||
|
st.error(f"Не удалось определить личное пространство: {exc}")
|
||||||
|
space_id = None
|
||||||
|
else:
|
||||||
|
space_id = drive["id"]
|
||||||
|
st.info(
|
||||||
|
f"Найдено личное пространство: id={drive['id']!r}, "
|
||||||
|
f"webDavUrl={drive.get('web_dav_url')!r}"
|
||||||
|
)
|
||||||
|
|
||||||
|
if space_id:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||||
|
zip_path = Path(tmp_dir) / uploaded_zip.name
|
||||||
|
zip_path.write_bytes(uploaded_zip.getvalue())
|
||||||
|
with st.spinner("Распаковываем, обрабатываем, грузим в OpenCloud..."):
|
||||||
|
result = run_local_zip_batch(
|
||||||
|
zip_path=zip_path,
|
||||||
|
launcher=launcher or "local-test",
|
||||||
|
opencloud_space_id=space_id,
|
||||||
|
opencloud_access_token=access_token,
|
||||||
|
)
|
||||||
|
|
||||||
|
if result.status == "success":
|
||||||
|
st.success("Обработка завершена успешно.")
|
||||||
|
elif result.status == "partial":
|
||||||
|
st.warning("Обработка завершена частично.")
|
||||||
|
else:
|
||||||
|
st.error("Обработка завершена с ошибками.")
|
||||||
|
|
||||||
|
st.write(
|
||||||
|
{
|
||||||
|
"processed": result.processed,
|
||||||
|
"partial": result.partial,
|
||||||
|
"errors": result.errors,
|
||||||
|
"report_path": result.report_path,
|
||||||
|
"protocol_path": result.protocol_path,
|
||||||
|
"log_path": result.log_path,
|
||||||
|
"duration_seconds": result.duration_seconds,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if result.messages:
|
||||||
|
st.caption("Сообщения обработки:")
|
||||||
|
for message in result.messages:
|
||||||
|
st.write(f"- {message}")
|
||||||
@ -86,6 +86,13 @@ def _valid_xml() -> str:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def _fake_opencloud_upload(
|
||||||
|
local_path, space_id, folder_segments, file_name, access_token
|
||||||
|
) -> str:
|
||||||
|
_ = (local_path, space_id, access_token)
|
||||||
|
return f"https://opencloud.test/{'/'.join(folder_segments)}/{file_name}"
|
||||||
|
|
||||||
|
|
||||||
def test_run_batch_smoke_and_idempotency(tmp_path: Path, monkeypatch) -> None:
|
def test_run_batch_smoke_and_idempotency(tmp_path: Path, monkeypatch) -> None:
|
||||||
samba = FakeSamba(tmp_path)
|
samba = FakeSamba(tmp_path)
|
||||||
in_dir_local = tmp_path / "Exchange" / "SMBDEMO" / "test"
|
in_dir_local = tmp_path / "Exchange" / "SMBDEMO" / "test"
|
||||||
@ -100,6 +107,9 @@ def test_run_batch_smoke_and_idempotency(tmp_path: Path, monkeypatch) -> None:
|
|||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
service, "connect_samba", lambda smb_base_path, user_params: samba
|
service, "connect_samba", lambda smb_base_path, user_params: samba
|
||||||
)
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
service, "upload_local_file_to_opencloud", _fake_opencloud_upload
|
||||||
|
)
|
||||||
|
|
||||||
first = run_batch(
|
first = run_batch(
|
||||||
smb_base_path="//test-server/share",
|
smb_base_path="//test-server/share",
|
||||||
@ -107,6 +117,8 @@ def test_run_batch_smoke_and_idempotency(tmp_path: Path, monkeypatch) -> None:
|
|||||||
samba_user="user",
|
samba_user="user",
|
||||||
samba_password="pass",
|
samba_password="pass",
|
||||||
launcher="tester",
|
launcher="tester",
|
||||||
|
opencloud_space_id="space-1",
|
||||||
|
opencloud_access_token="token-123",
|
||||||
)
|
)
|
||||||
assert first.status in {"success", "partial"}
|
assert first.status in {"success", "partial"}
|
||||||
assert first.protocol_path
|
assert first.protocol_path
|
||||||
@ -118,6 +130,8 @@ def test_run_batch_smoke_and_idempotency(tmp_path: Path, monkeypatch) -> None:
|
|||||||
samba_user="user",
|
samba_user="user",
|
||||||
samba_password="pass",
|
samba_password="pass",
|
||||||
launcher="tester",
|
launcher="tester",
|
||||||
|
opencloud_space_id="space-1",
|
||||||
|
opencloud_access_token="token-123",
|
||||||
)
|
)
|
||||||
assert second.status == "success"
|
assert second.status == "success"
|
||||||
assert second.processed == 0
|
assert second.processed == 0
|
||||||
@ -137,6 +151,9 @@ def test_run_batch_success_when_delete_denied_after_copy(
|
|||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
service, "connect_samba", lambda smb_base_path, user_params: samba
|
service, "connect_samba", lambda smb_base_path, user_params: samba
|
||||||
)
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
service, "upload_local_file_to_opencloud", _fake_opencloud_upload
|
||||||
|
)
|
||||||
|
|
||||||
result = run_batch(
|
result = run_batch(
|
||||||
smb_base_path="//test-server/share",
|
smb_base_path="//test-server/share",
|
||||||
@ -144,6 +161,8 @@ def test_run_batch_success_when_delete_denied_after_copy(
|
|||||||
samba_user="user",
|
samba_user="user",
|
||||||
samba_password="pass",
|
samba_password="pass",
|
||||||
launcher="tester",
|
launcher="tester",
|
||||||
|
opencloud_space_id="space-1",
|
||||||
|
opencloud_access_token="token-123",
|
||||||
)
|
)
|
||||||
|
|
||||||
assert result.status == "success"
|
assert result.status == "success"
|
||||||
|
|||||||
128
tests/smoke/test_local_zip_batch_smoke.py
Normal file
128
tests/smoke/test_local_zip_batch_smoke.py
Normal file
@ -0,0 +1,128 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import zipfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from app.pipeline import service
|
||||||
|
from app.pipeline.service import run_local_zip_batch
|
||||||
|
|
||||||
|
|
||||||
|
def _valid_xml() -> str:
|
||||||
|
return """<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<СообщОперКО xmlns="urn:test">
|
||||||
|
<СлужЧасть><ИдФайл>ID1</ИдФайл></СлужЧасть>
|
||||||
|
<ИнформЧасть>
|
||||||
|
<СведКО>
|
||||||
|
<Операция>
|
||||||
|
<ИдентификаторЗаписи>OP1</ИдентификаторЗаписи>
|
||||||
|
<УчастникОп><Тип>01</Тип></УчастникОп>
|
||||||
|
</Операция>
|
||||||
|
</СведКО>
|
||||||
|
</ИнформЧасть>
|
||||||
|
</СообщОперКО>
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def _fake_opencloud_upload(
|
||||||
|
local_path: Path, space_id, folder_segments, file_name, access_token
|
||||||
|
) -> str:
|
||||||
|
assert local_path.exists()
|
||||||
|
return f"https://opencloud.test/{'/'.join(folder_segments)}/{file_name}"
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_local_zip_batch_success(tmp_path: Path, monkeypatch) -> None:
|
||||||
|
monkeypatch.setattr(
|
||||||
|
service, "upload_local_file_to_opencloud", _fake_opencloud_upload
|
||||||
|
)
|
||||||
|
|
||||||
|
zip_path = tmp_path / "batch.zip"
|
||||||
|
with zipfile.ZipFile(zip_path, "w") as archive:
|
||||||
|
archive.writestr("SKO115FZ_01_123456789_20260616_X00001.xml", _valid_xml())
|
||||||
|
|
||||||
|
result = run_local_zip_batch(
|
||||||
|
zip_path=zip_path,
|
||||||
|
launcher="tester",
|
||||||
|
opencloud_space_id="space-1",
|
||||||
|
opencloud_access_token="token-123",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.status == "success"
|
||||||
|
assert result.processed == 1
|
||||||
|
assert result.errors == 0
|
||||||
|
assert result.report_path
|
||||||
|
assert result.protocol_path
|
||||||
|
assert result.log_path
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_local_zip_batch_bad_file_name_is_error(
|
||||||
|
tmp_path: Path, monkeypatch
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setattr(
|
||||||
|
service, "upload_local_file_to_opencloud", _fake_opencloud_upload
|
||||||
|
)
|
||||||
|
|
||||||
|
zip_path = tmp_path / "batch.zip"
|
||||||
|
with zipfile.ZipFile(zip_path, "w") as archive:
|
||||||
|
archive.writestr("bad_name.xml", _valid_xml())
|
||||||
|
|
||||||
|
result = run_local_zip_batch(
|
||||||
|
zip_path=zip_path,
|
||||||
|
launcher="tester",
|
||||||
|
opencloud_space_id="space-1",
|
||||||
|
opencloud_access_token="token-123",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.status == "error"
|
||||||
|
assert result.processed == 0
|
||||||
|
assert result.errors == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_local_zip_batch_empty_archive(tmp_path: Path, monkeypatch) -> None:
|
||||||
|
monkeypatch.setattr(
|
||||||
|
service, "upload_local_file_to_opencloud", _fake_opencloud_upload
|
||||||
|
)
|
||||||
|
|
||||||
|
zip_path = tmp_path / "empty.zip"
|
||||||
|
with zipfile.ZipFile(zip_path, "w") as archive:
|
||||||
|
archive.writestr("readme.txt", "no xml here")
|
||||||
|
|
||||||
|
result = run_local_zip_batch(
|
||||||
|
zip_path=zip_path,
|
||||||
|
launcher="tester",
|
||||||
|
opencloud_space_id="space-1",
|
||||||
|
opencloud_access_token="token-123",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.status == "success"
|
||||||
|
assert result.processed == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_local_zip_batch_missing_token(tmp_path: Path) -> None:
|
||||||
|
zip_path = tmp_path / "batch.zip"
|
||||||
|
with zipfile.ZipFile(zip_path, "w") as archive:
|
||||||
|
archive.writestr("SKO115FZ_01_123456789_20260616_X00001.xml", _valid_xml())
|
||||||
|
|
||||||
|
result = run_local_zip_batch(
|
||||||
|
zip_path=zip_path,
|
||||||
|
launcher="tester",
|
||||||
|
opencloud_space_id="space-1",
|
||||||
|
opencloud_access_token="",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.status == "error"
|
||||||
|
assert not result.report_path
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_local_zip_batch_bad_zip_file(tmp_path: Path) -> None:
|
||||||
|
zip_path = tmp_path / "not_a_zip.zip"
|
||||||
|
zip_path.write_bytes(b"this is not a zip file")
|
||||||
|
|
||||||
|
result = run_local_zip_batch(
|
||||||
|
zip_path=zip_path,
|
||||||
|
launcher="tester",
|
||||||
|
opencloud_space_id="space-1",
|
||||||
|
opencloud_access_token="token-123",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.status == "error"
|
||||||
|
assert any("распаковать" in message for message in result.messages)
|
||||||
158
tests/unit/test_opencloud_access.py
Normal file
158
tests/unit/test_opencloud_access.py
Normal file
@ -0,0 +1,158 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import Mock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import requests
|
||||||
|
|
||||||
|
from app.opencloud_access import (
|
||||||
|
OpenCloudDriveResolutionError,
|
||||||
|
OpenCloudUploadError,
|
||||||
|
build_opencloud_url,
|
||||||
|
resolve_personal_drive,
|
||||||
|
upload_local_file_to_opencloud,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_opencloud_url_encodes_cyrillic_and_spaces() -> None:
|
||||||
|
url = build_opencloud_url(
|
||||||
|
space_id="storage123$space456",
|
||||||
|
folder_segments=("Райков М.С.",),
|
||||||
|
file_name="report 2026.xlsx",
|
||||||
|
)
|
||||||
|
assert url == (
|
||||||
|
"https://docs.raisa.go.rshbank.ru/dav/spaces/storage123$space456/"
|
||||||
|
"%D0%A0%D0%B0%D0%B9%D0%BA%D0%BE%D0%B2%20%D0%9C.%D0%A1./"
|
||||||
|
"report%202026.xlsx"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_opencloud_url_without_folder() -> None:
|
||||||
|
url = build_opencloud_url(
|
||||||
|
space_id="abc", folder_segments=(), file_name="report.xlsx"
|
||||||
|
)
|
||||||
|
assert url == "https://docs.raisa.go.rshbank.ru/dav/spaces/abc/report.xlsx"
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_opencloud_url_requires_space_id() -> None:
|
||||||
|
with pytest.raises(OpenCloudUploadError):
|
||||||
|
build_opencloud_url(space_id="", folder_segments=(), file_name="report.xlsx")
|
||||||
|
|
||||||
|
|
||||||
|
def test_upload_local_file_to_opencloud_success(tmp_path: Path) -> None:
|
||||||
|
local_file = tmp_path / "report.xlsx"
|
||||||
|
local_file.write_bytes(b"fake-xlsx-bytes")
|
||||||
|
|
||||||
|
fake_response = Mock(status_code=201, text="")
|
||||||
|
with patch("app.opencloud_access.requests.put", return_value=fake_response) as put:
|
||||||
|
url = upload_local_file_to_opencloud(
|
||||||
|
local_file,
|
||||||
|
space_id="abc$def",
|
||||||
|
folder_segments=("Райков М.С.",),
|
||||||
|
file_name="report.xlsx",
|
||||||
|
access_token="token-123",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert url.endswith("/report.xlsx")
|
||||||
|
_, kwargs = put.call_args
|
||||||
|
assert kwargs["headers"]["Authorization"] == "Bearer token-123"
|
||||||
|
|
||||||
|
|
||||||
|
def test_upload_local_file_to_opencloud_missing_token(tmp_path: Path) -> None:
|
||||||
|
local_file = tmp_path / "report.xlsx"
|
||||||
|
local_file.write_bytes(b"data")
|
||||||
|
with pytest.raises(OpenCloudUploadError):
|
||||||
|
upload_local_file_to_opencloud(
|
||||||
|
local_file,
|
||||||
|
space_id="abc",
|
||||||
|
folder_segments=(),
|
||||||
|
file_name="report.xlsx",
|
||||||
|
access_token="",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_upload_local_file_to_opencloud_missing_local_file(tmp_path: Path) -> None:
|
||||||
|
missing_file = tmp_path / "does-not-exist.xlsx"
|
||||||
|
with pytest.raises(OpenCloudUploadError):
|
||||||
|
upload_local_file_to_opencloud(
|
||||||
|
missing_file,
|
||||||
|
space_id="abc",
|
||||||
|
folder_segments=(),
|
||||||
|
file_name="report.xlsx",
|
||||||
|
access_token="token-123",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_upload_local_file_to_opencloud_http_error(tmp_path: Path) -> None:
|
||||||
|
local_file = tmp_path / "report.xlsx"
|
||||||
|
local_file.write_bytes(b"data")
|
||||||
|
|
||||||
|
fake_response = Mock(status_code=401, text="Unauthorized")
|
||||||
|
with patch("app.opencloud_access.requests.put", return_value=fake_response):
|
||||||
|
with pytest.raises(OpenCloudUploadError):
|
||||||
|
upload_local_file_to_opencloud(
|
||||||
|
local_file,
|
||||||
|
space_id="abc",
|
||||||
|
folder_segments=(),
|
||||||
|
file_name="report.xlsx",
|
||||||
|
access_token="token-123",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_personal_drive_success() -> None:
|
||||||
|
fake_response = Mock(
|
||||||
|
status_code=200,
|
||||||
|
json=lambda: {
|
||||||
|
"value": [
|
||||||
|
{"driveType": "project", "id": "irrelevant$irrelevant"},
|
||||||
|
{
|
||||||
|
"driveType": "personal",
|
||||||
|
"id": "storage123$space456",
|
||||||
|
"root": {
|
||||||
|
"webDavUrl": "https://example.test/dav/spaces/storage123$space456"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]
|
||||||
|
},
|
||||||
|
)
|
||||||
|
with patch("app.opencloud_access.requests.get", return_value=fake_response) as get:
|
||||||
|
drive = resolve_personal_drive("token-123")
|
||||||
|
|
||||||
|
assert drive == {
|
||||||
|
"id": "storage123$space456",
|
||||||
|
"web_dav_url": "https://example.test/dav/spaces/storage123$space456",
|
||||||
|
}
|
||||||
|
_, kwargs = get.call_args
|
||||||
|
assert kwargs["headers"]["Authorization"] == "Bearer token-123"
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_personal_drive_missing_token() -> None:
|
||||||
|
with pytest.raises(OpenCloudDriveResolutionError):
|
||||||
|
resolve_personal_drive("")
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_personal_drive_unauthorized() -> None:
|
||||||
|
fake_response = Mock(status_code=401, text="Unauthorized")
|
||||||
|
with patch("app.opencloud_access.requests.get", return_value=fake_response):
|
||||||
|
with pytest.raises(OpenCloudDriveResolutionError):
|
||||||
|
resolve_personal_drive("token-123")
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_personal_drive_no_personal_drive_found() -> None:
|
||||||
|
fake_response = Mock(
|
||||||
|
status_code=200,
|
||||||
|
json=lambda: {"value": [{"driveType": "project", "id": "abc$def"}]},
|
||||||
|
)
|
||||||
|
with patch("app.opencloud_access.requests.get", return_value=fake_response):
|
||||||
|
with pytest.raises(OpenCloudDriveResolutionError):
|
||||||
|
resolve_personal_drive("token-123")
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_personal_drive_network_error() -> None:
|
||||||
|
with patch(
|
||||||
|
"app.opencloud_access.requests.get",
|
||||||
|
side_effect=requests.RequestException("boom"),
|
||||||
|
):
|
||||||
|
with pytest.raises(OpenCloudDriveResolutionError):
|
||||||
|
resolve_personal_drive("token-123")
|
||||||
Loading…
x
Reference in New Issue
Block a user