Raw version migrate smb to openclaude

This commit is contained in:
Raykov-MS 2026-09-14 17:25:05 +03:00
parent 44f23947a3
commit 79a35edcf4
10 changed files with 1011 additions and 32 deletions

View File

@ -1,5 +1,6 @@
from __future__ import annotations from __future__ import annotations
import os
from pathlib import Path from pathlib import Path
from tempfile import gettempdir from tempfile import gettempdir
@ -9,6 +10,18 @@ SFM_SMB_BASE_PATH = "//sgo-fc01-r06/sfm/ОТЧЕТ_7081У"
SFM_DEFAULT_INPUT_DIR = r"//sgo-fc01-r06/sfm/ОТЧЕТ_7081У" SFM_DEFAULT_INPUT_DIR = r"//sgo-fc01-r06/sfm/ОТЧЕТ_7081У"
ACCESS_SUPPORT_CONTACT = "администратору СФМ" ACCESS_SUPPORT_CONTACT = "администратору СФМ"
# TODO(BIQ-36593): подтвердить SPACE_ID для прода (сейчас — значение,
# проверенное вручную на тестовом пространстве "AURORA"). ID составной
# ("storageId$spaceId") — храним как есть, без разбора, через
# env-переменную, чтобы не хардкодить прод-значение в коде. В проде
# SPACE_ID личного пространства пользователя определяется динамически
# через opencloud_access.resolve_personal_drive(access_token), это
# значение используется только как тестовый дефолт (AURORA).
OPENCLOUD_SPACE_ID = os.getenv(
"SFM_OPENCLOUD_SPACE_ID",
"28c0cecf-1e20-4664-b165-5c9d5515e70f$95e71bcd-3c89-4b77-b271-63c6d553f4fd",
)
EDIT_PATH_OPEN_KEY = "edit_path_open" EDIT_PATH_OPEN_KEY = "edit_path_open"
EDIT_PATH_VALUE_KEY = "edit_path_value" EDIT_PATH_VALUE_KEY = "edit_path_value"
# Runtime-only user paths storage. In bank contour we use tmpfs. # Runtime-only user paths storage. In bank contour we use tmpfs.
@ -25,3 +38,23 @@ def resolve_user_id() -> str:
) )
full_name = str(profile.get("name") or profile.get("full_name") or "anonymous") full_name = str(profile.get("name") or profile.get("full_name") or "anonymous")
return f"{email}|{full_name}" return f"{email}|{full_name}"
def resolve_opencloud_access_token() -> str | None:
"""Достаёт access_token для OpenCloud из token_data в сессии.
НЕ ПОДТВЕРЖДЕНО: в проде (raisa_streamlit_oauth) неизвестно, содержит ли
token_data ключ "access_token" — в локальном mock_provider.py его нет
(только email/preferred_username/name). Нужно проверить на проде через
st.write(token_data.keys()).
Для локального теста (LOCATION=local, mock-провайдер) — fallback на
переменную окружения SFM_OPENCLOUD_ACCESS_TOKEN из .env. В проде эта
переменная не задана, так что fallback не сработает и не помешает.
"""
token_data = st.session_state.get("token_data", {})
if isinstance(token_data, dict):
value = token_data.get("access_token")
if value:
return str(value)
return os.getenv("SFM_OPENCLOUD_ACCESS_TOKEN") or None

173
app/opencloud_access.py Normal file
View File

@ -0,0 +1,173 @@
from __future__ import annotations
from pathlib import Path
from urllib.parse import quote
import requests
OPENCLOUD_DOMAIN = "docs.raisa.go.rshbank.ru"
OPENCLOUD_UPLOAD_TIMEOUT_SECONDS = 30
OPENCLOUD_GRAPH_TIMEOUT_SECONDS = 30
# Content-Type подобран по расширению — сервер OpenCloud (как и обычный WebDAV)
# не требует конкретного значения для PUT, но так удобнее просматривать файлы
# в интерфейсе.
_CONTENT_TYPES = {
".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
".xlsm": "application/vnd.ms-excel.sheet.macroEnabled.12",
".log": "text/plain; charset=utf-8",
}
_DEFAULT_CONTENT_TYPE = "application/octet-stream"
class OpenCloudUploadError(RuntimeError):
"""Ошибка загрузки файла в OpenCloud по WebDAV."""
class OpenCloudDriveResolutionError(RuntimeError):
"""Ошибка получения личного пространства (drive) пользователя в OpenCloud."""
def resolve_personal_drive(access_token: str) -> dict:
"""Находит личное пространство пользователя через Graph API OpenCloud.
Личное пространство в OpenCloud — не подпапка внутри общего пространства,
а отдельный drive (driveType == "personal") со своим составным id вида
"{storageId}${spaceId}". Эта функция делает GET /graph/v1.0/me/drives,
выбирает запись с driveType == "personal" и возвращает словарь с ключами
"id" (SPACE_ID для загрузки) и "web_dav_url" (готовый базовый WebDAV URL
из root.webDavUrl, на случай если он понадобится вместо ручной сборки).
"""
if not access_token:
raise OpenCloudDriveResolutionError(
"Отсутствует access_token для запроса личного пространства OpenCloud."
)
url = f"https://{OPENCLOUD_DOMAIN}/graph/v1.0/me/drives"
headers = {"Authorization": f"Bearer {access_token}"}
try:
response = requests.get(
url, headers=headers, timeout=OPENCLOUD_GRAPH_TIMEOUT_SECONDS
)
except requests.RequestException as exc:
raise OpenCloudDriveResolutionError(
f"Ошибка сети при запросе списка пространств OpenCloud: {exc}"
) from exc
if response.status_code == 401:
raise OpenCloudDriveResolutionError(
"OpenCloud отклонил access_token (HTTP 401) при запросе "
"списка пространств."
)
if response.status_code != 200:
raise OpenCloudDriveResolutionError(
"Не удалось получить список пространств OpenCloud: "
f"HTTP {response.status_code}, body={response.text[:500]!r}"
)
try:
drives = response.json().get("value", [])
except ValueError as exc:
raise OpenCloudDriveResolutionError(
f"Некорректный JSON-ответ при запросе списка пространств OpenCloud: {exc}"
) from exc
for drive in drives:
if not isinstance(drive, dict) or drive.get("driveType") != "personal":
continue
drive_id = drive.get("id")
if not drive_id:
continue
web_dav_url = (drive.get("root") or {}).get("webDavUrl")
return {"id": drive_id, "web_dav_url": web_dav_url}
raise OpenCloudDriveResolutionError(
'Личное пространство (driveType="personal") не найдено среди '
"пространств пользователя в OpenCloud."
)
def build_opencloud_url(
space_id: str, folder_segments: tuple[str, ...] | list[str], file_name: str
) -> str:
"""Собирает URL вида /dav/spaces/{SPACE_ID}/{FOLDER}/{FILENAME}.
Каждый сегмент папки и имя файла кодируются ОТДЕЛЬНО через urllib.parse.quote —
requests не кодирует кириллицу/пробелы в уже собранном URL так, как нужно
серверу (проверено вручную, см. open_cloud_upload_fixed.py).
space_id передаётся как есть — по факту наблюдений это составной идентификатор
"storageId$spaceId" (обе части через "$"), но эта функция его не разбирает
и не проверяет формат, чтобы не завязываться на недокументированное поведение.
"""
if not space_id:
raise OpenCloudUploadError("Не задан SPACE_ID для OpenCloud.")
if not file_name:
raise OpenCloudUploadError("Не задано имя файла для загрузки в OpenCloud.")
encoded_folder = "/".join(
quote(segment, safe="") for segment in folder_segments if segment
)
encoded_file_name = quote(file_name, safe="")
path = f"/dav/spaces/{space_id}"
if encoded_folder:
path += f"/{encoded_folder}"
path += f"/{encoded_file_name}"
return f"https://{OPENCLOUD_DOMAIN}{path}"
def upload_local_file_to_opencloud(
local_path: Path,
space_id: str,
folder_segments: tuple[str, ...] | list[str],
file_name: str,
access_token: str,
) -> str:
"""Загружает локальный файл в OpenCloud через WebDAV PUT (просто attach).
Возвращает URL, по которому файл загружен (для отображения пользователю /
записи в протокол).
ВРЕМЕННО (тестовый прогон): папку НЕ создаём (без MKCOL) — просто
кладём файл прямо в личную папку участника (folder_segments — как
правило один сегмент, например "Райков М.С."). Структуру Out/год/
месяц/день/номер_запуска вернём отдельным шагом после того, как
подтвердим на тесте базовый сценарий "файл долетает в нужную папку".
"""
if not access_token:
raise OpenCloudUploadError(
"Отсутствует access_token для загрузки в OpenCloud. "
"Проверьте наличие ключа 'access_token' в st.session_state['token_data']."
)
if not local_path.exists():
raise OpenCloudUploadError(f"Локальный файл не найден: {local_path}")
url = build_opencloud_url(space_id, folder_segments, file_name)
content_type = _CONTENT_TYPES.get(local_path.suffix.lower(), _DEFAULT_CONTENT_TYPE)
headers = {
"Authorization": f"Bearer {access_token}",
"Content-Type": content_type,
}
try:
with local_path.open("rb") as file_data:
response = requests.put(
url,
data=file_data,
headers=headers,
timeout=OPENCLOUD_UPLOAD_TIMEOUT_SECONDS,
)
except requests.RequestException as exc:
raise OpenCloudUploadError(
f"Ошибка сети при загрузке в OpenCloud: url='{url}', {exc}"
) from exc
if response.status_code not in (200, 201, 204):
raise OpenCloudUploadError(
"Загрузка в OpenCloud не удалась: "
f"HTTP {response.status_code}, url='{url}', "
f"body={response.text[:500]!r}"
)
return url

View File

@ -2,15 +2,17 @@ from __future__ import annotations
import tempfile import tempfile
import time import time
import zipfile
from dataclasses import dataclass, field from dataclasses import dataclass, field
from datetime import datetime from datetime import datetime
from pathlib import Path from pathlib import Path
from ..opencloud_access import OpenCloudUploadError, upload_local_file_to_opencloud
from .config import ( from .config import (
build_date_parts,
build_process_log_file_name, build_process_log_file_name,
build_protocol_file_name, build_protocol_file_name,
build_report_file_name, build_report_file_name,
join_path,
) )
from .file_manager import ( from .file_manager import (
ProcessedCopySucceededDeleteFailedError, ProcessedCopySucceededDeleteFailedError,
@ -23,7 +25,6 @@ from .file_manager import (
move_file_to_processed, move_file_to_processed,
read_remote_file_bytes, read_remote_file_bytes,
scan_input_files, scan_input_files,
upload_local_file,
) )
from .parser import FileParseResult, parse_xml_content from .parser import FileParseResult, parse_xml_content
from .protocol import ProtocolRow, write_protocol from .protocol import ProtocolRow, write_protocol
@ -51,6 +52,8 @@ def run_batch(
samba_user: str, samba_user: str,
samba_password: str, samba_password: str,
launcher: str, launcher: str,
opencloud_space_id: str | None,
opencloud_access_token: str | None,
artifacts_dir: str | None = None, artifacts_dir: str | None = None,
) -> BatchResult: ) -> BatchResult:
started_at = time.perf_counter() started_at = time.perf_counter()
@ -66,6 +69,33 @@ def run_batch(
log_path="", log_path="",
messages=["Введите логин SAMBA."], messages=["Введите логин SAMBA."],
) )
if not opencloud_access_token:
return BatchResult(
status="error",
processed=0,
partial=0,
errors=1,
report_path="",
protocol_path="",
log_path="",
messages=[
"Не удалось получить access_token для OpenCloud "
"(проверьте авторизацию)."
],
)
if not opencloud_space_id:
return BatchResult(
status="error",
processed=0,
partial=0,
errors=1,
report_path="",
protocol_path="",
log_path="",
messages=[
"Не удалось определить личное пространство пользователя в OpenCloud."
],
)
samba_conn = connect_samba( samba_conn = connect_samba(
smb_base_path=smb_base_path, smb_base_path=smb_base_path,
@ -157,9 +187,6 @@ def run_batch(
run_context.date_key, run_context.run_number run_context.date_key, run_context.run_number
) )
report_path = join_path(run_context.run_dir, report_file_name)
protocol_path = join_path(run_context.run_dir, protocol_file_name)
log_path = join_path(run_context.run_dir, log_file_name)
with tempfile.TemporaryDirectory() as temp_dir: with tempfile.TemporaryDirectory() as temp_dir:
temp_dir_path = Path(temp_dir) temp_dir_path = Path(temp_dir)
local_report = temp_dir_path / "report.xlsx" local_report = temp_dir_path / "report.xlsx"
@ -210,15 +237,32 @@ def run_batch(
# Use CRLF to keep one-record-per-line in Windows viewers. # Use CRLF to keep one-record-per-line in Windows viewers.
local_log.write_text("\r\n".join(log_lines) + "\r\n", encoding="utf-8") local_log.write_text("\r\n".join(log_lines) + "\r\n", encoding="utf-8")
_upload_artifacts( try:
local_report=local_report, report_path, protocol_path, log_path = _upload_artifacts(
local_protocol=local_protocol, local_report=local_report,
local_log=local_log, local_protocol=local_protocol,
report_path=report_path, local_log=local_log,
protocol_path=protocol_path, report_file_name=report_file_name,
log_path=log_path, protocol_file_name=protocol_file_name,
samba_conn=samba_conn, log_file_name=log_file_name,
) opencloud_space_id=opencloud_space_id,
opencloud_access_token=opencloud_access_token,
)
except OpenCloudUploadError as exc:
return BatchResult(
status="error",
processed=0,
partial=0,
errors=1,
report_path="",
protocol_path="",
log_path="",
messages=[
*messages,
f"Файлы обработаны, но не удалось загрузить результат в "
f"OpenCloud: {exc}",
],
)
processed = sum(1 for row in protocol_rows if row.status == "success") processed = sum(1 for row in protocol_rows if row.status == "success")
partial = sum(1 for row in protocol_rows if row.status == "partial") partial = sum(1 for row in protocol_rows if row.status == "partial")
@ -252,6 +296,210 @@ def run_batch(
) )
def run_local_zip_batch(
zip_path: Path,
launcher: str,
opencloud_space_id: str | None,
opencloud_access_token: str | None,
) -> BatchResult:
"""Локальный тестовый прогон: zip с XML -> отчёт/протокол/лог -> OpenCloud.
В отличие от run_batch, НЕ использует SMB вообще — ни для чтения входных
файлов, ни для перемещения в Processed/Err. Предназначена для ручного
теста всей цепочки (парсинг + загрузка в облако) без сетевой шары.
Файлы с некорректным именем/битым XML просто попадают в протокол со
статусом "error" — никуда не перемещаются (перемещать их и некуда,
это временная распакованная копия).
"""
started_at = time.perf_counter()
if not opencloud_access_token:
return BatchResult(
status="error",
processed=0,
partial=0,
errors=1,
report_path="",
protocol_path="",
log_path="",
messages=[
"Не удалось получить access_token для OpenCloud "
"(проверьте авторизацию)."
],
)
if not opencloud_space_id:
return BatchResult(
status="error",
processed=0,
partial=0,
errors=1,
report_path="",
protocol_path="",
log_path="",
messages=[
"Не удалось определить личное пространство пользователя в OpenCloud."
],
)
protocol_rows: list[ProtocolRow] = []
log_lines: list[str] = [f"{_now()} | local zip batch start: {zip_path.name}"]
with tempfile.TemporaryDirectory() as extract_dir:
extract_dir_path = Path(extract_dir)
try:
with zipfile.ZipFile(zip_path) as archive:
archive.extractall(extract_dir_path)
except zipfile.BadZipFile as exc:
return BatchResult(
status="error",
processed=0,
partial=0,
errors=1,
report_path="",
protocol_path="",
log_path="",
messages=[f"Не удалось распаковать архив: {exc}"],
)
xml_paths = sorted(extract_dir_path.rglob("*.xml"))
log_lines.append(
f"{_now()} | распаковано, найдено XML-файлов: {len(xml_paths)}"
)
report_writer = StreamingReportWriter()
for xml_path in xml_paths:
file_name = xml_path.name
try:
content = xml_path.read_bytes()
except Exception as exc: # noqa: BLE001
protocol_rows.append(
ProtocolRow(
file_name=file_name,
status="error",
message=f"Ошибка чтения файла из архива: {exc}",
operations_total=0,
rows_written=0,
launcher=launcher,
processed_at=datetime.now(),
)
)
log_lines.append(f"{_now()} | read failed: {file_name}. {exc}")
continue
# parse_xml_content сам проверяет имя файла и проставляет
# fatal_error, если оно не соответствует шаблону — отдельная
# валидация здесь не нужна.
parse_result = parse_xml_content(file_name=file_name, xml_content=content)
for row in parse_result.rows:
report_writer.append_row(
ReportRow(
file_name=row.file_name,
record_id=row.record_id,
operation_index=row.operation_index,
operation_fields=row.operation_fields,
participant_fields=row.participant_fields,
is_continuation=row.is_continuation,
)
)
status, message = _resolve_parse_status(parse_result)
protocol_rows.append(
ProtocolRow(
file_name=file_name,
status=status,
message=message,
operations_total=parse_result.total_operations,
rows_written=len(parse_result.rows),
launcher=launcher,
processed_at=datetime.now(),
)
)
log_lines.append(f"{_now()} | parsed {file_name}: {status}")
if not protocol_rows:
protocol_rows.append(
ProtocolRow(
file_name="-",
status=EMPTY_BATCH_STATUS,
message="В архиве не найдено XML-файлов.",
operations_total=0,
rows_written=0,
launcher=launcher,
processed_at=datetime.now(),
)
)
log_lines.append(f"{_now()} | empty batch")
_, _, _, date_key = build_date_parts()
run_number = 1
report_file_name = build_report_file_name(date_key, run_number)
protocol_file_name = build_protocol_file_name(date_key, run_number)
log_file_name = build_process_log_file_name(date_key, run_number)
with tempfile.TemporaryDirectory() as artifacts_dir:
artifacts_dir_path = Path(artifacts_dir)
local_report = artifacts_dir_path / "report.xlsx"
local_protocol = artifacts_dir_path / "protocol.xlsx"
local_log = artifacts_dir_path / "process.log"
report_writer.save(local_report)
write_protocol(protocol_rows, local_protocol)
local_log.write_text("\r\n".join(log_lines) + "\r\n", encoding="utf-8")
try:
report_path, protocol_path, log_path = _upload_artifacts(
local_report=local_report,
local_protocol=local_protocol,
local_log=local_log,
report_file_name=report_file_name,
protocol_file_name=protocol_file_name,
log_file_name=log_file_name,
opencloud_space_id=opencloud_space_id,
opencloud_access_token=opencloud_access_token,
)
except OpenCloudUploadError as exc:
return BatchResult(
status="error",
processed=0,
partial=0,
errors=1,
report_path="",
protocol_path="",
log_path="",
messages=[
*log_lines,
f"Файлы обработаны, но не удалось загрузить результат "
f"в OpenCloud: {exc}",
],
)
processed = sum(1 for row in protocol_rows if row.status == "success")
partial = sum(1 for row in protocol_rows if row.status == "partial")
errors = sum(1 for row in protocol_rows if row.status == "error")
status = (
"success"
if errors == 0 and partial == 0
else ("partial" if partial > 0 else "error")
)
messages = [
_build_result_message(
status=status, processed=processed, partial=partial, errors=errors
)
]
duration_seconds = round(time.perf_counter() - started_at, 2)
messages.append(f"Время выполнения: {_format_duration(duration_seconds)}.")
messages.extend(log_lines[-5:])
return BatchResult(
status=status,
processed=processed,
partial=partial,
errors=errors,
report_path=report_path,
protocol_path=protocol_path,
log_path=log_path,
messages=messages,
duration_seconds=duration_seconds,
)
def _parse_remote_xml( def _parse_remote_xml(
file_name: str, remote_path: str, samba_conn, stage_timings: dict[str, float] file_name: str, remote_path: str, samba_conn, stage_timings: dict[str, float]
) -> FileParseResult: ) -> FileParseResult:
@ -414,14 +662,40 @@ def _upload_artifacts(
local_report: Path, local_report: Path,
local_protocol: Path, local_protocol: Path,
local_log: Path, local_log: Path,
report_path: str, report_file_name: str,
protocol_path: str, protocol_file_name: str,
log_path: str, log_file_name: str,
samba_conn, opencloud_space_id: str,
) -> None: opencloud_access_token: str,
upload_local_file(local_report, report_path, samba_conn=samba_conn) ) -> tuple[str, str, str]:
upload_local_file(local_protocol, protocol_path, samba_conn=samba_conn) # ВРЕМЕННО (тестовый прогон): грузим плоско, прямо в корень личного
upload_local_file(local_log, log_path, samba_conn=samba_conn) # пространства пользователя (найденного через resolve_personal_drive),
# без MKCOL и без структуры Out/год/месяц/день/номер запуска. Сначала
# проверяем на тесте, что файл вообще долетает — структуру подпапок
# вернём отдельным шагом.
folder_segments: tuple[str, ...] = ()
report_url = upload_local_file_to_opencloud(
local_report,
space_id=opencloud_space_id,
folder_segments=folder_segments,
file_name=report_file_name,
access_token=opencloud_access_token,
)
protocol_url = upload_local_file_to_opencloud(
local_protocol,
space_id=opencloud_space_id,
folder_segments=folder_segments,
file_name=protocol_file_name,
access_token=opencloud_access_token,
)
log_url = upload_local_file_to_opencloud(
local_log,
space_id=opencloud_space_id,
folder_segments=folder_segments,
file_name=log_file_name,
access_token=opencloud_access_token,
)
return report_url, protocol_url, log_url
def _now() -> str: def _now() -> str:

48
main.py
View File

@ -1,8 +1,14 @@
import streamlit as st import streamlit as st
from dotenv import load_dotenv from dotenv import load_dotenv
from app.app_config import ACCESS_SUPPORT_CONTACT, SFM_SMB_BASE_PATH, resolve_user_id from app.app_config import (
ACCESS_SUPPORT_CONTACT,
SFM_SMB_BASE_PATH,
resolve_opencloud_access_token,
resolve_user_id,
)
from app.auth.service import AuthService, clear_session from app.auth.service import AuthService, clear_session
from app.opencloud_access import OpenCloudDriveResolutionError, resolve_personal_drive
from app.path_settings import ( from app.path_settings import (
close_path_editor, close_path_editor,
ensure_user_path_state, ensure_user_path_state,
@ -12,7 +18,7 @@ from app.path_settings import (
) )
from app.path_store import set_user_path from app.path_store import set_user_path
from app.pipeline.config import join_path from app.pipeline.config import join_path
from app.pipeline.service import run_batch from app.pipeline.service import BatchResult, run_batch
from app.samba_access import check_access_and_list_files, diagnose_smb_access from app.samba_access import check_access_and_list_files, diagnose_smb_access
@ -73,14 +79,36 @@ def main() -> None:
): ):
with st.spinner("Выполняется обработка..."): with st.spinner("Выполняется обработка..."):
launcher_name = samba_user.strip() or user_id launcher_name = samba_user.strip() or user_id
batch_result = run_batch( access_token = resolve_opencloud_access_token()
smb_base_path=SFM_SMB_BASE_PATH, opencloud_space_id = None
input_dir=processing_input_dir, opencloud_error = None
samba_user=samba_user, if access_token:
samba_password=samba_password, try:
launcher=launcher_name, opencloud_space_id = resolve_personal_drive(access_token)["id"]
artifacts_dir=resolved_input_dir, except OpenCloudDriveResolutionError as exc:
) opencloud_error = str(exc)
if opencloud_error:
batch_result = BatchResult(
status="error",
processed=0,
partial=0,
errors=1,
report_path="",
protocol_path="",
log_path="",
messages=[opencloud_error],
)
else:
batch_result = run_batch(
smb_base_path=SFM_SMB_BASE_PATH,
input_dir=processing_input_dir,
samba_user=samba_user,
samba_password=samba_password,
launcher=launcher_name,
opencloud_space_id=opencloud_space_id,
opencloud_access_token=access_token,
artifacts_dir=resolved_input_dir,
)
if batch_result.status == "success": if batch_result.status == "success":
st.success("Обработка завершена успешно.") st.success("Обработка завершена успешно.")
elif batch_result.status == "partial": elif batch_result.status == "partial":

View File

@ -9,3 +9,4 @@ openpyxl
xlsxwriter==3.2.9 xlsxwriter==3.2.9
pytest pytest
smbprotocol==1.15.0 smbprotocol==1.15.0
requests==2.32.3

View File

@ -0,0 +1,53 @@
"""Локальный ручной тест: чисто attach в OpenCloud, без Streamlit.
Использование:
1. Впишите ниже ACCESS_TOKEN, SPACE_ID (если отличается от дефолта),
FOLDER и путь к тестовому файлу.
2. Запустите: python scripts/local_opencloud_smoke.py
Использует тот же app/opencloud_access.py, что и продакшн-код в
app/pipeline/service.py — так что если тут сработает, значит и в
приложении сработает (сама логика загрузки не отличается).
"""
from __future__ import annotations
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from app.opencloud_access import OpenCloudUploadError, upload_local_file_to_opencloud
# --- Заполните перед запуском ---
ACCESS_TOKEN = "PASTE_TOKEN_HERE"
SPACE_ID = "28c0cecf-1e20-4664-b165-5c9d5515e70f" # тестовое пространство AURORA
FOLDER = "Райков М.С." # ваша личная папка в OpenCloud
LOCAL_FILE = Path(__file__).resolve().parent / "test_upload.xlsx"
REMOTE_FILE_NAME = "test_upload.xlsx"
# ---------------------------------
def main() -> None:
if not LOCAL_FILE.exists():
LOCAL_FILE.write_bytes(b"local opencloud smoke test")
print(f"Тестовый файл создан: {LOCAL_FILE}")
try:
url = upload_local_file_to_opencloud(
LOCAL_FILE,
space_id=SPACE_ID,
folder_segments=(FOLDER,),
file_name=REMOTE_FILE_NAME,
access_token=ACCESS_TOKEN,
)
except OpenCloudUploadError as exc:
print(f"ОШИБКА: {exc}")
raise SystemExit(1) from exc
print(f"OK, загружено: {url}")
print("Проверьте файл в веб-интерфейсе OpenCloud, в папке", FOLDER)
if __name__ == "__main__":
main()

View File

@ -0,0 +1,112 @@
"""Тестовая страница: прикрепляете .zip с XML, он распаковывается,
обрабатывается тем же парсером, что и прод, а готовый отчёт/протокол/лог
грузятся в OpenCloud. Без SMB на входе — специально для локального теста
всей цепочки целиком.
Не трогает main.py и основной SMB-пайплайн.
Запуск:
streamlit run scripts/local_zip_test_app.py
Токен/папку можно один раз прописать в .env (см. .env.example) — поля
на странице предзаполнятся, но их всегда можно поменять руками.
"""
from __future__ import annotations
import os
import sys
import tempfile
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import streamlit as st
from dotenv import load_dotenv
from app.opencloud_access import OpenCloudDriveResolutionError, resolve_personal_drive
from app.pipeline.service import run_local_zip_batch
load_dotenv()
st.set_page_config(page_title="СФМ — локальный тест", page_icon="📦")
st.title("Локальный тест: zip → обработка → OpenCloud")
st.caption(
"Прикрепляете .zip с XML-файлами, он распаковывается и обрабатывается "
"тем же кодом парсинга, что и прод (app/pipeline/parser.py), а отчёт, "
"протокол и лог грузятся в OpenCloud (app/opencloud_access.py). "
"SMB тут не используется вообще."
)
access_token = st.text_input(
"Access token",
value=os.getenv("SFM_OPENCLOUD_ACCESS_TOKEN", ""),
type="password",
help="Bearer-токен из Keycloak. Можно прописать в .env как SFM_OPENCLOUD_ACCESS_TOKEN.",
)
space_id_override = st.text_input(
"SPACE_ID (переопределить вручную, необязательно)",
value=os.getenv("SFM_OPENCLOUD_SPACE_ID", ""),
help=(
"Если оставить пустым — личное пространство определится "
"автоматически по токену через resolve_personal_drive()."
),
)
launcher = st.text_input("Запустил (для протокола)", value="local-test")
uploaded_zip = st.file_uploader("Архив с XML-файлами (.zip)", type=["zip"])
if st.button("Начать обработку", type="primary", use_container_width=True):
if not access_token:
st.error("Укажите access token.")
elif uploaded_zip is None:
st.error("Прикрепите .zip-архив.")
else:
space_id = space_id_override
if not space_id:
try:
drive = resolve_personal_drive(access_token)
except OpenCloudDriveResolutionError as exc:
st.error(f"Не удалось определить личное пространство: {exc}")
space_id = None
else:
space_id = drive["id"]
st.info(
f"Найдено личное пространство: id={drive['id']!r}, "
f"webDavUrl={drive.get('web_dav_url')!r}"
)
if space_id:
with tempfile.TemporaryDirectory() as tmp_dir:
zip_path = Path(tmp_dir) / uploaded_zip.name
zip_path.write_bytes(uploaded_zip.getvalue())
with st.spinner("Распаковываем, обрабатываем, грузим в OpenCloud..."):
result = run_local_zip_batch(
zip_path=zip_path,
launcher=launcher or "local-test",
opencloud_space_id=space_id,
opencloud_access_token=access_token,
)
if result.status == "success":
st.success("Обработка завершена успешно.")
elif result.status == "partial":
st.warning("Обработка завершена частично.")
else:
st.error("Обработка завершена с ошибками.")
st.write(
{
"processed": result.processed,
"partial": result.partial,
"errors": result.errors,
"report_path": result.report_path,
"protocol_path": result.protocol_path,
"log_path": result.log_path,
"duration_seconds": result.duration_seconds,
}
)
if result.messages:
st.caption("Сообщения обработки:")
for message in result.messages:
st.write(f"- {message}")

View File

@ -86,6 +86,13 @@ def _valid_xml() -> str:
""" """
def _fake_opencloud_upload(
local_path, space_id, folder_segments, file_name, access_token
) -> str:
_ = (local_path, space_id, access_token)
return f"https://opencloud.test/{'/'.join(folder_segments)}/{file_name}"
def test_run_batch_smoke_and_idempotency(tmp_path: Path, monkeypatch) -> None: def test_run_batch_smoke_and_idempotency(tmp_path: Path, monkeypatch) -> None:
samba = FakeSamba(tmp_path) samba = FakeSamba(tmp_path)
in_dir_local = tmp_path / "Exchange" / "SMBDEMO" / "test" in_dir_local = tmp_path / "Exchange" / "SMBDEMO" / "test"
@ -100,6 +107,9 @@ def test_run_batch_smoke_and_idempotency(tmp_path: Path, monkeypatch) -> None:
monkeypatch.setattr( monkeypatch.setattr(
service, "connect_samba", lambda smb_base_path, user_params: samba service, "connect_samba", lambda smb_base_path, user_params: samba
) )
monkeypatch.setattr(
service, "upload_local_file_to_opencloud", _fake_opencloud_upload
)
first = run_batch( first = run_batch(
smb_base_path="//test-server/share", smb_base_path="//test-server/share",
@ -107,6 +117,8 @@ def test_run_batch_smoke_and_idempotency(tmp_path: Path, monkeypatch) -> None:
samba_user="user", samba_user="user",
samba_password="pass", samba_password="pass",
launcher="tester", launcher="tester",
opencloud_space_id="space-1",
opencloud_access_token="token-123",
) )
assert first.status in {"success", "partial"} assert first.status in {"success", "partial"}
assert first.protocol_path assert first.protocol_path
@ -118,6 +130,8 @@ def test_run_batch_smoke_and_idempotency(tmp_path: Path, monkeypatch) -> None:
samba_user="user", samba_user="user",
samba_password="pass", samba_password="pass",
launcher="tester", launcher="tester",
opencloud_space_id="space-1",
opencloud_access_token="token-123",
) )
assert second.status == "success" assert second.status == "success"
assert second.processed == 0 assert second.processed == 0
@ -137,6 +151,9 @@ def test_run_batch_success_when_delete_denied_after_copy(
monkeypatch.setattr( monkeypatch.setattr(
service, "connect_samba", lambda smb_base_path, user_params: samba service, "connect_samba", lambda smb_base_path, user_params: samba
) )
monkeypatch.setattr(
service, "upload_local_file_to_opencloud", _fake_opencloud_upload
)
result = run_batch( result = run_batch(
smb_base_path="//test-server/share", smb_base_path="//test-server/share",
@ -144,6 +161,8 @@ def test_run_batch_success_when_delete_denied_after_copy(
samba_user="user", samba_user="user",
samba_password="pass", samba_password="pass",
launcher="tester", launcher="tester",
opencloud_space_id="space-1",
opencloud_access_token="token-123",
) )
assert result.status == "success" assert result.status == "success"

View File

@ -0,0 +1,128 @@
from __future__ import annotations
import zipfile
from pathlib import Path
from app.pipeline import service
from app.pipeline.service import run_local_zip_batch
def _valid_xml() -> str:
return """<?xml version="1.0" encoding="UTF-8"?>
<СообщОперКО xmlns="urn:test">
<СлужЧасть><ИдФайл>ID1</ИдФайл></СлужЧасть>
<ИнформЧасть>
<СведКО>
<Операция>
<ИдентификаторЗаписи>OP1</ИдентификаторЗаписи>
<УчастникОп><Тип>01</Тип></УчастникОп>
</Операция>
</СведКО>
</ИнформЧасть>
</СообщОперКО>
"""
def _fake_opencloud_upload(
local_path: Path, space_id, folder_segments, file_name, access_token
) -> str:
assert local_path.exists()
return f"https://opencloud.test/{'/'.join(folder_segments)}/{file_name}"
def test_run_local_zip_batch_success(tmp_path: Path, monkeypatch) -> None:
monkeypatch.setattr(
service, "upload_local_file_to_opencloud", _fake_opencloud_upload
)
zip_path = tmp_path / "batch.zip"
with zipfile.ZipFile(zip_path, "w") as archive:
archive.writestr("SKO115FZ_01_123456789_20260616_X00001.xml", _valid_xml())
result = run_local_zip_batch(
zip_path=zip_path,
launcher="tester",
opencloud_space_id="space-1",
opencloud_access_token="token-123",
)
assert result.status == "success"
assert result.processed == 1
assert result.errors == 0
assert result.report_path
assert result.protocol_path
assert result.log_path
def test_run_local_zip_batch_bad_file_name_is_error(
tmp_path: Path, monkeypatch
) -> None:
monkeypatch.setattr(
service, "upload_local_file_to_opencloud", _fake_opencloud_upload
)
zip_path = tmp_path / "batch.zip"
with zipfile.ZipFile(zip_path, "w") as archive:
archive.writestr("bad_name.xml", _valid_xml())
result = run_local_zip_batch(
zip_path=zip_path,
launcher="tester",
opencloud_space_id="space-1",
opencloud_access_token="token-123",
)
assert result.status == "error"
assert result.processed == 0
assert result.errors == 1
def test_run_local_zip_batch_empty_archive(tmp_path: Path, monkeypatch) -> None:
monkeypatch.setattr(
service, "upload_local_file_to_opencloud", _fake_opencloud_upload
)
zip_path = tmp_path / "empty.zip"
with zipfile.ZipFile(zip_path, "w") as archive:
archive.writestr("readme.txt", "no xml here")
result = run_local_zip_batch(
zip_path=zip_path,
launcher="tester",
opencloud_space_id="space-1",
opencloud_access_token="token-123",
)
assert result.status == "success"
assert result.processed == 0
def test_run_local_zip_batch_missing_token(tmp_path: Path) -> None:
zip_path = tmp_path / "batch.zip"
with zipfile.ZipFile(zip_path, "w") as archive:
archive.writestr("SKO115FZ_01_123456789_20260616_X00001.xml", _valid_xml())
result = run_local_zip_batch(
zip_path=zip_path,
launcher="tester",
opencloud_space_id="space-1",
opencloud_access_token="",
)
assert result.status == "error"
assert not result.report_path
def test_run_local_zip_batch_bad_zip_file(tmp_path: Path) -> None:
zip_path = tmp_path / "not_a_zip.zip"
zip_path.write_bytes(b"this is not a zip file")
result = run_local_zip_batch(
zip_path=zip_path,
launcher="tester",
opencloud_space_id="space-1",
opencloud_access_token="token-123",
)
assert result.status == "error"
assert any("распаковать" in message for message in result.messages)

View File

@ -0,0 +1,158 @@
from __future__ import annotations
from pathlib import Path
from unittest.mock import Mock, patch
import pytest
import requests
from app.opencloud_access import (
OpenCloudDriveResolutionError,
OpenCloudUploadError,
build_opencloud_url,
resolve_personal_drive,
upload_local_file_to_opencloud,
)
def test_build_opencloud_url_encodes_cyrillic_and_spaces() -> None:
url = build_opencloud_url(
space_id="storage123$space456",
folder_segments=("Райков М.С.",),
file_name="report 2026.xlsx",
)
assert url == (
"https://docs.raisa.go.rshbank.ru/dav/spaces/storage123$space456/"
"%D0%A0%D0%B0%D0%B9%D0%BA%D0%BE%D0%B2%20%D0%9C.%D0%A1./"
"report%202026.xlsx"
)
def test_build_opencloud_url_without_folder() -> None:
url = build_opencloud_url(
space_id="abc", folder_segments=(), file_name="report.xlsx"
)
assert url == "https://docs.raisa.go.rshbank.ru/dav/spaces/abc/report.xlsx"
def test_build_opencloud_url_requires_space_id() -> None:
with pytest.raises(OpenCloudUploadError):
build_opencloud_url(space_id="", folder_segments=(), file_name="report.xlsx")
def test_upload_local_file_to_opencloud_success(tmp_path: Path) -> None:
local_file = tmp_path / "report.xlsx"
local_file.write_bytes(b"fake-xlsx-bytes")
fake_response = Mock(status_code=201, text="")
with patch("app.opencloud_access.requests.put", return_value=fake_response) as put:
url = upload_local_file_to_opencloud(
local_file,
space_id="abc$def",
folder_segments=("Райков М.С.",),
file_name="report.xlsx",
access_token="token-123",
)
assert url.endswith("/report.xlsx")
_, kwargs = put.call_args
assert kwargs["headers"]["Authorization"] == "Bearer token-123"
def test_upload_local_file_to_opencloud_missing_token(tmp_path: Path) -> None:
local_file = tmp_path / "report.xlsx"
local_file.write_bytes(b"data")
with pytest.raises(OpenCloudUploadError):
upload_local_file_to_opencloud(
local_file,
space_id="abc",
folder_segments=(),
file_name="report.xlsx",
access_token="",
)
def test_upload_local_file_to_opencloud_missing_local_file(tmp_path: Path) -> None:
missing_file = tmp_path / "does-not-exist.xlsx"
with pytest.raises(OpenCloudUploadError):
upload_local_file_to_opencloud(
missing_file,
space_id="abc",
folder_segments=(),
file_name="report.xlsx",
access_token="token-123",
)
def test_upload_local_file_to_opencloud_http_error(tmp_path: Path) -> None:
local_file = tmp_path / "report.xlsx"
local_file.write_bytes(b"data")
fake_response = Mock(status_code=401, text="Unauthorized")
with patch("app.opencloud_access.requests.put", return_value=fake_response):
with pytest.raises(OpenCloudUploadError):
upload_local_file_to_opencloud(
local_file,
space_id="abc",
folder_segments=(),
file_name="report.xlsx",
access_token="token-123",
)
def test_resolve_personal_drive_success() -> None:
fake_response = Mock(
status_code=200,
json=lambda: {
"value": [
{"driveType": "project", "id": "irrelevant$irrelevant"},
{
"driveType": "personal",
"id": "storage123$space456",
"root": {
"webDavUrl": "https://example.test/dav/spaces/storage123$space456"
},
},
]
},
)
with patch("app.opencloud_access.requests.get", return_value=fake_response) as get:
drive = resolve_personal_drive("token-123")
assert drive == {
"id": "storage123$space456",
"web_dav_url": "https://example.test/dav/spaces/storage123$space456",
}
_, kwargs = get.call_args
assert kwargs["headers"]["Authorization"] == "Bearer token-123"
def test_resolve_personal_drive_missing_token() -> None:
with pytest.raises(OpenCloudDriveResolutionError):
resolve_personal_drive("")
def test_resolve_personal_drive_unauthorized() -> None:
fake_response = Mock(status_code=401, text="Unauthorized")
with patch("app.opencloud_access.requests.get", return_value=fake_response):
with pytest.raises(OpenCloudDriveResolutionError):
resolve_personal_drive("token-123")
def test_resolve_personal_drive_no_personal_drive_found() -> None:
fake_response = Mock(
status_code=200,
json=lambda: {"value": [{"driveType": "project", "id": "abc$def"}]},
)
with patch("app.opencloud_access.requests.get", return_value=fake_response):
with pytest.raises(OpenCloudDriveResolutionError):
resolve_personal_drive("token-123")
def test_resolve_personal_drive_network_error() -> None:
with patch(
"app.opencloud_access.requests.get",
side_effect=requests.RequestException("boom"),
):
with pytest.raises(OpenCloudDriveResolutionError):
resolve_personal_drive("token-123")