721712d86a
The uploader took a single .zip, so a day split across several archives, or a handful of loose XML files, needed one run each. - main.py and scripts/local_zip_test_app.py now use accept_multiple_files and take .xml alongside .zip and the tar family. Each attachment is written to its own temp subdirectory, because two attachments may share a name. - run_batch takes input_paths instead of zip_path. - New app/pipeline/attachments.py flattens the attachments into a list of XML files. Archive format is detected from the content rather than the suffix, since exports arrive with the wrong extension often enough. .rar and .7z are recognised only to explain why they are refused. - A broken attachment no longer aborts the run. It becomes one protocol row and the remaining attachments are still processed; the reason is also shown in the UI, not only in the protocol file. - Files repeated across attachments are processed once and the repeats are recorded with status "skip". Attaching the same archive twice would otherwise double every row of the report. - Archives are unpacked member by member with the target path checked against the extraction directory. tarfile.data_filter only exists from Python 3.11.4 and extractall would otherwise let a "../" member escape. - The protocol gains a "Вложение" column, so it is clear which attachment a file came from when names repeat across archives. Checked end to end on the 03.02.2023 export: six attachments (two archives, two loose XML files, one broken archive, one duplicate archive) give 57 files processed, 9 repeats skipped, one attachment refused with its reason, and the report is still produced.
109 lines
4.7 KiB
Python
109 lines
4.7 KiB
Python
"""Тестовая страница: та же логика прикрепления .zip с XML, что и в проде
|
||
(main.py), но с ручным вводом username/App Token вместо Keycloak-сессии —
|
||
чтобы гонять всю цепочку (парсинг + загрузка в OpenCloud) локально, без
|
||
поднятого raisa_streamlit_oauth.
|
||
|
||
Запуск:
|
||
streamlit run scripts/local_zip_test_app.py
|
||
|
||
Username/токен можно один раз прописать в .env (см. .env.example) — поля
|
||
на странице предзаполнятся, но их всегда можно поменять руками. Личное
|
||
пространство ищется автоматически через GET /graph/v1.0/me/drives, см.
|
||
app/opencloud_access.py.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
import sys
|
||
import tempfile
|
||
from pathlib import Path
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||
|
||
import streamlit as st
|
||
from dotenv import load_dotenv
|
||
|
||
from app.pipeline.service import run_batch
|
||
|
||
load_dotenv()
|
||
|
||
st.set_page_config(page_title="СФМ — локальный тест", page_icon="📦")
|
||
st.title("Локальный тест: вложения → обработка → OpenCloud")
|
||
st.caption(
|
||
"Прикрепляете XML-файлы и/или архивы с ними, они распаковываются и обрабатываются "
|
||
"тем же кодом парсинга, что и прод (app/pipeline/parser.py), а отчёт, "
|
||
"протокол и лог грузятся в OpenCloud (app/opencloud_access.py). Та же "
|
||
"логика, что и в main.py, только username/token вводятся руками."
|
||
)
|
||
|
||
username = st.text_input(
|
||
"Username (для Basic Auth в OpenCloud)",
|
||
value=os.getenv("SFM_OPENCLOUD_USERNAME", ""),
|
||
help="Логин, под которым заказчик создал App Token в OpenCloud. Можно прописать в .env как SFM_OPENCLOUD_USERNAME.",
|
||
)
|
||
access_token = st.text_input(
|
||
"App Token",
|
||
value=os.getenv("SFM_OPENCLOUD_ACCESS_TOKEN", ""),
|
||
type="password",
|
||
help=(
|
||
"Долгоживущий App Token из веб-интерфейса OpenCloud (не Bearer из Keycloak — "
|
||
"авторизация теперь через HTTP Basic Auth). Можно прописать в .env как "
|
||
"SFM_OPENCLOUD_ACCESS_TOKEN."
|
||
),
|
||
)
|
||
launcher = st.text_input("Запустил (для протокола)", value="local-test")
|
||
|
||
uploaded_files = st.file_uploader(
|
||
"XML-файлы и архивы",
|
||
type=["xml", "zip", "tar", "gz", "tgz", "bz2", "xz"],
|
||
accept_multiple_files=True,
|
||
)
|
||
|
||
if st.button("Начать обработку", type="primary", use_container_width=True):
|
||
if not username:
|
||
st.error("Укажите username.")
|
||
elif not access_token:
|
||
st.error("Укажите App Token.")
|
||
elif not uploaded_files:
|
||
st.error("Прикрепите XML-файлы или архивы.")
|
||
else:
|
||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||
input_paths = []
|
||
for index, uploaded in enumerate(uploaded_files):
|
||
holder = Path(tmp_dir) / str(index)
|
||
holder.mkdir(parents=True, exist_ok=True)
|
||
attachment_path = holder / uploaded.name
|
||
attachment_path.write_bytes(uploaded.getvalue())
|
||
input_paths.append(attachment_path)
|
||
with st.spinner("Распаковываем, обрабатываем, грузим в OpenCloud..."):
|
||
result = run_batch(
|
||
input_paths=input_paths,
|
||
launcher=launcher or "local-test",
|
||
opencloud_username=username,
|
||
opencloud_access_token=access_token,
|
||
)
|
||
|
||
if result.status == "success":
|
||
st.success("Обработка завершена успешно.")
|
||
elif result.status == "partial":
|
||
st.warning("Обработка завершена частично.")
|
||
else:
|
||
st.error("Обработка завершена с ошибками.")
|
||
|
||
st.write(
|
||
{
|
||
"processed": result.processed,
|
||
"partial": result.partial,
|
||
"errors": result.errors,
|
||
"report_path": result.report_path,
|
||
"protocol_path": result.protocol_path,
|
||
"log_path": result.log_path,
|
||
"duration_seconds": result.duration_seconds,
|
||
}
|
||
)
|
||
if result.messages:
|
||
st.caption("Сообщения обработки:")
|
||
for message in result.messages:
|
||
st.write(f"- {message}")
|