export-optimize-refresh: очередь загрузки при экспорте #184
19
api/alembic/versions/0043_summary2_go_export.py
Normal file
19
api/alembic/versions/0043_summary2_go_export.py
Normal file
@ -0,0 +1,19 @@
|
||||
from alembic import op
|
||||
|
||||
from sql_migration import run_sql_file_migration
|
||||
|
||||
revision = "0043"
|
||||
down_revision = "0042"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
run_sql_file_migration("0043_summary2_go_export.sql")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute(
|
||||
"DROP FUNCTION IF EXISTS "
|
||||
"v3.v_svod2_go_export(integer, integer[], varchar, varchar)"
|
||||
)
|
||||
35
api/alembic/versions/0044_export_jobs.py
Normal file
35
api/alembic/versions/0044_export_jobs.py
Normal file
@ -0,0 +1,35 @@
|
||||
from alembic import op
|
||||
|
||||
revision = "0044"
|
||||
down_revision = "0043"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.execute("""
|
||||
CREATE TABLE v3.export_job (
|
||||
id uuid PRIMARY KEY,
|
||||
user_id integer NOT NULL REFERENCES v3.app_user(id),
|
||||
request jsonb NOT NULL,
|
||||
status varchar(20) NOT NULL,
|
||||
created_at timestamptz NOT NULL,
|
||||
started_at timestamptz,
|
||||
finished_at timestamptz,
|
||||
filename text,
|
||||
media_type text,
|
||||
file_path text,
|
||||
error text,
|
||||
CONSTRAINT export_job_status CHECK
|
||||
(status IN ('queued', 'processing', 'completed', 'failed'))
|
||||
)
|
||||
""")
|
||||
op.execute("CREATE INDEX export_job_user ON v3.export_job(user_id, created_at)")
|
||||
op.execute("""
|
||||
CREATE UNIQUE INDEX export_job_active_user ON v3.export_job(user_id)
|
||||
WHERE status IN ('queued', 'processing')
|
||||
""")
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.execute("DROP TABLE v3.export_job")
|
||||
209
api/alembic/versions/sql/0043_summary2_go_export.sql
Normal file
209
api/alembic/versions/sql/0043_summary2_go_export.sql
Normal file
@ -0,0 +1,209 @@
|
||||
DROP FUNCTION IF EXISTS v3.v_svod2_go_export(integer, integer[], varchar, varchar);
|
||||
|
||||
CREATE OR REPLACE FUNCTION v3.v_svod2_go_export(
|
||||
p_year integer,
|
||||
p_org_unit_ids integer[] DEFAULT NULL,
|
||||
p_sort_by varchar DEFAULT 'org_name',
|
||||
p_sort_direction varchar DEFAULT 'asc'
|
||||
)
|
||||
RETURNS TABLE(sort_order bigint, row_values text[])
|
||||
LANGUAGE sql
|
||||
STABLE
|
||||
SET search_path TO 'v3', 'pg_catalog'
|
||||
AS $function$
|
||||
WITH eligible_lines AS (
|
||||
SELECT
|
||||
bl.id AS line_id,
|
||||
bf.form_type_code,
|
||||
bf.id AS form_id,
|
||||
bf.org_unit_id,
|
||||
ou.title AS org_name,
|
||||
ei.section_code,
|
||||
ei.item_id,
|
||||
ei.num_group_id,
|
||||
COALESCE(bl.name, ei.name) AS name,
|
||||
bl.justification,
|
||||
CASE bl.direction
|
||||
WHEN 'Support' THEN 'Поддержка'
|
||||
WHEN 'Development' THEN 'Развитие'
|
||||
END::varchar AS smeta_direction,
|
||||
CASE ei.sheet
|
||||
WHEN 'AHR' THEN 'АХР'
|
||||
WHEN 'CAP' THEN 'КВ'
|
||||
WHEN 'OPER' THEN 'Операц'
|
||||
END::varchar AS smeta_type,
|
||||
cd.counterparty,
|
||||
cd.reference,
|
||||
cd.subject,
|
||||
cd.currency,
|
||||
cd.ceiling_amount,
|
||||
cd.expenses_q1,
|
||||
cd.expenses_q2,
|
||||
cd.expenses_q3,
|
||||
cd.expenses_q4,
|
||||
cd.vat_rate,
|
||||
cd.deadline
|
||||
FROM v3.budget_form bf
|
||||
JOIN v3.budget_line bl ON bl.budget_form_id = bf.id
|
||||
JOIN v3.expense_item ei ON ei.id = bl.expense_item_id
|
||||
JOIN v3.contract_detail cd ON cd.line_id = bl.id
|
||||
LEFT JOIN v3.org_unit ou ON ou.id = bf.org_unit_id
|
||||
WHERE bf.year = p_year
|
||||
AND bf.form_type_code IN ('FORM_1', 'FORM_4')
|
||||
AND ei.sheet IN ('AHR', 'CAP', 'OPER')
|
||||
AND (
|
||||
p_org_unit_ids IS NULL
|
||||
OR cardinality(p_org_unit_ids) = 0
|
||||
OR bf.org_unit_id = ANY(p_org_unit_ids)
|
||||
)
|
||||
), page AS MATERIALIZED (
|
||||
SELECT
|
||||
el.*,
|
||||
row_number() OVER (
|
||||
ORDER BY
|
||||
CASE WHEN p_sort_by = 'org_name' AND lower(p_sort_direction) = 'asc' THEN el.org_name END ASC NULLS LAST,
|
||||
CASE WHEN p_sort_by = 'org_name' AND lower(p_sort_direction) = 'desc' THEN el.org_name END DESC NULLS LAST,
|
||||
CASE WHEN p_sort_by = 'org_name' AND lower(p_sort_direction) = 'asc' THEN el.item_id END ASC NULLS LAST,
|
||||
CASE WHEN p_sort_by = 'org_name' AND lower(p_sort_direction) = 'desc' THEN el.item_id END DESC NULLS LAST,
|
||||
CASE WHEN p_sort_by = 'section_code' AND lower(p_sort_direction) = 'asc' THEN el.section_code END ASC NULLS LAST,
|
||||
CASE WHEN p_sort_by = 'section_code' AND lower(p_sort_direction) = 'desc' THEN el.section_code END DESC NULLS LAST,
|
||||
el.form_type_code,
|
||||
el.form_id,
|
||||
el.line_id
|
||||
) AS sort_order
|
||||
FROM eligible_lines el
|
||||
)
|
||||
SELECT
|
||||
pg.sort_order,
|
||||
ARRAY[
|
||||
pg.smeta_type::text,
|
||||
pg.smeta_direction::text,
|
||||
pg.org_name::text,
|
||||
pg.name::text,
|
||||
pg.item_id::text,
|
||||
pg.num_group_id::text,
|
||||
pg.justification::text,
|
||||
pl.plan_q1::text,
|
||||
pl.plan_q2::text,
|
||||
pl.plan_q3::text,
|
||||
pl.plan_q4::text,
|
||||
al.internal_order::text,
|
||||
al.property_object::text,
|
||||
sd.adj_q1::text,
|
||||
sd.adj_q2::text,
|
||||
sd.adj_q3::text,
|
||||
sd.adj_q4::text,
|
||||
rv.amount_q1::text,
|
||||
rv.amount_q2::text,
|
||||
rv.amount_q3::text,
|
||||
rv.amount_q4::text,
|
||||
(CASE WHEN pl.line_id IS NULL AND sd.id IS NULL AND rv.line_id IS NULL THEN NULL ELSE COALESCE(pl.plan_q1, 0) + COALESCE(sd.adj_q1, 0) + COALESCE(rv.amount_q1, 0) END)::text,
|
||||
(CASE WHEN pl.line_id IS NULL AND sd.id IS NULL AND rv.line_id IS NULL THEN NULL ELSE COALESCE(pl.plan_q2, 0) + COALESCE(sd.adj_q2, 0) + COALESCE(rv.amount_q2, 0) END)::text,
|
||||
(CASE WHEN pl.line_id IS NULL AND sd.id IS NULL AND rv.line_id IS NULL THEN NULL ELSE COALESCE(pl.plan_q3, 0) + COALESCE(sd.adj_q3, 0) + COALESCE(rv.amount_q3, 0) END)::text,
|
||||
(CASE WHEN pl.line_id IS NULL AND sd.id IS NULL AND rv.line_id IS NULL THEN NULL ELSE COALESCE(pl.plan_q4, 0) + COALESCE(sd.adj_q4, 0) + COALESCE(rv.amount_q4, 0) END)::text,
|
||||
pg.counterparty::text,
|
||||
pg.reference::text,
|
||||
pg.subject::text,
|
||||
pg.currency::text,
|
||||
pg.ceiling_amount::text,
|
||||
pg.expenses_q1::text,
|
||||
pg.expenses_q2::text,
|
||||
pg.expenses_q3::text,
|
||||
pg.expenses_q4::text,
|
||||
pg.vat_rate::text,
|
||||
pg.deadline::text,
|
||||
q1.adj_current::text,
|
||||
q1.adj_ssp::text,
|
||||
q1.adj_rf::text,
|
||||
q1.adj_reserve::text,
|
||||
q1.payment_date::text,
|
||||
q1.payment_amount::text,
|
||||
q1.payment_amount_ho::text,
|
||||
q1.payment_amount_rf::text,
|
||||
q1.payment_act::text,
|
||||
q1.booking_amount::text,
|
||||
q1.actual_m1::text,
|
||||
q1.actual_m2::text,
|
||||
q1.actual_m3::text,
|
||||
q1.transfer_to_q2::text,
|
||||
q1.transfer_to_q3::text,
|
||||
q1.transfer_to_q4::text,
|
||||
q1.transfer_to_economy::text,
|
||||
q2.adj_current::text,
|
||||
q2.adj_ssp::text,
|
||||
q2.adj_rf::text,
|
||||
q2.adj_reserve::text,
|
||||
q2.payment_date::text,
|
||||
q2.payment_amount::text,
|
||||
q2.payment_amount_ho::text,
|
||||
q2.payment_amount_rf::text,
|
||||
q2.payment_act::text,
|
||||
q2.booking_amount::text,
|
||||
q2.actual_m1::text,
|
||||
q2.actual_m2::text,
|
||||
q2.actual_m3::text,
|
||||
q2.plan_revision_eco_change::text,
|
||||
q2.plan_revision_item_adj::text,
|
||||
q2.plan_revision_increase::text,
|
||||
q2.plan_revision_sequester::text,
|
||||
q2.transfer_to_q3::text,
|
||||
q2.transfer_to_q4::text,
|
||||
q2.transfer_to_economy::text,
|
||||
q3.adj_current::text,
|
||||
q3.adj_ssp::text,
|
||||
q3.adj_rf::text,
|
||||
q3.adj_reserve::text,
|
||||
q3.payment_date::text,
|
||||
q3.payment_amount::text,
|
||||
q3.payment_amount_ho::text,
|
||||
q3.payment_amount_rf::text,
|
||||
q3.payment_act::text,
|
||||
q3.booking_amount::text,
|
||||
q3.actual_m1::text,
|
||||
q3.actual_m2::text,
|
||||
q3.actual_m3::text,
|
||||
q3.plan_revision_eco_change::text,
|
||||
q3.plan_revision_item_adj::text,
|
||||
q3.plan_revision_increase::text,
|
||||
q3.plan_revision_sequester::text,
|
||||
q3.transfer_to_q4::text,
|
||||
q3.transfer_to_economy::text,
|
||||
q4.adj_current::text,
|
||||
q4.adj_ssp::text,
|
||||
q4.adj_rf::text,
|
||||
q4.adj_reserve::text,
|
||||
q4.payment_date::text,
|
||||
q4.payment_amount::text,
|
||||
q4.payment_amount_ho::text,
|
||||
q4.payment_amount_rf::text,
|
||||
q4.payment_act::text,
|
||||
q4.booking_amount::text,
|
||||
q4.actual_m1::text,
|
||||
q4.actual_m2::text,
|
||||
q4.actual_m3::text,
|
||||
q4.actual_spod::text,
|
||||
q4.plan_revision_eco_change::text,
|
||||
q4.plan_revision_item_adj::text,
|
||||
q4.plan_revision_increase::text,
|
||||
q4.plan_revision_sequester::text
|
||||
]::text[] AS row_values
|
||||
FROM page pg
|
||||
LEFT JOIN v3.plan pl ON pl.line_id = pg.line_id
|
||||
LEFT JOIN v3.sequestration sd
|
||||
ON sd.line_id = pg.line_id
|
||||
AND sd.actor = 'DFIP'
|
||||
LEFT JOIN v3.reserve rv ON rv.line_id = pg.line_id
|
||||
LEFT JOIN v3.allocation al ON al.line_id = pg.line_id
|
||||
LEFT JOIN v3.budget_line_quarter q1
|
||||
ON q1.line_id = pg.line_id AND q1.quarter = 1
|
||||
LEFT JOIN v3.budget_line_quarter q2
|
||||
ON q2.line_id = pg.line_id AND q2.quarter = 2
|
||||
LEFT JOIN v3.budget_line_quarter q3
|
||||
ON q3.line_id = pg.line_id AND q3.quarter = 3
|
||||
LEFT JOIN v3.budget_line_quarter q4
|
||||
ON q4.line_id = pg.line_id AND q4.quarter = 4
|
||||
ORDER BY pg.sort_order;
|
||||
$function$;
|
||||
|
||||
COMMENT ON FUNCTION v3.v_svod2_go_export(integer, integer[], varchar, varchar) IS
|
||||
'Плоская выборка свода 2 / ГО для XLSX-экспорта без построения JSONB';
|
||||
@ -15,3 +15,4 @@ asyncpg==0.30.0
|
||||
pytest==8.3.2
|
||||
pytest-asyncio==0.24.0
|
||||
openpyxl==3.1.5
|
||||
XlsxWriter>=3.2,<4
|
||||
|
||||
@ -13,3 +13,4 @@ alembic==1.16.1
|
||||
asyncpg==0.30.0
|
||||
raisa-fastapi-protected-api==1.0.0
|
||||
openpyxl
|
||||
XlsxWriter>=3.2,<4
|
||||
|
||||
@ -1,15 +1,80 @@
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from fastapi.responses import StreamingResponse
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from typing import Annotated, Literal
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Path, status
|
||||
from fastapi.responses import FileResponse
|
||||
from starlette.background import BackgroundTask
|
||||
|
||||
from src.api.v1.deps import get_current_active_user_with_set_db
|
||||
from src.core.config import settings
|
||||
from src.db.models.app_user import AppUser
|
||||
from src.db.session import get_db
|
||||
from src.domain.schemas import DirectionSchemaEnum, ExportBulkRequest
|
||||
from src.services.export_service import ExportService
|
||||
from src.services.export_service.export_job_service import (
|
||||
ExportJobAlreadyRunningError,
|
||||
export_job_manager,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/export", tags=["export"])
|
||||
FORM3_ALLOWED_SECTIONS = {"q1", "q2", "q3", "q4", "year"}
|
||||
SUMMARY1_ALLOWED_SHEETS = {"FORM_1", "FORM_2", "FORM_3", "FORM_4", "MAIN"}
|
||||
SUMMARY2_ALLOWED_SHEETS = {"GO", "RF"}
|
||||
|
||||
|
||||
async def _enqueue_export(
|
||||
background_tasks: BackgroundTasks,
|
||||
current_user: AppUser,
|
||||
request: dict,
|
||||
):
|
||||
try:
|
||||
job = await export_job_manager.create(current_user.id, request)
|
||||
except ExportJobAlreadyRunningError:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail="Экспорт уже выполняется. Дождитесь завершения предыдущего экспорта.",
|
||||
)
|
||||
background_tasks.add_task(export_job_manager.run, job.id)
|
||||
return job.as_dict()
|
||||
|
||||
|
||||
@router.get("/jobs")
|
||||
async def list_export_jobs(current_user: AppUser = Depends(get_current_active_user_with_set_db)):
|
||||
return [job.as_dict() for job in await export_job_manager.list_jobs(current_user.id)]
|
||||
|
||||
|
||||
@router.get("/jobs/{job_id}")
|
||||
async def get_export_job(
|
||||
job_id: UUID,
|
||||
current_user: AppUser = Depends(get_current_active_user_with_set_db),
|
||||
):
|
||||
job = await export_job_manager.get(job_id, current_user.id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Экспорт не найден")
|
||||
return job.as_dict()
|
||||
|
||||
|
||||
@router.get("/jobs/{job_id}/download")
|
||||
async def download_export_job(
|
||||
job_id: UUID,
|
||||
current_user: AppUser = Depends(get_current_active_user_with_set_db),
|
||||
):
|
||||
job = await export_job_manager.get(job_id, current_user.id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Экспорт не найден")
|
||||
if job.status != "completed" or job.file_path is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail="Файл экспорта ещё не готов",
|
||||
)
|
||||
if not job.file_path.is_file():
|
||||
await export_job_manager.mark_file_missing(job)
|
||||
raise HTTPException(status_code=status.HTTP_410_GONE, detail="Файл экспорта удалён")
|
||||
return FileResponse(
|
||||
job.file_path,
|
||||
media_type=job.media_type,
|
||||
filename=job.filename,
|
||||
content_disposition_type="attachment",
|
||||
background=BackgroundTask(job.file_path.unlink, missing_ok=True),
|
||||
)
|
||||
|
||||
|
||||
def _parse_sections_csv(sections: str | None) -> list[str] | None:
|
||||
@ -47,130 +112,138 @@ def _parse_form3_sections_csv(sections: str | None) -> list[str] | None:
|
||||
return parsed
|
||||
|
||||
|
||||
@router.get("/form/{form_id}")
|
||||
def _parse_org_ids_csv(org_ids: str | None) -> list[int] | None:
|
||||
if not org_ids:
|
||||
return None
|
||||
try:
|
||||
parsed = [int(value.strip()) for value in org_ids.split(",") if value.strip()]
|
||||
except ValueError:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="org_ids должен быть списком целых чисел",
|
||||
)
|
||||
return parsed or None
|
||||
|
||||
|
||||
@router.get("/form/{form_id}", status_code=status.HTTP_202_ACCEPTED)
|
||||
async def export_form(
|
||||
background_tasks: BackgroundTasks,
|
||||
form_id: int,
|
||||
direction: DirectionSchemaEnum | None = None,
|
||||
sections: str | None = None,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: AppUser = Depends(get_current_active_user_with_set_db),
|
||||
):
|
||||
export_service = ExportService(db)
|
||||
try:
|
||||
stream, filename, media_type = await export_service.export_form_payload(
|
||||
form_id=form_id,
|
||||
current_user=current_user,
|
||||
direction=direction.value if direction else None,
|
||||
sections=_parse_sections_csv(sections),
|
||||
ignore_non_applicable_query_params=True,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc))
|
||||
return StreamingResponse(
|
||||
stream,
|
||||
media_type=media_type,
|
||||
headers={"Content-Disposition": export_service.build_content_disposition(filename)},
|
||||
return await _enqueue_export(
|
||||
background_tasks,
|
||||
current_user,
|
||||
{
|
||||
"kind": "form", "form_id": form_id,
|
||||
"direction": direction.value if direction else None,
|
||||
"sections": _parse_sections_csv(sections),
|
||||
"ignore_unsupported_sheet_params": True,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/form/{form_id}/sheet/{sheet}")
|
||||
@router.get("/form/{form_id}/sheet/{sheet}", status_code=status.HTTP_202_ACCEPTED)
|
||||
async def export_form_sheet(
|
||||
background_tasks: BackgroundTasks,
|
||||
form_id: int,
|
||||
sheet: str,
|
||||
direction: DirectionSchemaEnum | None = None,
|
||||
sections: str | None = None,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: AppUser = Depends(get_current_active_user_with_set_db),
|
||||
):
|
||||
export_service = ExportService(db)
|
||||
try:
|
||||
stream, filename, media_type = await export_service.export_form_payload(
|
||||
form_id=form_id,
|
||||
current_user=current_user,
|
||||
sheet=sheet,
|
||||
direction=direction.value if direction else None,
|
||||
sections=_parse_sections_csv(sections),
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc))
|
||||
return StreamingResponse(
|
||||
stream,
|
||||
media_type=media_type,
|
||||
headers={"Content-Disposition": export_service.build_content_disposition(filename)},
|
||||
return await _enqueue_export(
|
||||
background_tasks, current_user,
|
||||
{"kind": "form", "form_id": form_id, "sheet": sheet,
|
||||
"direction": direction.value if direction else None,
|
||||
"sections": _parse_sections_csv(sections)},
|
||||
)
|
||||
|
||||
|
||||
@router.post("/bulk")
|
||||
@router.post("/bulk", status_code=status.HTTP_202_ACCEPTED)
|
||||
async def export_forms_bulk(
|
||||
background_tasks: BackgroundTasks,
|
||||
payload: ExportBulkRequest,
|
||||
direction: DirectionSchemaEnum | None = None,
|
||||
sections: str | None = None,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: AppUser = Depends(get_current_active_user_with_set_db),
|
||||
):
|
||||
export_service = ExportService(db)
|
||||
try:
|
||||
stream, filename, media_type = await export_service.export_bulk_payload(
|
||||
form_ids=payload.form_ids,
|
||||
current_user=current_user,
|
||||
skip_failed=payload.skip_failed,
|
||||
direction=direction.value if direction else None,
|
||||
sections=_parse_sections_csv(sections),
|
||||
if len(payload.form_ids) > settings.EXPORT_BULK_MAX_FORMS:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=(
|
||||
"В пакетном экспорте разрешено не более "
|
||||
f"{settings.EXPORT_BULK_MAX_FORMS} форм"
|
||||
),
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc))
|
||||
return StreamingResponse(
|
||||
stream,
|
||||
media_type=media_type,
|
||||
headers={"Content-Disposition": export_service.build_content_disposition(filename)},
|
||||
return await _enqueue_export(
|
||||
background_tasks, current_user,
|
||||
{"kind": "bulk", "form_ids": payload.form_ids,
|
||||
"skip_failed": payload.skip_failed,
|
||||
"direction": direction.value if direction else None,
|
||||
"sections": _parse_sections_csv(sections)},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/project/{project_id}")
|
||||
@router.get("/summary/{summary_number}/{sheet}/{year}", status_code=status.HTTP_202_ACCEPTED)
|
||||
async def export_summary(
|
||||
background_tasks: BackgroundTasks,
|
||||
summary_number: Annotated[int, Path(ge=1, le=2)],
|
||||
sheet: str,
|
||||
year: int,
|
||||
org_ids: str | None = None,
|
||||
sort_by: Literal["org_name", "section_code"] = "org_name",
|
||||
sort_direction: Literal["asc", "desc"] = "asc",
|
||||
current_user: AppUser = Depends(get_current_active_user_with_set_db),
|
||||
):
|
||||
sheet = sheet.upper()
|
||||
allowed_sheets = (
|
||||
SUMMARY1_ALLOWED_SHEETS
|
||||
if summary_number == 1
|
||||
else SUMMARY2_ALLOWED_SHEETS
|
||||
)
|
||||
if sheet not in allowed_sheets:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Неизвестный лист свода {summary_number}: {sheet}",
|
||||
)
|
||||
return await _enqueue_export(
|
||||
background_tasks, current_user,
|
||||
{"kind": "summary", "summary_number": summary_number, "year": year,
|
||||
"sheet": sheet, "org_ids": _parse_org_ids_csv(org_ids),
|
||||
"sort_by": sort_by, "sort_direction": sort_direction},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/project/{project_id}", status_code=status.HTTP_202_ACCEPTED)
|
||||
async def export_project(
|
||||
background_tasks: BackgroundTasks,
|
||||
project_id: int,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: AppUser = Depends(get_current_active_user_with_set_db),
|
||||
):
|
||||
export_service = ExportService(db)
|
||||
try:
|
||||
stream, filename, media_type = await export_service.export_project_payload(
|
||||
project_id=project_id,
|
||||
current_user=current_user,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc))
|
||||
return StreamingResponse(
|
||||
stream,
|
||||
media_type=media_type,
|
||||
headers={"Content-Disposition": export_service.build_content_disposition(filename)},
|
||||
return await _enqueue_export(
|
||||
background_tasks, current_user,
|
||||
{"kind": "project", "project_id": project_id},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/project/{project_id}/report/{year}/{report_type}")
|
||||
@router.get("/project/{project_id}/report/{year}/{report_type}", status_code=status.HTTP_202_ACCEPTED)
|
||||
async def export_project_report(
|
||||
background_tasks: BackgroundTasks,
|
||||
project_id: int,
|
||||
year: int,
|
||||
report_type: str,
|
||||
sections: str | None = None,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: AppUser = Depends(get_current_active_user_with_set_db),
|
||||
):
|
||||
export_service = ExportService(db)
|
||||
try:
|
||||
stream, filename, media_type = await export_service.export_project_report_payload(
|
||||
project_id=project_id,
|
||||
year=year,
|
||||
report_type=report_type,
|
||||
current_user=current_user,
|
||||
sections=_parse_form3_sections_csv(sections),
|
||||
)
|
||||
parsed_sections = _parse_form3_sections_csv(sections)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc))
|
||||
return StreamingResponse(
|
||||
stream,
|
||||
media_type=media_type,
|
||||
headers={"Content-Disposition": export_service.build_content_disposition(filename)},
|
||||
return await _enqueue_export(
|
||||
background_tasks, current_user,
|
||||
{"kind": "project_report", "project_id": project_id, "year": year,
|
||||
"report_type": report_type, "sections": parsed_sections},
|
||||
)
|
||||
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -108,6 +108,36 @@ class Settings(BaseSettings):
|
||||
alias="AUDIT_LOG_CLEANUP_INTERVAL_SECONDS",
|
||||
)
|
||||
|
||||
EXPORT_DIR: str = Field(
|
||||
default="tmp/exports",
|
||||
description="Каталог временных файлов фонового экспорта",
|
||||
alias="EXPORT_DIR",
|
||||
)
|
||||
EXPORT_PROCESS_WORKERS: int = Field(
|
||||
default=2,
|
||||
ge=1,
|
||||
description="Число отдельных процессов для формирования файлов экспорта",
|
||||
alias="EXPORT_PROCESS_WORKERS",
|
||||
)
|
||||
EXPORT_BULK_MAX_FORMS: int = Field(
|
||||
default=10,
|
||||
ge=1,
|
||||
description="Максимальное число форм в одном пакетном экспорте",
|
||||
alias="EXPORT_BULK_MAX_FORMS",
|
||||
)
|
||||
EXPORT_FILE_TTL_SECONDS: int = Field(
|
||||
default=86400,
|
||||
ge=60,
|
||||
description="Срок хранения готовых файлов экспорта в секундах",
|
||||
alias="EXPORT_FILE_TTL_SECONDS",
|
||||
)
|
||||
EXPORT_CLEANUP_INTERVAL_SECONDS: int = Field(
|
||||
default=3600,
|
||||
ge=60,
|
||||
description="Интервал очистки устаревших файлов экспорта в секундах",
|
||||
alias="EXPORT_CLEANUP_INTERVAL_SECONDS",
|
||||
)
|
||||
|
||||
model_config = SettingsConfigDict(
|
||||
env_file=".env",
|
||||
extra="ignore",
|
||||
|
||||
@ -10,6 +10,7 @@ from src.db.models.collegial_approval import CollegialApproval
|
||||
from src.db.models.contract_detail import ContractDetail
|
||||
from src.db.models.contract_summary import ContractSummary
|
||||
from src.db.models.expense_item import ExpenseItem, ExpenseItemFormType
|
||||
from src.db.models.export_job import ExportJob
|
||||
from src.db.models.form3_phase import Form3Phase
|
||||
from src.db.models.form4_project import Form4Project
|
||||
from src.db.models.form_limit import FormLimit
|
||||
|
||||
82
api/src/db/models/export_job.py
Normal file
82
api/src/db/models/export_job.py
Normal file
@ -0,0 +1,82 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
from uuid import UUID as PythonUUID, uuid4
|
||||
|
||||
from sqlalchemy import DateTime, ForeignKey, JSON, String, Text, Uuid
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from src.db.base import Base
|
||||
|
||||
ExportJobStatus = Literal["queued", "processing", "completed", "failed"]
|
||||
|
||||
|
||||
class ExportJob(Base):
|
||||
__tablename__ = "export_job"
|
||||
__table_args__ = {"schema": "v3"}
|
||||
|
||||
id: Mapped[PythonUUID] = mapped_column(Uuid(as_uuid=True), primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("v3.app_user.id"))
|
||||
request: Mapped[dict[str, Any]] = mapped_column(JSON)
|
||||
status: Mapped[ExportJobStatus] = mapped_column(String(20), default="queued")
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
filename: Mapped[str | None] = mapped_column(Text)
|
||||
media_type: Mapped[str | None] = mapped_column(Text)
|
||||
_file_path: Mapped[str | None] = mapped_column("file_path", Text)
|
||||
error: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
user_id: int,
|
||||
request: dict[str, Any],
|
||||
id: PythonUUID | None = None,
|
||||
status: ExportJobStatus = "queued",
|
||||
created_at: datetime | None = None,
|
||||
started_at: datetime | None = None,
|
||||
finished_at: datetime | None = None,
|
||||
filename: str | None = None,
|
||||
media_type: str | None = None,
|
||||
file_path: str | Path | None = None,
|
||||
error: str | None = None,
|
||||
) -> None:
|
||||
self.id = id or uuid4()
|
||||
self.user_id = user_id
|
||||
self.request = request
|
||||
self.status = status
|
||||
self.created_at = created_at or datetime.now(timezone.utc)
|
||||
self.started_at = started_at
|
||||
self.finished_at = finished_at
|
||||
self.filename = filename
|
||||
self.media_type = media_type
|
||||
self.file_path = file_path
|
||||
self.error = error
|
||||
|
||||
@property
|
||||
def file_path(self) -> Path | None:
|
||||
return Path(self._file_path) if self._file_path else None
|
||||
|
||||
@file_path.setter
|
||||
def file_path(self, value: str | Path | None) -> None:
|
||||
self._file_path = str(value) if value else None
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"id": str(self.id),
|
||||
"status": self.status,
|
||||
"created_at": self.created_at,
|
||||
"started_at": self.started_at,
|
||||
"finished_at": self.finished_at,
|
||||
"filename": self.filename,
|
||||
"media_type": self.media_type,
|
||||
"error": self.error,
|
||||
"status_url": f"/api/v1/export/jobs/{self.id}",
|
||||
"download_url": (
|
||||
f"/api/v1/export/jobs/{self.id}/download"
|
||||
if self.status == "completed"
|
||||
else None
|
||||
),
|
||||
}
|
||||
@ -1,16 +1,17 @@
|
||||
import asyncio
|
||||
from typing import AsyncGenerator
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import AsyncGenerator, AsyncIterator
|
||||
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from src.core.config import settings
|
||||
from typing import AsyncGenerator
|
||||
|
||||
from src.db.base import SessionLocal
|
||||
|
||||
|
||||
async def get_db() -> AsyncGenerator:
|
||||
@asynccontextmanager
|
||||
async def db_session() -> AsyncGenerator[AsyncSession, None]:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
@ -22,6 +23,11 @@ async def get_db() -> AsyncGenerator:
|
||||
await db.close()
|
||||
|
||||
|
||||
async def get_db() -> AsyncGenerator[AsyncSession, None]:
|
||||
async with db_session() as db:
|
||||
yield db
|
||||
|
||||
|
||||
async def run_migrations() -> None:
|
||||
alembic_cfg = Config("alembic.ini")
|
||||
await asyncio.to_thread(command.upgrade, alembic_cfg, "head")
|
||||
|
||||
@ -20,8 +20,7 @@ from src.core.config import settings
|
||||
from src.core.exception_handlers import register_exception_handlers
|
||||
from src.db.base import engine
|
||||
from src.db.session import create_tables
|
||||
|
||||
|
||||
from src.services.export_service.export_job_service import export_job_manager
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
@ -99,6 +98,17 @@ async def _audit_cleanup_loop() -> None:
|
||||
await asyncio.sleep(max(60, settings.AUDIT_LOG_CLEANUP_INTERVAL_SECONDS))
|
||||
|
||||
|
||||
async def _export_cleanup_loop() -> None:
|
||||
while True:
|
||||
try:
|
||||
removed = await export_job_manager.cleanup_expired()
|
||||
if removed:
|
||||
logging.info("Removed %s expired export file(s)", removed)
|
||||
except Exception:
|
||||
logging.exception("Failed to clean up expired export files")
|
||||
await asyncio.sleep(settings.EXPORT_CLEANUP_INTERVAL_SECONDS)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
cleanup_task: asyncio.Task | None = None
|
||||
@ -112,6 +122,8 @@ async def lifespan(app: FastAPI):
|
||||
# ),
|
||||
# )
|
||||
cleanup_task = asyncio.create_task(_audit_cleanup_loop())
|
||||
await export_job_manager.recover_interrupted()
|
||||
export_cleanup_task = asyncio.create_task(_export_cleanup_loop())
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
@ -119,6 +131,10 @@ async def lifespan(app: FastAPI):
|
||||
cleanup_task.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await cleanup_task
|
||||
export_cleanup_task.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await export_cleanup_task
|
||||
await export_job_manager.shutdown()
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
|
||||
51
api/src/repository/export_job_repository.py
Normal file
51
api/src/repository/export_job_repository.py
Normal file
@ -0,0 +1,51 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
|
||||
from src.db.models.export_job import ExportJob
|
||||
from src.db.session import db_session
|
||||
|
||||
|
||||
class ExportJobRepository:
|
||||
async def save(self, job: ExportJob) -> None:
|
||||
async with db_session() as db:
|
||||
await db.merge(job)
|
||||
|
||||
async def get(self, job_id) -> ExportJob | None:
|
||||
async with db_session() as db:
|
||||
return await db.get(ExportJob, job_id)
|
||||
|
||||
async def list(self, user_id, cutoff) -> list[ExportJob]:
|
||||
async with db_session() as db:
|
||||
result = await db.scalars(
|
||||
select(ExportJob)
|
||||
.where(
|
||||
ExportJob.user_id == user_id,
|
||||
(ExportJob.finished_at.is_(None))
|
||||
| (ExportJob.finished_at > cutoff),
|
||||
)
|
||||
.order_by(ExportJob.created_at.desc())
|
||||
)
|
||||
return list(result)
|
||||
|
||||
async def recover_interrupted(self) -> list[ExportJob]:
|
||||
async with db_session() as db:
|
||||
result = await db.scalars(
|
||||
select(ExportJob).where(
|
||||
ExportJob.status.in_(("queued", "processing"))
|
||||
)
|
||||
)
|
||||
jobs = list(result)
|
||||
for job in jobs:
|
||||
job.status = "queued"
|
||||
job.started_at = None
|
||||
job.finished_at = None
|
||||
job.filename = None
|
||||
job.media_type = None
|
||||
job.file_path = None
|
||||
job.error = None
|
||||
return jobs
|
||||
|
||||
async def cleanup(self, cutoff) -> None:
|
||||
async with db_session() as db:
|
||||
await db.execute(delete(ExportJob).where(ExportJob.finished_at <= cutoff))
|
||||
@ -1,3 +1,5 @@
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
@ -58,7 +60,6 @@ class SummaryRepository:
|
||||
CAST(:sort_by AS VARCHAR),
|
||||
CAST(:sort_direction AS VARCHAR)
|
||||
)
|
||||
ORDER BY sort_order
|
||||
"""
|
||||
)
|
||||
return (
|
||||
@ -74,3 +75,120 @@ class SummaryRepository:
|
||||
},
|
||||
)
|
||||
).all()
|
||||
|
||||
async def get_svod2_go_export_rows(
|
||||
self,
|
||||
year: int,
|
||||
org_ids: list[int] | None,
|
||||
sort_by: str,
|
||||
sort_direction: str,
|
||||
) -> list[tuple]:
|
||||
"""Load the compact flat representation produced for XLSX export."""
|
||||
common_quarter_fields = (
|
||||
"adj_current", "adj_ssp", "adj_rf", "adj_reserve",
|
||||
"payment_date", "payment_amount", "payment_amount_ho",
|
||||
"payment_amount_rf", "payment_act", "booking_amount",
|
||||
"actual_m1", "actual_m2", "actual_m3",
|
||||
)
|
||||
export_keys = [
|
||||
"header.smeta_type",
|
||||
"header.smeta_direction",
|
||||
"header.org_name",
|
||||
"header.name",
|
||||
"header.item_id",
|
||||
"header.num_group",
|
||||
"header.justification",
|
||||
*(f"plan.plan_q{quarter}" for quarter in range(1, 5)),
|
||||
"allocation.internal_order",
|
||||
"allocation.property_object",
|
||||
*(f"sequestration.DFIP.adj_q{quarter}" for quarter in range(1, 5)),
|
||||
*(f"reserve.amount_q{quarter}" for quarter in range(1, 5)),
|
||||
*(f"approved.approved_q{quarter}" for quarter in range(1, 5)),
|
||||
*(
|
||||
f"contract_detail.{field}"
|
||||
for field in (
|
||||
"counterparty", "reference", "subject", "currency",
|
||||
"ceiling_amount", "expenses_q1", "expenses_q2",
|
||||
"expenses_q3", "expenses_q4", "vat_rate", "deadline",
|
||||
)
|
||||
),
|
||||
]
|
||||
for quarter in range(1, 5):
|
||||
quarter_fields = list(common_quarter_fields)
|
||||
if quarter == 4:
|
||||
quarter_fields.append("actual_spod")
|
||||
if quarter > 1:
|
||||
quarter_fields.extend(
|
||||
(
|
||||
"plan_revision_eco_change",
|
||||
"plan_revision_item_adj",
|
||||
"plan_revision_increase",
|
||||
"plan_revision_sequester",
|
||||
)
|
||||
)
|
||||
quarter_fields.extend(
|
||||
f"transfer_to_q{target_quarter}"
|
||||
for target_quarter in range(quarter + 1, 5)
|
||||
)
|
||||
if quarter < 4:
|
||||
quarter_fields.append("transfer_to_economy")
|
||||
export_keys.extend(f"q{quarter}.{field}" for field in quarter_fields)
|
||||
|
||||
query = text(
|
||||
"""
|
||||
SELECT sort_order, row_values
|
||||
FROM v3.v_svod2_go_export(
|
||||
CAST(:year AS INT),
|
||||
CAST(:org_ids AS INT[]),
|
||||
CAST(:sort_by AS VARCHAR),
|
||||
CAST(:sort_direction AS VARCHAR)
|
||||
)
|
||||
"""
|
||||
)
|
||||
result = await self.db.execute(
|
||||
query,
|
||||
{
|
||||
"year": year,
|
||||
"org_ids": org_ids,
|
||||
"sort_by": sort_by,
|
||||
"sort_direction": sort_direction.lower(),
|
||||
},
|
||||
)
|
||||
rows = []
|
||||
for row in result.mappings():
|
||||
data = {
|
||||
key: self._parse_svod2_export_value(key, value)
|
||||
for key, value in zip(export_keys, row["row_values"])
|
||||
}
|
||||
rows.append(("INPUT", 3, row["sort_order"], data))
|
||||
return rows
|
||||
|
||||
@staticmethod
|
||||
def _parse_svod2_export_value(key: str, value: str | None) -> Any:
|
||||
if value is None:
|
||||
return None
|
||||
numeric_prefixes = (
|
||||
"plan.",
|
||||
"sequestration.",
|
||||
"reserve.",
|
||||
"approved.",
|
||||
)
|
||||
numeric_contract_fields = {
|
||||
"contract_detail.ceiling_amount",
|
||||
"contract_detail.expenses_q1",
|
||||
"contract_detail.expenses_q2",
|
||||
"contract_detail.expenses_q3",
|
||||
"contract_detail.expenses_q4",
|
||||
}
|
||||
quarter_text_suffixes = (".payment_date", ".payment_act")
|
||||
is_numeric_quarter = (
|
||||
key.startswith(("q1.", "q2.", "q3.", "q4."))
|
||||
and not key.endswith(quarter_text_suffixes)
|
||||
)
|
||||
if (
|
||||
key.startswith(numeric_prefixes)
|
||||
or key in numeric_contract_fields
|
||||
or is_numeric_quarter
|
||||
):
|
||||
return float(value)
|
||||
return value
|
||||
|
||||
265
api/src/services/export_service/export_job_service.py
Normal file
265
api/src/services/export_service/export_job_service.py
Normal file
@ -0,0 +1,265 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import multiprocessing
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from src.core.config import settings
|
||||
from src.db.base import engine
|
||||
from src.db.models.export_job import ExportJob
|
||||
from src.db.session import db_session
|
||||
from src.repository.export_job_repository import ExportJobRepository
|
||||
from src.repository.user_repository import UserRepository
|
||||
from src.services.export_service.export_service import ExportService
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ExportJobAlreadyRunningError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class ExportJobManager:
|
||||
def __init__(self) -> None:
|
||||
self._jobs: dict[UUID, ExportJob] = {}
|
||||
self._active_by_user: dict[int, UUID] = {}
|
||||
self._lock = asyncio.Lock()
|
||||
self.repository = ExportJobRepository()
|
||||
self._subscribers: dict[int, set[asyncio.Queue]] = {}
|
||||
self.export_dir = Path(settings.EXPORT_DIR).resolve()
|
||||
self._executor: ProcessPoolExecutor | None = None
|
||||
|
||||
def _get_executor(self) -> ProcessPoolExecutor:
|
||||
if self._executor is None:
|
||||
self._executor = ProcessPoolExecutor(
|
||||
max_workers=settings.EXPORT_PROCESS_WORKERS,
|
||||
mp_context=multiprocessing.get_context("spawn"),
|
||||
)
|
||||
return self._executor
|
||||
|
||||
async def _execute_export(self, service, current_user, request: dict[str, Any]):
|
||||
request = request.copy()
|
||||
kind = request.pop("kind")
|
||||
if kind == "form":
|
||||
return await service.export_form_payload(current_user=current_user, **request)
|
||||
if kind == "bulk":
|
||||
return await service.export_bulk_payload(current_user=current_user, **request)
|
||||
if kind == "summary":
|
||||
return await service.export_summary_payload(
|
||||
current_user=current_user, **request
|
||||
)
|
||||
if kind == "project":
|
||||
return await service.export_project_payload(
|
||||
current_user=current_user, **request
|
||||
)
|
||||
if kind == "project_report":
|
||||
return await service.export_project_report_payload(
|
||||
current_user=current_user, **request
|
||||
)
|
||||
raise ValueError(f"Неизвестный тип экспорта: {kind}")
|
||||
|
||||
async def _generate_export_file(
|
||||
self,
|
||||
job_id: str,
|
||||
user_id: int,
|
||||
request: dict[str, Any],
|
||||
export_dir_value: str,
|
||||
) -> dict[str, str]:
|
||||
try:
|
||||
async with db_session() as db:
|
||||
user_repo = UserRepository(db)
|
||||
current_user = await user_repo.get(user_id)
|
||||
if current_user is None or not current_user.is_active:
|
||||
raise RuntimeError("Пользователь не найден или заблокирован")
|
||||
await user_repo.set_app_user_id(current_user.id)
|
||||
stream, filename, media_type = await self._execute_export(
|
||||
ExportService(db), current_user, request
|
||||
)
|
||||
|
||||
export_dir = Path(export_dir_value)
|
||||
export_dir.mkdir(parents=True, exist_ok=True)
|
||||
suffix = Path(filename).suffix or ".bin"
|
||||
file_path = export_dir / f"{job_id}{suffix}"
|
||||
try:
|
||||
stream.seek(0)
|
||||
file_path.write_bytes(stream.read())
|
||||
finally:
|
||||
stream.close()
|
||||
return {
|
||||
"filename": filename,
|
||||
"media_type": media_type,
|
||||
"file_path": str(file_path),
|
||||
}
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
@staticmethod
|
||||
def _run_export_process(
|
||||
job_id: str,
|
||||
user_id: int,
|
||||
request: dict[str, Any],
|
||||
export_dir: str,
|
||||
) -> dict[str, str]:
|
||||
manager = ExportJobManager()
|
||||
return asyncio.run(
|
||||
manager._generate_export_file(job_id, user_id, request, export_dir)
|
||||
)
|
||||
|
||||
async def create(self, user_id: int, request: dict[str, Any]) -> ExportJob:
|
||||
async with self._lock:
|
||||
active_id = self._active_by_user.get(user_id)
|
||||
if active_id is not None:
|
||||
active = self._jobs.get(active_id)
|
||||
if active and active.status in {"queued", "processing"}:
|
||||
raise ExportJobAlreadyRunningError
|
||||
job = ExportJob(id=uuid4(), user_id=user_id, request=request)
|
||||
try:
|
||||
await self.repository.save(job)
|
||||
except IntegrityError as exc:
|
||||
if "export_job_active_user" not in str(exc):
|
||||
raise
|
||||
raise ExportJobAlreadyRunningError from exc
|
||||
self._jobs[job.id] = job
|
||||
self._active_by_user[user_id] = job.id
|
||||
self.notify(user_id)
|
||||
return job
|
||||
|
||||
async def list_jobs(self, user_id: int) -> list[ExportJob]:
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(
|
||||
seconds=settings.EXPORT_FILE_TTL_SECONDS
|
||||
)
|
||||
return await self.repository.list(user_id, cutoff)
|
||||
|
||||
async def get(self, job_id: UUID, user_id: int) -> ExportJob | None:
|
||||
return next(
|
||||
(job for job in await self.list_jobs(user_id) if job.id == job_id), None
|
||||
)
|
||||
|
||||
async def mark_file_missing(self, job: ExportJob) -> None:
|
||||
job.status = "failed"
|
||||
job.error = "Файл экспорта удалён"
|
||||
job.file_path = None
|
||||
if job.finished_at is None:
|
||||
job.finished_at = datetime.now(timezone.utc)
|
||||
await self.repository.save(job)
|
||||
self._jobs[job.id] = job
|
||||
self.notify(job.user_id)
|
||||
|
||||
def subscribe(self, user_id):
|
||||
queue = asyncio.Queue(maxsize=1)
|
||||
self._subscribers.setdefault(user_id, set()).add(queue)
|
||||
return queue
|
||||
|
||||
def unsubscribe(self, user_id, queue):
|
||||
queues = self._subscribers.get(user_id, set())
|
||||
queues.discard(queue)
|
||||
if not queues:
|
||||
self._subscribers.pop(user_id, None)
|
||||
|
||||
def notify(self, user_id):
|
||||
for queue in self._subscribers.get(user_id, ()):
|
||||
if queue.empty():
|
||||
queue.put_nowait(True)
|
||||
|
||||
async def run(self, job_id: UUID) -> None:
|
||||
job = self._jobs.get(job_id) or await self.repository.get(job_id)
|
||||
if job is None:
|
||||
return
|
||||
self._jobs[job.id] = job
|
||||
job.status = "processing"
|
||||
job.started_at = datetime.now(timezone.utc)
|
||||
try:
|
||||
await self.repository.save(job)
|
||||
self.notify(job.user_id)
|
||||
loop = asyncio.get_running_loop()
|
||||
result = await loop.run_in_executor(
|
||||
self._get_executor(),
|
||||
ExportJobManager._run_export_process,
|
||||
str(job.id),
|
||||
job.user_id,
|
||||
job.request,
|
||||
str(self.export_dir),
|
||||
)
|
||||
job.filename = result["filename"]
|
||||
job.media_type = result["media_type"]
|
||||
job.file_path = Path(result["file_path"])
|
||||
job.status = "completed"
|
||||
except Exception as exc:
|
||||
logger.exception("Export job %s failed", job.id)
|
||||
job.status = "failed"
|
||||
job.error = str(exc) or "Не удалось сформировать файл"
|
||||
finally:
|
||||
job.finished_at = datetime.now(timezone.utc)
|
||||
await self.repository.save(job)
|
||||
self.notify(job.user_id)
|
||||
async with self._lock:
|
||||
if self._active_by_user.get(job.user_id) == job.id:
|
||||
self._active_by_user.pop(job.user_id, None)
|
||||
|
||||
async def recover_interrupted(self) -> None:
|
||||
jobs = await self.repository.recover_interrupted()
|
||||
async with self._lock:
|
||||
for job in jobs:
|
||||
self._jobs[job.id] = job
|
||||
self._active_by_user[job.user_id] = job.id
|
||||
for job in jobs:
|
||||
self.notify(job.user_id)
|
||||
asyncio.create_task(self.run(job.id))
|
||||
|
||||
async def shutdown(self) -> None:
|
||||
if self._executor is None:
|
||||
return
|
||||
executor, self._executor = self._executor, None
|
||||
await asyncio.to_thread(executor.shutdown, wait=True, cancel_futures=True)
|
||||
|
||||
async def cleanup_expired(self) -> int:
|
||||
now = datetime.now(timezone.utc)
|
||||
cutoff = now - timedelta(seconds=settings.EXPORT_FILE_TTL_SECONDS)
|
||||
async with self._lock:
|
||||
expired_ids = [
|
||||
job_id
|
||||
for job_id, job in self._jobs.items()
|
||||
if job.status not in {"queued", "processing"}
|
||||
and job.finished_at is not None
|
||||
and job.finished_at <= cutoff
|
||||
]
|
||||
for job_id in expired_ids:
|
||||
job = self._jobs.pop(job_id, None)
|
||||
if job and self._active_by_user.get(job.user_id) == job_id:
|
||||
self._active_by_user.pop(job.user_id, None)
|
||||
|
||||
removed = self._cleanup_export_files(cutoff)
|
||||
await self.repository.cleanup(cutoff)
|
||||
return removed
|
||||
|
||||
def _cleanup_export_files(self, cutoff: datetime) -> int:
|
||||
if not self.export_dir.is_dir():
|
||||
return 0
|
||||
removed = 0
|
||||
for file_path in self.export_dir.iterdir():
|
||||
if not file_path.is_file():
|
||||
continue
|
||||
try:
|
||||
UUID(file_path.stem)
|
||||
except ValueError:
|
||||
continue
|
||||
modified_at = datetime.fromtimestamp(
|
||||
file_path.stat().st_mtime, tz=timezone.utc
|
||||
)
|
||||
if modified_at > cutoff:
|
||||
continue
|
||||
try:
|
||||
file_path.unlink()
|
||||
removed += 1
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
return removed
|
||||
|
||||
|
||||
export_job_manager = ExportJobManager()
|
||||
@ -1,3 +1,5 @@
|
||||
import copy
|
||||
|
||||
|
||||
|
||||
FORM1_FLAT_COLUMNS: list[tuple[str, str]] = [
|
||||
@ -1423,6 +1425,7 @@ PALETTE = {
|
||||
"GROUP": "D7E3BC",
|
||||
"ITEM": "EAF0DD",
|
||||
"SUB_ITEM": "FFFFFF",
|
||||
"SUB_SUB_ITEM": "FFFFFF",
|
||||
"INPUT": "FFFFFF",
|
||||
"COLOR": "000000",
|
||||
},
|
||||
@ -1430,8 +1433,8 @@ PALETTE = {
|
||||
"ROOT": "C2D59A",
|
||||
"GROUP": "D7E3BC",
|
||||
"ITEM": "EAF0DD",
|
||||
"SUB_ITEM": "FFFFFF",
|
||||
"SUB_ITEM": "EAF0DD",
|
||||
"SUB_SUB_ITEM": "EAF0DD",
|
||||
"INPUT": "FFFFFF",
|
||||
"COLOR": "000000",
|
||||
},
|
||||
@ -1440,6 +1443,7 @@ PALETTE = {
|
||||
"GROUP": "F6D6B9",
|
||||
"ITEM": "FAEBDC",
|
||||
"SUB_ITEM": "FFFFFF",
|
||||
"SUB_SUB_ITEM": "FFFFFF",
|
||||
"INPUT": "FFFFFF",
|
||||
"COLOR": "000000",
|
||||
},
|
||||
@ -1448,6 +1452,7 @@ PALETTE = {
|
||||
"GROUP": "FFFFFF",
|
||||
"ITEM": "ffffff",
|
||||
"SUB_ITEM": "FFFFFF",
|
||||
"SUB_SUB_ITEM": "FFFFFF",
|
||||
"INPUT": "FFFFFF",
|
||||
"COLOR": "000000",
|
||||
},
|
||||
@ -1456,6 +1461,7 @@ PALETTE = {
|
||||
"GROUP": "D89493",
|
||||
"ITEM": "E5B7B6",
|
||||
"SUB_ITEM": "FFFFFF",
|
||||
"SUB_SUB_ITEM": "FFFFFF",
|
||||
"INPUT": "FFFFFF",
|
||||
"COLOR": "ffffff",
|
||||
},
|
||||
@ -1464,6 +1470,7 @@ PALETTE = {
|
||||
"GROUP": "8DB3E2",
|
||||
"ITEM": "C5D8F0",
|
||||
"SUB_ITEM": "FFFFFF",
|
||||
"SUB_SUB_ITEM": "FFFFFF",
|
||||
"INPUT": "FFFFFF",
|
||||
"COLOR": "000000",
|
||||
},
|
||||
@ -1472,6 +1479,7 @@ PALETTE = {
|
||||
"GROUP": "FFFF99",
|
||||
"ITEM": "FFFFCC",
|
||||
"SUB_ITEM": "FFFFFF",
|
||||
"SUB_SUB_ITEM": "FFFFFF",
|
||||
"INPUT": "FFFFFF",
|
||||
"COLOR": "000000",
|
||||
},
|
||||
@ -1504,7 +1512,7 @@ FORM1_COLORS: dict = {
|
||||
"palette": "blue",
|
||||
},
|
||||
"upper_ssp_smeta.contract_summary.deadline": {
|
||||
"palette": None,
|
||||
"palette": "blue",
|
||||
},
|
||||
"allocation": {
|
||||
"background": "bfbfbf",
|
||||
@ -1542,6 +1550,10 @@ FORM1_COLORS: dict = {
|
||||
"empty_upper.ckk.q4": {
|
||||
"background": "748c42",
|
||||
},
|
||||
"empty_upper.ckk.deadline": {
|
||||
"background": "fabf8f",
|
||||
"palette": "orange",
|
||||
},
|
||||
"empty_upper.contract_detail": {
|
||||
"background": "fabf8f",
|
||||
"palette": "orange",
|
||||
@ -1558,6 +1570,10 @@ FORM1_COLORS: dict = {
|
||||
"empty_upper.contract_detail.q4": {
|
||||
"background": "748c42",
|
||||
},
|
||||
"empty_upper.contract_detail.deadline": {
|
||||
"background": "fabf8f",
|
||||
"palette": "orange",
|
||||
},
|
||||
"q1.cur_correct": {
|
||||
"background": "FFFF66",
|
||||
"palette": "yellow",
|
||||
@ -2043,7 +2059,7 @@ FORM1_COLORS: dict = {
|
||||
"palette": "green",
|
||||
},
|
||||
"upper_ssp_smeta.contract_summary.deadline": {
|
||||
"palette": None,
|
||||
"palette": "green",
|
||||
},
|
||||
"allocation": {
|
||||
"background": "bfbfbf",
|
||||
@ -2081,6 +2097,10 @@ FORM1_COLORS: dict = {
|
||||
"empty_upper.ckk.q4": {
|
||||
"background": "748c42",
|
||||
},
|
||||
"empty_upper.ckk.deadline": {
|
||||
"background": "fabf8f",
|
||||
"palette": "orange",
|
||||
},
|
||||
"empty_upper.contract_detail": {
|
||||
"background": "fabf8f",
|
||||
"palette": "orange",
|
||||
@ -2097,6 +2117,10 @@ FORM1_COLORS: dict = {
|
||||
"empty_upper.contract_detail.q4": {
|
||||
"background": "748c42",
|
||||
},
|
||||
"empty_upper.contract_detail.deadline": {
|
||||
"background": "fabf8f",
|
||||
"palette": "orange",
|
||||
},
|
||||
"q1.cur_correct": {
|
||||
"background": "FFFF66",
|
||||
"palette": "yellow",
|
||||
@ -2582,7 +2606,7 @@ FORM1_COLORS: dict = {
|
||||
"palette": "orange",
|
||||
},
|
||||
"upper_ssp_smeta.contract_summary.deadline": {
|
||||
"palette": None,
|
||||
"palette": "orange",
|
||||
},
|
||||
"allocation": {
|
||||
"background": "bfbfbf",
|
||||
@ -2620,6 +2644,10 @@ FORM1_COLORS: dict = {
|
||||
"empty_upper.ckk.q4": {
|
||||
"background": "748c42",
|
||||
},
|
||||
"empty_upper.ckk.deadline": {
|
||||
"background": "fabf8f",
|
||||
"palette": "orange",
|
||||
},
|
||||
"empty_upper.contract_detail": {
|
||||
"background": "fabf8f",
|
||||
"palette": "orange",
|
||||
@ -2636,6 +2664,10 @@ FORM1_COLORS: dict = {
|
||||
"empty_upper.contract_detail.q4": {
|
||||
"background": "748c42",
|
||||
},
|
||||
"empty_upper.contract_detail.deadline": {
|
||||
"background": "fabf8f",
|
||||
"palette": "orange",
|
||||
},
|
||||
"q1.cur_correct": {
|
||||
"background": "FFFF66",
|
||||
"palette": "yellow",
|
||||
@ -3098,6 +3130,262 @@ FORM1_COLORS: dict = {
|
||||
}
|
||||
|
||||
|
||||
FORM4_COLUMN_RENAMES = {
|
||||
"data": "header",
|
||||
"header.section": "header.section_code",
|
||||
"sequestration": "seq_dfip",
|
||||
"allocation.order": "allocation.contract_ref",
|
||||
"allocation.property": "allocation.allocation_purpose",
|
||||
"empty_upper.contract_detail": "empty_upper.contract",
|
||||
"contract_detail.payment_scheme": "contract.scheme",
|
||||
"empty_upper.contract.payment_scheme": "empty_upper.contract.scheme",
|
||||
"q1.rem_correct.rem_booking": "q1.rem_correct.residual_after_booking",
|
||||
"q1.rem_actual_correct.rem_actual": "q1.rem_actual_correct.residual_after_actual",
|
||||
"q1.close_q1.economy": "q1.close_q1.transfer_econ",
|
||||
"q2.rev_eco_correct.rev_eco": "q2.rev_eco_correct.target_change",
|
||||
"q2.rev_item_correct.rev_item": "q2.rev_item_correct.base_correction",
|
||||
"q2.rev_inc_correct.rev_inc": "q2.rev_inc_correct.revision_inc",
|
||||
"q2.rev_seq_correct.rev_seq": "q2.rev_seq_correct.revision_seq",
|
||||
"q2.rev_correct.rev_comment": "q2.rev_correct.revision_comment",
|
||||
"q2.rem_correct.rem_booking": "q2.rem_correct.residual_after_booking",
|
||||
"q2.rem_actual_correct.rem_actual": "q2.rem_actual_correct.residual_after_actual",
|
||||
"q2.close_q2.economy": "q2.close_q2.transfer_econ",
|
||||
"q3.rev_eco_correct.rev_eco": "q3.rev_eco_correct.target_change",
|
||||
"q3.rev_item_correct.rev_item": "q3.rev_item_correct.base_correction",
|
||||
"q3.rev_inc_correct.rev_inc": "q3.rev_inc_correct.revision_inc",
|
||||
"q3.rev_seq_correct.rev_seq": "q3.rev_seq_correct.revision_seq",
|
||||
"q3.rev_correct.rev_comment": "q3.rev_correct.revision_comment",
|
||||
"q3.rem_correct.rem_booking": "q3.rem_correct.residual_after_booking",
|
||||
"q3.rem_actual_correct.rem_actual": "q3.rem_actual_correct.residual_after_actual",
|
||||
"q3.close_q3.economy": "q3.close_q3.transfer_econ",
|
||||
"q4.rev_eco_correct.rev_eco": "q4.rev_eco_correct.target_change",
|
||||
"q4.rev_item_correct.rev_item": "q4.rev_item_correct.base_correction",
|
||||
"q4.rev_inc_correct.rev_inc": "q4.rev_inc_correct.revision_inc",
|
||||
"q4.rev_seq_correct.rev_seq": "q4.rev_seq_correct.revision_seq",
|
||||
"q4.rev_correct.rev_comment": "q4.rev_correct.revision_comment",
|
||||
"q4.rem_correct.rem_booking": "q4.rem_correct.residual_after_booking",
|
||||
"q4.rem_actual_correct.rem_actual": "q4.rem_actual_correct.residual_after_actual",
|
||||
"q4.close_q4.economy": "q4.close_q4.transfer_econ",
|
||||
}
|
||||
|
||||
FORM4_STRUCTURE_COLUMN_RENAMES = {
|
||||
**FORM4_COLUMN_RENAMES,
|
||||
"upper_ssp_smeta.contract_summary.total": "upper_ssp_smeta.contract_summary.amount",
|
||||
"upper_ssp_smeta.contract_summary.future_y1": "upper_ssp_smeta.contract_summary.future_payments_y1",
|
||||
"upper_ssp_smeta.contract_summary.future_y2": "upper_ssp_smeta.contract_summary.future_payments_y2",
|
||||
"upper_ssp_smeta.contract_summary.other_ssp": "upper_ssp_smeta.contract_summary.other_ssp_amount",
|
||||
"empty_upper.collegial.approved": "empty_upper.collegial.amount",
|
||||
"empty_upper.collegial.protocol": "empty_upper.collegial.protocol_reference",
|
||||
"empty_upper.ckk.q1": "empty_upper.ckk.expenses_q1",
|
||||
"empty_upper.ckk.q2": "empty_upper.ckk.expenses_q2",
|
||||
"empty_upper.ckk.q3": "empty_upper.ckk.expenses_q3",
|
||||
"empty_upper.ckk.q4": "empty_upper.ckk.expenses_q4",
|
||||
"empty_upper.ckk.deadline": "empty_upper.ckk.delivery_deadline",
|
||||
"empty_upper.ckk.proc_plan": "empty_upper.ckk.procurement_plan",
|
||||
"empty_upper.ckk.proc_method": "empty_upper.ckk.procurement_method",
|
||||
"empty_upper.contract_detail.ceiling": "empty_upper.contract.ceiling",
|
||||
"empty_upper.contract_detail.q1": "empty_upper.contract.expenses_q1",
|
||||
"empty_upper.contract_detail.q2": "empty_upper.contract.expenses_q2",
|
||||
"empty_upper.contract_detail.q3": "empty_upper.contract.expenses_q3",
|
||||
"empty_upper.contract_detail.q4": "empty_upper.contract.expenses_q4",
|
||||
"empty_upper.contract_detail.payment_scheme": "empty_upper.contract.scheme",
|
||||
"empty_upper.contract.q1": "empty_upper.contract.expenses_q1",
|
||||
"empty_upper.contract.q2": "empty_upper.contract.expenses_q2",
|
||||
"empty_upper.contract.q3": "empty_upper.contract.expenses_q3",
|
||||
"empty_upper.contract.q4": "empty_upper.contract.expenses_q4",
|
||||
"empty_upper.contract.payment_scheme": "empty_upper.contract.scheme",
|
||||
}
|
||||
|
||||
|
||||
def _rename_depth_columns(columns: dict, renames: dict[str, str]) -> dict:
|
||||
def convert_node(node: dict, path: str) -> dict:
|
||||
converted = copy.deepcopy(node)
|
||||
children = converted.get("children")
|
||||
if isinstance(children, dict):
|
||||
converted["children"] = convert_children(children, path)
|
||||
return converted
|
||||
|
||||
def convert_children(children: dict, parent_path: str = "") -> dict:
|
||||
converted: dict = {}
|
||||
for key, value in children.items():
|
||||
old_path = f"{parent_path}.{key}" if parent_path else key
|
||||
new_path = renames.get(old_path, old_path)
|
||||
new_key = new_path.rsplit(".", 1)[-1]
|
||||
converted[new_key] = convert_node(value, new_path)
|
||||
return converted
|
||||
|
||||
return convert_children(columns)
|
||||
|
||||
|
||||
def _rename_path(path: str, renames: dict[str, str]) -> str:
|
||||
result = path
|
||||
for _ in range(4):
|
||||
changed = False
|
||||
for old, new in sorted(renames.items(), key=lambda item: len(item[0]), reverse=True):
|
||||
if result == old:
|
||||
result = new
|
||||
changed = True
|
||||
break
|
||||
prefix = f"{old}."
|
||||
if result.startswith(prefix):
|
||||
result = f"{new}.{result[len(prefix):]}"
|
||||
changed = True
|
||||
break
|
||||
if not changed:
|
||||
return result
|
||||
return result
|
||||
|
||||
|
||||
def _rename_color_paths(colors: dict, renames: dict[str, str]) -> dict:
|
||||
renamed: dict = {}
|
||||
for path, value in colors.items():
|
||||
renamed[_rename_path(path, renames)] = value
|
||||
return renamed
|
||||
|
||||
|
||||
FORM4_DEPTH_COLUMNS: dict = _rename_depth_columns(
|
||||
FORM1_DEPTH_COLUMNS,
|
||||
FORM4_COLUMN_RENAMES,
|
||||
)
|
||||
FORM4_STRUCTURE_DEPTH_COLUMNS: dict = _rename_depth_columns(
|
||||
FORM1_DEPTH_COLUMNS,
|
||||
FORM4_STRUCTURE_COLUMN_RENAMES,
|
||||
)
|
||||
FORM4_COLORS: dict = {
|
||||
sheet: _rename_color_paths(colors, FORM4_COLUMN_RENAMES)
|
||||
for sheet, colors in FORM1_COLORS.items()
|
||||
}
|
||||
FORM4_STRUCTURE_COLORS: dict = _rename_color_paths(
|
||||
FORM1_COLORS["AHR"],
|
||||
FORM4_STRUCTURE_COLUMN_RENAMES,
|
||||
)
|
||||
|
||||
|
||||
def _reorder_children(children: dict, order: tuple[str, ...]) -> dict:
|
||||
return {
|
||||
**{key: children[key] for key in order if key in children},
|
||||
**{key: value for key, value in children.items() if key not in order},
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def _patch_form4_regular_columns() -> None:
|
||||
for columns in (FORM4_DEPTH_COLUMNS, FORM4_STRUCTURE_DEPTH_COLUMNS):
|
||||
columns["header"].pop("key", None)
|
||||
|
||||
for columns in (FORM4_DEPTH_COLUMNS, FORM4_STRUCTURE_DEPTH_COLUMNS):
|
||||
for quarter in ("q1", "q2", "q3", "q4"):
|
||||
columns[quarter]["children"].pop("shift", None)
|
||||
columns["empty_upper"]["children"]["collegial"]["children"].pop("empty", None)
|
||||
|
||||
for key in ("item_id", "num_group", "internal_order"):
|
||||
FORM4_DEPTH_COLUMNS["header"]["children"].pop(key, None)
|
||||
FORM4_STRUCTURE_DEPTH_COLUMNS["header"]["children"].pop("justification", None)
|
||||
|
||||
for quarter in ("q2", "q3"):
|
||||
FORM4_DEPTH_COLUMNS[quarter]["children"].setdefault(
|
||||
"comment_correct",
|
||||
{
|
||||
"key": None,
|
||||
"name": "Комментарий",
|
||||
"children": {"adj_comment": {"name": "Сумма"}},
|
||||
},
|
||||
)
|
||||
FORM4_STRUCTURE_DEPTH_COLUMNS[quarter]["children"].setdefault(
|
||||
"comment_correct",
|
||||
{
|
||||
"key": None,
|
||||
"name": "Комментарий",
|
||||
"children": {"adj_comment": {"name": "Сумма"}},
|
||||
},
|
||||
)
|
||||
|
||||
for quarter in (1, 2, 3):
|
||||
FORM4_DEPTH_COLUMNS.pop(f"actual_rsbu_spod_q{quarter}", None)
|
||||
FORM4_DEPTH_COLUMNS["totals"]["children"].pop("pay_year", None)
|
||||
|
||||
|
||||
def _patch_form4_header_columns() -> None:
|
||||
for columns in (FORM4_DEPTH_COLUMNS, FORM4_STRUCTURE_DEPTH_COLUMNS):
|
||||
header = columns["header"]["children"]
|
||||
if "justification" not in header:
|
||||
header["justification"] = {"name": "Обоснование расхода"}
|
||||
|
||||
for colors in FORM4_COLORS.values():
|
||||
source_key = "header.name" if "header.name" in colors else "header.section_code"
|
||||
if source_key in colors:
|
||||
colors["header.justification"] = copy.deepcopy(colors[source_key])
|
||||
if "header.name" in FORM4_STRUCTURE_COLORS:
|
||||
FORM4_STRUCTURE_COLORS["header.justification"] = copy.deepcopy(
|
||||
FORM4_STRUCTURE_COLORS["header.name"]
|
||||
)
|
||||
|
||||
|
||||
def _patch_form4_contract_summary_columns() -> None:
|
||||
summary_parent = FORM4_DEPTH_COLUMNS["upper_ssp_smeta"]["children"]["contract_summary"]
|
||||
summary = summary_parent["children"]
|
||||
summary["reference"] = {"name": "Реквизиты договора"}
|
||||
summary["future_y2"] = {"name": "Платежи в 2026 году"}
|
||||
summary["centralized_flag"] = {"name": "Централизованный договор"}
|
||||
summary_parent["children"] = _reorder_children(
|
||||
summary,
|
||||
(
|
||||
"total", "reference", "counterparty", "deadline", "comment",
|
||||
"future_y1", "future_y2", "other_ssp", "centralized_flag",
|
||||
),
|
||||
)
|
||||
|
||||
structure_summary_parent = FORM4_STRUCTURE_DEPTH_COLUMNS["upper_ssp_smeta"]["children"]["contract_summary"]
|
||||
structure_summary = structure_summary_parent["children"]
|
||||
structure_summary["future_payments_y2"] = {"name": "Платежи в 2026 году"}
|
||||
structure_summary_parent["children"] = _reorder_children(
|
||||
structure_summary,
|
||||
(
|
||||
"amount", "counterparty", "deadline", "comment",
|
||||
"future_payments_y1", "future_payments_y2", "other_ssp_amount",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _patch_form4_actual_rsbu_spod_columns() -> None:
|
||||
totals = FORM4_DEPTH_COLUMNS.pop("totals")
|
||||
for quarter in range(1, 5):
|
||||
FORM4_DEPTH_COLUMNS[f"actual_rsbu_spod_q{quarter}"] = {
|
||||
"key": None,
|
||||
"name": f"Факт {quarter} кв" if quarter < 4 else "Факт 4 кв + СПОД",
|
||||
"children": {
|
||||
"actual_rsbu_spod": {
|
||||
"key": f"q{quarter}.actual_rsbu_spod",
|
||||
"name": "",
|
||||
},
|
||||
},
|
||||
}
|
||||
FORM4_DEPTH_COLUMNS["totals"] = totals
|
||||
|
||||
|
||||
_patch_form4_regular_columns()
|
||||
_patch_form4_header_columns()
|
||||
_patch_form4_contract_summary_columns()
|
||||
_patch_form4_actual_rsbu_spod_columns()
|
||||
_patch_form4_regular_columns()
|
||||
|
||||
FORM4_OPER_DEPTH_COLUMNS: dict = copy.deepcopy(FORM4_DEPTH_COLUMNS)
|
||||
FORM4_OPER_DEPTH_COLUMNS["empty_upper"]["children"]["ckk"]["children"]["procurement_plan"] = (
|
||||
FORM4_OPER_DEPTH_COLUMNS["empty_upper"]["children"]["ckk"]["children"].pop("proc_plan")
|
||||
)
|
||||
FORM4_OPER_DEPTH_COLUMNS["empty_upper"]["children"]["ckk"]["children"]["procurement_method"] = (
|
||||
FORM4_OPER_DEPTH_COLUMNS["empty_upper"]["children"]["ckk"]["children"].pop("proc_method")
|
||||
)
|
||||
FORM4_OPER_COLORS: dict = _rename_color_paths(
|
||||
FORM4_COLORS["OPER"],
|
||||
{
|
||||
"empty_upper.ckk.proc_plan": "empty_upper.ckk.procurement_plan",
|
||||
"empty_upper.ckk.proc_method": "empty_upper.ckk.procurement_method",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
SMETA_COLUMNS: list[tuple[str, str]] = [
|
||||
("section_code", "Код раздела"),
|
||||
("name", "Наименование"),
|
||||
@ -3527,3 +3815,173 @@ FORM3_COLORS: dict = {
|
||||
"q4.ekonomiya": {"background": "C2D69B", "palette": "green"},
|
||||
"q4.ekonomiya.economy": {"background": "C2D69B", "palette": "green"},
|
||||
}
|
||||
|
||||
|
||||
# Маппинги свода
|
||||
def build_summary1_columns(sheet: str) -> dict:
|
||||
metric_labels = {
|
||||
"plan": "План",
|
||||
"approved": "Утверждено",
|
||||
"fact": "Факт",
|
||||
"corrected": "Скорректированный план",
|
||||
}
|
||||
direction_labels = {"support": "Поддержка", "development": "Развитие"}
|
||||
period_labels = {
|
||||
"q1": "I кв.", "q2": "II кв.", "q3": "III кв.",
|
||||
"q4": "IV кв.", "year": "Год",
|
||||
}
|
||||
directions = ["support", "development"]
|
||||
if sheet == "FORM_2":
|
||||
directions = ["support"]
|
||||
elif sheet in {"FORM_3", "FORM_4"}:
|
||||
directions = ["development"]
|
||||
|
||||
columns: dict = {
|
||||
"section_code": {"name": "Код"},
|
||||
"name": {"name": "Статья расходов"},
|
||||
}
|
||||
for metric, metric_label in metric_labels.items():
|
||||
periods = ["q2", "q3", "q4"] if metric == "corrected" else list(period_labels)
|
||||
children = {
|
||||
direction: {
|
||||
"name": direction_labels[direction],
|
||||
"children": {
|
||||
period: {"name": period_labels[period]} for period in periods
|
||||
},
|
||||
}
|
||||
for direction in directions
|
||||
}
|
||||
if metric != "corrected" and sheet in {"FORM_1", "MAIN"}:
|
||||
children["total_year"] = {"name": "Итого за год"}
|
||||
columns[metric] = {"name": metric_label, "children": children}
|
||||
return columns
|
||||
|
||||
|
||||
def _quarter_columns() -> dict:
|
||||
months = {
|
||||
1: ("Январь", "Февраль", "Март"),
|
||||
2: ("Апрель", "Май", "Июнь"),
|
||||
3: ("Июль", "Август", "Сентябрь"),
|
||||
4: ("Октябрь", "Ноябрь", "Декабрь"),
|
||||
}
|
||||
result: dict = {}
|
||||
for quarter in range(1, 5):
|
||||
children = {
|
||||
"adj_current": {"name": "Текущие корректировки (= 0)"},
|
||||
"adj_ssp": {"name": "Корректировки с ССП / сметой развития"},
|
||||
"adj_rf": {"name": "Корректировки с РФ"},
|
||||
"adj_reserve": {"name": "Корректировки из резерва"},
|
||||
"payment_date": {"name": "Дата платежа"},
|
||||
"payment_amount": {"name": "Сумма платежа, тыс. руб. (без НДС)"},
|
||||
"payment_amount_ho": {"name": "в т.ч. сумма ГО (без НДС)"},
|
||||
"payment_amount_rf": {"name": "в т.ч. сумма РФ (без НДС)"},
|
||||
"payment_act": {"name": "Предоставление акта"},
|
||||
"booking_amount": {"name": f"Бронь {quarter} кв."},
|
||||
}
|
||||
for month_number, month in enumerate(months[quarter], start=1):
|
||||
children[f"actual_m{month_number}"] = {"name": f"Факт {month}"}
|
||||
if quarter == 4:
|
||||
children["actual_spod"] = {"name": "Факт СПОД"}
|
||||
if quarter > 1:
|
||||
children.update(
|
||||
{
|
||||
"plan_revision_eco_change": {
|
||||
"name": "Изменение целевого назначения перенесенной экономии (= 0)"
|
||||
},
|
||||
"plan_revision_item_adj": {
|
||||
"name": "Корректировка статей базового плана (= 0)"
|
||||
},
|
||||
"plan_revision_increase": {
|
||||
"name": "Увеличение базового плана (> 0)"
|
||||
},
|
||||
"plan_revision_sequester": {
|
||||
"name": "Секвестр базового плана (< 0)"
|
||||
},
|
||||
}
|
||||
)
|
||||
for target_quarter in range(quarter + 1, 5):
|
||||
children[f"transfer_to_q{target_quarter}"] = {
|
||||
"name": f"Перенос в {target_quarter} кв."
|
||||
}
|
||||
if quarter < 4:
|
||||
children["transfer_to_economy"] = {"name": "Перенос в фонд экономии"}
|
||||
result[f"q{quarter}"] = {
|
||||
"name": f"{quarter} квартал",
|
||||
"children": children,
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def build_summary2_columns(sheet: str) -> dict:
|
||||
estimate_title = (
|
||||
"Смета расходов ГО в разрезе договоров"
|
||||
if sheet == "GO"
|
||||
else "Смета расходов РФ в разрезе договоров"
|
||||
)
|
||||
columns: dict = {
|
||||
"smeta_type": {"key": "header.smeta_type", "name": "Вид сметы"},
|
||||
"smeta_direction": {"key": "header.smeta_direction", "name": "Направление"},
|
||||
"org_name": {"key": "header.org_name", "name": "ССП"},
|
||||
"name": {"key": "header.name", "name": "Наименование статьи"},
|
||||
"estimate": {
|
||||
"key": None,
|
||||
"name": estimate_title,
|
||||
"children": {
|
||||
"item_id": {"key": "header.item_id", "name": "ID статьи"},
|
||||
"num_group": {"key": "header.num_group", "name": "ID группы номенклатуры"},
|
||||
"justification": {"key": "header.justification", "name": "Конкретный вид расхода"},
|
||||
**{
|
||||
f"plan_q{q}": {"key": f"plan.plan_q{q}", "name": f"{q} кв."}
|
||||
for q in range(1, 5)
|
||||
},
|
||||
},
|
||||
},
|
||||
"allocation": {
|
||||
"name": "Аллокация расходов",
|
||||
"children": {
|
||||
"internal_order": {"name": "Внутренний заказ"},
|
||||
"property_object": {"name": "Объект недвижимости"},
|
||||
},
|
||||
},
|
||||
"sequestration": {
|
||||
"name": "Секвестирование ДФиП",
|
||||
"children": {
|
||||
"DFIP": {
|
||||
"name": "ДФиП",
|
||||
"children": {
|
||||
f"adj_q{q}": {"name": f"{q} кв."} for q in range(1, 5)
|
||||
},
|
||||
}
|
||||
},
|
||||
},
|
||||
"reserve": {
|
||||
"name": "Отнесение в резерв",
|
||||
"children": {
|
||||
f"amount_q{q}": {"name": f"{q} кв."} for q in range(1, 5)
|
||||
},
|
||||
},
|
||||
"approved": {
|
||||
"name": "Итоговая смета расходов",
|
||||
"children": {
|
||||
f"approved_q{q}": {"name": f"{q} кв."} for q in range(1, 5)
|
||||
},
|
||||
},
|
||||
"contract_detail": {
|
||||
"name": "Договор",
|
||||
"children": {
|
||||
"counterparty": {"name": "Контрагент"},
|
||||
"reference": {"name": "Договор (реквизиты)"},
|
||||
"subject": {"name": "Предмет"},
|
||||
"currency": {"name": "Валюта договора"},
|
||||
"ceiling_amount": {"name": "Предельная стоимость, тыс. руб. (без НДС)"},
|
||||
**{
|
||||
f"expenses_q{q}": {"name": f"Расходы {q} кв., тыс. руб. (без НДС)"}
|
||||
for q in range(1, 5)
|
||||
},
|
||||
"vat_rate": {"name": "Ставка НДС (%)"},
|
||||
"deadline": {"name": "Срок"},
|
||||
},
|
||||
},
|
||||
}
|
||||
columns.update(_quarter_columns())
|
||||
return columns
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
SMETA_VALUE_PATHS: list[tuple[str, tuple[str, ...]]] = [
|
||||
@ -72,6 +73,8 @@ FORM1_METRIC_BLOCK_KEYS = (
|
||||
|
||||
class ExcelValueSerializer:
|
||||
def to_excel_value(self, value: Any) -> Any:
|
||||
if isinstance(value, Decimal):
|
||||
return float(value)
|
||||
if value is None or isinstance(value, (str, int, float, bool)):
|
||||
return value
|
||||
if isinstance(value, dict):
|
||||
|
||||
@ -1,19 +1,28 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import zipfile
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from functools import partial
|
||||
from time import perf_counter
|
||||
from typing import Any, BinaryIO
|
||||
from urllib.parse import quote
|
||||
|
||||
from openpyxl import Workbook
|
||||
from openpyxl.worksheet.worksheet import Worksheet
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from xlsxwriter import Workbook as XlsxWorkbook
|
||||
|
||||
from src.db.models.app_user import AppUser
|
||||
from src.db.models.form_type import FormTypeEnum
|
||||
from src.db.session import db_session
|
||||
from src.services.budget_form_service import BudgetFormService
|
||||
from src.services.export_service.export_normalizers import (
|
||||
ExcelValueSerializer,
|
||||
@ -26,19 +35,32 @@ from src.services.export_service.export_mappings import (
|
||||
FORM1_DEPTH_COLUMNS,
|
||||
FORM2_DEPTH_AHR_SUB_COLUMNS,
|
||||
FORM3_COLORS,
|
||||
FORM4_COLORS,
|
||||
FORM4_DEPTH_COLUMNS,
|
||||
FORM4_OPER_COLORS,
|
||||
FORM4_OPER_DEPTH_COLUMNS,
|
||||
FORM4_STRUCTURE_COLORS,
|
||||
FORM4_STRUCTURE_DEPTH_COLUMNS,
|
||||
FORM3_CURRENT_EXPENSES_DEPTH_COLUMNS,
|
||||
FORM3_LIMIT_DEPTH_COLUMNS,
|
||||
PALETTE,
|
||||
SMETA_COLUMNS,
|
||||
build_summary1_columns,
|
||||
build_summary2_columns,
|
||||
)
|
||||
from src.services.export_service.export_writers import (
|
||||
DepthSheetWriter,
|
||||
TabularSheetWriter,
|
||||
XlsxDepthSheetWriter,
|
||||
XlsxTabularSheetWriter,
|
||||
)
|
||||
from src.services.project_service import ProjectService
|
||||
from src.services.sheet_service import SheetService
|
||||
from src.services.summary_service import SummaryService
|
||||
|
||||
XLSX_MEDIA_TYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
||||
ZIP_MEDIA_TYPE = "application/zip"
|
||||
SUMMARY_EXPORT_COLUMN_WIDTH = 15
|
||||
FORM1_DIRECTION_REQUIRED_SHEETS = {"AHR", "CAP"}
|
||||
SHEETS_WITH_SECTIONS = {"AHR", "CAP", "OPER"}
|
||||
|
||||
@ -56,6 +78,11 @@ SHEET_NAMES = {
|
||||
'CURRENT_EXPENSES': 'Текущие расходы',
|
||||
}
|
||||
SKIP_SHEETS = ('OTCH9F',)
|
||||
BULK_EXPORT_CONCURRENCY = 2
|
||||
EXPORT_TMPDIR = os.getenv("EXPORT_TMPDIR") or tempfile.gettempdir()
|
||||
|
||||
logger = logging.getLogger()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ExportSheetPlan:
|
||||
@ -71,6 +98,7 @@ class ExportService:
|
||||
self.budget_form_service = BudgetFormService(db)
|
||||
self.sheet_service = SheetService(db)
|
||||
self.project_service = ProjectService(db)
|
||||
self.summary_service = SummaryService(db)
|
||||
self.value_serializer = ExcelValueSerializer()
|
||||
self.form1_row_normalizer = Form1DepthRowNormalizer(self.value_serializer)
|
||||
self.form3_row_normalizer = Form3DepthRowNormalizer(self.value_serializer)
|
||||
@ -120,10 +148,10 @@ class ExportService:
|
||||
value_serializer=self.value_serializer.to_excel_value,
|
||||
),
|
||||
"STRUCTURE": DepthSheetWriter(
|
||||
columns_dict=FORM1_DEPTH_COLUMNS,
|
||||
columns_dict=FORM4_STRUCTURE_DEPTH_COLUMNS,
|
||||
row_normalizer=self.form1_row_normalizer.normalize,
|
||||
value_serializer=self.value_serializer.to_excel_value,
|
||||
colors_config=FORM1_COLORS["AHR"],
|
||||
colors_config=FORM4_STRUCTURE_COLORS,
|
||||
),
|
||||
}
|
||||
|
||||
@ -134,15 +162,15 @@ class ExportService:
|
||||
sheet: str | None = None,
|
||||
direction: str | None = None,
|
||||
sections: list[str] | None = None,
|
||||
ignore_non_applicable_query_params: bool = False,
|
||||
) -> tuple[io.BytesIO, str, str]:
|
||||
ignore_unsupported_sheet_params: bool = False,
|
||||
) -> tuple[BinaryIO, str, str]:
|
||||
stream, filename = await self.export_form_to_xlsx(
|
||||
form_id=form_id,
|
||||
current_user=current_user,
|
||||
sheet=sheet,
|
||||
direction=direction,
|
||||
sections=sections,
|
||||
ignore_non_applicable_query_params=ignore_non_applicable_query_params,
|
||||
ignore_unsupported_sheet_params=ignore_unsupported_sheet_params,
|
||||
)
|
||||
return stream, filename, XLSX_MEDIA_TYPE
|
||||
|
||||
@ -153,7 +181,7 @@ class ExportService:
|
||||
report_type: str,
|
||||
current_user: AppUser,
|
||||
sections: list[str] | None = None,
|
||||
) -> tuple[io.BytesIO, str, str]:
|
||||
) -> tuple[BinaryIO, str, str]:
|
||||
stream, filename = await self.export_project_report_to_xlsx(
|
||||
project_id=project_id,
|
||||
year=year,
|
||||
@ -167,13 +195,74 @@ class ExportService:
|
||||
self,
|
||||
project_id: int,
|
||||
current_user: AppUser,
|
||||
) -> tuple[io.BytesIO, str, str]:
|
||||
) -> tuple[BinaryIO, str, str]:
|
||||
stream, filename = await self.export_project_to_xlsx(
|
||||
project_id=project_id,
|
||||
current_user=current_user,
|
||||
)
|
||||
return stream, filename, XLSX_MEDIA_TYPE
|
||||
|
||||
async def export_summary_payload(
|
||||
self,
|
||||
summary_number: int,
|
||||
year: int,
|
||||
sheet: str,
|
||||
current_user: AppUser,
|
||||
org_ids: list[int] | None = None,
|
||||
sort_by: str = "org_name",
|
||||
sort_direction: str = "asc",
|
||||
) -> tuple[BinaryIO, str, str]:
|
||||
export_started_at = perf_counter()
|
||||
load_started_at = perf_counter()
|
||||
if summary_number == 1:
|
||||
rows = await self.summary_service.get_svod1_rows(
|
||||
year=year,
|
||||
sheet=sheet,
|
||||
user=current_user,
|
||||
org_ids=org_ids,
|
||||
)
|
||||
columns = build_summary1_columns(sheet)
|
||||
elif sheet == "GO":
|
||||
rows = await self.summary_service.get_svod2_go_export_rows(
|
||||
year=year,
|
||||
user=current_user,
|
||||
org_ids=org_ids,
|
||||
sort_by=sort_by,
|
||||
sort_direction=sort_direction,
|
||||
)
|
||||
columns = build_summary2_columns(sheet)
|
||||
else:
|
||||
rows = await self.summary_service.get_svod2_rows(
|
||||
year=year,
|
||||
sheet=sheet,
|
||||
user=current_user,
|
||||
org_ids=org_ids,
|
||||
offset=None,
|
||||
limit=None,
|
||||
sort_by=sort_by,
|
||||
sort_direction=sort_direction,
|
||||
)
|
||||
columns = build_summary2_columns(sheet)
|
||||
logger.info(
|
||||
f"[export timing] summary={summary_number} sheet={sheet} year={year}: "
|
||||
f"load_rows={perf_counter() - load_started_at:.3f}s, rows={len(rows)}",
|
||||
)
|
||||
stream = await self._export_summary_rows(
|
||||
rows=rows,
|
||||
columns=columns,
|
||||
metadata=(("Свод", summary_number), ("Лист", sheet), ("Год", year)),
|
||||
worksheet_name=f"Свод {summary_number} {sheet}",
|
||||
summary_number=summary_number,
|
||||
year=year,
|
||||
sheet=sheet,
|
||||
)
|
||||
logger.info(
|
||||
f"[export timing] summary={summary_number} sheet={sheet} year={year}: "
|
||||
f"total={perf_counter() - export_started_at:.3f}s",
|
||||
)
|
||||
filename = self._build_summary_filename(summary_number, sheet, year)
|
||||
return stream, filename, XLSX_MEDIA_TYPE
|
||||
|
||||
async def export_bulk_payload(
|
||||
self,
|
||||
form_ids: list[int],
|
||||
@ -181,9 +270,10 @@ class ExportService:
|
||||
skip_failed: bool = True,
|
||||
direction: str | None = None,
|
||||
sections: list[str] | None = None,
|
||||
ignore_non_applicable_query_params: bool = False,
|
||||
) -> tuple[io.BytesIO, str, str]:
|
||||
archive_stream = io.BytesIO()
|
||||
ignore_unsupported_sheet_params: bool = False,
|
||||
) -> tuple[BinaryIO, str, str]:
|
||||
bulk_started_at = perf_counter()
|
||||
archive_stream = tempfile.TemporaryFile(mode="w+b", dir=EXPORT_TMPDIR)
|
||||
exported_count = 0
|
||||
errors: list[dict[str, Any]] = []
|
||||
used_names: set[str] = set()
|
||||
@ -193,22 +283,48 @@ class ExportService:
|
||||
mode="w",
|
||||
compression=zipfile.ZIP_DEFLATED,
|
||||
) as archive:
|
||||
for form_id in self._deduplicate_ids(form_ids):
|
||||
try:
|
||||
form_stream, filename = await self.export_form_to_xlsx(
|
||||
form_id=form_id,
|
||||
current_user=current_user,
|
||||
direction=direction,
|
||||
sections=sections,
|
||||
ignore_non_applicable_query_params=ignore_non_applicable_query_params,
|
||||
)
|
||||
unique_form_ids = self._deduplicate_ids(form_ids)
|
||||
for batch_start in range(0, len(unique_form_ids), BULK_EXPORT_CONCURRENCY):
|
||||
batch_ids = unique_form_ids[
|
||||
batch_start:batch_start + BULK_EXPORT_CONCURRENCY
|
||||
]
|
||||
batch_results = await asyncio.gather(
|
||||
*[
|
||||
self._export_bulk_form(
|
||||
form_id=form_id,
|
||||
current_user=current_user,
|
||||
direction=direction,
|
||||
sections=sections,
|
||||
ignore_unsupported_sheet_params=ignore_unsupported_sheet_params,
|
||||
)
|
||||
for form_id in batch_ids
|
||||
],
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
for form_id, result in zip(batch_ids, batch_results):
|
||||
if isinstance(result, BaseException):
|
||||
if not isinstance(result, ValueError) or not skip_failed:
|
||||
archive_stream.close()
|
||||
raise result
|
||||
errors.append({"form_id": form_id, "error": str(result)})
|
||||
continue
|
||||
|
||||
form_stream, filename, xlsx_elapsed = result
|
||||
xlsx_finished_at = perf_counter()
|
||||
archive_name = self._make_unique_filename(filename, used_names)
|
||||
archive.writestr(archive_name, form_stream.getvalue())
|
||||
try:
|
||||
form_stream.seek(0)
|
||||
with archive.open(archive_name, mode="w") as archive_entry:
|
||||
shutil.copyfileobj(form_stream, archive_entry, length=1024 * 1024)
|
||||
finally:
|
||||
form_stream.close()
|
||||
logger.info(
|
||||
f"[export timing] bulk form_id={form_id}: "
|
||||
f"xlsx={xlsx_elapsed:.3f}s, "
|
||||
f"zip_write={perf_counter() - xlsx_finished_at:.3f}s",
|
||||
)
|
||||
exported_count += 1
|
||||
except ValueError as exc:
|
||||
if not skip_failed:
|
||||
raise
|
||||
errors.append({"form_id": form_id, "error": str(exc)})
|
||||
|
||||
if errors:
|
||||
archive.writestr(
|
||||
@ -217,12 +333,37 @@ class ExportService:
|
||||
)
|
||||
|
||||
if exported_count == 0:
|
||||
archive_stream.close()
|
||||
raise ValueError("Не удалось экспортировать ни одной формы.")
|
||||
|
||||
archive_stream.seek(0)
|
||||
filename = f"export_forms_{datetime.now(timezone.utc).strftime('%Y-%m-%d')}.zip"
|
||||
logger.info(
|
||||
f"[export timing] bulk total: forms={exported_count}, "
|
||||
f"elapsed={perf_counter() - bulk_started_at:.3f}s",
|
||||
)
|
||||
return archive_stream, filename, ZIP_MEDIA_TYPE
|
||||
|
||||
async def _export_bulk_form(
|
||||
self,
|
||||
form_id: int,
|
||||
current_user: AppUser,
|
||||
direction: str | None,
|
||||
sections: list[str] | None,
|
||||
ignore_unsupported_sheet_params: bool,
|
||||
) -> tuple[BinaryIO, str, float]:
|
||||
started_at = perf_counter()
|
||||
async with db_session() as db:
|
||||
export_service = ExportService(db)
|
||||
stream, filename = await export_service.export_form_to_xlsx(
|
||||
form_id=form_id,
|
||||
current_user=current_user,
|
||||
direction=direction,
|
||||
sections=sections,
|
||||
ignore_unsupported_sheet_params=ignore_unsupported_sheet_params,
|
||||
)
|
||||
return stream, filename, perf_counter() - started_at
|
||||
|
||||
async def export_form_to_xlsx(
|
||||
self,
|
||||
form_id: int,
|
||||
@ -230,9 +371,17 @@ class ExportService:
|
||||
sheet: str | None = None,
|
||||
direction: str | None = None,
|
||||
sections: list[str] | None = None,
|
||||
ignore_non_applicable_query_params: bool = False,
|
||||
) -> tuple[io.BytesIO, str]:
|
||||
ignore_unsupported_sheet_params: bool = False,
|
||||
) -> tuple[BinaryIO, str]:
|
||||
export_started_at = perf_counter()
|
||||
|
||||
stage_started_at = perf_counter()
|
||||
form = await self._get_form_for_export(form_id=form_id, current_user=current_user)
|
||||
logger.info(
|
||||
f"[export timing] form_id={form_id}: load_form={perf_counter() - stage_started_at:.3f}s",
|
||||
)
|
||||
|
||||
stage_started_at = perf_counter()
|
||||
sheets = self._resolve_requested_sheets(form=form, requested_sheet=sheet)
|
||||
self._validate_sections_for_form(form=form, sections=sections)
|
||||
|
||||
@ -242,7 +391,7 @@ class ExportService:
|
||||
sheets=sheets,
|
||||
direction=direction,
|
||||
sections=sections,
|
||||
ignore_non_applicable_query_params=ignore_non_applicable_query_params,
|
||||
ignore_unsupported_sheet_params=ignore_unsupported_sheet_params,
|
||||
)
|
||||
else:
|
||||
sheets_by_directions = self._resolve_sheets_by_directions(form=form, requested_sheet=sheet)
|
||||
@ -253,12 +402,24 @@ class ExportService:
|
||||
sheets=shts,
|
||||
direction=dr,
|
||||
sections=sections,
|
||||
ignore_non_applicable_query_params=ignore_non_applicable_query_params,
|
||||
ignore_unsupported_sheet_params=ignore_unsupported_sheet_params,
|
||||
)
|
||||
logger.info(
|
||||
f"[export timing] form_id={form_id}: build_plan={perf_counter() - stage_started_at:.3f}s, "
|
||||
f"sheets={len(plans)}",
|
||||
)
|
||||
|
||||
|
||||
|
||||
workbook = self._create_workbook_with_metadata(form=form)
|
||||
# XlsxWriter хранит XML-части и итоговую книгу во временных файлах вместо RAM.
|
||||
stream = tempfile.TemporaryFile(mode="w+b", dir=EXPORT_TMPDIR)
|
||||
workbook = XlsxWorkbook(
|
||||
stream,
|
||||
{
|
||||
"tmpdir": EXPORT_TMPDIR,
|
||||
"strings_to_urls": False,
|
||||
},
|
||||
)
|
||||
self._write_xlsxwriter_metadata(workbook.add_worksheet("metadata"), form)
|
||||
sheet_writers = self._create_xlsxwriter_sheet_writers(workbook, form.form_type.code)
|
||||
exported_sheets = 0
|
||||
for plan in plans:
|
||||
if not plan.validation_direction and plan.sheet_name in SKIP_SHEETS:
|
||||
@ -270,27 +431,44 @@ class ExportService:
|
||||
direction=plan.validation_direction,
|
||||
sections=plan.validation_sections,
|
||||
)
|
||||
|
||||
# Этап 3: отдельно измеряем запрос данных каждого листа к БД.
|
||||
sheet_load_started_at = perf_counter()
|
||||
rows = await self.sheet_service.get(
|
||||
form_id=form_id,
|
||||
sheet=plan.sheet_name,
|
||||
direction=plan.effective_direction,
|
||||
sections=plan.effective_sections,
|
||||
)
|
||||
worksheet = workbook.create_sheet(title=self._safe_sheet_name(plan.sheet_name, plan.effective_direction))
|
||||
self._write_sheet_rows(
|
||||
worksheet=worksheet,
|
||||
rows=rows,
|
||||
sheet_name=plan.sheet_name,
|
||||
sheet_loaded_at = perf_counter()
|
||||
|
||||
# Этап 4: рендер листа уводим с event loop, чтобы экспорт не блокировал API.
|
||||
worksheet = workbook.add_worksheet(
|
||||
self._safe_sheet_name(plan.sheet_name, plan.effective_direction)
|
||||
)
|
||||
writer = sheet_writers.get(plan.sheet_name, sheet_writers["__default__"])
|
||||
await asyncio.to_thread(writer.write, worksheet, rows)
|
||||
logger.info(
|
||||
f"[export timing] form_id={form_id} sheet={plan.sheet_name} "
|
||||
f"direction={plan.effective_direction or '-'} rows={len(rows)}: "
|
||||
f"load_rows={sheet_loaded_at - sheet_load_started_at:.3f}s, "
|
||||
f"write_sheet={perf_counter() - sheet_loaded_at:.3f}s",
|
||||
)
|
||||
exported_sheets += 1
|
||||
|
||||
if exported_sheets == 0:
|
||||
raise ValueError("Не удалось экспортировать листы формы.")
|
||||
|
||||
stream = io.BytesIO()
|
||||
workbook.save(stream)
|
||||
workbook.close()
|
||||
# Этап 5: сериализация всей книги в XLSX обычно заметна на больших формах.
|
||||
save_started_at = perf_counter()
|
||||
# Формирование XML и ZIP-контейнера XLSX также не должно блокировать event loop.
|
||||
await asyncio.to_thread(workbook.close)
|
||||
xlsx_size = stream.seek(0, io.SEEK_END)
|
||||
stream.seek(0)
|
||||
logger.info(
|
||||
f"[export timing] form_id={form_id}: save_xlsx={perf_counter() - save_started_at:.3f}s, "
|
||||
f"total={perf_counter() - export_started_at:.3f}s, size={xlsx_size} bytes",
|
||||
)
|
||||
filename = self._build_filename(form_id=form.id, form_type_code=form.form_type.code, year=form.year, sheet=sheet)
|
||||
return stream, filename
|
||||
|
||||
@ -482,20 +660,303 @@ class ExportService:
|
||||
self._fill_metadata(meta_sheet, form)
|
||||
return workbook
|
||||
|
||||
def _write_xlsxwriter_metadata(self, sheet: Any, form: Any) -> None:
|
||||
metadata_rows = [
|
||||
("Форма ID", form.id),
|
||||
("Тип формы", form.form_type_code),
|
||||
("Год", form.year),
|
||||
("ID ССП.РФ", form.org_unit_id),
|
||||
("ССП/РФ", form.org_unit.title if form.org_unit else None),
|
||||
("Экспортировано", datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")),
|
||||
]
|
||||
for row, (key, value) in enumerate(metadata_rows):
|
||||
sheet.write(row, 0, key)
|
||||
sheet.write(row, 1, value)
|
||||
|
||||
async def _export_summary_rows(
|
||||
self,
|
||||
rows: list[tuple],
|
||||
columns: dict,
|
||||
metadata: tuple[tuple[str, Any], ...],
|
||||
worksheet_name: str,
|
||||
summary_number: int,
|
||||
year: int,
|
||||
sheet: str,
|
||||
) -> BinaryIO:
|
||||
workbook_started_at = perf_counter()
|
||||
stream = tempfile.TemporaryFile(mode="w+b", dir=EXPORT_TMPDIR)
|
||||
workbook = XlsxWorkbook(
|
||||
stream,
|
||||
{"tmpdir": EXPORT_TMPDIR, "strings_to_urls": False},
|
||||
)
|
||||
try:
|
||||
metadata_sheet = workbook.add_worksheet("metadata")
|
||||
metadata_rows = (
|
||||
*metadata,
|
||||
("Строк", len(rows)),
|
||||
(
|
||||
"Экспортировано",
|
||||
datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S"),
|
||||
),
|
||||
)
|
||||
for row_number, (key, value) in enumerate(metadata_rows):
|
||||
metadata_sheet.write(row_number, 0, key)
|
||||
metadata_sheet.write(row_number, 1, value)
|
||||
|
||||
worksheet = workbook.add_worksheet(self._safe_sheet_name(worksheet_name))
|
||||
(
|
||||
header_format_resolver,
|
||||
row_format_resolver,
|
||||
column_format_resolver,
|
||||
) = (
|
||||
self._summary_format_resolvers(columns, summary_number)
|
||||
)
|
||||
writer = XlsxDepthSheetWriter(
|
||||
workbook=workbook,
|
||||
columns_dict=columns,
|
||||
row_normalizer=(
|
||||
self._identity_row
|
||||
if summary_number == 2 and sheet == "GO"
|
||||
else self.form1_row_normalizer.normalize
|
||||
),
|
||||
value_serializer=self.value_serializer.to_excel_value,
|
||||
header_format_resolver=header_format_resolver,
|
||||
row_format_resolver=row_format_resolver,
|
||||
column_format_resolver=column_format_resolver,
|
||||
fixed_column_width=SUMMARY_EXPORT_COLUMN_WIDTH,
|
||||
)
|
||||
logger.info(
|
||||
f"[export timing] summary={summary_number} sheet={sheet} year={year}: "
|
||||
f"build_workbook={perf_counter() - workbook_started_at:.3f}s",
|
||||
)
|
||||
write_started_at = perf_counter()
|
||||
await asyncio.to_thread(writer.write, worksheet, rows)
|
||||
logger.info(
|
||||
f"[export timing] summary={summary_number} sheet={sheet} year={year}: "
|
||||
f"write_sheet={perf_counter() - write_started_at:.3f}s",
|
||||
)
|
||||
save_started_at = perf_counter()
|
||||
await asyncio.to_thread(workbook.close)
|
||||
logger.info(
|
||||
f"[export timing] summary={summary_number} sheet={sheet} year={year}: "
|
||||
f"save_xlsx={perf_counter() - save_started_at:.3f}s",
|
||||
)
|
||||
except BaseException:
|
||||
stream.close()
|
||||
raise
|
||||
stream.seek(0)
|
||||
return stream
|
||||
|
||||
@staticmethod
|
||||
def _identity_row(row: dict[str, Any]) -> dict[str, Any]:
|
||||
return row
|
||||
|
||||
@staticmethod
|
||||
def _summary_format_resolvers(
|
||||
columns: dict,
|
||||
summary_number: int,
|
||||
) -> tuple[Any, Any, Any]:
|
||||
if summary_number == 1:
|
||||
return (
|
||||
ExportService._summary1_header_format,
|
||||
ExportService._summary1_row_format,
|
||||
None,
|
||||
)
|
||||
|
||||
roots = list(columns)
|
||||
band_by_root = {root: index % 2 for index, root in enumerate(roots)}
|
||||
leaf_bands: list[int] = []
|
||||
group_end_columns: set[int] = set()
|
||||
group_end_header_keys: set[str] = set()
|
||||
|
||||
for root in roots:
|
||||
ExportService._add_summary_leaf_bands(
|
||||
config={root: columns[root]},
|
||||
band=band_by_root[root],
|
||||
leaf_bands=leaf_bands,
|
||||
)
|
||||
group_end_columns.add(len(leaf_bands) - 1)
|
||||
group_end_header_keys.add(
|
||||
ExportService._summary_last_leaf_key(root, columns[root])
|
||||
)
|
||||
|
||||
return (
|
||||
partial(
|
||||
ExportService._summary2_header_format,
|
||||
band_by_root=band_by_root,
|
||||
roots=set(roots),
|
||||
group_end_header_keys=group_end_header_keys,
|
||||
),
|
||||
None,
|
||||
partial(
|
||||
ExportService._summary2_column_format,
|
||||
leaf_bands=leaf_bands,
|
||||
group_end_columns=group_end_columns,
|
||||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _summary1_header_format(_full_key: str) -> dict[str, Any]:
|
||||
return {
|
||||
"bg_color": "#F8F9FA",
|
||||
"pattern": 1,
|
||||
"font_color": "#4A5565",
|
||||
"bold": True,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _summary1_row_format(
|
||||
row: tuple,
|
||||
_column_key: str,
|
||||
_column: int,
|
||||
) -> dict[str, Any]:
|
||||
thin_border = {"right": 1, "right_color": "#EDF0F2"}
|
||||
data = row[3] if len(row) > 3 and isinstance(row[3], dict) else {}
|
||||
section_code = data.get("section_code")
|
||||
if section_code in (None, "") and isinstance(data.get("header"), dict):
|
||||
section_code = data["header"].get("section_code")
|
||||
if section_code in (None, ""):
|
||||
return thin_border
|
||||
|
||||
palette_name = {"1": "green", "3": "orange", "4": "blue"}.get(
|
||||
str(section_code)[:1],
|
||||
"blue",
|
||||
)
|
||||
item_type = row[0] if row else "INPUT"
|
||||
background = PALETTE[palette_name].get(item_type, "FFFFFF")
|
||||
red, green, blue = (
|
||||
int(background[index:index + 2], 16) for index in (0, 2, 4)
|
||||
)
|
||||
is_dark = 0.299 * red + 0.587 * green + 0.114 * blue < 128
|
||||
return {
|
||||
"bg_color": f"#{background}",
|
||||
"pattern": 1,
|
||||
"font_color": "#FFFFFF" if is_dark else "#000000",
|
||||
**thin_border,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _add_summary_leaf_bands(
|
||||
config: dict,
|
||||
band: int,
|
||||
leaf_bands: list[int],
|
||||
) -> None:
|
||||
for child in config.values():
|
||||
if "children" in child:
|
||||
ExportService._add_summary_leaf_bands(
|
||||
child["children"],
|
||||
band,
|
||||
leaf_bands,
|
||||
)
|
||||
else:
|
||||
leaf_bands.append(band)
|
||||
|
||||
@staticmethod
|
||||
def _summary_last_leaf_key(key: str, config: dict) -> str:
|
||||
if "children" not in config:
|
||||
return key
|
||||
child_key, child_config = next(reversed(config["children"].items()))
|
||||
return ExportService._summary_last_leaf_key(
|
||||
f"{key}.{child_key}",
|
||||
child_config,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _summary2_header_format(
|
||||
full_key: str,
|
||||
*,
|
||||
band_by_root: dict[str, int],
|
||||
roots: set[str],
|
||||
group_end_header_keys: set[str],
|
||||
) -> dict[str, Any]:
|
||||
band = band_by_root.get(full_key.split(".", 1)[0], 0)
|
||||
result = {
|
||||
"bg_color": ("#F8F9FA", "#F1F7F3")[band],
|
||||
"pattern": 1,
|
||||
"font_color": "#4A5565",
|
||||
"bold": True,
|
||||
}
|
||||
if full_key in roots or full_key in group_end_header_keys:
|
||||
result.update({"right": 2, "right_color": "#9EAFA3"})
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _summary2_column_format(
|
||||
_column_key: str,
|
||||
column: int,
|
||||
*,
|
||||
leaf_bands: list[int],
|
||||
group_end_columns: set[int],
|
||||
) -> dict[str, Any]:
|
||||
is_group_end = column in group_end_columns
|
||||
return {
|
||||
"bg_color": ("#FFFFFF", "#F8FBF9")[leaf_bands[column]],
|
||||
"pattern": 1,
|
||||
"right": 2 if is_group_end else 1,
|
||||
"right_color": "#9EAFA3" if is_group_end else "#EDF0F2",
|
||||
}
|
||||
|
||||
def _create_xlsxwriter_sheet_writers(
|
||||
self,
|
||||
workbook: XlsxWorkbook,
|
||||
form_type_code: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
default_writer = XlsxDepthSheetWriter(
|
||||
workbook=workbook,
|
||||
columns_dict=FORM1_DEPTH_COLUMNS,
|
||||
row_normalizer=self.form1_row_normalizer.normalize,
|
||||
value_serializer=self.value_serializer.to_excel_value,
|
||||
)
|
||||
writers: dict[str, Any] = {
|
||||
"__default__": default_writer,
|
||||
"SMETA": XlsxTabularSheetWriter(
|
||||
workbook=workbook,
|
||||
columns=SMETA_COLUMNS,
|
||||
row_normalizer=self.smeta_row_normalizer.normalize,
|
||||
value_serializer=self.value_serializer.to_excel_value,
|
||||
),
|
||||
}
|
||||
if form_type_code == FormTypeEnum.FORM_4.value:
|
||||
writer_configs = {
|
||||
"OPER": (FORM4_OPER_DEPTH_COLUMNS, FORM4_OPER_COLORS),
|
||||
"AHR": (FORM4_DEPTH_COLUMNS, FORM4_COLORS["AHR"]),
|
||||
"CAP": (FORM4_DEPTH_COLUMNS, FORM4_COLORS["CAP"]),
|
||||
"STRUCTURE": (FORM4_STRUCTURE_DEPTH_COLUMNS, FORM4_STRUCTURE_COLORS),
|
||||
}
|
||||
else:
|
||||
writer_configs = {
|
||||
"OPER": (FORM1_DEPTH_COLUMNS, FORM1_COLORS["OPER"]),
|
||||
"AHR": (FORM1_DEPTH_COLUMNS, FORM1_COLORS["AHR"]),
|
||||
"CAP": (FORM1_DEPTH_COLUMNS, FORM1_COLORS["CAP"]),
|
||||
"AHR_RENT": (FORM2_DEPTH_AHR_SUB_COLUMNS, None),
|
||||
"AHR_UTILITY": (FORM2_DEPTH_AHR_SUB_COLUMNS, None),
|
||||
"AHR_SECURITY": (FORM2_DEPTH_AHR_SUB_COLUMNS, None),
|
||||
"STRUCTURE": (FORM4_STRUCTURE_DEPTH_COLUMNS, FORM4_STRUCTURE_COLORS),
|
||||
}
|
||||
for sheet_name, (columns, colors) in writer_configs.items():
|
||||
writers[sheet_name] = XlsxDepthSheetWriter(
|
||||
workbook=workbook,
|
||||
columns_dict=columns,
|
||||
row_normalizer=self.form1_row_normalizer.normalize,
|
||||
value_serializer=self.value_serializer.to_excel_value,
|
||||
colors_config=colors,
|
||||
)
|
||||
return writers
|
||||
|
||||
def _build_sheet_plans(
|
||||
self,
|
||||
form: Any,
|
||||
sheets: list[str],
|
||||
direction: str | None,
|
||||
sections: list[str] | None,
|
||||
ignore_non_applicable_query_params: bool,
|
||||
ignore_unsupported_sheet_params: bool,
|
||||
) -> list[ExportSheetPlan]:
|
||||
plans: list[ExportSheetPlan] = []
|
||||
for sheet_name in sheets:
|
||||
effective_direction = direction if self._requires_direction(form.form_type.code, sheet_name) else None
|
||||
effective_sections = sections if sheet_name in SHEETS_WITH_SECTIONS else None
|
||||
validation_direction = effective_direction if ignore_non_applicable_query_params else direction
|
||||
validation_sections = effective_sections if ignore_non_applicable_query_params else sections
|
||||
validation_direction = effective_direction if ignore_unsupported_sheet_params else direction
|
||||
validation_sections = effective_sections if ignore_unsupported_sheet_params else sections
|
||||
plans.append(
|
||||
ExportSheetPlan(
|
||||
sheet_name=sheet_name,
|
||||
@ -549,6 +1010,11 @@ class ExportService:
|
||||
return f"{year}_{form_type_code}_{form_id}_{sheet}_{date_str}.xlsx"
|
||||
return f"{year}_{form_type_code}_{form_id}_{date_str}.xlsx"
|
||||
|
||||
@staticmethod
|
||||
def _build_summary_filename(summary: int, sheet: str, year: int) -> str:
|
||||
date_str = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
return f"{year}_SVOD_{summary}_{sheet}_{date_str}.xlsx"
|
||||
|
||||
@staticmethod
|
||||
def _deduplicate_ids(form_ids: list[int]) -> list[int]:
|
||||
seen: set[int] = set()
|
||||
|
||||
@ -7,6 +7,8 @@ from typing import Any, Callable
|
||||
from openpyxl.styles import Alignment, Border, Color, Font, PatternFill, Side
|
||||
from openpyxl.worksheet.worksheet import Worksheet
|
||||
from openpyxl.utils import get_column_letter
|
||||
from xlsxwriter import Workbook as XlsxWorkbook
|
||||
from xlsxwriter.worksheet import Worksheet as XlsxWorksheet
|
||||
|
||||
from src.services.export_service.export_mappings import PALETTE
|
||||
|
||||
@ -14,23 +16,28 @@ from src.services.export_service.export_mappings import PALETTE
|
||||
|
||||
RowNormalizer = Callable[[dict[str, Any]], dict[str, Any]]
|
||||
ValueSerializer = Callable[[Any], Any]
|
||||
HeaderFormatResolver = Callable[[str], dict[str, Any] | None]
|
||||
RowFormatResolver = Callable[[tuple, str, int], dict[str, Any] | None]
|
||||
ColumnFormatResolver = Callable[[str, int], dict[str, Any] | None]
|
||||
|
||||
|
||||
class AbstractWriter(abc.ABC):
|
||||
|
||||
def _set_widths(self, worksheet: Worksheet):
|
||||
for col in worksheet.columns:
|
||||
max_len = 0
|
||||
col_letter = get_column_letter(col[0].column) # Get alphabetical column name (e.g., 'A')
|
||||
|
||||
for cell in col:
|
||||
if cell.value is not None:
|
||||
cell_len = len(str(cell.value))
|
||||
if cell_len > max_len:
|
||||
max_len = cell_len
|
||||
|
||||
adjusted_width = max(max_len + 3, 10)
|
||||
worksheet.column_dimensions[col_letter].width = adjusted_width
|
||||
@staticmethod
|
||||
def _track_width(column_widths: dict[int, int], column: int, value: Any) -> None:
|
||||
if value is None:
|
||||
return
|
||||
column_widths[column] = max(column_widths.get(column, 0), len(str(value)))
|
||||
|
||||
def _track_row_widths(self, column_widths: dict[int, int], values: list[Any]) -> None:
|
||||
for column, value in enumerate(values, start=1):
|
||||
self._track_width(column_widths, column, value)
|
||||
|
||||
@staticmethod
|
||||
def _apply_widths(worksheet: Worksheet, column_widths: dict[int, int]) -> None:
|
||||
for column, max_len in column_widths.items():
|
||||
column_letter = get_column_letter(column)
|
||||
worksheet.column_dimensions[column_letter].width = max(max_len + 3, 10)
|
||||
|
||||
@abc.abstractmethod
|
||||
def write(self, *args, **kwargs):
|
||||
@ -49,14 +56,17 @@ class TabularSheetWriter(AbstractWriter):
|
||||
self.value_serializer = value_serializer
|
||||
|
||||
def write(self, worksheet: Worksheet, rows: list[tuple]) -> None:
|
||||
worksheet.append([title for _, title in self.columns])
|
||||
column_widths: dict[int, int] = {}
|
||||
headers = [title for _, title in self.columns]
|
||||
worksheet.append(headers)
|
||||
self._track_row_widths(column_widths, headers)
|
||||
for row in rows:
|
||||
raw_data = row[3] if len(row) > 3 and isinstance(row[3], dict) else {}
|
||||
normalized = self.row_normalizer(raw_data)
|
||||
worksheet.append(
|
||||
[self.value_serializer(normalized.get(key)) for key, _ in self.columns]
|
||||
)
|
||||
self._set_widths(worksheet)
|
||||
values = [self.value_serializer(normalized.get(key)) for key, _ in self.columns]
|
||||
worksheet.append(values)
|
||||
self._track_row_widths(column_widths, values)
|
||||
self._apply_widths(worksheet, column_widths)
|
||||
|
||||
|
||||
class DepthSheetWriter(AbstractWriter):
|
||||
@ -82,6 +92,7 @@ class DepthSheetWriter(AbstractWriter):
|
||||
self.value_serializer = value_serializer
|
||||
self.colors_config = colors_config
|
||||
self.palette = None
|
||||
self._row_fill_cache: dict[tuple[str, str], PatternFill] = {}
|
||||
if colors_config:
|
||||
self._set_palette()
|
||||
|
||||
@ -138,17 +149,21 @@ class DepthSheetWriter(AbstractWriter):
|
||||
current_column: int = 1,
|
||||
current_row: int = 1,
|
||||
max_row: int | None = None,
|
||||
full_key: str | None = None
|
||||
full_key: str | None = None,
|
||||
column_widths: dict[int, int] | None = None,
|
||||
) -> int:
|
||||
if columns_dict is None:
|
||||
columns_dict = self.columns_dict
|
||||
max_row = self._get_header_max_row(self.columns_dict)
|
||||
if column_widths is None:
|
||||
column_widths = {}
|
||||
for key, sub_dict in columns_dict.items():
|
||||
if full_key:
|
||||
key = f"{full_key}.{key}"
|
||||
|
||||
if "children" not in sub_dict:
|
||||
cell = worksheet.cell(row=current_row, column=current_column, value=sub_dict["name"])
|
||||
self._track_width(column_widths, current_column, sub_dict["name"])
|
||||
cell.border = self.thin_border
|
||||
if max_row != current_row:
|
||||
worksheet.merge_cells(
|
||||
@ -166,8 +181,10 @@ class DepthSheetWriter(AbstractWriter):
|
||||
current_row=current_row+1,
|
||||
max_row=max_row,
|
||||
full_key=key,
|
||||
column_widths=column_widths,
|
||||
)
|
||||
cell = worksheet.cell(row=current_row, column=current_column, value=sub_dict["name"])
|
||||
self._track_width(column_widths, current_column, sub_dict["name"])
|
||||
if new_column - current_column > 1:
|
||||
worksheet.merge_cells(
|
||||
start_row=current_row,
|
||||
@ -189,24 +206,27 @@ class DepthSheetWriter(AbstractWriter):
|
||||
cell.font = Font(color=color["color"])
|
||||
return current_column
|
||||
|
||||
def _set_color_to_row(self, ws: Worksheet, item_type: str) -> None:
|
||||
def _set_color_to_row(self, ws: Worksheet, row: int, item_type: str) -> None:
|
||||
if not self.palette or item_type in ("SUB_ITEM", "INPUT"):
|
||||
return
|
||||
|
||||
row = ws.max_row
|
||||
|
||||
for i, key in enumerate(self.columns):
|
||||
if key not in self.palette:
|
||||
continue
|
||||
cell = ws.cell(column=i+1, row=row)
|
||||
if (palette := self.palette.get(key)):
|
||||
cell.fill = PatternFill(
|
||||
start_color=Color(rgb=PALETTE[palette][item_type]),
|
||||
fill_type="solid",
|
||||
)
|
||||
cache_key = (palette, item_type)
|
||||
if cache_key not in self._row_fill_cache:
|
||||
self._row_fill_cache[cache_key] = PatternFill(
|
||||
start_color=Color(rgb=PALETTE[palette][item_type]),
|
||||
fill_type="solid",
|
||||
)
|
||||
cell.fill = self._row_fill_cache[cache_key]
|
||||
|
||||
def write(self, worksheet: Worksheet, rows: list[tuple]) -> None:
|
||||
self._write_header(worksheet)
|
||||
column_widths: dict[int, int] = {}
|
||||
self._write_header(worksheet, column_widths=column_widths)
|
||||
next_row = worksheet.max_row + 1
|
||||
for row in rows:
|
||||
if len(row) > 3 and isinstance(row[3], dict):
|
||||
if row[0] == 'META':
|
||||
@ -215,9 +235,308 @@ class DepthSheetWriter(AbstractWriter):
|
||||
else:
|
||||
raw_data = {}
|
||||
normalized = self.row_normalizer(raw_data)
|
||||
worksheet.append(
|
||||
[self.value_serializer(normalized.get(key)) for key in self.columns]
|
||||
)
|
||||
values = [self.value_serializer(normalized.get(key)) for key in self.columns]
|
||||
worksheet.append(values)
|
||||
self._track_row_widths(column_widths, values)
|
||||
if len(row) > 3:
|
||||
self._set_color_to_row(ws=worksheet, item_type=row[0])
|
||||
self._set_widths(worksheet)
|
||||
self._set_color_to_row(ws=worksheet, row=next_row, item_type=row[0])
|
||||
next_row += 1
|
||||
self._apply_widths(worksheet, column_widths)
|
||||
|
||||
|
||||
class XlsxTabularSheetWriter(AbstractWriter):
|
||||
def __init__(
|
||||
self,
|
||||
workbook: XlsxWorkbook,
|
||||
columns: list[tuple[str, str]],
|
||||
row_normalizer: RowNormalizer,
|
||||
value_serializer: ValueSerializer,
|
||||
):
|
||||
self.workbook = workbook
|
||||
self.columns = columns
|
||||
self.row_normalizer = row_normalizer
|
||||
self.value_serializer = value_serializer
|
||||
|
||||
@staticmethod
|
||||
def _apply_widths(
|
||||
worksheet: XlsxWorksheet,
|
||||
column_widths: dict[int, int],
|
||||
) -> None:
|
||||
for column, max_len in column_widths.items():
|
||||
zero_based_column = column - 1
|
||||
worksheet.set_column(
|
||||
zero_based_column,
|
||||
zero_based_column,
|
||||
max(max_len + 3, 10),
|
||||
)
|
||||
|
||||
def write(self, worksheet: XlsxWorksheet, rows: list[tuple]) -> None:
|
||||
column_widths: dict[int, int] = {}
|
||||
headers = [title for _, title in self.columns]
|
||||
worksheet.write_row(0, 0, headers)
|
||||
self._track_row_widths(column_widths, headers)
|
||||
|
||||
output_row = 1
|
||||
for row in rows:
|
||||
raw_data = row[3] if len(row) > 3 and isinstance(row[3], dict) else {}
|
||||
normalized = self.row_normalizer(raw_data)
|
||||
values = [self.value_serializer(normalized.get(key)) for key, _ in self.columns]
|
||||
worksheet.write_row(output_row, 0, values)
|
||||
self._track_row_widths(column_widths, values)
|
||||
output_row += 1
|
||||
|
||||
self._apply_widths(worksheet, column_widths)
|
||||
|
||||
|
||||
class XlsxDepthSheetWriter(AbstractWriter):
|
||||
def __init__(
|
||||
self,
|
||||
workbook: XlsxWorkbook,
|
||||
columns_dict: dict,
|
||||
row_normalizer: RowNormalizer,
|
||||
value_serializer: ValueSerializer,
|
||||
colors_config: dict | None = None,
|
||||
header_format_resolver: HeaderFormatResolver | None = None,
|
||||
row_format_resolver: RowFormatResolver | None = None,
|
||||
column_format_resolver: ColumnFormatResolver | None = None,
|
||||
fixed_column_width: int | None = None,
|
||||
):
|
||||
self.workbook = workbook
|
||||
self.columns_dict = columns_dict
|
||||
self._flat_columns_to_depth: dict[str, str] = {}
|
||||
self.columns = self._columns_dict_to_flat(columns_dict)
|
||||
self.row_normalizer = row_normalizer
|
||||
self.value_serializer = value_serializer
|
||||
self.colors_config = colors_config
|
||||
self.header_format_resolver = header_format_resolver
|
||||
self.row_format_resolver = row_format_resolver
|
||||
self.column_format_resolver = column_format_resolver
|
||||
self.fixed_column_width = fixed_column_width
|
||||
self.palette: dict[str, str] = {}
|
||||
self._format_cache: dict[tuple[Any, ...], Any] = {}
|
||||
if colors_config:
|
||||
self._set_palette()
|
||||
|
||||
def _apply_widths(
|
||||
self,
|
||||
worksheet: XlsxWorksheet,
|
||||
column_widths: dict[int, int],
|
||||
column_formats: list[Any] | None = None,
|
||||
) -> None:
|
||||
if self.fixed_column_width is not None:
|
||||
for column in range(len(self.columns)):
|
||||
worksheet.set_column(
|
||||
column,
|
||||
column,
|
||||
self.fixed_column_width,
|
||||
column_formats[column] if column_formats else None,
|
||||
)
|
||||
return
|
||||
|
||||
for column, max_len in column_widths.items():
|
||||
zero_based_column = column - 1
|
||||
worksheet.set_column(
|
||||
zero_based_column,
|
||||
zero_based_column,
|
||||
max(max_len + 3, 10),
|
||||
column_formats[zero_based_column] if column_formats else None,
|
||||
)
|
||||
|
||||
def _resolved_format(self, properties: dict[str, Any]) -> Any:
|
||||
cache_key = ("resolved", tuple(sorted(properties.items())))
|
||||
if cache_key not in self._format_cache:
|
||||
self._format_cache[cache_key] = self.workbook.add_format(properties)
|
||||
return self._format_cache[cache_key]
|
||||
|
||||
def _columns_dict_to_flat(
|
||||
self,
|
||||
columns_dict: dict | None = None,
|
||||
prefix: str = "",
|
||||
depth_prefix: str = "",
|
||||
) -> list[str]:
|
||||
if columns_dict is None:
|
||||
columns_dict = self.columns_dict
|
||||
result: list[str] = []
|
||||
for key, sub_dict in columns_dict.items():
|
||||
new_key = sub_dict.get("key", key)
|
||||
if "children" not in sub_dict:
|
||||
if new_key:
|
||||
flat_key = f"{prefix}{new_key}"
|
||||
result.append(flat_key)
|
||||
self._flat_columns_to_depth[flat_key] = f"{depth_prefix}{key}"
|
||||
else:
|
||||
result += self._columns_dict_to_flat(
|
||||
sub_dict["children"],
|
||||
prefix=f"{prefix}{new_key}." if new_key else prefix,
|
||||
depth_prefix=f"{depth_prefix}{key}.",
|
||||
)
|
||||
return result
|
||||
|
||||
def _set_palette(self) -> None:
|
||||
for key in self.columns:
|
||||
full_key = None
|
||||
for sub_key in self._flat_columns_to_depth[key].split("."):
|
||||
full_key = sub_key if full_key is None else f"{full_key}.{sub_key}"
|
||||
color = self.colors_config.get(full_key)
|
||||
if color and "palette" in color:
|
||||
self.palette[key] = color["palette"]
|
||||
|
||||
def _get_header_max_row(self, column_dict: dict) -> int:
|
||||
max_depth = 0
|
||||
for sub_dict in column_dict.values():
|
||||
if "children" in sub_dict:
|
||||
max_depth = max(max_depth, self._get_header_max_row(sub_dict["children"]))
|
||||
return max_depth + 1
|
||||
|
||||
def _header_format(self, full_key: str):
|
||||
properties: dict[str, Any] = {
|
||||
"border": 1,
|
||||
"align": "center",
|
||||
"valign": "vcenter",
|
||||
}
|
||||
if self.colors_config and (color := self.colors_config.get(full_key)):
|
||||
if "background" in color:
|
||||
properties["bg_color"] = f"#{color['background']}"
|
||||
properties["pattern"] = 1
|
||||
if "color" in color:
|
||||
properties["font_color"] = f"#{color['color']}"
|
||||
if self.header_format_resolver:
|
||||
properties.update(self.header_format_resolver(full_key) or {})
|
||||
return self.workbook.add_format(properties)
|
||||
|
||||
def _write_header(
|
||||
self,
|
||||
worksheet: XlsxWorksheet,
|
||||
column_widths: dict[int, int],
|
||||
columns_dict: dict | None = None,
|
||||
current_column: int = 0,
|
||||
current_row: int = 0,
|
||||
max_row: int | None = None,
|
||||
full_key: str | None = None,
|
||||
) -> int:
|
||||
if columns_dict is None:
|
||||
columns_dict = self.columns_dict
|
||||
max_row = self._get_header_max_row(columns_dict) - 1
|
||||
|
||||
for key, sub_dict in columns_dict.items():
|
||||
current_key = f"{full_key}.{key}" if full_key else key
|
||||
title = sub_dict["name"]
|
||||
start_column = current_column
|
||||
|
||||
if "children" in sub_dict:
|
||||
current_column = self._write_header(
|
||||
worksheet=worksheet,
|
||||
column_widths=column_widths,
|
||||
columns_dict=sub_dict["children"],
|
||||
current_column=current_column,
|
||||
current_row=current_row + 1,
|
||||
max_row=max_row,
|
||||
full_key=current_key,
|
||||
)
|
||||
end_column = current_column - 1
|
||||
if end_column > start_column:
|
||||
worksheet.merge_range(
|
||||
current_row,
|
||||
start_column,
|
||||
current_row,
|
||||
end_column,
|
||||
title,
|
||||
self._header_format(current_key),
|
||||
)
|
||||
else:
|
||||
worksheet.write(
|
||||
current_row,
|
||||
start_column,
|
||||
title,
|
||||
self._header_format(current_key),
|
||||
)
|
||||
else:
|
||||
if current_row != max_row:
|
||||
worksheet.merge_range(
|
||||
current_row,
|
||||
current_column,
|
||||
max_row,
|
||||
current_column,
|
||||
title,
|
||||
self._header_format(current_key),
|
||||
)
|
||||
else:
|
||||
worksheet.write(
|
||||
current_row,
|
||||
current_column,
|
||||
title,
|
||||
self._header_format(current_key),
|
||||
)
|
||||
current_column += 1
|
||||
|
||||
if self.fixed_column_width is None:
|
||||
self._track_width(column_widths, start_column + 1, title)
|
||||
|
||||
return current_column
|
||||
|
||||
def _row_format(self, palette: str, item_type: str):
|
||||
cache_key = (palette, item_type)
|
||||
if cache_key not in self._format_cache:
|
||||
self._format_cache[cache_key] = self.workbook.add_format(
|
||||
{
|
||||
"bg_color": f"#{PALETTE[palette][item_type]}",
|
||||
"pattern": 1,
|
||||
}
|
||||
)
|
||||
return self._format_cache[cache_key]
|
||||
|
||||
def write(self, worksheet: XlsxWorksheet, rows: list[tuple]) -> None:
|
||||
column_widths: dict[int, int] = {}
|
||||
header_rows = self._get_header_max_row(self.columns_dict)
|
||||
self._write_header(worksheet, column_widths=column_widths)
|
||||
output_row = header_rows
|
||||
column_formats = None
|
||||
if self.column_format_resolver:
|
||||
column_formats = [
|
||||
self._resolved_format(properties)
|
||||
if (properties := self.column_format_resolver(column_key, column))
|
||||
else None
|
||||
for column, column_key in enumerate(self.columns)
|
||||
]
|
||||
|
||||
for row in rows:
|
||||
if len(row) > 3 and isinstance(row[3], dict):
|
||||
if row[0] == "META":
|
||||
continue
|
||||
raw_data = row[3]
|
||||
else:
|
||||
raw_data = {}
|
||||
|
||||
normalized = self.row_normalizer(raw_data)
|
||||
values = [self.value_serializer(normalized.get(key)) for key in self.columns]
|
||||
if column_formats is not None:
|
||||
worksheet.write_row(output_row, 0, values)
|
||||
if self.fixed_column_width is None:
|
||||
self._track_row_widths(column_widths, values)
|
||||
output_row += 1
|
||||
continue
|
||||
|
||||
item_type = row[0] if len(row) > 3 else None
|
||||
for column, value in enumerate(values):
|
||||
cell_format = None
|
||||
column_key = self.columns[column]
|
||||
resolved_format = (
|
||||
self.row_format_resolver(row, column_key, column)
|
||||
if self.row_format_resolver
|
||||
else None
|
||||
)
|
||||
if resolved_format:
|
||||
cell_format = self._resolved_format(resolved_format)
|
||||
palette = self.palette.get(column_key)
|
||||
if (
|
||||
cell_format is None
|
||||
and palette
|
||||
and item_type not in (None, "SUB_ITEM", "INPUT")
|
||||
):
|
||||
cell_format = self._row_format(palette, item_type)
|
||||
worksheet.write(output_row, column, value, cell_format)
|
||||
if self.fixed_column_width is None:
|
||||
self._track_row_widths(column_widths, values)
|
||||
output_row += 1
|
||||
|
||||
self._apply_widths(worksheet, column_widths, column_formats)
|
||||
|
||||
@ -52,6 +52,24 @@ class SummaryService:
|
||||
sort_direction=sort_direction,
|
||||
)
|
||||
|
||||
async def get_svod2_go_export_rows(
|
||||
self,
|
||||
year: int,
|
||||
user: AppUser,
|
||||
org_ids: list[int] | None = None,
|
||||
sort_by: str = "org_name",
|
||||
sort_direction: str = "asc",
|
||||
) -> list[tuple]:
|
||||
org_ids = await self._allowed_org_ids(user, org_ids)
|
||||
if org_ids == []:
|
||||
return []
|
||||
return await self.summary_repo.get_svod2_go_export_rows(
|
||||
year=year,
|
||||
org_ids=org_ids,
|
||||
sort_by=sort_by,
|
||||
sort_direction=sort_direction,
|
||||
)
|
||||
|
||||
async def _allowed_org_ids(
|
||||
self,
|
||||
user: AppUser,
|
||||
|
||||
@ -1,38 +0,0 @@
|
||||
import pytest
|
||||
|
||||
|
||||
def _get_any_project_report(client, admin_tokens, auth_headers) -> tuple[int, int, str]:
|
||||
list_response = client.get(
|
||||
"/api/v1/projects?limit=50",
|
||||
headers=auth_headers(admin_tokens),
|
||||
)
|
||||
assert list_response.status_code == 200
|
||||
projects = list_response.json().get("result") or []
|
||||
for project in projects:
|
||||
project_id = int(project["id"])
|
||||
details_response = client.get(
|
||||
f"/api/v1/projects/{project_id}",
|
||||
headers=auth_headers(admin_tokens),
|
||||
)
|
||||
if details_response.status_code != 200:
|
||||
continue
|
||||
reports = (details_response.json().get("result") or {}).get("reports") or []
|
||||
if reports:
|
||||
report = reports[0]
|
||||
return project_id, int(report["year"]), str(report["report_type"])
|
||||
pytest.skip("Не найден проект FORM_3 с отчетами для smoke-экспорта")
|
||||
|
||||
|
||||
def test_export_project_report_smoke(client, admin_tokens, auth_headers):
|
||||
project_id, report_year, report_type = _get_any_project_report(client, admin_tokens, auth_headers)
|
||||
|
||||
response = client.get(
|
||||
f"/api/v1/export/project/{project_id}/report/{report_year}/{report_type}",
|
||||
headers=auth_headers(admin_tokens),
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.headers["content-type"] == (
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
||||
)
|
||||
assert "attachment; filename*=UTF-8''" in response.headers["content-disposition"]
|
||||
assert len(response.content) > 0
|
||||
@ -2,6 +2,7 @@ import { useEffect } from 'react';
|
||||
import { AppRoutes } from './app/Routes';
|
||||
import './App.css';
|
||||
import { isTestMode } from './conf/config';
|
||||
import ExportJobs from './components/ExportJobs';
|
||||
|
||||
const BaseTag = () => {
|
||||
useEffect(() => {
|
||||
@ -27,6 +28,7 @@ function App() {
|
||||
<div className='App'>
|
||||
<BaseTag />
|
||||
<AppRoutes />
|
||||
<ExportJobs />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
101
web/src/api/exports.js
Normal file
101
web/src/api/exports.js
Normal file
@ -0,0 +1,101 @@
|
||||
import api from './client';
|
||||
|
||||
let jobs = [];
|
||||
const listeners = new Set();
|
||||
let creating = 0;
|
||||
const filenames = new Map();
|
||||
|
||||
export const getExportFilename = (job) => {
|
||||
let saved = filenames.get(job.id);
|
||||
try { saved ||= localStorage.getItem(`export-filename:${job.id}`); } catch { /* Memory fallback. */ }
|
||||
return saved || job.filename?.replace(/\.[^.]+$/, '').replace(/_\d{4}-\d{2}-\d{2}$/, '') || 'Экспорт';
|
||||
};
|
||||
|
||||
export const createNamedExport = async (createJob, filename) => {
|
||||
creating++;
|
||||
try {
|
||||
const job = await createJob();
|
||||
if (filename) {
|
||||
filenames.set(job.id, filename);
|
||||
try { localStorage.setItem(`export-filename:${job.id}`, filename); } catch { /* Memory fallback. */ }
|
||||
}
|
||||
registerExport(job);
|
||||
} finally {
|
||||
creating--;
|
||||
if (!creating) receiveJobs(jobs);
|
||||
}
|
||||
};
|
||||
|
||||
export const registerExport = (job) => {
|
||||
// The WebSocket can deliver completion before the creation response arrives.
|
||||
if (!jobs.some((item) => item.id === job.id)) receiveJobs([job, ...jobs]);
|
||||
};
|
||||
|
||||
export const subscribeExports = (listener) => {
|
||||
listeners.add(listener);
|
||||
listener(jobs);
|
||||
return () => listeners.delete(listener);
|
||||
};
|
||||
|
||||
function receiveJobs(next) {
|
||||
jobs = next;
|
||||
if (creating) return;
|
||||
for (const listener of listeners) listener(jobs);
|
||||
}
|
||||
|
||||
export const downloadExport = (id) => api.get(`/export/jobs/${id}/download`, { responseType: 'blob' });
|
||||
|
||||
export function startExportUpdates() {
|
||||
let stopped = false;
|
||||
let socket;
|
||||
let retry;
|
||||
let watchdog;
|
||||
let attempts = 0;
|
||||
const connect = () => {
|
||||
if (stopped) return;
|
||||
const url = new URL(`${api.defaults.baseURL}/ws/export`, window.location.origin);
|
||||
url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:';
|
||||
socket = new WebSocket(url);
|
||||
const armWatchdog = () => {
|
||||
clearTimeout(watchdog);
|
||||
watchdog = setTimeout(() => socket.close(), 45000);
|
||||
};
|
||||
socket.onopen = () => {
|
||||
socket.send(JSON.stringify({ token: localStorage.getItem('access_token') }));
|
||||
armWatchdog();
|
||||
};
|
||||
socket.onmessage = (event) => {
|
||||
if (stopped) return;
|
||||
try {
|
||||
const message = JSON.parse(event.data);
|
||||
if (message.event === 'exports') {
|
||||
attempts = 0;
|
||||
receiveJobs(message.jobs);
|
||||
armWatchdog();
|
||||
}
|
||||
} catch (error) { console.error('Ошибка статуса экспорта', error); }
|
||||
};
|
||||
socket.onclose = (event) => {
|
||||
clearTimeout(watchdog);
|
||||
if (stopped || event.code === 4401) return;
|
||||
retry = setTimeout(connect, Math.min(30000, 2000 * 2 ** attempts++));
|
||||
};
|
||||
};
|
||||
// HTTP fallback only when the WebSocket is unavailable.
|
||||
const fallback = setInterval(async () => {
|
||||
if (socket?.readyState === WebSocket.OPEN || stopped) return;
|
||||
try {
|
||||
const response = await api.get('/export/jobs');
|
||||
if (!stopped) receiveJobs(response.data);
|
||||
} catch { /* Retry on the next fallback interval. */ }
|
||||
}, 30000);
|
||||
connect();
|
||||
return () => {
|
||||
stopped = true;
|
||||
clearTimeout(retry);
|
||||
clearTimeout(watchdog);
|
||||
clearInterval(fallback);
|
||||
socket?.close();
|
||||
receiveJobs([]);
|
||||
};
|
||||
}
|
||||
@ -7,8 +7,7 @@ export const FormsSheetApi = {
|
||||
export(formId, sheetName, params = {}) {
|
||||
return api.get(`export/form/${formId}/sheet/${sheetName}`, {
|
||||
params,
|
||||
responseType: 'blob',
|
||||
});
|
||||
}).then((r) => r.data);
|
||||
},
|
||||
getExpenseItem(itemId) {
|
||||
return api.get(`/expense-item/${itemId}`).then((r) => r.data);
|
||||
|
||||
@ -39,13 +39,9 @@ export const ProjectsApi = {
|
||||
return api.get(`/projects/${id}/summary`).then((r) => r.data);
|
||||
},
|
||||
export: (id, year, sheetName) => {
|
||||
return api.get(`/export/project/${id}/report/${year}/${sheetName}`, {
|
||||
responseType: 'blob',
|
||||
});
|
||||
return api.get(`/export/project/${id}/report/${year}/${sheetName}`).then((r) => r.data);
|
||||
},
|
||||
exportProject: (id) => {
|
||||
return api.get(`/export/project/${id}`, {
|
||||
responseType: 'blob',
|
||||
});
|
||||
return api.get(`/export/project/${id}`).then((r) => r.data);
|
||||
},
|
||||
};
|
||||
|
||||
@ -28,8 +28,7 @@ export const TasksApi = {
|
||||
exportTask(taskId, params = {}) {
|
||||
return api.get(`export/form/${taskId}`, {
|
||||
params,
|
||||
responseType: 'blob',
|
||||
});
|
||||
}).then((r) => r.data);
|
||||
},
|
||||
markForDeletion(taskId) {
|
||||
return api.post(`/form/${taskId}/mark-delete`).then((r) => r.data);
|
||||
@ -43,7 +42,6 @@ export const TasksApi = {
|
||||
exportTasks(payload, params = {}) {
|
||||
return api.post(`export/bulk`, payload, {
|
||||
params,
|
||||
responseType: 'blob',
|
||||
});
|
||||
}).then((r) => r.data);
|
||||
},
|
||||
};
|
||||
|
||||
63
web/src/components/ExportJobs.jsx
Normal file
63
web/src/components/ExportJobs.jsx
Normal file
@ -0,0 +1,63 @@
|
||||
import { useEffect } from 'react';
|
||||
import { toast } from 'react-toastify';
|
||||
import { useAuth } from '../app/context/AuthProvider';
|
||||
import { downloadExport, getExportFilename, startExportUpdates, subscribeExports } from '../api/exports';
|
||||
import { downloadFile } from '../utils/downloadFile';
|
||||
import { getApiErrorMessage } from '../utils/getApiErrorMessage';
|
||||
|
||||
export default function ExportJobs() {
|
||||
const { user, isAuthenticated } = useAuth();
|
||||
useEffect(() => {
|
||||
if (!isAuthenticated || !user?.id) return;
|
||||
let stopped = false;
|
||||
const processing = new Set();
|
||||
const handled = new Set();
|
||||
const toastIds = new Set();
|
||||
const key = (id) => `export-handled:${user.id}:${id}`;
|
||||
const wasHandled = (id) => {
|
||||
try { return handled.has(id) || Boolean(localStorage.getItem(key(id))); }
|
||||
catch { return handled.has(id); }
|
||||
};
|
||||
const markHandled = (id) => {
|
||||
handled.add(id);
|
||||
try { localStorage.setItem(key(id), String(Date.now())); } catch { /* Memory fallback. */ }
|
||||
};
|
||||
const showLoading = (id, message) => {
|
||||
toastIds.add(id);
|
||||
if (!toast.isActive(id)) toast.loading(message, { toastId: id, autoClose: false });
|
||||
else toast.update(id, { render: message, isLoading: true, autoClose: false });
|
||||
};
|
||||
const observe = async (job) => {
|
||||
if (stopped || wasHandled(job.id) || processing.has(job.id)) return;
|
||||
const toastId = `export:${user.id}:${job.id}`;
|
||||
if (['queued', 'processing'].includes(job.status)) {
|
||||
showLoading(toastId, 'Идёт формирование Excel…');
|
||||
return;
|
||||
}
|
||||
processing.add(job.id);
|
||||
showLoading(toastId, 'Скачивание Excel…');
|
||||
try {
|
||||
if (job.status === 'failed') throw new Error(job.error || 'Ошибка экспорта');
|
||||
const response = await downloadExport(job.id);
|
||||
if (stopped) return;
|
||||
await downloadFile(response, getExportFilename(job), 'xlsx');
|
||||
markHandled(job.id);
|
||||
toast.update(toastId, { render: 'Excel сформирован, скачивание началось', type: 'success', isLoading: false, autoClose: 3000 });
|
||||
} catch (error) {
|
||||
if (stopped) return;
|
||||
if (job.status === 'failed' || error?.response?.status === 410) markHandled(job.id);
|
||||
const message = await getApiErrorMessage(error, 'Не удалось скачать файл');
|
||||
if (!stopped) toast.update(toastId, { render: message, type: 'error', isLoading: false, autoClose: 5000 });
|
||||
} finally { processing.delete(job.id); }
|
||||
};
|
||||
const unsubscribe = subscribeExports((jobs) => { for (const job of jobs) void observe(job); });
|
||||
const stop = startExportUpdates();
|
||||
return () => {
|
||||
stopped = true;
|
||||
unsubscribe();
|
||||
stop();
|
||||
for (const id of toastIds) toast.dismiss(id);
|
||||
};
|
||||
}, [isAuthenticated, user?.id]);
|
||||
return null;
|
||||
}
|
||||
@ -2,58 +2,33 @@ import { toast } from 'react-toastify';
|
||||
import { FormsSheetApi } from '../api/form_sheet';
|
||||
import { ProjectsApi } from '../api/projects';
|
||||
import { TasksApi } from '../api/tasks';
|
||||
import { downloadFile } from './downloadFile';
|
||||
import { createNamedExport } from '../api/exports';
|
||||
import { getApiErrorMessage } from './getApiErrorMessage';
|
||||
|
||||
export const exportSingleForm = async (taskId, taskTitle) => {
|
||||
const runExport = async (createJob, defaultFilename, rethrow = false) => {
|
||||
try {
|
||||
const response = await TasksApi.exportTask(taskId);
|
||||
await downloadFile(response, taskTitle, 'xlsx');
|
||||
await createNamedExport(createJob, defaultFilename);
|
||||
} catch (error) {
|
||||
const message = await getApiErrorMessage(error, 'Ошибка при скачивании файла');
|
||||
toast.error(message);
|
||||
if (rethrow) throw error;
|
||||
}
|
||||
};
|
||||
|
||||
export const exportSheet = async (formId, sheetName, direction) => {
|
||||
try {
|
||||
const normalizedDirection = direction && direction !== 'null' ? direction : undefined;
|
||||
const params = normalizedDirection ? { direction: normalizedDirection } : {};
|
||||
const response = await FormsSheetApi.export(formId, sheetName, params);
|
||||
await downloadFile(response, sheetName, 'xlsx');
|
||||
} catch (error) {
|
||||
const message = await getApiErrorMessage(error, 'Ошибка при скачивании файла');
|
||||
toast.error(message);
|
||||
}
|
||||
export const exportSingleForm = (taskId, taskTitle) =>
|
||||
runExport(() => TasksApi.exportTask(taskId), taskTitle);
|
||||
|
||||
export const exportSheet = (formId, sheetName, direction) => {
|
||||
const normalizedDirection = direction && direction !== 'null' ? direction : undefined;
|
||||
const params = normalizedDirection ? { direction: normalizedDirection } : {};
|
||||
return runExport(() => FormsSheetApi.export(formId, sheetName, params), sheetName);
|
||||
};
|
||||
|
||||
export const exportSheetProject = async (formId, sheetName, year) => {
|
||||
try {
|
||||
const response = await ProjectsApi.export(formId, year, sheetName);
|
||||
await downloadFile(response, sheetName, 'xlsx');
|
||||
} catch (error) {
|
||||
const message = await getApiErrorMessage(error, 'Ошибка при скачивании файла');
|
||||
toast.error(message);
|
||||
}
|
||||
};
|
||||
export const exportSheetProject = (formId, sheetName, year) =>
|
||||
runExport(() => ProjectsApi.export(formId, year, sheetName), sheetName);
|
||||
|
||||
export const exportProject = async (projectId, projectTitle) => {
|
||||
try {
|
||||
const response = await ProjectsApi.exportProject(projectId);
|
||||
await downloadFile(response, projectTitle, 'xlsx');
|
||||
} catch (error) {
|
||||
const message = await getApiErrorMessage(error, 'Ошибка при скачивании файла');
|
||||
toast.error(message);
|
||||
}
|
||||
};
|
||||
export const exportProject = (projectId, projectTitle) =>
|
||||
runExport(() => ProjectsApi.exportProject(projectId), projectTitle);
|
||||
|
||||
export const exportBulkForms = async (formIds, fileName = 'Задачи') => {
|
||||
try {
|
||||
const response = await TasksApi.exportTasks({ form_ids: formIds });
|
||||
await downloadFile(response, fileName, 'xlsx');
|
||||
} catch (error) {
|
||||
const message = await getApiErrorMessage(error, 'Ошибка при скачивании файла');
|
||||
toast.error(message);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
export const exportBulkForms = (formIds, fileName = 'Задачи') =>
|
||||
runExport(() => TasksApi.exportTasks({ form_ids: formIds }), fileName, true);
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user