diff --git a/api/alembic/versions/0043_summary2_go_export.py b/api/alembic/versions/0043_summary2_go_export.py new file mode 100644 index 0000000..6ab8e00 --- /dev/null +++ b/api/alembic/versions/0043_summary2_go_export.py @@ -0,0 +1,19 @@ +from alembic import op + +from sql_migration import run_sql_file_migration + +revision = "0043" +down_revision = "0042" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + run_sql_file_migration("0043_summary2_go_export.sql") + + +def downgrade() -> None: + op.execute( + "DROP FUNCTION IF EXISTS " + "v3.v_svod2_go_export(integer, integer[], varchar, varchar)" + ) diff --git a/api/alembic/versions/0044_export_jobs.py b/api/alembic/versions/0044_export_jobs.py new file mode 100644 index 0000000..29aec54 --- /dev/null +++ b/api/alembic/versions/0044_export_jobs.py @@ -0,0 +1,35 @@ +from alembic import op + +revision = "0044" +down_revision = "0043" +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute(""" + CREATE TABLE v3.export_job ( + id uuid PRIMARY KEY, + user_id integer NOT NULL REFERENCES v3.app_user(id), + request jsonb NOT NULL, + status varchar(20) NOT NULL, + created_at timestamptz NOT NULL, + started_at timestamptz, + finished_at timestamptz, + filename text, + media_type text, + file_path text, + error text, + CONSTRAINT export_job_status CHECK + (status IN ('queued', 'processing', 'completed', 'failed')) + ) + """) + op.execute("CREATE INDEX export_job_user ON v3.export_job(user_id, created_at)") + op.execute(""" + CREATE UNIQUE INDEX export_job_active_user ON v3.export_job(user_id) + WHERE status IN ('queued', 'processing') + """) + + +def downgrade(): + op.execute("DROP TABLE v3.export_job") diff --git a/api/alembic/versions/sql/0043_summary2_go_export.sql b/api/alembic/versions/sql/0043_summary2_go_export.sql new file mode 100644 index 0000000..c25cadc --- /dev/null +++ b/api/alembic/versions/sql/0043_summary2_go_export.sql @@ -0,0 +1,209 @@ +DROP FUNCTION IF EXISTS v3.v_svod2_go_export(integer, integer[], varchar, varchar); + +CREATE OR REPLACE FUNCTION v3.v_svod2_go_export( + p_year integer, + p_org_unit_ids integer[] DEFAULT NULL, + p_sort_by varchar DEFAULT 'org_name', + p_sort_direction varchar DEFAULT 'asc' +) +RETURNS TABLE(sort_order bigint, row_values text[]) +LANGUAGE sql +STABLE +SET search_path TO 'v3', 'pg_catalog' +AS $function$ +WITH eligible_lines AS ( + SELECT + bl.id AS line_id, + bf.form_type_code, + bf.id AS form_id, + bf.org_unit_id, + ou.title AS org_name, + ei.section_code, + ei.item_id, + ei.num_group_id, + COALESCE(bl.name, ei.name) AS name, + bl.justification, + CASE bl.direction + WHEN 'Support' THEN 'Поддержка' + WHEN 'Development' THEN 'Развитие' + END::varchar AS smeta_direction, + CASE ei.sheet + WHEN 'AHR' THEN 'АХР' + WHEN 'CAP' THEN 'КВ' + WHEN 'OPER' THEN 'Операц' + END::varchar AS smeta_type, + cd.counterparty, + cd.reference, + cd.subject, + cd.currency, + cd.ceiling_amount, + cd.expenses_q1, + cd.expenses_q2, + cd.expenses_q3, + cd.expenses_q4, + cd.vat_rate, + cd.deadline + FROM v3.budget_form bf + JOIN v3.budget_line bl ON bl.budget_form_id = bf.id + JOIN v3.expense_item ei ON ei.id = bl.expense_item_id + JOIN v3.contract_detail cd ON cd.line_id = bl.id + LEFT JOIN v3.org_unit ou ON ou.id = bf.org_unit_id + WHERE bf.year = p_year + AND bf.form_type_code IN ('FORM_1', 'FORM_4') + AND ei.sheet IN ('AHR', 'CAP', 'OPER') + AND ( + p_org_unit_ids IS NULL + OR cardinality(p_org_unit_ids) = 0 + OR bf.org_unit_id = ANY(p_org_unit_ids) + ) +), page AS MATERIALIZED ( + SELECT + el.*, + row_number() OVER ( + ORDER BY + CASE WHEN p_sort_by = 'org_name' AND lower(p_sort_direction) = 'asc' THEN el.org_name END ASC NULLS LAST, + CASE WHEN p_sort_by = 'org_name' AND lower(p_sort_direction) = 'desc' THEN el.org_name END DESC NULLS LAST, + CASE WHEN p_sort_by = 'org_name' AND lower(p_sort_direction) = 'asc' THEN el.item_id END ASC NULLS LAST, + CASE WHEN p_sort_by = 'org_name' AND lower(p_sort_direction) = 'desc' THEN el.item_id END DESC NULLS LAST, + CASE WHEN p_sort_by = 'section_code' AND lower(p_sort_direction) = 'asc' THEN el.section_code END ASC NULLS LAST, + CASE WHEN p_sort_by = 'section_code' AND lower(p_sort_direction) = 'desc' THEN el.section_code END DESC NULLS LAST, + el.form_type_code, + el.form_id, + el.line_id + ) AS sort_order + FROM eligible_lines el +) +SELECT + pg.sort_order, + ARRAY[ + pg.smeta_type::text, + pg.smeta_direction::text, + pg.org_name::text, + pg.name::text, + pg.item_id::text, + pg.num_group_id::text, + pg.justification::text, + pl.plan_q1::text, + pl.plan_q2::text, + pl.plan_q3::text, + pl.plan_q4::text, + al.internal_order::text, + al.property_object::text, + sd.adj_q1::text, + sd.adj_q2::text, + sd.adj_q3::text, + sd.adj_q4::text, + rv.amount_q1::text, + rv.amount_q2::text, + rv.amount_q3::text, + rv.amount_q4::text, + (CASE WHEN pl.line_id IS NULL AND sd.id IS NULL AND rv.line_id IS NULL THEN NULL ELSE COALESCE(pl.plan_q1, 0) + COALESCE(sd.adj_q1, 0) + COALESCE(rv.amount_q1, 0) END)::text, + (CASE WHEN pl.line_id IS NULL AND sd.id IS NULL AND rv.line_id IS NULL THEN NULL ELSE COALESCE(pl.plan_q2, 0) + COALESCE(sd.adj_q2, 0) + COALESCE(rv.amount_q2, 0) END)::text, + (CASE WHEN pl.line_id IS NULL AND sd.id IS NULL AND rv.line_id IS NULL THEN NULL ELSE COALESCE(pl.plan_q3, 0) + COALESCE(sd.adj_q3, 0) + COALESCE(rv.amount_q3, 0) END)::text, + (CASE WHEN pl.line_id IS NULL AND sd.id IS NULL AND rv.line_id IS NULL THEN NULL ELSE COALESCE(pl.plan_q4, 0) + COALESCE(sd.adj_q4, 0) + COALESCE(rv.amount_q4, 0) END)::text, + pg.counterparty::text, + pg.reference::text, + pg.subject::text, + pg.currency::text, + pg.ceiling_amount::text, + pg.expenses_q1::text, + pg.expenses_q2::text, + pg.expenses_q3::text, + pg.expenses_q4::text, + pg.vat_rate::text, + pg.deadline::text, + q1.adj_current::text, + q1.adj_ssp::text, + q1.adj_rf::text, + q1.adj_reserve::text, + q1.payment_date::text, + q1.payment_amount::text, + q1.payment_amount_ho::text, + q1.payment_amount_rf::text, + q1.payment_act::text, + q1.booking_amount::text, + q1.actual_m1::text, + q1.actual_m2::text, + q1.actual_m3::text, + q1.transfer_to_q2::text, + q1.transfer_to_q3::text, + q1.transfer_to_q4::text, + q1.transfer_to_economy::text, + q2.adj_current::text, + q2.adj_ssp::text, + q2.adj_rf::text, + q2.adj_reserve::text, + q2.payment_date::text, + q2.payment_amount::text, + q2.payment_amount_ho::text, + q2.payment_amount_rf::text, + q2.payment_act::text, + q2.booking_amount::text, + q2.actual_m1::text, + q2.actual_m2::text, + q2.actual_m3::text, + q2.plan_revision_eco_change::text, + q2.plan_revision_item_adj::text, + q2.plan_revision_increase::text, + q2.plan_revision_sequester::text, + q2.transfer_to_q3::text, + q2.transfer_to_q4::text, + q2.transfer_to_economy::text, + q3.adj_current::text, + q3.adj_ssp::text, + q3.adj_rf::text, + q3.adj_reserve::text, + q3.payment_date::text, + q3.payment_amount::text, + q3.payment_amount_ho::text, + q3.payment_amount_rf::text, + q3.payment_act::text, + q3.booking_amount::text, + q3.actual_m1::text, + q3.actual_m2::text, + q3.actual_m3::text, + q3.plan_revision_eco_change::text, + q3.plan_revision_item_adj::text, + q3.plan_revision_increase::text, + q3.plan_revision_sequester::text, + q3.transfer_to_q4::text, + q3.transfer_to_economy::text, + q4.adj_current::text, + q4.adj_ssp::text, + q4.adj_rf::text, + q4.adj_reserve::text, + q4.payment_date::text, + q4.payment_amount::text, + q4.payment_amount_ho::text, + q4.payment_amount_rf::text, + q4.payment_act::text, + q4.booking_amount::text, + q4.actual_m1::text, + q4.actual_m2::text, + q4.actual_m3::text, + q4.actual_spod::text, + q4.plan_revision_eco_change::text, + q4.plan_revision_item_adj::text, + q4.plan_revision_increase::text, + q4.plan_revision_sequester::text + ]::text[] AS row_values +FROM page pg +LEFT JOIN v3.plan pl ON pl.line_id = pg.line_id +LEFT JOIN v3.sequestration sd + ON sd.line_id = pg.line_id + AND sd.actor = 'DFIP' +LEFT JOIN v3.reserve rv ON rv.line_id = pg.line_id +LEFT JOIN v3.allocation al ON al.line_id = pg.line_id +LEFT JOIN v3.budget_line_quarter q1 + ON q1.line_id = pg.line_id AND q1.quarter = 1 +LEFT JOIN v3.budget_line_quarter q2 + ON q2.line_id = pg.line_id AND q2.quarter = 2 +LEFT JOIN v3.budget_line_quarter q3 + ON q3.line_id = pg.line_id AND q3.quarter = 3 +LEFT JOIN v3.budget_line_quarter q4 + ON q4.line_id = pg.line_id AND q4.quarter = 4 +ORDER BY pg.sort_order; +$function$; + +COMMENT ON FUNCTION v3.v_svod2_go_export(integer, integer[], varchar, varchar) IS +'Плоская выборка свода 2 / ГО для XLSX-экспорта без построения JSONB'; diff --git a/api/requirements-dev.txt b/api/requirements-dev.txt index faa293a..34552d5 100644 --- a/api/requirements-dev.txt +++ b/api/requirements-dev.txt @@ -15,3 +15,4 @@ asyncpg==0.30.0 pytest==8.3.2 pytest-asyncio==0.24.0 openpyxl==3.1.5 +XlsxWriter>=3.2,<4 diff --git a/api/requirements.txt b/api/requirements.txt index 38e1601..60160d0 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -13,3 +13,4 @@ alembic==1.16.1 asyncpg==0.30.0 raisa-fastapi-protected-api==1.0.0 openpyxl +XlsxWriter>=3.2,<4 diff --git a/api/src/api/v1/export.py b/api/src/api/v1/export.py index 9f8966b..01119ad 100644 --- a/api/src/api/v1/export.py +++ b/api/src/api/v1/export.py @@ -1,15 +1,80 @@ -from fastapi import APIRouter, Depends, HTTPException, status -from fastapi.responses import StreamingResponse -from sqlalchemy.ext.asyncio import AsyncSession +from typing import Annotated, Literal +from uuid import UUID + +from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Path, status +from fastapi.responses import FileResponse +from starlette.background import BackgroundTask from src.api.v1.deps import get_current_active_user_with_set_db +from src.core.config import settings from src.db.models.app_user import AppUser -from src.db.session import get_db from src.domain.schemas import DirectionSchemaEnum, ExportBulkRequest -from src.services.export_service import ExportService +from src.services.export_service.export_job_service import ( + ExportJobAlreadyRunningError, + export_job_manager, +) router = APIRouter(prefix="/export", tags=["export"]) FORM3_ALLOWED_SECTIONS = {"q1", "q2", "q3", "q4", "year"} +SUMMARY1_ALLOWED_SHEETS = {"FORM_1", "FORM_2", "FORM_3", "FORM_4", "MAIN"} +SUMMARY2_ALLOWED_SHEETS = {"GO", "RF"} + + +async def _enqueue_export( + background_tasks: BackgroundTasks, + current_user: AppUser, + request: dict, +): + try: + job = await export_job_manager.create(current_user.id, request) + except ExportJobAlreadyRunningError: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Экспорт уже выполняется. Дождитесь завершения предыдущего экспорта.", + ) + background_tasks.add_task(export_job_manager.run, job.id) + return job.as_dict() + + +@router.get("/jobs") +async def list_export_jobs(current_user: AppUser = Depends(get_current_active_user_with_set_db)): + return [job.as_dict() for job in await export_job_manager.list_jobs(current_user.id)] + + +@router.get("/jobs/{job_id}") +async def get_export_job( + job_id: UUID, + current_user: AppUser = Depends(get_current_active_user_with_set_db), +): + job = await export_job_manager.get(job_id, current_user.id) + if job is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Экспорт не найден") + return job.as_dict() + + +@router.get("/jobs/{job_id}/download") +async def download_export_job( + job_id: UUID, + current_user: AppUser = Depends(get_current_active_user_with_set_db), +): + job = await export_job_manager.get(job_id, current_user.id) + if job is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Экспорт не найден") + if job.status != "completed" or job.file_path is None: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Файл экспорта ещё не готов", + ) + if not job.file_path.is_file(): + await export_job_manager.mark_file_missing(job) + raise HTTPException(status_code=status.HTTP_410_GONE, detail="Файл экспорта удалён") + return FileResponse( + job.file_path, + media_type=job.media_type, + filename=job.filename, + content_disposition_type="attachment", + background=BackgroundTask(job.file_path.unlink, missing_ok=True), + ) def _parse_sections_csv(sections: str | None) -> list[str] | None: @@ -47,130 +112,138 @@ def _parse_form3_sections_csv(sections: str | None) -> list[str] | None: return parsed -@router.get("/form/{form_id}") +def _parse_org_ids_csv(org_ids: str | None) -> list[int] | None: + if not org_ids: + return None + try: + parsed = [int(value.strip()) for value in org_ids.split(",") if value.strip()] + except ValueError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="org_ids должен быть списком целых чисел", + ) + return parsed or None + + +@router.get("/form/{form_id}", status_code=status.HTTP_202_ACCEPTED) async def export_form( + background_tasks: BackgroundTasks, form_id: int, direction: DirectionSchemaEnum | None = None, sections: str | None = None, - db: AsyncSession = Depends(get_db), current_user: AppUser = Depends(get_current_active_user_with_set_db), ): - export_service = ExportService(db) - try: - stream, filename, media_type = await export_service.export_form_payload( - form_id=form_id, - current_user=current_user, - direction=direction.value if direction else None, - sections=_parse_sections_csv(sections), - ignore_non_applicable_query_params=True, - ) - except ValueError as exc: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) - return StreamingResponse( - stream, - media_type=media_type, - headers={"Content-Disposition": export_service.build_content_disposition(filename)}, + return await _enqueue_export( + background_tasks, + current_user, + { + "kind": "form", "form_id": form_id, + "direction": direction.value if direction else None, + "sections": _parse_sections_csv(sections), + "ignore_unsupported_sheet_params": True, + }, ) -@router.get("/form/{form_id}/sheet/{sheet}") +@router.get("/form/{form_id}/sheet/{sheet}", status_code=status.HTTP_202_ACCEPTED) async def export_form_sheet( + background_tasks: BackgroundTasks, form_id: int, sheet: str, direction: DirectionSchemaEnum | None = None, sections: str | None = None, - db: AsyncSession = Depends(get_db), current_user: AppUser = Depends(get_current_active_user_with_set_db), ): - export_service = ExportService(db) - try: - stream, filename, media_type = await export_service.export_form_payload( - form_id=form_id, - current_user=current_user, - sheet=sheet, - direction=direction.value if direction else None, - sections=_parse_sections_csv(sections), - ) - except ValueError as exc: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) - return StreamingResponse( - stream, - media_type=media_type, - headers={"Content-Disposition": export_service.build_content_disposition(filename)}, + return await _enqueue_export( + background_tasks, current_user, + {"kind": "form", "form_id": form_id, "sheet": sheet, + "direction": direction.value if direction else None, + "sections": _parse_sections_csv(sections)}, ) -@router.post("/bulk") +@router.post("/bulk", status_code=status.HTTP_202_ACCEPTED) async def export_forms_bulk( + background_tasks: BackgroundTasks, payload: ExportBulkRequest, direction: DirectionSchemaEnum | None = None, sections: str | None = None, - db: AsyncSession = Depends(get_db), current_user: AppUser = Depends(get_current_active_user_with_set_db), ): - export_service = ExportService(db) - try: - stream, filename, media_type = await export_service.export_bulk_payload( - form_ids=payload.form_ids, - current_user=current_user, - skip_failed=payload.skip_failed, - direction=direction.value if direction else None, - sections=_parse_sections_csv(sections), + if len(payload.form_ids) > settings.EXPORT_BULK_MAX_FORMS: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "В пакетном экспорте разрешено не более " + f"{settings.EXPORT_BULK_MAX_FORMS} форм" + ), ) - except ValueError as exc: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) - return StreamingResponse( - stream, - media_type=media_type, - headers={"Content-Disposition": export_service.build_content_disposition(filename)}, + return await _enqueue_export( + background_tasks, current_user, + {"kind": "bulk", "form_ids": payload.form_ids, + "skip_failed": payload.skip_failed, + "direction": direction.value if direction else None, + "sections": _parse_sections_csv(sections)}, ) -@router.get("/project/{project_id}") +@router.get("/summary/{summary_number}/{sheet}/{year}", status_code=status.HTTP_202_ACCEPTED) +async def export_summary( + background_tasks: BackgroundTasks, + summary_number: Annotated[int, Path(ge=1, le=2)], + sheet: str, + year: int, + org_ids: str | None = None, + sort_by: Literal["org_name", "section_code"] = "org_name", + sort_direction: Literal["asc", "desc"] = "asc", + current_user: AppUser = Depends(get_current_active_user_with_set_db), +): + sheet = sheet.upper() + allowed_sheets = ( + SUMMARY1_ALLOWED_SHEETS + if summary_number == 1 + else SUMMARY2_ALLOWED_SHEETS + ) + if sheet not in allowed_sheets: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Неизвестный лист свода {summary_number}: {sheet}", + ) + return await _enqueue_export( + background_tasks, current_user, + {"kind": "summary", "summary_number": summary_number, "year": year, + "sheet": sheet, "org_ids": _parse_org_ids_csv(org_ids), + "sort_by": sort_by, "sort_direction": sort_direction}, + ) + + +@router.get("/project/{project_id}", status_code=status.HTTP_202_ACCEPTED) async def export_project( + background_tasks: BackgroundTasks, project_id: int, - db: AsyncSession = Depends(get_db), current_user: AppUser = Depends(get_current_active_user_with_set_db), ): - export_service = ExportService(db) - try: - stream, filename, media_type = await export_service.export_project_payload( - project_id=project_id, - current_user=current_user, - ) - except ValueError as exc: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) - return StreamingResponse( - stream, - media_type=media_type, - headers={"Content-Disposition": export_service.build_content_disposition(filename)}, + return await _enqueue_export( + background_tasks, current_user, + {"kind": "project", "project_id": project_id}, ) -@router.get("/project/{project_id}/report/{year}/{report_type}") +@router.get("/project/{project_id}/report/{year}/{report_type}", status_code=status.HTTP_202_ACCEPTED) async def export_project_report( + background_tasks: BackgroundTasks, project_id: int, year: int, report_type: str, sections: str | None = None, - db: AsyncSession = Depends(get_db), current_user: AppUser = Depends(get_current_active_user_with_set_db), ): - export_service = ExportService(db) try: - stream, filename, media_type = await export_service.export_project_report_payload( - project_id=project_id, - year=year, - report_type=report_type, - current_user=current_user, - sections=_parse_form3_sections_csv(sections), - ) + parsed_sections = _parse_form3_sections_csv(sections) except ValueError as exc: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) - return StreamingResponse( - stream, - media_type=media_type, - headers={"Content-Disposition": export_service.build_content_disposition(filename)}, + return await _enqueue_export( + background_tasks, current_user, + {"kind": "project_report", "project_id": project_id, "year": year, + "report_type": report_type, "sections": parsed_sections}, ) - - diff --git a/api/src/api/v1/websocket.py b/api/src/api/v1/websocket.py index 4389254..2f805c9 100644 --- a/api/src/api/v1/websocket.py +++ b/api/src/api/v1/websocket.py @@ -1,773 +1,775 @@ -import asyncio -from contextlib import asynccontextmanager -from dataclasses import asdict -import dataclasses -import enum -from json import dumps, loads -import logging -import time -from typing import Any, Optional - -# +import asyncio +from contextlib import asynccontextmanager, suppress +from dataclasses import asdict +from datetime import datetime +import dataclasses +import enum +from json import dumps, loads +import logging +import time +from typing import Any, Optional + +# from sqlalchemy import text -from sqlalchemy.exc import SQLAlchemyError -from fastapi import APIRouter, WebSocket, WebSocketDisconnect, status -from sqlalchemy.ext.asyncio import AsyncSession - -from src.repository.sheet_repository import SheetValidationError -from src.services.rf_project_report_service import RfProjectReportService -from src.services.rf_project_report_line_service import RfProjectReportLineService -from src.repository.user_repository import UserRepository -from src.services.budget_line_service import BudgetLineService -from src.api.v1.deps import get_user_by_token -from src.db.models.app_user import AppUser -from src.db.models.form_type import FormTypeEnum -from src.services.budget_form_service import BudgetFormService -from src.services.project_service import ProjectService -from src.services.sheet_service import SheetService -from src.core.errors import BasicAppException, ValidationsError -from src.core.exception_handlers import map_sqlalchemy_error -from src.db.session import SessionLocal - -from src.services.user_service import UserService - - -logger = logging.getLogger() - -FORM_2_AHR_SATELLITE_SHEETS = { - "AHR_LIMIT", - "AHR_RENT", - "AHR_UTILITY", - "AHR_SECURITY", -} - - -@asynccontextmanager -async def get_db_session(user_id: int | None = None): - """Контекстный менеджер для получения сессии базы данных.""" - db = SessionLocal() - try: - if user_id is not None: - user_repo = UserRepository(db) - await user_repo.set_app_user_id(user_id=user_id) - yield db - finally: - await db.close() - - -router = APIRouter(prefix="/ws", tags=["websocket"]) - - -@dataclasses.dataclass -class ConnectionInfo: - ws: WebSocket - user_id: int | None - - -@dataclasses.dataclass -class FormConnectionInfo(ConnectionInfo): - form_id: int | None = None - sheet: str | None = None - direction: str | None = None - - -@dataclasses.dataclass -class ProjectConnectionInfo(ConnectionInfo): - project_id: int | None = None - year: int | None = None - report_type: str | None = None - - -class ConnectionKeyEnum(str, enum.Enum): - FORM = "FORM" - PROJECT = "PROJECT" - - -class ConnectionManager: - con_info_mapping = { - ConnectionKeyEnum.FORM: FormConnectionInfo, - ConnectionKeyEnum.PROJECT: ProjectConnectionInfo, - } - - def __init__(self): - self.connections: dict[int, dict[ConnectionKeyEnum, list[ConnectionInfo]]] = { - ConnectionKeyEnum.FORM: {}, - ConnectionKeyEnum.PROJECT: {}, - } - self.cell_locks: dict[Any, int] = {} - - - async def connect( - self, - websocket: WebSocket, - con_key: ConnectionKeyEnum, - **kwargs, - # form_id: int, - # sheet: str, - # direction: str | None = None, - ): - await websocket.accept() - - key = frozenset(kwargs.items()) - cls = self.con_info_mapping[con_key] - - if key not in self.connections[con_key]: - self.connections[con_key][key] = [ - cls( - ws=websocket, - user_id=None, - **kwargs, - ) - ] - else: - self.connections[con_key][key].append( - cls( - ws=websocket, - user_id=None, - **kwargs, - ) - - ) - - def set_user( - self, - websocket: WebSocket, - user_id: int, - con_key: ConnectionKeyEnum, - **kwargs, - ): - key = frozenset(kwargs.items()) - - if key not in self.connections[con_key]: - return - for con_info in self.connections[con_key][key]: - if con_info.ws == websocket: - con_info.user_id = user_id - - async def disconnect( - self, - websocket: WebSocket, - con_key: ConnectionKeyEnum, - code: int = status.WS_1008_POLICY_VIOLATION, - reason: str = "Ошибка", - **kwargs, - ): - key = frozenset(kwargs.items()) - - if key not in self.connections[con_key]: - return - for con_info in self.connections[con_key][key]: - if con_info.ws == websocket: - self.connections[con_key][key].remove(con_info) - if not len(self.connections[con_key][key]): - del self.connections[con_key][key] - - self.release_locks_for_user(con_key=con_key, user_id=con_info.user_id, form_key=kwargs) - await websocket.close( - code=code, reason=reason, - ) - - async def broadcast_to_other( - self, - data: dict, - websocket: WebSocket, - con_key: ConnectionKeyEnum, - **kwargs, - ): - key = frozenset(kwargs.items()) - - if key not in self.connections[con_key]: - return - - data = data.copy() - data["is_self"] = False - message = dumps(data, default=_convert_error, ensure_ascii=False) - - for con_info in self.connections[con_key][key]: - if con_info.ws != websocket: - try: - await con_info.ws.send_text(message) - except: - pass # Игнорируем недоступные соединения - - async def broadcast_to_all( - self, - data: dict, - con_key: ConnectionKeyEnum, - websocket: WebSocket | None = None, - **kwargs, - ): - key = frozenset(kwargs.items()) - if key not in self.connections[con_key]: - return - - data = data.copy() - if websocket is None: - data["is_self"] = None - default_message = dumps(data, default=_convert_error, ensure_ascii=False) - self_message = None - else: - data["is_self"] = False - default_message = dumps(data, default=_convert_error, ensure_ascii=False) - data["is_self"] = True - self_message = dumps(data, default=_convert_error, ensure_ascii=False) - - for con_info in self.connections[con_key][key]: - try: - if websocket is not None and con_info.ws == websocket: - await con_info.ws.send_text(self_message) - else: - await con_info.ws.send_text(default_message) - - - except: - pass # Игнорируем недоступные соединения - - async def broadcast_to_form_sheet( - self, - data: dict, - form_id: int, - sheet: str, - ): - data = data.copy() - data["is_self"] = None - message = dumps(data, default=_convert_error, ensure_ascii=False) - - for key, connections in self.connections[ConnectionKeyEnum.FORM].items(): - key_data = dict(key) - if key_data.get("form_id") != form_id or key_data.get("sheet") != sheet: - continue - - for con_info in connections: - try: - await con_info.ws.send_text(message) - except: - pass # Игнорируем недоступные соединения - - async def send_back(self, data: dict, websocket: WebSocket): - data = data.copy() - data["is_self"] = True - message = dumps(data, default=_convert_error, ensure_ascii=False) - await websocket.send_text(message) - - async def get_data(self, websocket: WebSocket) -> dict: - return loads(await websocket.receive_text()) - - def acquire_cell_lock( - self, - con_key: ConnectionKeyEnum, - user_id: int, - cell_key: dict, - form_key: dict, - ) -> bool: - if not cell_key: - return False - cell_key = frozenset(cell_key.items()) - - lock_key = (con_key, frozenset(form_key.items()), cell_key) - lock_owner_id = self.cell_locks.get(lock_key) - if lock_owner_id is not None and lock_owner_id != user_id: - return False - self.cell_locks[lock_key] = user_id - return True - - def release_cell_lock( - self, - con_key: ConnectionKeyEnum, - user_id: int, - cell_key: dict, - form_key: dict, - ): - if not cell_key: - return - cell_key = frozenset(cell_key.items()) - - lock_key = (con_key, frozenset(form_key.items()), cell_key) - lock_owner_id = self.cell_locks.get(lock_key) - if lock_owner_id == user_id: - self.cell_locks.pop(lock_key, None) - - def release_locks_for_user( - self, - con_key: ConnectionKeyEnum, - user_id: int, - form_key: dict - ): - filter_key = (con_key, frozenset(form_key.items())) - lock_keys = [ - lock_key - for lock_key, lock_owner_id in self.cell_locks.items() - if lock_key[:-1] == filter_key and lock_owner_id == user_id - ] - for lock_key in lock_keys: - self.cell_locks.pop(lock_key, None) - - def is_cell_available_for_edit( - self, - con_key: ConnectionKeyEnum, - user_id: int, - cell_key: dict, - form_key, - ) -> bool: - if not cell_key: - return True - cell_key = frozenset(cell_key.items()) - lock_owner_id = self.cell_locks.get((con_key, frozenset(form_key.items()), cell_key)) - return lock_owner_id is not None and lock_owner_id == user_id - - def is_row_locked_by_other( - self, - con_key: ConnectionKeyEnum, - user_id: int, - line_id: int, - ) -> bool: - if line_id is None: - return False - - for (dict_con_key, form_key, cell_key), lock_owner_id in self.cell_locks.items(): - if dict_con_key != con_key: - continue - - if form_key != form_key or lock_owner_id == user_id: - continue - - for k, v in cell_key: - if k == "line_id" and v == line_id: - return True - return False - - def release_locks_for_row( - self, - con_key: ConnectionKeyEnum, - user_id: int, - line_id: int, - ) -> bool: - - if line_id is None: - return False - - lock_keys = [] - for lock_key, lock_owner_id in self.cell_locks.items(): - dict_con_key, form_key, cell_key = lock_key - if dict_con_key != con_key: - continue - - if form_key != form_key or lock_owner_id == user_id: - continue - - for k, v in cell_key: - if k == "line_id" and v == line_id: - lock_keys.append(lock_key) - - for lock_key in lock_keys: - self.cell_locks.pop(lock_key, None) - - -class FormEventProcess: - def __init__(self, db: AsyncSession): - self.sheet_service: SheetService = SheetService(db) - self.bf_service: BudgetFormService = BudgetFormService(db) - self.bl_service: BudgetLineService = BudgetLineService(db) - self.user_service: UserService = UserService(db) - - - async def process( - self, - event_data: dict, - form_id: int, - user_id: int, - sheet: str, - direction: str | None = None, - ) -> int | bool | dict: - curr_user: AppUser = await self.user_service.get(user_id) - - match event_data["event"]: - case "cell_updated": - return await self.__update_cell( - event_data=event_data["data"], - form_id=form_id, - user=curr_user, - sheet=sheet, - direction=direction, - ) - case "row_added": - return await self.__add_row( - event_data=event_data["data"], - form_id=form_id, - user=curr_user, - sheet=sheet, - direction=direction, - ) - case "row_deleted": - return await self.__del_row( - event_data=event_data["data"], - form_id=form_id, - user=curr_user, - sheet=sheet, - direction=direction, - ) - case "program_added": - return await self.__add_program( - event_data=event_data["data"], - form_id=form_id, - user=curr_user, - sheet=sheet, - ) - case "project_added": - return await self.__add_project( - event_data=event_data["data"], - user=curr_user, - ) - case "program_updated": - return await self.__update_form4_program( - event_data=event_data["data"], - user=curr_user, - ) - case "program_deleted": - return await self.__delete_form4_program( - event_data=event_data["data"], - user=curr_user, - ) - case "project_updated": - return await self.__update_form4_project( - event_data=event_data["data"], - user=curr_user, - ) - case "project_deleted": - return await self.__delete_form4_project( - event_data=event_data["data"], - user=curr_user, - ) - case _: - pass - - async def __update_cell( - self, - event_data: dict, - form_id: int, - sheet: str, - user: AppUser, - direction: str | None = None, - ) -> list[tuple]: - """ - event_data: { - "line_id": int, - "column": str, - "value": any, - } - """ - form = await self.bf_service.get(budget_form_id=form_id, user=user) - if not form: - return None - - return await self.sheet_service.update_cell( - form_id=form_id, - sheet=sheet, - direction=direction, - sections=None, - line_id=event_data["line_id"], - column=event_data["column"], - value=event_data["value"], - user=user, - ) - - async def __add_row( - self, - event_data: dict, - form_id: int, - sheet: str, - user: AppUser, - direction: str | None, - ) -> list[tuple]: - """ - data = { - expense_item_id: Optional[int] = None - item_id: Optional[str] = None - section_code: Optional[str] = None - name: Optional[str] = None - internal_order: Optional[str] = None - vsp_id: Optional[int] = None - project_id: Optional[int] = None - justification: Optional[str] = None - contract_number: Optional[str] = None - contract_end_date: Optional[datetime] = None - } - """ - form = await self.bf_service.get(budget_form_id=form_id, user=user) - if not form: - return None - - ids = await self.bl_service.get_ids(budget_form_id=form_id) - result = await self.sheet_service.add_line( - form_id=form_id, - sheet=sheet, - expense_item_id=event_data.get("expense_item_id"), - item_id=event_data.get("item_id"), - section_code=event_data.get("section_code"), - direction=direction, - name=event_data.get("name"), - internal_order=event_data.get("internal_order"), - vsp_id=event_data.get("vsp_id"), - project_id=event_data.get("project_id"), - justification=event_data.get("justification"), - contract_number=event_data.get("contract_number"), - contract_end_date=event_data.get("contract_end_date"), - user=user, - ) - final_result = { - "data": result, - "new_line_id": None, - } - for el in result: - if el[3]["line_id"] and el[3]["line_id"] not in ids: - final_result["new_line_id"] = el[3]["line_id"] - return final_result - return final_result - - async def __add_program( - self, - event_data: dict, - form_id: int, - sheet: str, - user: AppUser, - ) -> list[tuple]: - """ - data = { - name: str - } - """ - return await self.sheet_service.add_program( - form_id=form_id, - sheet=sheet, - name=event_data["name"], - user=user, - ) - - async def __add_project( - self, - event_data: dict, - user: AppUser, - ) -> list[tuple]: - """ - data = { - name: str, - program_id: int - } - """ - return await self.sheet_service.add_project( - name=event_data["name"], - program_id=event_data["program_id"], - user=user, - ) - - async def __update_form4_program( - self, - event_data: dict, - user: AppUser, - ) -> tuple[int, list[tuple]] | None: - return await self.sheet_service.update_form4_program( - program_id=event_data["program_id"], - name=event_data["name"], - user=user, - ) - - async def __delete_form4_program( - self, - event_data: dict, - user: AppUser, - ) -> tuple[int, list[tuple]] | None: - return await self.sheet_service.delete_form4_program( - program_id=event_data["program_id"], - user=user, - ) - - async def __update_form4_project( - self, - event_data: dict, - user: AppUser, - ) -> tuple[int, list[tuple]] | None: - return await self.sheet_service.update_form4_project( - project_id=event_data["project_id"], - name=event_data["name"], - user=user, - ) - - async def __delete_form4_project( - self, - event_data: dict, - user: AppUser, - ) -> tuple[int, list[tuple]] | None: - return await self.sheet_service.delete_form4_project( - project_id=event_data["project_id"], - user=user, - ) - - async def __del_row( - self, - event_data: dict, - form_id: int, - sheet: str, - user: AppUser, - direction: str | None, - ) -> list[tuple]: - """ - data = { - "row_id": int, - } - """ - form = await self.bf_service.get(budget_form_id=form_id, user=user) - if not form: - return None - result = await self.sheet_service.delete_line( - form_id=form_id, - sheet=sheet, - row_id=event_data["row_id"], - direction=direction, - user=user, - ) - return result - - -class ProjectEventProcess: - def __init__(self, db: AsyncSession): - self.project_service: ProjectService = ProjectService(db) - self.prl_service: RfProjectReportLineService = RfProjectReportLineService(db) - self.pr_service: RfProjectReportService = RfProjectReportService(db) - self.user_service: UserService = UserService(db) - - async def process( - self, - event_data: dict, - project_id: int, - user_id: int, - year: int, - report_type: str, - ) -> int | bool | dict: - curr_user: AppUser = await self.user_service.get(user_id) - - match event_data["event"]: - case "cell_updated": - return await self.__update_cell( - event_data=event_data["data"], - project_id=project_id, - year=year, - report_type=report_type, - user=curr_user, - ) - case "row_added": - return await self.__add_row( - event_data=event_data["data"], - project_id=project_id, - year=year, - report_type=report_type, - user=curr_user, - ) - case "row_deleted": - return await self.__del_row( - event_data=event_data["data"], - project_id=project_id, - year=year, - report_type=report_type, - user=curr_user, - ) - case _: - pass - - async def __update_cell( - self, - event_data: dict, - project_id: int, - year: int, - report_type: str, - user: AppUser, - ) -> list[tuple]: - """ - event_data: { - "line_id": int, - "column": str, - "value": any, - } - """ - project = await self.project_service.get(project_id=project_id, user=user) - if not project: - return None - - return await self.project_service.upd_form3_cell( - project_id=project_id, - year=year, - report_type=report_type, - line_id=event_data["line_id"], - column=event_data["column"], - value=event_data["value"], - user=user, - ) - - async def __add_row( - self, - event_data: dict, - project_id: int, - year: int, - report_type: str, - user: AppUser, - ) -> list[tuple]: - """ - data = { - expense_item_id: Optional[int] = None - } - """ - project_report = await self.pr_service.get( - year=year, - report_type=report_type, - project_id=project_id, - ) - if not project_report: - return None - - ids = await self.prl_service.get_ids(project_report_id=project_report.id) - result = await self.project_service.add_form3_line( - project_id=project_id, - year=year, - report_type=report_type, - expense_item_id=event_data["expense_item_id"], - user=user, - ) - - final_result = { - "data": result, - "new_line_id": None, - } - for el in result: - if el[3]["line_id"] and el[3]["line_id"] not in ids: - final_result["new_line_id"] = el[3]["line_id"] - return final_result - return final_result - - async def __del_row( - self, - event_data: dict, - project_id: int, - year: int, - report_type: str, - user: AppUser, - ) -> list[tuple]: - """ - data = { - "row_id": int, - } - """ - project = await self.project_service.get(project_id=project_id, user=user) - if not project: - return None - - return await self.project_service.del_form3_line( - project_id=project_id, - year=year, - report_type=report_type, - line_id=event_data["row_id"], - user=user, - ) - - +from sqlalchemy.exc import SQLAlchemyError +from fastapi import APIRouter, HTTPException, WebSocket, WebSocketDisconnect, status +from sqlalchemy.ext.asyncio import AsyncSession + +from src.repository.sheet_repository import SheetValidationError +from src.services.rf_project_report_service import RfProjectReportService +from src.services.rf_project_report_line_service import RfProjectReportLineService +from src.repository.user_repository import UserRepository +from src.services.budget_line_service import BudgetLineService +from src.api.v1.deps import get_user_by_token +from src.db.models.app_user import AppUser +from src.db.models.form_type import FormTypeEnum +from src.services.budget_form_service import BudgetFormService +from src.services.project_service import ProjectService +from src.services.sheet_service import SheetService +from src.core.errors import BasicAppException, ValidationsError +from src.core.exception_handlers import map_sqlalchemy_error +from src.services.export_service.export_job_service import export_job_manager +from src.db.session import SessionLocal, db_session + +from src.services.user_service import UserService + + +logger = logging.getLogger() + +FORM_2_AHR_SATELLITE_SHEETS = { + "AHR_LIMIT", + "AHR_RENT", + "AHR_UTILITY", + "AHR_SECURITY", +} + + +@asynccontextmanager +async def get_db_session(user_id: int | None = None): + """Контекстный менеджер для получения сессии базы данных.""" + db = SessionLocal() + try: + if user_id is not None: + user_repo = UserRepository(db) + await user_repo.set_app_user_id(user_id=user_id) + yield db + finally: + await db.close() + + +router = APIRouter(prefix="/ws", tags=["websocket"]) + + +@dataclasses.dataclass +class ConnectionInfo: + ws: WebSocket + user_id: int | None + + +@dataclasses.dataclass +class FormConnectionInfo(ConnectionInfo): + form_id: int | None = None + sheet: str | None = None + direction: str | None = None + + +@dataclasses.dataclass +class ProjectConnectionInfo(ConnectionInfo): + project_id: int | None = None + year: int | None = None + report_type: str | None = None + + +class ConnectionKeyEnum(str, enum.Enum): + FORM = "FORM" + PROJECT = "PROJECT" + + +class ConnectionManager: + con_info_mapping = { + ConnectionKeyEnum.FORM: FormConnectionInfo, + ConnectionKeyEnum.PROJECT: ProjectConnectionInfo, + } + + def __init__(self): + self.connections: dict[int, dict[ConnectionKeyEnum, list[ConnectionInfo]]] = { + ConnectionKeyEnum.FORM: {}, + ConnectionKeyEnum.PROJECT: {}, + } + self.cell_locks: dict[Any, int] = {} + + + async def connect( + self, + websocket: WebSocket, + con_key: ConnectionKeyEnum, + **kwargs, + # form_id: int, + # sheet: str, + # direction: str | None = None, + ): + await websocket.accept() + + key = frozenset(kwargs.items()) + cls = self.con_info_mapping[con_key] + + if key not in self.connections[con_key]: + self.connections[con_key][key] = [ + cls( + ws=websocket, + user_id=None, + **kwargs, + ) + ] + else: + self.connections[con_key][key].append( + cls( + ws=websocket, + user_id=None, + **kwargs, + ) + + ) + + def set_user( + self, + websocket: WebSocket, + user_id: int, + con_key: ConnectionKeyEnum, + **kwargs, + ): + key = frozenset(kwargs.items()) + + if key not in self.connections[con_key]: + return + for con_info in self.connections[con_key][key]: + if con_info.ws == websocket: + con_info.user_id = user_id + + async def disconnect( + self, + websocket: WebSocket, + con_key: ConnectionKeyEnum, + code: int = status.WS_1008_POLICY_VIOLATION, + reason: str = "Ошибка", + **kwargs, + ): + key = frozenset(kwargs.items()) + + if key not in self.connections[con_key]: + return + for con_info in self.connections[con_key][key]: + if con_info.ws == websocket: + self.connections[con_key][key].remove(con_info) + if not len(self.connections[con_key][key]): + del self.connections[con_key][key] + + self.release_locks_for_user(con_key=con_key, user_id=con_info.user_id, form_key=kwargs) + await websocket.close( + code=code, reason=reason, + ) + + async def broadcast_to_other( + self, + data: dict, + websocket: WebSocket, + con_key: ConnectionKeyEnum, + **kwargs, + ): + key = frozenset(kwargs.items()) + + if key not in self.connections[con_key]: + return + + data = data.copy() + data["is_self"] = False + message = dumps(data, default=_convert_error, ensure_ascii=False) + + for con_info in self.connections[con_key][key]: + if con_info.ws != websocket: + try: + await con_info.ws.send_text(message) + except: + pass # Игнорируем недоступные соединения + + async def broadcast_to_all( + self, + data: dict, + con_key: ConnectionKeyEnum, + websocket: WebSocket | None = None, + **kwargs, + ): + key = frozenset(kwargs.items()) + if key not in self.connections[con_key]: + return + + data = data.copy() + if websocket is None: + data["is_self"] = None + default_message = dumps(data, default=_convert_error, ensure_ascii=False) + self_message = None + else: + data["is_self"] = False + default_message = dumps(data, default=_convert_error, ensure_ascii=False) + data["is_self"] = True + self_message = dumps(data, default=_convert_error, ensure_ascii=False) + + for con_info in self.connections[con_key][key]: + try: + if websocket is not None and con_info.ws == websocket: + await con_info.ws.send_text(self_message) + else: + await con_info.ws.send_text(default_message) + + + except: + pass # Игнорируем недоступные соединения + + async def broadcast_to_form_sheet( + self, + data: dict, + form_id: int, + sheet: str, + ): + data = data.copy() + data["is_self"] = None + message = dumps(data, default=_convert_error, ensure_ascii=False) + + for key, connections in self.connections[ConnectionKeyEnum.FORM].items(): + key_data = dict(key) + if key_data.get("form_id") != form_id or key_data.get("sheet") != sheet: + continue + + for con_info in connections: + try: + await con_info.ws.send_text(message) + except: + pass # Игнорируем недоступные соединения + + async def send_back(self, data: dict, websocket: WebSocket): + data = data.copy() + data["is_self"] = True + message = dumps(data, default=_convert_error, ensure_ascii=False) + await websocket.send_text(message) + + async def get_data(self, websocket: WebSocket) -> dict: + return loads(await websocket.receive_text()) + + def acquire_cell_lock( + self, + con_key: ConnectionKeyEnum, + user_id: int, + cell_key: dict, + form_key: dict, + ) -> bool: + if not cell_key: + return False + cell_key = frozenset(cell_key.items()) + + lock_key = (con_key, frozenset(form_key.items()), cell_key) + lock_owner_id = self.cell_locks.get(lock_key) + if lock_owner_id is not None and lock_owner_id != user_id: + return False + self.cell_locks[lock_key] = user_id + return True + + def release_cell_lock( + self, + con_key: ConnectionKeyEnum, + user_id: int, + cell_key: dict, + form_key: dict, + ): + if not cell_key: + return + cell_key = frozenset(cell_key.items()) + + lock_key = (con_key, frozenset(form_key.items()), cell_key) + lock_owner_id = self.cell_locks.get(lock_key) + if lock_owner_id == user_id: + self.cell_locks.pop(lock_key, None) + + def release_locks_for_user( + self, + con_key: ConnectionKeyEnum, + user_id: int, + form_key: dict + ): + filter_key = (con_key, frozenset(form_key.items())) + lock_keys = [ + lock_key + for lock_key, lock_owner_id in self.cell_locks.items() + if lock_key[:-1] == filter_key and lock_owner_id == user_id + ] + for lock_key in lock_keys: + self.cell_locks.pop(lock_key, None) + + def is_cell_available_for_edit( + self, + con_key: ConnectionKeyEnum, + user_id: int, + cell_key: dict, + form_key, + ) -> bool: + if not cell_key: + return True + cell_key = frozenset(cell_key.items()) + lock_owner_id = self.cell_locks.get((con_key, frozenset(form_key.items()), cell_key)) + return lock_owner_id is not None and lock_owner_id == user_id + + def is_row_locked_by_other( + self, + con_key: ConnectionKeyEnum, + user_id: int, + line_id: int, + ) -> bool: + if line_id is None: + return False + + for (dict_con_key, form_key, cell_key), lock_owner_id in self.cell_locks.items(): + if dict_con_key != con_key: + continue + + if form_key != form_key or lock_owner_id == user_id: + continue + + for k, v in cell_key: + if k == "line_id" and v == line_id: + return True + return False + + def release_locks_for_row( + self, + con_key: ConnectionKeyEnum, + user_id: int, + line_id: int, + ) -> bool: + + if line_id is None: + return False + + lock_keys = [] + for lock_key, lock_owner_id in self.cell_locks.items(): + dict_con_key, form_key, cell_key = lock_key + if dict_con_key != con_key: + continue + + if form_key != form_key or lock_owner_id == user_id: + continue + + for k, v in cell_key: + if k == "line_id" and v == line_id: + lock_keys.append(lock_key) + + for lock_key in lock_keys: + self.cell_locks.pop(lock_key, None) + + +class FormEventProcess: + def __init__(self, db: AsyncSession): + self.sheet_service: SheetService = SheetService(db) + self.bf_service: BudgetFormService = BudgetFormService(db) + self.bl_service: BudgetLineService = BudgetLineService(db) + self.user_service: UserService = UserService(db) + + + async def process( + self, + event_data: dict, + form_id: int, + user_id: int, + sheet: str, + direction: str | None = None, + ) -> int | bool | dict: + curr_user: AppUser = await self.user_service.get(user_id) + + match event_data["event"]: + case "cell_updated": + return await self.__update_cell( + event_data=event_data["data"], + form_id=form_id, + user=curr_user, + sheet=sheet, + direction=direction, + ) + case "row_added": + return await self.__add_row( + event_data=event_data["data"], + form_id=form_id, + user=curr_user, + sheet=sheet, + direction=direction, + ) + case "row_deleted": + return await self.__del_row( + event_data=event_data["data"], + form_id=form_id, + user=curr_user, + sheet=sheet, + direction=direction, + ) + case "program_added": + return await self.__add_program( + event_data=event_data["data"], + form_id=form_id, + user=curr_user, + sheet=sheet, + ) + case "project_added": + return await self.__add_project( + event_data=event_data["data"], + user=curr_user, + ) + case "program_updated": + return await self.__update_form4_program( + event_data=event_data["data"], + user=curr_user, + ) + case "program_deleted": + return await self.__delete_form4_program( + event_data=event_data["data"], + user=curr_user, + ) + case "project_updated": + return await self.__update_form4_project( + event_data=event_data["data"], + user=curr_user, + ) + case "project_deleted": + return await self.__delete_form4_project( + event_data=event_data["data"], + user=curr_user, + ) + case _: + pass + + async def __update_cell( + self, + event_data: dict, + form_id: int, + sheet: str, + user: AppUser, + direction: str | None = None, + ) -> list[tuple]: + """ + event_data: { + "line_id": int, + "column": str, + "value": any, + } + """ + form = await self.bf_service.get(budget_form_id=form_id, user=user) + if not form: + return None + + return await self.sheet_service.update_cell( + form_id=form_id, + sheet=sheet, + direction=direction, + sections=None, + line_id=event_data["line_id"], + column=event_data["column"], + value=event_data["value"], + user=user, + ) + + async def __add_row( + self, + event_data: dict, + form_id: int, + sheet: str, + user: AppUser, + direction: str | None, + ) -> list[tuple]: + """ + data = { + expense_item_id: Optional[int] = None + item_id: Optional[str] = None + section_code: Optional[str] = None + name: Optional[str] = None + internal_order: Optional[str] = None + vsp_id: Optional[int] = None + project_id: Optional[int] = None + justification: Optional[str] = None + contract_number: Optional[str] = None + contract_end_date: Optional[datetime] = None + } + """ + form = await self.bf_service.get(budget_form_id=form_id, user=user) + if not form: + return None + + ids = await self.bl_service.get_ids(budget_form_id=form_id) + result = await self.sheet_service.add_line( + form_id=form_id, + sheet=sheet, + expense_item_id=event_data.get("expense_item_id"), + item_id=event_data.get("item_id"), + section_code=event_data.get("section_code"), + direction=direction, + name=event_data.get("name"), + internal_order=event_data.get("internal_order"), + vsp_id=event_data.get("vsp_id"), + project_id=event_data.get("project_id"), + justification=event_data.get("justification"), + contract_number=event_data.get("contract_number"), + contract_end_date=event_data.get("contract_end_date"), + user=user, + ) + final_result = { + "data": result, + "new_line_id": None, + } + for el in result: + if el[3]["line_id"] and el[3]["line_id"] not in ids: + final_result["new_line_id"] = el[3]["line_id"] + return final_result + return final_result + + async def __add_program( + self, + event_data: dict, + form_id: int, + sheet: str, + user: AppUser, + ) -> list[tuple]: + """ + data = { + name: str + } + """ + return await self.sheet_service.add_program( + form_id=form_id, + sheet=sheet, + name=event_data["name"], + user=user, + ) + + async def __add_project( + self, + event_data: dict, + user: AppUser, + ) -> list[tuple]: + """ + data = { + name: str, + program_id: int + } + """ + return await self.sheet_service.add_project( + name=event_data["name"], + program_id=event_data["program_id"], + user=user, + ) + + async def __update_form4_program( + self, + event_data: dict, + user: AppUser, + ) -> tuple[int, list[tuple]] | None: + return await self.sheet_service.update_form4_program( + program_id=event_data["program_id"], + name=event_data["name"], + user=user, + ) + + async def __delete_form4_program( + self, + event_data: dict, + user: AppUser, + ) -> tuple[int, list[tuple]] | None: + return await self.sheet_service.delete_form4_program( + program_id=event_data["program_id"], + user=user, + ) + + async def __update_form4_project( + self, + event_data: dict, + user: AppUser, + ) -> tuple[int, list[tuple]] | None: + return await self.sheet_service.update_form4_project( + project_id=event_data["project_id"], + name=event_data["name"], + user=user, + ) + + async def __delete_form4_project( + self, + event_data: dict, + user: AppUser, + ) -> tuple[int, list[tuple]] | None: + return await self.sheet_service.delete_form4_project( + project_id=event_data["project_id"], + user=user, + ) + + async def __del_row( + self, + event_data: dict, + form_id: int, + sheet: str, + user: AppUser, + direction: str | None, + ) -> list[tuple]: + """ + data = { + "row_id": int, + } + """ + form = await self.bf_service.get(budget_form_id=form_id, user=user) + if not form: + return None + result = await self.sheet_service.delete_line( + form_id=form_id, + sheet=sheet, + row_id=event_data["row_id"], + direction=direction, + user=user, + ) + return result + + +class ProjectEventProcess: + def __init__(self, db: AsyncSession): + self.project_service: ProjectService = ProjectService(db) + self.prl_service: RfProjectReportLineService = RfProjectReportLineService(db) + self.pr_service: RfProjectReportService = RfProjectReportService(db) + self.user_service: UserService = UserService(db) + + async def process( + self, + event_data: dict, + project_id: int, + user_id: int, + year: int, + report_type: str, + ) -> int | bool | dict: + curr_user: AppUser = await self.user_service.get(user_id) + + match event_data["event"]: + case "cell_updated": + return await self.__update_cell( + event_data=event_data["data"], + project_id=project_id, + year=year, + report_type=report_type, + user=curr_user, + ) + case "row_added": + return await self.__add_row( + event_data=event_data["data"], + project_id=project_id, + year=year, + report_type=report_type, + user=curr_user, + ) + case "row_deleted": + return await self.__del_row( + event_data=event_data["data"], + project_id=project_id, + year=year, + report_type=report_type, + user=curr_user, + ) + case _: + pass + + async def __update_cell( + self, + event_data: dict, + project_id: int, + year: int, + report_type: str, + user: AppUser, + ) -> list[tuple]: + """ + event_data: { + "line_id": int, + "column": str, + "value": any, + } + """ + project = await self.project_service.get(project_id=project_id, user=user) + if not project: + return None + + return await self.project_service.upd_form3_cell( + project_id=project_id, + year=year, + report_type=report_type, + line_id=event_data["line_id"], + column=event_data["column"], + value=event_data["value"], + user=user, + ) + + async def __add_row( + self, + event_data: dict, + project_id: int, + year: int, + report_type: str, + user: AppUser, + ) -> list[tuple]: + """ + data = { + expense_item_id: Optional[int] = None + } + """ + project_report = await self.pr_service.get( + year=year, + report_type=report_type, + project_id=project_id, + ) + if not project_report: + return None + + ids = await self.prl_service.get_ids(project_report_id=project_report.id) + result = await self.project_service.add_form3_line( + project_id=project_id, + year=year, + report_type=report_type, + expense_item_id=event_data["expense_item_id"], + user=user, + ) + + final_result = { + "data": result, + "new_line_id": None, + } + for el in result: + if el[3]["line_id"] and el[3]["line_id"] not in ids: + final_result["new_line_id"] = el[3]["line_id"] + return final_result + return final_result + + async def __del_row( + self, + event_data: dict, + project_id: int, + year: int, + report_type: str, + user: AppUser, + ) -> list[tuple]: + """ + data = { + "row_id": int, + } + """ + project = await self.project_service.get(project_id=project_id, user=user) + if not project: + return None + + return await self.project_service.del_form3_line( + project_id=project_id, + year=year, + report_type=report_type, + line_id=event_data["row_id"], + user=user, + ) + + manager = ConnectionManager() @@ -883,192 +885,194 @@ async def notify_ahr_about_satellite_change( "new_line_id": new_line_id, }, } - - await manager.broadcast_to_form_sheet( - data=ahr_event, - form_id=form_id, - sheet="AHR", - ) - except Exception: - logger.exception( - "Ошибка отправки на лист АХР", - extra={ + + await manager.broadcast_to_form_sheet( + data=ahr_event, + form_id=form_id, + sheet="AHR", + ) + except Exception: + logger.exception( + "Ошибка отправки на лист АХР", + extra={ "form_id": form_id, "source_sheet": source_sheet, "source_row_id": source_row_id, "source_event": source_event, }, ) - - -def _convert_error(o): - try: - return asdict(o) - except TypeError: - return o - - -async def login(websocket: WebSocket, **kwargs) -> int: - user_data = await manager.get_data(websocket=websocket) - if user_data.get("event") != "user_login": - return None - - async with get_db_session() as db: - user = await get_user_by_token(token=user_data["data"].get("token"), db=db) - if not user: - return None - manager.set_user( - websocket=websocket, - user_id=user.id, - **kwargs, - ) - return user.id - - -def resolve_cell_key(event_data: dict) -> dict | None: - data = event_data.get("data") - - if not data or ("line_id" not in data and "line_id_code" not in data) or "column" not in data: - return None - return { - "line_id": data.get("line_id_code") or data.get("line_id"), - "column": data["column"], - } - - -async def process_websocket( - websocket: WebSocket, - processor_cls, - **kwargs, -): - await manager.connect( - websocket=websocket, - **kwargs, - ) - - - kwargs_process = kwargs.copy() - if "con_key" in kwargs_process: - kwargs_process.pop("con_key") - con_key = kwargs.get("con_key") - - try: - user_id = await login( - websocket=websocket, - **kwargs, - ) - if user_id is None: - - await manager.disconnect( - websocket=websocket, - reason="Ошибка авторизации", - **kwargs, - ) - return - - while True: - data = loads(await websocket.receive_text()) - - try: - event_name = data.get("event") - cell_key = resolve_cell_key(data) - - match event_name: - case "cell_edit_start": - if not manager.acquire_cell_lock( - con_key=con_key, - user_id=user_id, - cell_key=cell_key, - form_key=kwargs_process, - ): - data["error"] = "Ячейка уже редактируется другим пользователем" - await manager.send_back( - data, - websocket, - ) - continue - await manager.send_back( - data, - websocket, - ) - await manager.broadcast_to_other( - data=data, - # user_id=user_id, - websocket=websocket, - con_key=con_key, - **kwargs_process, - ) - continue - case "cell_edit_end": - manager.release_cell_lock( - con_key=con_key, - user_id=user_id, - cell_key=cell_key, - form_key=kwargs_process, - ) - await manager.send_back( - data, - websocket, - ) - await manager.broadcast_to_other( - data=data, - # user_id=user_id, - websocket=websocket, - con_key=con_key, - **kwargs_process, - ) - continue - case "cell_updated": - if not manager.is_cell_available_for_edit( - con_key=con_key, - user_id=user_id, - cell_key=cell_key, - form_key=kwargs_process, - ): - data["error"] = "Ячейка уже редактируется другим пользователем" - await manager.send_back( - data, - websocket, - ) - continue - # if cell_key and not manager.acquire_cell_lock( - # con_key=con_key, - # user_id=user_id, - # cell_key=cell_key, - # form_key=kwargs_process, - # ): - # data["error"] = "Ячейка уже редактируется другим пользователем" - # await manager.send_back( - # data, - # websocket, - # ) - # continue - case "row_deleted": - if manager.is_row_locked_by_other( - con_key=con_key, - user_id=user_id, - line_id=data["data"]["row_id"] - ): - data["error"] = ( - "Нельзя удалить строку: в этой строке есть редактируемые ячейки" - ) - await manager.send_back( - data, - websocket, - ) - continue - case _: - pass - - async with get_db_session(user_id=user_id) as db: - processor = processor_cls(db) - t0 = time.perf_counter() - data_old = data.copy() - data["result"] = await processor.process( - event_data=data, - user_id=user_id, - **kwargs_process - ) + + +def _convert_error(o): + if isinstance(o, datetime): + return o.isoformat() + try: + return asdict(o) + except TypeError: + return o + + +async def login(websocket: WebSocket, **kwargs) -> int: + user_data = await manager.get_data(websocket=websocket) + if user_data.get("event") != "user_login": + return None + + async with get_db_session() as db: + user = await get_user_by_token(token=user_data["data"].get("token"), db=db) + if not user: + return None + manager.set_user( + websocket=websocket, + user_id=user.id, + **kwargs, + ) + return user.id + + +def resolve_cell_key(event_data: dict) -> dict | None: + data = event_data.get("data") + + if not data or ("line_id" not in data and "line_id_code" not in data) or "column" not in data: + return None + return { + "line_id": data.get("line_id_code") or data.get("line_id"), + "column": data["column"], + } + + +async def process_websocket( + websocket: WebSocket, + processor_cls, + **kwargs, +): + await manager.connect( + websocket=websocket, + **kwargs, + ) + + + kwargs_process = kwargs.copy() + if "con_key" in kwargs_process: + kwargs_process.pop("con_key") + con_key = kwargs.get("con_key") + + try: + user_id = await login( + websocket=websocket, + **kwargs, + ) + if user_id is None: + + await manager.disconnect( + websocket=websocket, + reason="Ошибка авторизации", + **kwargs, + ) + return + + while True: + data = loads(await websocket.receive_text()) + + try: + event_name = data.get("event") + cell_key = resolve_cell_key(data) + + match event_name: + case "cell_edit_start": + if not manager.acquire_cell_lock( + con_key=con_key, + user_id=user_id, + cell_key=cell_key, + form_key=kwargs_process, + ): + data["error"] = "Ячейка уже редактируется другим пользователем" + await manager.send_back( + data, + websocket, + ) + continue + await manager.send_back( + data, + websocket, + ) + await manager.broadcast_to_other( + data=data, + # user_id=user_id, + websocket=websocket, + con_key=con_key, + **kwargs_process, + ) + continue + case "cell_edit_end": + manager.release_cell_lock( + con_key=con_key, + user_id=user_id, + cell_key=cell_key, + form_key=kwargs_process, + ) + await manager.send_back( + data, + websocket, + ) + await manager.broadcast_to_other( + data=data, + # user_id=user_id, + websocket=websocket, + con_key=con_key, + **kwargs_process, + ) + continue + case "cell_updated": + if not manager.is_cell_available_for_edit( + con_key=con_key, + user_id=user_id, + cell_key=cell_key, + form_key=kwargs_process, + ): + data["error"] = "Ячейка уже редактируется другим пользователем" + await manager.send_back( + data, + websocket, + ) + continue + # if cell_key and not manager.acquire_cell_lock( + # con_key=con_key, + # user_id=user_id, + # cell_key=cell_key, + # form_key=kwargs_process, + # ): + # data["error"] = "Ячейка уже редактируется другим пользователем" + # await manager.send_back( + # data, + # websocket, + # ) + # continue + case "row_deleted": + if manager.is_row_locked_by_other( + con_key=con_key, + user_id=user_id, + line_id=data["data"]["row_id"] + ): + data["error"] = ( + "Нельзя удалить строку: в этой строке есть редактируемые ячейки" + ) + await manager.send_back( + data, + websocket, + ) + continue + case _: + pass + + async with get_db_session(user_id=user_id) as db: + processor = processor_cls(db) + t0 = time.perf_counter() + data_old = data.copy() + data["result"] = await processor.process( + event_data=data, + user_id=user_id, + **kwargs_process + ) should_refresh_ahr = ( con_key == ConnectionKeyEnum.FORM and kwargs_process.get("sheet") in FORM_2_AHR_SATELLITE_SHEETS @@ -1080,51 +1084,51 @@ async def process_websocket( else None ) await db.commit() - db_ms = (time.perf_counter() - t0) * 1000 - - extra = { - "type": "db_time", - "time_ms": f"{db_ms:.2f}", - "event": data.get("event"), - } - extra.update(data_old) - logger.info( - f"db_time: {extra}", - extra=extra, - ) - - if data.get("event") == "row_deleted": - manager.release_locks_for_row( - con_key=con_key, - user_id=user_id, - line_id=data["data"]["row_id"] - ) - - if data.get("event") in [ - "cell_updated", - "row_added", - "program_added", - "project_added", - "program_updated", - "project_updated", - "program_deleted", - "project_deleted", - "row_deleted", - ]: - await manager.broadcast_to_all( - data=data, - websocket=websocket, - # user_id=user_id, - **kwargs - ) - else: - await manager.broadcast_to_other( - data=data, - # user_id=user_id, - websocket=websocket, - **kwargs, - ) - + db_ms = (time.perf_counter() - t0) * 1000 + + extra = { + "type": "db_time", + "time_ms": f"{db_ms:.2f}", + "event": data.get("event"), + } + extra.update(data_old) + logger.info( + f"db_time: {extra}", + extra=extra, + ) + + if data.get("event") == "row_deleted": + manager.release_locks_for_row( + con_key=con_key, + user_id=user_id, + line_id=data["data"]["row_id"] + ) + + if data.get("event") in [ + "cell_updated", + "row_added", + "program_added", + "project_added", + "program_updated", + "project_updated", + "program_deleted", + "project_deleted", + "row_deleted", + ]: + await manager.broadcast_to_all( + data=data, + websocket=websocket, + # user_id=user_id, + **kwargs + ) + else: + await manager.broadcast_to_other( + data=data, + # user_id=user_id, + websocket=websocket, + **kwargs, + ) + if should_refresh_ahr and ahr_source_row_id is not None: asyncio.create_task( notify_ahr_about_satellite_change( @@ -1135,87 +1139,137 @@ async def process_websocket( user_id=user_id, ) ) - - except BasicAppException as e: - data["error"] = e.description or "Неизвестная ошибка" - await manager.send_back( - data=data, - websocket=websocket, - ) - except SQLAlchemyError as e: - _, code, message, field = map_sqlalchemy_error(e) - data["error"] = { - "code": code, - "field": field, - "message": message, - } - await manager.send_back( - data=data, - websocket=websocket, - ) - except SheetValidationError as e: - data["error"] = e.message - await manager.send_back( - data=data, - websocket=websocket, - ) - - except Exception as e: - tp = type(e) - handlers = websocket.app.exception_handlers - if tp in handlers: - data["error"] = loads((await handlers[tp](request=None, exc=e)).body) - await manager.send_back( - data=data, - websocket=websocket, - ) - else: - raise e - - except WebSocketDisconnect: - await manager.disconnect( - websocket=websocket, - **kwargs - ) - - except Exception as e: - # Логируем ошибку, но не бросаем HTTPException — это WebSocket - print(f"Error: {e}") - await manager.disconnect( - websocket=websocket, - **kwargs, - ) - - -@router.websocket("/form/{form_id}/sheet/{sheet}") -async def websocket_form( - websocket: WebSocket, - form_id: int, - sheet: str, - direction: Optional[str] = None, -): - await process_websocket( - websocket=websocket, - form_id=form_id, - sheet=sheet, - direction=direction, - processor_cls=FormEventProcess, - con_key=ConnectionKeyEnum.FORM, - ) - - -@router.websocket("/projects/{project_id}/report/{year}/{report_type}") -async def websocket_project( - websocket: WebSocket, - project_id: int, - year: int, - report_type: str, -): - await process_websocket( - websocket=websocket, - project_id=project_id, - year=year, - report_type=report_type, - processor_cls=ProjectEventProcess, - con_key=ConnectionKeyEnum.PROJECT, - ) + + except BasicAppException as e: + data["error"] = e.description or "Неизвестная ошибка" + await manager.send_back( + data=data, + websocket=websocket, + ) + except SQLAlchemyError as e: + _, code, message, field = map_sqlalchemy_error(e) + data["error"] = { + "code": code, + "field": field, + "message": message, + } + await manager.send_back( + data=data, + websocket=websocket, + ) + except SheetValidationError as e: + data["error"] = e.message + await manager.send_back( + data=data, + websocket=websocket, + ) + + except Exception as e: + tp = type(e) + handlers = websocket.app.exception_handlers + if tp in handlers: + data["error"] = loads((await handlers[tp](request=None, exc=e)).body) + await manager.send_back( + data=data, + websocket=websocket, + ) + else: + raise e + + except WebSocketDisconnect: + await manager.disconnect( + websocket=websocket, + **kwargs + ) + + except Exception as e: + # Логируем ошибку, но не бросаем HTTPException — это WebSocket + print(f"Error: {e}") + await manager.disconnect( + websocket=websocket, + **kwargs, + ) + + +@router.websocket("/form/{form_id}/sheet/{sheet}") +async def websocket_form( + websocket: WebSocket, + form_id: int, + sheet: str, + direction: Optional[str] = None, +): + await process_websocket( + websocket=websocket, + form_id=form_id, + sheet=sheet, + direction=direction, + processor_cls=FormEventProcess, + con_key=ConnectionKeyEnum.FORM, + ) + + +@router.websocket("/projects/{project_id}/report/{year}/{report_type}") +async def websocket_project( + websocket: WebSocket, + project_id: int, + year: int, + report_type: str, +): + await process_websocket( + websocket=websocket, + project_id=project_id, + year=year, + report_type=report_type, + processor_cls=ProjectEventProcess, + con_key=ConnectionKeyEnum.PROJECT, + ) + + +@router.websocket("/export") +async def export_socket(websocket: WebSocket): + await websocket.accept() + queue = None + user_id = None + receiver = None + try: + data = loads(await websocket.receive_text()) + token = data.get('token') if isinstance(data, dict) else None + if not isinstance(token, str): + await websocket.close(code=4401) + return + async with db_session() as db: + user = await get_user_by_token(token, db) + user_id = user.id + queue = export_job_manager.subscribe(user_id) + receiver = asyncio.create_task(websocket.receive_text()) + while True: + jobs = await export_job_manager.list_jobs(user_id) + data = {'event': 'exports', 'jobs': [j.as_dict() for j in jobs]} + message = dumps(data, default=_convert_error, ensure_ascii=False) + await websocket.send_text(message) + update = asyncio.create_task(queue.get()) + try: + done, _ = await asyncio.wait( + {receiver, update}, + timeout=10, + return_when=asyncio.FIRST_COMPLETED, + ) + if receiver in done: + receiver = asyncio.create_task(websocket.receive_text()) + finally: + update.cancel() + try: + await update + except asyncio.CancelledError: + pass + except (WebSocketDisconnect, OSError): + pass + except (HTTPException, ValueError, asyncio.TimeoutError): + await websocket.close(code=4401) + finally: + if receiver is not None: + receiver.cancel() + with suppress(asyncio.CancelledError, WebSocketDisconnect): + await receiver + if queue is not None: + export_job_manager.unsubscribe(user_id, queue) diff --git a/api/src/core/config.py b/api/src/core/config.py index dc8e269..0d62061 100644 --- a/api/src/core/config.py +++ b/api/src/core/config.py @@ -108,6 +108,36 @@ class Settings(BaseSettings): alias="AUDIT_LOG_CLEANUP_INTERVAL_SECONDS", ) + EXPORT_DIR: str = Field( + default="tmp/exports", + description="Каталог временных файлов фонового экспорта", + alias="EXPORT_DIR", + ) + EXPORT_PROCESS_WORKERS: int = Field( + default=2, + ge=1, + description="Число отдельных процессов для формирования файлов экспорта", + alias="EXPORT_PROCESS_WORKERS", + ) + EXPORT_BULK_MAX_FORMS: int = Field( + default=10, + ge=1, + description="Максимальное число форм в одном пакетном экспорте", + alias="EXPORT_BULK_MAX_FORMS", + ) + EXPORT_FILE_TTL_SECONDS: int = Field( + default=86400, + ge=60, + description="Срок хранения готовых файлов экспорта в секундах", + alias="EXPORT_FILE_TTL_SECONDS", + ) + EXPORT_CLEANUP_INTERVAL_SECONDS: int = Field( + default=3600, + ge=60, + description="Интервал очистки устаревших файлов экспорта в секундах", + alias="EXPORT_CLEANUP_INTERVAL_SECONDS", + ) + model_config = SettingsConfigDict( env_file=".env", extra="ignore", diff --git a/api/src/db/models/__init__.py b/api/src/db/models/__init__.py index 85c2fb7..f304900 100644 --- a/api/src/db/models/__init__.py +++ b/api/src/db/models/__init__.py @@ -10,6 +10,7 @@ from src.db.models.collegial_approval import CollegialApproval from src.db.models.contract_detail import ContractDetail from src.db.models.contract_summary import ContractSummary from src.db.models.expense_item import ExpenseItem, ExpenseItemFormType +from src.db.models.export_job import ExportJob from src.db.models.form3_phase import Form3Phase from src.db.models.form4_project import Form4Project from src.db.models.form_limit import FormLimit diff --git a/api/src/db/models/export_job.py b/api/src/db/models/export_job.py new file mode 100644 index 0000000..7208e0a --- /dev/null +++ b/api/src/db/models/export_job.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Literal +from uuid import UUID as PythonUUID, uuid4 + +from sqlalchemy import DateTime, ForeignKey, JSON, String, Text, Uuid +from sqlalchemy.orm import Mapped, mapped_column + +from src.db.base import Base + +ExportJobStatus = Literal["queued", "processing", "completed", "failed"] + + +class ExportJob(Base): + __tablename__ = "export_job" + __table_args__ = {"schema": "v3"} + + id: Mapped[PythonUUID] = mapped_column(Uuid(as_uuid=True), primary_key=True) + user_id: Mapped[int] = mapped_column(ForeignKey("v3.app_user.id")) + request: Mapped[dict[str, Any]] = mapped_column(JSON) + status: Mapped[ExportJobStatus] = mapped_column(String(20), default="queued") + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + filename: Mapped[str | None] = mapped_column(Text) + media_type: Mapped[str | None] = mapped_column(Text) + _file_path: Mapped[str | None] = mapped_column("file_path", Text) + error: Mapped[str | None] = mapped_column(Text) + + def __init__( + self, + user_id: int, + request: dict[str, Any], + id: PythonUUID | None = None, + status: ExportJobStatus = "queued", + created_at: datetime | None = None, + started_at: datetime | None = None, + finished_at: datetime | None = None, + filename: str | None = None, + media_type: str | None = None, + file_path: str | Path | None = None, + error: str | None = None, + ) -> None: + self.id = id or uuid4() + self.user_id = user_id + self.request = request + self.status = status + self.created_at = created_at or datetime.now(timezone.utc) + self.started_at = started_at + self.finished_at = finished_at + self.filename = filename + self.media_type = media_type + self.file_path = file_path + self.error = error + + @property + def file_path(self) -> Path | None: + return Path(self._file_path) if self._file_path else None + + @file_path.setter + def file_path(self, value: str | Path | None) -> None: + self._file_path = str(value) if value else None + + def as_dict(self) -> dict[str, Any]: + return { + "id": str(self.id), + "status": self.status, + "created_at": self.created_at, + "started_at": self.started_at, + "finished_at": self.finished_at, + "filename": self.filename, + "media_type": self.media_type, + "error": self.error, + "status_url": f"/api/v1/export/jobs/{self.id}", + "download_url": ( + f"/api/v1/export/jobs/{self.id}/download" + if self.status == "completed" + else None + ), + } diff --git a/api/src/db/session.py b/api/src/db/session.py index 83da0bc..1c6e36c 100644 --- a/api/src/db/session.py +++ b/api/src/db/session.py @@ -1,16 +1,17 @@ import asyncio -from typing import AsyncGenerator +from contextlib import asynccontextmanager +from typing import AsyncGenerator, AsyncIterator from alembic import command from alembic.config import Config +from sqlalchemy.ext.asyncio import AsyncSession from src.core.config import settings -from typing import AsyncGenerator - from src.db.base import SessionLocal -async def get_db() -> AsyncGenerator: +@asynccontextmanager +async def db_session() -> AsyncGenerator[AsyncSession, None]: db = SessionLocal() try: yield db @@ -22,6 +23,11 @@ async def get_db() -> AsyncGenerator: await db.close() +async def get_db() -> AsyncGenerator[AsyncSession, None]: + async with db_session() as db: + yield db + + async def run_migrations() -> None: alembic_cfg = Config("alembic.ini") await asyncio.to_thread(command.upgrade, alembic_cfg, "head") diff --git a/api/src/main.py b/api/src/main.py index 1fc02e3..e0e4cc7 100644 --- a/api/src/main.py +++ b/api/src/main.py @@ -20,8 +20,7 @@ from src.core.config import settings from src.core.exception_handlers import register_exception_handlers from src.db.base import engine from src.db.session import create_tables - - +from src.services.export_service.export_job_service import export_job_manager logging.basicConfig(level=logging.INFO) @@ -99,6 +98,17 @@ async def _audit_cleanup_loop() -> None: await asyncio.sleep(max(60, settings.AUDIT_LOG_CLEANUP_INTERVAL_SECONDS)) +async def _export_cleanup_loop() -> None: + while True: + try: + removed = await export_job_manager.cleanup_expired() + if removed: + logging.info("Removed %s expired export file(s)", removed) + except Exception: + logging.exception("Failed to clean up expired export files") + await asyncio.sleep(settings.EXPORT_CLEANUP_INTERVAL_SECONDS) + + @asynccontextmanager async def lifespan(app: FastAPI): cleanup_task: asyncio.Task | None = None @@ -112,6 +122,8 @@ async def lifespan(app: FastAPI): # ), # ) cleanup_task = asyncio.create_task(_audit_cleanup_loop()) + await export_job_manager.recover_interrupted() + export_cleanup_task = asyncio.create_task(_export_cleanup_loop()) try: yield finally: @@ -119,6 +131,10 @@ async def lifespan(app: FastAPI): cleanup_task.cancel() with suppress(asyncio.CancelledError): await cleanup_task + export_cleanup_task.cancel() + with suppress(asyncio.CancelledError): + await export_cleanup_task + await export_job_manager.shutdown() await engine.dispose() diff --git a/api/src/repository/export_job_repository.py b/api/src/repository/export_job_repository.py new file mode 100644 index 0000000..ecb9128 --- /dev/null +++ b/api/src/repository/export_job_repository.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from sqlalchemy import delete, select + +from src.db.models.export_job import ExportJob +from src.db.session import db_session + + +class ExportJobRepository: + async def save(self, job: ExportJob) -> None: + async with db_session() as db: + await db.merge(job) + + async def get(self, job_id) -> ExportJob | None: + async with db_session() as db: + return await db.get(ExportJob, job_id) + + async def list(self, user_id, cutoff) -> list[ExportJob]: + async with db_session() as db: + result = await db.scalars( + select(ExportJob) + .where( + ExportJob.user_id == user_id, + (ExportJob.finished_at.is_(None)) + | (ExportJob.finished_at > cutoff), + ) + .order_by(ExportJob.created_at.desc()) + ) + return list(result) + + async def recover_interrupted(self) -> list[ExportJob]: + async with db_session() as db: + result = await db.scalars( + select(ExportJob).where( + ExportJob.status.in_(("queued", "processing")) + ) + ) + jobs = list(result) + for job in jobs: + job.status = "queued" + job.started_at = None + job.finished_at = None + job.filename = None + job.media_type = None + job.file_path = None + job.error = None + return jobs + + async def cleanup(self, cutoff) -> None: + async with db_session() as db: + await db.execute(delete(ExportJob).where(ExportJob.finished_at <= cutoff)) diff --git a/api/src/repository/summary_repository.py b/api/src/repository/summary_repository.py index 781b477..88285bb 100644 --- a/api/src/repository/summary_repository.py +++ b/api/src/repository/summary_repository.py @@ -1,3 +1,5 @@ +from typing import Any + from sqlalchemy import text from sqlalchemy.ext.asyncio import AsyncSession @@ -58,7 +60,6 @@ class SummaryRepository: CAST(:sort_by AS VARCHAR), CAST(:sort_direction AS VARCHAR) ) - ORDER BY sort_order """ ) return ( @@ -74,3 +75,120 @@ class SummaryRepository: }, ) ).all() + + async def get_svod2_go_export_rows( + self, + year: int, + org_ids: list[int] | None, + sort_by: str, + sort_direction: str, + ) -> list[tuple]: + """Load the compact flat representation produced for XLSX export.""" + common_quarter_fields = ( + "adj_current", "adj_ssp", "adj_rf", "adj_reserve", + "payment_date", "payment_amount", "payment_amount_ho", + "payment_amount_rf", "payment_act", "booking_amount", + "actual_m1", "actual_m2", "actual_m3", + ) + export_keys = [ + "header.smeta_type", + "header.smeta_direction", + "header.org_name", + "header.name", + "header.item_id", + "header.num_group", + "header.justification", + *(f"plan.plan_q{quarter}" for quarter in range(1, 5)), + "allocation.internal_order", + "allocation.property_object", + *(f"sequestration.DFIP.adj_q{quarter}" for quarter in range(1, 5)), + *(f"reserve.amount_q{quarter}" for quarter in range(1, 5)), + *(f"approved.approved_q{quarter}" for quarter in range(1, 5)), + *( + f"contract_detail.{field}" + for field in ( + "counterparty", "reference", "subject", "currency", + "ceiling_amount", "expenses_q1", "expenses_q2", + "expenses_q3", "expenses_q4", "vat_rate", "deadline", + ) + ), + ] + for quarter in range(1, 5): + quarter_fields = list(common_quarter_fields) + if quarter == 4: + quarter_fields.append("actual_spod") + if quarter > 1: + quarter_fields.extend( + ( + "plan_revision_eco_change", + "plan_revision_item_adj", + "plan_revision_increase", + "plan_revision_sequester", + ) + ) + quarter_fields.extend( + f"transfer_to_q{target_quarter}" + for target_quarter in range(quarter + 1, 5) + ) + if quarter < 4: + quarter_fields.append("transfer_to_economy") + export_keys.extend(f"q{quarter}.{field}" for field in quarter_fields) + + query = text( + """ + SELECT sort_order, row_values + FROM v3.v_svod2_go_export( + CAST(:year AS INT), + CAST(:org_ids AS INT[]), + CAST(:sort_by AS VARCHAR), + CAST(:sort_direction AS VARCHAR) + ) + """ + ) + result = await self.db.execute( + query, + { + "year": year, + "org_ids": org_ids, + "sort_by": sort_by, + "sort_direction": sort_direction.lower(), + }, + ) + rows = [] + for row in result.mappings(): + data = { + key: self._parse_svod2_export_value(key, value) + for key, value in zip(export_keys, row["row_values"]) + } + rows.append(("INPUT", 3, row["sort_order"], data)) + return rows + + @staticmethod + def _parse_svod2_export_value(key: str, value: str | None) -> Any: + if value is None: + return None + numeric_prefixes = ( + "plan.", + "sequestration.", + "reserve.", + "approved.", + ) + numeric_contract_fields = { + "contract_detail.ceiling_amount", + "contract_detail.expenses_q1", + "contract_detail.expenses_q2", + "contract_detail.expenses_q3", + "contract_detail.expenses_q4", + } + quarter_text_suffixes = (".payment_date", ".payment_act") + is_numeric_quarter = ( + key.startswith(("q1.", "q2.", "q3.", "q4.")) + and not key.endswith(quarter_text_suffixes) + ) + if ( + key.startswith(numeric_prefixes) + or key in numeric_contract_fields + or is_numeric_quarter + ): + return float(value) + return value diff --git a/api/src/services/export_service/export_job_service.py b/api/src/services/export_service/export_job_service.py new file mode 100644 index 0000000..c6c8efb --- /dev/null +++ b/api/src/services/export_service/export_job_service.py @@ -0,0 +1,265 @@ +from __future__ import annotations + +import asyncio +import logging +import multiprocessing +from concurrent.futures import ProcessPoolExecutor +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any +from uuid import UUID, uuid4 + +from src.core.config import settings +from src.db.base import engine +from src.db.models.export_job import ExportJob +from src.db.session import db_session +from src.repository.export_job_repository import ExportJobRepository +from src.repository.user_repository import UserRepository +from src.services.export_service.export_service import ExportService +from sqlalchemy.exc import IntegrityError + +logger = logging.getLogger(__name__) + + +class ExportJobAlreadyRunningError(RuntimeError): + pass + + +class ExportJobManager: + def __init__(self) -> None: + self._jobs: dict[UUID, ExportJob] = {} + self._active_by_user: dict[int, UUID] = {} + self._lock = asyncio.Lock() + self.repository = ExportJobRepository() + self._subscribers: dict[int, set[asyncio.Queue]] = {} + self.export_dir = Path(settings.EXPORT_DIR).resolve() + self._executor: ProcessPoolExecutor | None = None + + def _get_executor(self) -> ProcessPoolExecutor: + if self._executor is None: + self._executor = ProcessPoolExecutor( + max_workers=settings.EXPORT_PROCESS_WORKERS, + mp_context=multiprocessing.get_context("spawn"), + ) + return self._executor + + async def _execute_export(self, service, current_user, request: dict[str, Any]): + request = request.copy() + kind = request.pop("kind") + if kind == "form": + return await service.export_form_payload(current_user=current_user, **request) + if kind == "bulk": + return await service.export_bulk_payload(current_user=current_user, **request) + if kind == "summary": + return await service.export_summary_payload( + current_user=current_user, **request + ) + if kind == "project": + return await service.export_project_payload( + current_user=current_user, **request + ) + if kind == "project_report": + return await service.export_project_report_payload( + current_user=current_user, **request + ) + raise ValueError(f"Неизвестный тип экспорта: {kind}") + + async def _generate_export_file( + self, + job_id: str, + user_id: int, + request: dict[str, Any], + export_dir_value: str, + ) -> dict[str, str]: + try: + async with db_session() as db: + user_repo = UserRepository(db) + current_user = await user_repo.get(user_id) + if current_user is None or not current_user.is_active: + raise RuntimeError("Пользователь не найден или заблокирован") + await user_repo.set_app_user_id(current_user.id) + stream, filename, media_type = await self._execute_export( + ExportService(db), current_user, request + ) + + export_dir = Path(export_dir_value) + export_dir.mkdir(parents=True, exist_ok=True) + suffix = Path(filename).suffix or ".bin" + file_path = export_dir / f"{job_id}{suffix}" + try: + stream.seek(0) + file_path.write_bytes(stream.read()) + finally: + stream.close() + return { + "filename": filename, + "media_type": media_type, + "file_path": str(file_path), + } + finally: + await engine.dispose() + + @staticmethod + def _run_export_process( + job_id: str, + user_id: int, + request: dict[str, Any], + export_dir: str, + ) -> dict[str, str]: + manager = ExportJobManager() + return asyncio.run( + manager._generate_export_file(job_id, user_id, request, export_dir) + ) + + async def create(self, user_id: int, request: dict[str, Any]) -> ExportJob: + async with self._lock: + active_id = self._active_by_user.get(user_id) + if active_id is not None: + active = self._jobs.get(active_id) + if active and active.status in {"queued", "processing"}: + raise ExportJobAlreadyRunningError + job = ExportJob(id=uuid4(), user_id=user_id, request=request) + try: + await self.repository.save(job) + except IntegrityError as exc: + if "export_job_active_user" not in str(exc): + raise + raise ExportJobAlreadyRunningError from exc + self._jobs[job.id] = job + self._active_by_user[user_id] = job.id + self.notify(user_id) + return job + + async def list_jobs(self, user_id: int) -> list[ExportJob]: + cutoff = datetime.now(timezone.utc) - timedelta( + seconds=settings.EXPORT_FILE_TTL_SECONDS + ) + return await self.repository.list(user_id, cutoff) + + async def get(self, job_id: UUID, user_id: int) -> ExportJob | None: + return next( + (job for job in await self.list_jobs(user_id) if job.id == job_id), None + ) + + async def mark_file_missing(self, job: ExportJob) -> None: + job.status = "failed" + job.error = "Файл экспорта удалён" + job.file_path = None + if job.finished_at is None: + job.finished_at = datetime.now(timezone.utc) + await self.repository.save(job) + self._jobs[job.id] = job + self.notify(job.user_id) + + def subscribe(self, user_id): + queue = asyncio.Queue(maxsize=1) + self._subscribers.setdefault(user_id, set()).add(queue) + return queue + + def unsubscribe(self, user_id, queue): + queues = self._subscribers.get(user_id, set()) + queues.discard(queue) + if not queues: + self._subscribers.pop(user_id, None) + + def notify(self, user_id): + for queue in self._subscribers.get(user_id, ()): + if queue.empty(): + queue.put_nowait(True) + + async def run(self, job_id: UUID) -> None: + job = self._jobs.get(job_id) or await self.repository.get(job_id) + if job is None: + return + self._jobs[job.id] = job + job.status = "processing" + job.started_at = datetime.now(timezone.utc) + try: + await self.repository.save(job) + self.notify(job.user_id) + loop = asyncio.get_running_loop() + result = await loop.run_in_executor( + self._get_executor(), + ExportJobManager._run_export_process, + str(job.id), + job.user_id, + job.request, + str(self.export_dir), + ) + job.filename = result["filename"] + job.media_type = result["media_type"] + job.file_path = Path(result["file_path"]) + job.status = "completed" + except Exception as exc: + logger.exception("Export job %s failed", job.id) + job.status = "failed" + job.error = str(exc) or "Не удалось сформировать файл" + finally: + job.finished_at = datetime.now(timezone.utc) + await self.repository.save(job) + self.notify(job.user_id) + async with self._lock: + if self._active_by_user.get(job.user_id) == job.id: + self._active_by_user.pop(job.user_id, None) + + async def recover_interrupted(self) -> None: + jobs = await self.repository.recover_interrupted() + async with self._lock: + for job in jobs: + self._jobs[job.id] = job + self._active_by_user[job.user_id] = job.id + for job in jobs: + self.notify(job.user_id) + asyncio.create_task(self.run(job.id)) + + async def shutdown(self) -> None: + if self._executor is None: + return + executor, self._executor = self._executor, None + await asyncio.to_thread(executor.shutdown, wait=True, cancel_futures=True) + + async def cleanup_expired(self) -> int: + now = datetime.now(timezone.utc) + cutoff = now - timedelta(seconds=settings.EXPORT_FILE_TTL_SECONDS) + async with self._lock: + expired_ids = [ + job_id + for job_id, job in self._jobs.items() + if job.status not in {"queued", "processing"} + and job.finished_at is not None + and job.finished_at <= cutoff + ] + for job_id in expired_ids: + job = self._jobs.pop(job_id, None) + if job and self._active_by_user.get(job.user_id) == job_id: + self._active_by_user.pop(job.user_id, None) + + removed = self._cleanup_export_files(cutoff) + await self.repository.cleanup(cutoff) + return removed + + def _cleanup_export_files(self, cutoff: datetime) -> int: + if not self.export_dir.is_dir(): + return 0 + removed = 0 + for file_path in self.export_dir.iterdir(): + if not file_path.is_file(): + continue + try: + UUID(file_path.stem) + except ValueError: + continue + modified_at = datetime.fromtimestamp( + file_path.stat().st_mtime, tz=timezone.utc + ) + if modified_at > cutoff: + continue + try: + file_path.unlink() + removed += 1 + except FileNotFoundError: + pass + return removed + + +export_job_manager = ExportJobManager() diff --git a/api/src/services/export_service/export_mappings.py b/api/src/services/export_service/export_mappings.py index eb5bf2c..98344c7 100644 --- a/api/src/services/export_service/export_mappings.py +++ b/api/src/services/export_service/export_mappings.py @@ -1,3 +1,5 @@ +import copy + FORM1_FLAT_COLUMNS: list[tuple[str, str]] = [ @@ -1423,6 +1425,7 @@ PALETTE = { "GROUP": "D7E3BC", "ITEM": "EAF0DD", "SUB_ITEM": "FFFFFF", + "SUB_SUB_ITEM": "FFFFFF", "INPUT": "FFFFFF", "COLOR": "000000", }, @@ -1430,8 +1433,8 @@ PALETTE = { "ROOT": "C2D59A", "GROUP": "D7E3BC", "ITEM": "EAF0DD", - "SUB_ITEM": "FFFFFF", "SUB_ITEM": "EAF0DD", + "SUB_SUB_ITEM": "EAF0DD", "INPUT": "FFFFFF", "COLOR": "000000", }, @@ -1440,6 +1443,7 @@ PALETTE = { "GROUP": "F6D6B9", "ITEM": "FAEBDC", "SUB_ITEM": "FFFFFF", + "SUB_SUB_ITEM": "FFFFFF", "INPUT": "FFFFFF", "COLOR": "000000", }, @@ -1448,6 +1452,7 @@ PALETTE = { "GROUP": "FFFFFF", "ITEM": "ffffff", "SUB_ITEM": "FFFFFF", + "SUB_SUB_ITEM": "FFFFFF", "INPUT": "FFFFFF", "COLOR": "000000", }, @@ -1456,6 +1461,7 @@ PALETTE = { "GROUP": "D89493", "ITEM": "E5B7B6", "SUB_ITEM": "FFFFFF", + "SUB_SUB_ITEM": "FFFFFF", "INPUT": "FFFFFF", "COLOR": "ffffff", }, @@ -1464,6 +1470,7 @@ PALETTE = { "GROUP": "8DB3E2", "ITEM": "C5D8F0", "SUB_ITEM": "FFFFFF", + "SUB_SUB_ITEM": "FFFFFF", "INPUT": "FFFFFF", "COLOR": "000000", }, @@ -1472,6 +1479,7 @@ PALETTE = { "GROUP": "FFFF99", "ITEM": "FFFFCC", "SUB_ITEM": "FFFFFF", + "SUB_SUB_ITEM": "FFFFFF", "INPUT": "FFFFFF", "COLOR": "000000", }, @@ -1504,7 +1512,7 @@ FORM1_COLORS: dict = { "palette": "blue", }, "upper_ssp_smeta.contract_summary.deadline": { - "palette": None, + "palette": "blue", }, "allocation": { "background": "bfbfbf", @@ -1542,6 +1550,10 @@ FORM1_COLORS: dict = { "empty_upper.ckk.q4": { "background": "748c42", }, + "empty_upper.ckk.deadline": { + "background": "fabf8f", + "palette": "orange", + }, "empty_upper.contract_detail": { "background": "fabf8f", "palette": "orange", @@ -1558,6 +1570,10 @@ FORM1_COLORS: dict = { "empty_upper.contract_detail.q4": { "background": "748c42", }, + "empty_upper.contract_detail.deadline": { + "background": "fabf8f", + "palette": "orange", + }, "q1.cur_correct": { "background": "FFFF66", "palette": "yellow", @@ -2043,7 +2059,7 @@ FORM1_COLORS: dict = { "palette": "green", }, "upper_ssp_smeta.contract_summary.deadline": { - "palette": None, + "palette": "green", }, "allocation": { "background": "bfbfbf", @@ -2081,6 +2097,10 @@ FORM1_COLORS: dict = { "empty_upper.ckk.q4": { "background": "748c42", }, + "empty_upper.ckk.deadline": { + "background": "fabf8f", + "palette": "orange", + }, "empty_upper.contract_detail": { "background": "fabf8f", "palette": "orange", @@ -2097,6 +2117,10 @@ FORM1_COLORS: dict = { "empty_upper.contract_detail.q4": { "background": "748c42", }, + "empty_upper.contract_detail.deadline": { + "background": "fabf8f", + "palette": "orange", + }, "q1.cur_correct": { "background": "FFFF66", "palette": "yellow", @@ -2582,7 +2606,7 @@ FORM1_COLORS: dict = { "palette": "orange", }, "upper_ssp_smeta.contract_summary.deadline": { - "palette": None, + "palette": "orange", }, "allocation": { "background": "bfbfbf", @@ -2620,6 +2644,10 @@ FORM1_COLORS: dict = { "empty_upper.ckk.q4": { "background": "748c42", }, + "empty_upper.ckk.deadline": { + "background": "fabf8f", + "palette": "orange", + }, "empty_upper.contract_detail": { "background": "fabf8f", "palette": "orange", @@ -2636,6 +2664,10 @@ FORM1_COLORS: dict = { "empty_upper.contract_detail.q4": { "background": "748c42", }, + "empty_upper.contract_detail.deadline": { + "background": "fabf8f", + "palette": "orange", + }, "q1.cur_correct": { "background": "FFFF66", "palette": "yellow", @@ -3098,6 +3130,262 @@ FORM1_COLORS: dict = { } +FORM4_COLUMN_RENAMES = { + "data": "header", + "header.section": "header.section_code", + "sequestration": "seq_dfip", + "allocation.order": "allocation.contract_ref", + "allocation.property": "allocation.allocation_purpose", + "empty_upper.contract_detail": "empty_upper.contract", + "contract_detail.payment_scheme": "contract.scheme", + "empty_upper.contract.payment_scheme": "empty_upper.contract.scheme", + "q1.rem_correct.rem_booking": "q1.rem_correct.residual_after_booking", + "q1.rem_actual_correct.rem_actual": "q1.rem_actual_correct.residual_after_actual", + "q1.close_q1.economy": "q1.close_q1.transfer_econ", + "q2.rev_eco_correct.rev_eco": "q2.rev_eco_correct.target_change", + "q2.rev_item_correct.rev_item": "q2.rev_item_correct.base_correction", + "q2.rev_inc_correct.rev_inc": "q2.rev_inc_correct.revision_inc", + "q2.rev_seq_correct.rev_seq": "q2.rev_seq_correct.revision_seq", + "q2.rev_correct.rev_comment": "q2.rev_correct.revision_comment", + "q2.rem_correct.rem_booking": "q2.rem_correct.residual_after_booking", + "q2.rem_actual_correct.rem_actual": "q2.rem_actual_correct.residual_after_actual", + "q2.close_q2.economy": "q2.close_q2.transfer_econ", + "q3.rev_eco_correct.rev_eco": "q3.rev_eco_correct.target_change", + "q3.rev_item_correct.rev_item": "q3.rev_item_correct.base_correction", + "q3.rev_inc_correct.rev_inc": "q3.rev_inc_correct.revision_inc", + "q3.rev_seq_correct.rev_seq": "q3.rev_seq_correct.revision_seq", + "q3.rev_correct.rev_comment": "q3.rev_correct.revision_comment", + "q3.rem_correct.rem_booking": "q3.rem_correct.residual_after_booking", + "q3.rem_actual_correct.rem_actual": "q3.rem_actual_correct.residual_after_actual", + "q3.close_q3.economy": "q3.close_q3.transfer_econ", + "q4.rev_eco_correct.rev_eco": "q4.rev_eco_correct.target_change", + "q4.rev_item_correct.rev_item": "q4.rev_item_correct.base_correction", + "q4.rev_inc_correct.rev_inc": "q4.rev_inc_correct.revision_inc", + "q4.rev_seq_correct.rev_seq": "q4.rev_seq_correct.revision_seq", + "q4.rev_correct.rev_comment": "q4.rev_correct.revision_comment", + "q4.rem_correct.rem_booking": "q4.rem_correct.residual_after_booking", + "q4.rem_actual_correct.rem_actual": "q4.rem_actual_correct.residual_after_actual", + "q4.close_q4.economy": "q4.close_q4.transfer_econ", +} + +FORM4_STRUCTURE_COLUMN_RENAMES = { + **FORM4_COLUMN_RENAMES, + "upper_ssp_smeta.contract_summary.total": "upper_ssp_smeta.contract_summary.amount", + "upper_ssp_smeta.contract_summary.future_y1": "upper_ssp_smeta.contract_summary.future_payments_y1", + "upper_ssp_smeta.contract_summary.future_y2": "upper_ssp_smeta.contract_summary.future_payments_y2", + "upper_ssp_smeta.contract_summary.other_ssp": "upper_ssp_smeta.contract_summary.other_ssp_amount", + "empty_upper.collegial.approved": "empty_upper.collegial.amount", + "empty_upper.collegial.protocol": "empty_upper.collegial.protocol_reference", + "empty_upper.ckk.q1": "empty_upper.ckk.expenses_q1", + "empty_upper.ckk.q2": "empty_upper.ckk.expenses_q2", + "empty_upper.ckk.q3": "empty_upper.ckk.expenses_q3", + "empty_upper.ckk.q4": "empty_upper.ckk.expenses_q4", + "empty_upper.ckk.deadline": "empty_upper.ckk.delivery_deadline", + "empty_upper.ckk.proc_plan": "empty_upper.ckk.procurement_plan", + "empty_upper.ckk.proc_method": "empty_upper.ckk.procurement_method", + "empty_upper.contract_detail.ceiling": "empty_upper.contract.ceiling", + "empty_upper.contract_detail.q1": "empty_upper.contract.expenses_q1", + "empty_upper.contract_detail.q2": "empty_upper.contract.expenses_q2", + "empty_upper.contract_detail.q3": "empty_upper.contract.expenses_q3", + "empty_upper.contract_detail.q4": "empty_upper.contract.expenses_q4", + "empty_upper.contract_detail.payment_scheme": "empty_upper.contract.scheme", + "empty_upper.contract.q1": "empty_upper.contract.expenses_q1", + "empty_upper.contract.q2": "empty_upper.contract.expenses_q2", + "empty_upper.contract.q3": "empty_upper.contract.expenses_q3", + "empty_upper.contract.q4": "empty_upper.contract.expenses_q4", + "empty_upper.contract.payment_scheme": "empty_upper.contract.scheme", +} + + +def _rename_depth_columns(columns: dict, renames: dict[str, str]) -> dict: + def convert_node(node: dict, path: str) -> dict: + converted = copy.deepcopy(node) + children = converted.get("children") + if isinstance(children, dict): + converted["children"] = convert_children(children, path) + return converted + + def convert_children(children: dict, parent_path: str = "") -> dict: + converted: dict = {} + for key, value in children.items(): + old_path = f"{parent_path}.{key}" if parent_path else key + new_path = renames.get(old_path, old_path) + new_key = new_path.rsplit(".", 1)[-1] + converted[new_key] = convert_node(value, new_path) + return converted + + return convert_children(columns) + + +def _rename_path(path: str, renames: dict[str, str]) -> str: + result = path + for _ in range(4): + changed = False + for old, new in sorted(renames.items(), key=lambda item: len(item[0]), reverse=True): + if result == old: + result = new + changed = True + break + prefix = f"{old}." + if result.startswith(prefix): + result = f"{new}.{result[len(prefix):]}" + changed = True + break + if not changed: + return result + return result + + +def _rename_color_paths(colors: dict, renames: dict[str, str]) -> dict: + renamed: dict = {} + for path, value in colors.items(): + renamed[_rename_path(path, renames)] = value + return renamed + + +FORM4_DEPTH_COLUMNS: dict = _rename_depth_columns( + FORM1_DEPTH_COLUMNS, + FORM4_COLUMN_RENAMES, +) +FORM4_STRUCTURE_DEPTH_COLUMNS: dict = _rename_depth_columns( + FORM1_DEPTH_COLUMNS, + FORM4_STRUCTURE_COLUMN_RENAMES, +) +FORM4_COLORS: dict = { + sheet: _rename_color_paths(colors, FORM4_COLUMN_RENAMES) + for sheet, colors in FORM1_COLORS.items() +} +FORM4_STRUCTURE_COLORS: dict = _rename_color_paths( + FORM1_COLORS["AHR"], + FORM4_STRUCTURE_COLUMN_RENAMES, +) + + +def _reorder_children(children: dict, order: tuple[str, ...]) -> dict: + return { + **{key: children[key] for key in order if key in children}, + **{key: value for key, value in children.items() if key not in order}, + } + + + + +def _patch_form4_regular_columns() -> None: + for columns in (FORM4_DEPTH_COLUMNS, FORM4_STRUCTURE_DEPTH_COLUMNS): + columns["header"].pop("key", None) + + for columns in (FORM4_DEPTH_COLUMNS, FORM4_STRUCTURE_DEPTH_COLUMNS): + for quarter in ("q1", "q2", "q3", "q4"): + columns[quarter]["children"].pop("shift", None) + columns["empty_upper"]["children"]["collegial"]["children"].pop("empty", None) + + for key in ("item_id", "num_group", "internal_order"): + FORM4_DEPTH_COLUMNS["header"]["children"].pop(key, None) + FORM4_STRUCTURE_DEPTH_COLUMNS["header"]["children"].pop("justification", None) + + for quarter in ("q2", "q3"): + FORM4_DEPTH_COLUMNS[quarter]["children"].setdefault( + "comment_correct", + { + "key": None, + "name": "Комментарий", + "children": {"adj_comment": {"name": "Сумма"}}, + }, + ) + FORM4_STRUCTURE_DEPTH_COLUMNS[quarter]["children"].setdefault( + "comment_correct", + { + "key": None, + "name": "Комментарий", + "children": {"adj_comment": {"name": "Сумма"}}, + }, + ) + + for quarter in (1, 2, 3): + FORM4_DEPTH_COLUMNS.pop(f"actual_rsbu_spod_q{quarter}", None) + FORM4_DEPTH_COLUMNS["totals"]["children"].pop("pay_year", None) + + +def _patch_form4_header_columns() -> None: + for columns in (FORM4_DEPTH_COLUMNS, FORM4_STRUCTURE_DEPTH_COLUMNS): + header = columns["header"]["children"] + if "justification" not in header: + header["justification"] = {"name": "Обоснование расхода"} + + for colors in FORM4_COLORS.values(): + source_key = "header.name" if "header.name" in colors else "header.section_code" + if source_key in colors: + colors["header.justification"] = copy.deepcopy(colors[source_key]) + if "header.name" in FORM4_STRUCTURE_COLORS: + FORM4_STRUCTURE_COLORS["header.justification"] = copy.deepcopy( + FORM4_STRUCTURE_COLORS["header.name"] + ) + + +def _patch_form4_contract_summary_columns() -> None: + summary_parent = FORM4_DEPTH_COLUMNS["upper_ssp_smeta"]["children"]["contract_summary"] + summary = summary_parent["children"] + summary["reference"] = {"name": "Реквизиты договора"} + summary["future_y2"] = {"name": "Платежи в 2026 году"} + summary["centralized_flag"] = {"name": "Централизованный договор"} + summary_parent["children"] = _reorder_children( + summary, + ( + "total", "reference", "counterparty", "deadline", "comment", + "future_y1", "future_y2", "other_ssp", "centralized_flag", + ), + ) + + structure_summary_parent = FORM4_STRUCTURE_DEPTH_COLUMNS["upper_ssp_smeta"]["children"]["contract_summary"] + structure_summary = structure_summary_parent["children"] + structure_summary["future_payments_y2"] = {"name": "Платежи в 2026 году"} + structure_summary_parent["children"] = _reorder_children( + structure_summary, + ( + "amount", "counterparty", "deadline", "comment", + "future_payments_y1", "future_payments_y2", "other_ssp_amount", + ), + ) + + +def _patch_form4_actual_rsbu_spod_columns() -> None: + totals = FORM4_DEPTH_COLUMNS.pop("totals") + for quarter in range(1, 5): + FORM4_DEPTH_COLUMNS[f"actual_rsbu_spod_q{quarter}"] = { + "key": None, + "name": f"Факт {quarter} кв" if quarter < 4 else "Факт 4 кв + СПОД", + "children": { + "actual_rsbu_spod": { + "key": f"q{quarter}.actual_rsbu_spod", + "name": "", + }, + }, + } + FORM4_DEPTH_COLUMNS["totals"] = totals + + +_patch_form4_regular_columns() +_patch_form4_header_columns() +_patch_form4_contract_summary_columns() +_patch_form4_actual_rsbu_spod_columns() +_patch_form4_regular_columns() + +FORM4_OPER_DEPTH_COLUMNS: dict = copy.deepcopy(FORM4_DEPTH_COLUMNS) +FORM4_OPER_DEPTH_COLUMNS["empty_upper"]["children"]["ckk"]["children"]["procurement_plan"] = ( + FORM4_OPER_DEPTH_COLUMNS["empty_upper"]["children"]["ckk"]["children"].pop("proc_plan") +) +FORM4_OPER_DEPTH_COLUMNS["empty_upper"]["children"]["ckk"]["children"]["procurement_method"] = ( + FORM4_OPER_DEPTH_COLUMNS["empty_upper"]["children"]["ckk"]["children"].pop("proc_method") +) +FORM4_OPER_COLORS: dict = _rename_color_paths( + FORM4_COLORS["OPER"], + { + "empty_upper.ckk.proc_plan": "empty_upper.ckk.procurement_plan", + "empty_upper.ckk.proc_method": "empty_upper.ckk.procurement_method", + }, +) + + SMETA_COLUMNS: list[tuple[str, str]] = [ ("section_code", "Код раздела"), ("name", "Наименование"), @@ -3527,3 +3815,173 @@ FORM3_COLORS: dict = { "q4.ekonomiya": {"background": "C2D69B", "palette": "green"}, "q4.ekonomiya.economy": {"background": "C2D69B", "palette": "green"}, } + + +# Маппинги свода +def build_summary1_columns(sheet: str) -> dict: + metric_labels = { + "plan": "План", + "approved": "Утверждено", + "fact": "Факт", + "corrected": "Скорректированный план", + } + direction_labels = {"support": "Поддержка", "development": "Развитие"} + period_labels = { + "q1": "I кв.", "q2": "II кв.", "q3": "III кв.", + "q4": "IV кв.", "year": "Год", + } + directions = ["support", "development"] + if sheet == "FORM_2": + directions = ["support"] + elif sheet in {"FORM_3", "FORM_4"}: + directions = ["development"] + + columns: dict = { + "section_code": {"name": "Код"}, + "name": {"name": "Статья расходов"}, + } + for metric, metric_label in metric_labels.items(): + periods = ["q2", "q3", "q4"] if metric == "corrected" else list(period_labels) + children = { + direction: { + "name": direction_labels[direction], + "children": { + period: {"name": period_labels[period]} for period in periods + }, + } + for direction in directions + } + if metric != "corrected" and sheet in {"FORM_1", "MAIN"}: + children["total_year"] = {"name": "Итого за год"} + columns[metric] = {"name": metric_label, "children": children} + return columns + + +def _quarter_columns() -> dict: + months = { + 1: ("Январь", "Февраль", "Март"), + 2: ("Апрель", "Май", "Июнь"), + 3: ("Июль", "Август", "Сентябрь"), + 4: ("Октябрь", "Ноябрь", "Декабрь"), + } + result: dict = {} + for quarter in range(1, 5): + children = { + "adj_current": {"name": "Текущие корректировки (= 0)"}, + "adj_ssp": {"name": "Корректировки с ССП / сметой развития"}, + "adj_rf": {"name": "Корректировки с РФ"}, + "adj_reserve": {"name": "Корректировки из резерва"}, + "payment_date": {"name": "Дата платежа"}, + "payment_amount": {"name": "Сумма платежа, тыс. руб. (без НДС)"}, + "payment_amount_ho": {"name": "в т.ч. сумма ГО (без НДС)"}, + "payment_amount_rf": {"name": "в т.ч. сумма РФ (без НДС)"}, + "payment_act": {"name": "Предоставление акта"}, + "booking_amount": {"name": f"Бронь {quarter} кв."}, + } + for month_number, month in enumerate(months[quarter], start=1): + children[f"actual_m{month_number}"] = {"name": f"Факт {month}"} + if quarter == 4: + children["actual_spod"] = {"name": "Факт СПОД"} + if quarter > 1: + children.update( + { + "plan_revision_eco_change": { + "name": "Изменение целевого назначения перенесенной экономии (= 0)" + }, + "plan_revision_item_adj": { + "name": "Корректировка статей базового плана (= 0)" + }, + "plan_revision_increase": { + "name": "Увеличение базового плана (> 0)" + }, + "plan_revision_sequester": { + "name": "Секвестр базового плана (< 0)" + }, + } + ) + for target_quarter in range(quarter + 1, 5): + children[f"transfer_to_q{target_quarter}"] = { + "name": f"Перенос в {target_quarter} кв." + } + if quarter < 4: + children["transfer_to_economy"] = {"name": "Перенос в фонд экономии"} + result[f"q{quarter}"] = { + "name": f"{quarter} квартал", + "children": children, + } + return result + + +def build_summary2_columns(sheet: str) -> dict: + estimate_title = ( + "Смета расходов ГО в разрезе договоров" + if sheet == "GO" + else "Смета расходов РФ в разрезе договоров" + ) + columns: dict = { + "smeta_type": {"key": "header.smeta_type", "name": "Вид сметы"}, + "smeta_direction": {"key": "header.smeta_direction", "name": "Направление"}, + "org_name": {"key": "header.org_name", "name": "ССП"}, + "name": {"key": "header.name", "name": "Наименование статьи"}, + "estimate": { + "key": None, + "name": estimate_title, + "children": { + "item_id": {"key": "header.item_id", "name": "ID статьи"}, + "num_group": {"key": "header.num_group", "name": "ID группы номенклатуры"}, + "justification": {"key": "header.justification", "name": "Конкретный вид расхода"}, + **{ + f"plan_q{q}": {"key": f"plan.plan_q{q}", "name": f"{q} кв."} + for q in range(1, 5) + }, + }, + }, + "allocation": { + "name": "Аллокация расходов", + "children": { + "internal_order": {"name": "Внутренний заказ"}, + "property_object": {"name": "Объект недвижимости"}, + }, + }, + "sequestration": { + "name": "Секвестирование ДФиП", + "children": { + "DFIP": { + "name": "ДФиП", + "children": { + f"adj_q{q}": {"name": f"{q} кв."} for q in range(1, 5) + }, + } + }, + }, + "reserve": { + "name": "Отнесение в резерв", + "children": { + f"amount_q{q}": {"name": f"{q} кв."} for q in range(1, 5) + }, + }, + "approved": { + "name": "Итоговая смета расходов", + "children": { + f"approved_q{q}": {"name": f"{q} кв."} for q in range(1, 5) + }, + }, + "contract_detail": { + "name": "Договор", + "children": { + "counterparty": {"name": "Контрагент"}, + "reference": {"name": "Договор (реквизиты)"}, + "subject": {"name": "Предмет"}, + "currency": {"name": "Валюта договора"}, + "ceiling_amount": {"name": "Предельная стоимость, тыс. руб. (без НДС)"}, + **{ + f"expenses_q{q}": {"name": f"Расходы {q} кв., тыс. руб. (без НДС)"} + for q in range(1, 5) + }, + "vat_rate": {"name": "Ставка НДС (%)"}, + "deadline": {"name": "Срок"}, + }, + }, + } + columns.update(_quarter_columns()) + return columns diff --git a/api/src/services/export_service/export_normalizers.py b/api/src/services/export_service/export_normalizers.py index d2c99d5..dea1bbd 100644 --- a/api/src/services/export_service/export_normalizers.py +++ b/api/src/services/export_service/export_normalizers.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +from decimal import Decimal from typing import Any SMETA_VALUE_PATHS: list[tuple[str, tuple[str, ...]]] = [ @@ -72,6 +73,8 @@ FORM1_METRIC_BLOCK_KEYS = ( class ExcelValueSerializer: def to_excel_value(self, value: Any) -> Any: + if isinstance(value, Decimal): + return float(value) if value is None or isinstance(value, (str, int, float, bool)): return value if isinstance(value, dict): diff --git a/api/src/services/export_service/export_service.py b/api/src/services/export_service/export_service.py index 56b556c..fe0a638 100644 --- a/api/src/services/export_service/export_service.py +++ b/api/src/services/export_service/export_service.py @@ -1,19 +1,28 @@ from __future__ import annotations +import asyncio import io import json +import logging +import os +import shutil +import tempfile import zipfile from dataclasses import dataclass from datetime import datetime, timezone -from typing import Any +from functools import partial +from time import perf_counter +from typing import Any, BinaryIO from urllib.parse import quote from openpyxl import Workbook from openpyxl.worksheet.worksheet import Worksheet from sqlalchemy.ext.asyncio import AsyncSession +from xlsxwriter import Workbook as XlsxWorkbook from src.db.models.app_user import AppUser from src.db.models.form_type import FormTypeEnum +from src.db.session import db_session from src.services.budget_form_service import BudgetFormService from src.services.export_service.export_normalizers import ( ExcelValueSerializer, @@ -26,19 +35,32 @@ from src.services.export_service.export_mappings import ( FORM1_DEPTH_COLUMNS, FORM2_DEPTH_AHR_SUB_COLUMNS, FORM3_COLORS, + FORM4_COLORS, + FORM4_DEPTH_COLUMNS, + FORM4_OPER_COLORS, + FORM4_OPER_DEPTH_COLUMNS, + FORM4_STRUCTURE_COLORS, + FORM4_STRUCTURE_DEPTH_COLUMNS, FORM3_CURRENT_EXPENSES_DEPTH_COLUMNS, FORM3_LIMIT_DEPTH_COLUMNS, + PALETTE, SMETA_COLUMNS, + build_summary1_columns, + build_summary2_columns, ) from src.services.export_service.export_writers import ( DepthSheetWriter, TabularSheetWriter, + XlsxDepthSheetWriter, + XlsxTabularSheetWriter, ) from src.services.project_service import ProjectService from src.services.sheet_service import SheetService +from src.services.summary_service import SummaryService XLSX_MEDIA_TYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" ZIP_MEDIA_TYPE = "application/zip" +SUMMARY_EXPORT_COLUMN_WIDTH = 15 FORM1_DIRECTION_REQUIRED_SHEETS = {"AHR", "CAP"} SHEETS_WITH_SECTIONS = {"AHR", "CAP", "OPER"} @@ -56,6 +78,11 @@ SHEET_NAMES = { 'CURRENT_EXPENSES': 'Текущие расходы', } SKIP_SHEETS = ('OTCH9F',) +BULK_EXPORT_CONCURRENCY = 2 +EXPORT_TMPDIR = os.getenv("EXPORT_TMPDIR") or tempfile.gettempdir() + +logger = logging.getLogger() + @dataclass(frozen=True) class ExportSheetPlan: @@ -71,6 +98,7 @@ class ExportService: self.budget_form_service = BudgetFormService(db) self.sheet_service = SheetService(db) self.project_service = ProjectService(db) + self.summary_service = SummaryService(db) self.value_serializer = ExcelValueSerializer() self.form1_row_normalizer = Form1DepthRowNormalizer(self.value_serializer) self.form3_row_normalizer = Form3DepthRowNormalizer(self.value_serializer) @@ -120,10 +148,10 @@ class ExportService: value_serializer=self.value_serializer.to_excel_value, ), "STRUCTURE": DepthSheetWriter( - columns_dict=FORM1_DEPTH_COLUMNS, + columns_dict=FORM4_STRUCTURE_DEPTH_COLUMNS, row_normalizer=self.form1_row_normalizer.normalize, value_serializer=self.value_serializer.to_excel_value, - colors_config=FORM1_COLORS["AHR"], + colors_config=FORM4_STRUCTURE_COLORS, ), } @@ -134,15 +162,15 @@ class ExportService: sheet: str | None = None, direction: str | None = None, sections: list[str] | None = None, - ignore_non_applicable_query_params: bool = False, - ) -> tuple[io.BytesIO, str, str]: + ignore_unsupported_sheet_params: bool = False, + ) -> tuple[BinaryIO, str, str]: stream, filename = await self.export_form_to_xlsx( form_id=form_id, current_user=current_user, sheet=sheet, direction=direction, sections=sections, - ignore_non_applicable_query_params=ignore_non_applicable_query_params, + ignore_unsupported_sheet_params=ignore_unsupported_sheet_params, ) return stream, filename, XLSX_MEDIA_TYPE @@ -153,7 +181,7 @@ class ExportService: report_type: str, current_user: AppUser, sections: list[str] | None = None, - ) -> tuple[io.BytesIO, str, str]: + ) -> tuple[BinaryIO, str, str]: stream, filename = await self.export_project_report_to_xlsx( project_id=project_id, year=year, @@ -167,13 +195,74 @@ class ExportService: self, project_id: int, current_user: AppUser, - ) -> tuple[io.BytesIO, str, str]: + ) -> tuple[BinaryIO, str, str]: stream, filename = await self.export_project_to_xlsx( project_id=project_id, current_user=current_user, ) return stream, filename, XLSX_MEDIA_TYPE + async def export_summary_payload( + self, + summary_number: int, + year: int, + sheet: str, + current_user: AppUser, + org_ids: list[int] | None = None, + sort_by: str = "org_name", + sort_direction: str = "asc", + ) -> tuple[BinaryIO, str, str]: + export_started_at = perf_counter() + load_started_at = perf_counter() + if summary_number == 1: + rows = await self.summary_service.get_svod1_rows( + year=year, + sheet=sheet, + user=current_user, + org_ids=org_ids, + ) + columns = build_summary1_columns(sheet) + elif sheet == "GO": + rows = await self.summary_service.get_svod2_go_export_rows( + year=year, + user=current_user, + org_ids=org_ids, + sort_by=sort_by, + sort_direction=sort_direction, + ) + columns = build_summary2_columns(sheet) + else: + rows = await self.summary_service.get_svod2_rows( + year=year, + sheet=sheet, + user=current_user, + org_ids=org_ids, + offset=None, + limit=None, + sort_by=sort_by, + sort_direction=sort_direction, + ) + columns = build_summary2_columns(sheet) + logger.info( + f"[export timing] summary={summary_number} sheet={sheet} year={year}: " + f"load_rows={perf_counter() - load_started_at:.3f}s, rows={len(rows)}", + ) + stream = await self._export_summary_rows( + rows=rows, + columns=columns, + metadata=(("Свод", summary_number), ("Лист", sheet), ("Год", year)), + worksheet_name=f"Свод {summary_number} {sheet}", + summary_number=summary_number, + year=year, + sheet=sheet, + ) + logger.info( + f"[export timing] summary={summary_number} sheet={sheet} year={year}: " + f"total={perf_counter() - export_started_at:.3f}s", + ) + filename = self._build_summary_filename(summary_number, sheet, year) + return stream, filename, XLSX_MEDIA_TYPE + async def export_bulk_payload( self, form_ids: list[int], @@ -181,9 +270,10 @@ class ExportService: skip_failed: bool = True, direction: str | None = None, sections: list[str] | None = None, - ignore_non_applicable_query_params: bool = False, - ) -> tuple[io.BytesIO, str, str]: - archive_stream = io.BytesIO() + ignore_unsupported_sheet_params: bool = False, + ) -> tuple[BinaryIO, str, str]: + bulk_started_at = perf_counter() + archive_stream = tempfile.TemporaryFile(mode="w+b", dir=EXPORT_TMPDIR) exported_count = 0 errors: list[dict[str, Any]] = [] used_names: set[str] = set() @@ -193,22 +283,48 @@ class ExportService: mode="w", compression=zipfile.ZIP_DEFLATED, ) as archive: - for form_id in self._deduplicate_ids(form_ids): - try: - form_stream, filename = await self.export_form_to_xlsx( - form_id=form_id, - current_user=current_user, - direction=direction, - sections=sections, - ignore_non_applicable_query_params=ignore_non_applicable_query_params, - ) + unique_form_ids = self._deduplicate_ids(form_ids) + for batch_start in range(0, len(unique_form_ids), BULK_EXPORT_CONCURRENCY): + batch_ids = unique_form_ids[ + batch_start:batch_start + BULK_EXPORT_CONCURRENCY + ] + batch_results = await asyncio.gather( + *[ + self._export_bulk_form( + form_id=form_id, + current_user=current_user, + direction=direction, + sections=sections, + ignore_unsupported_sheet_params=ignore_unsupported_sheet_params, + ) + for form_id in batch_ids + ], + return_exceptions=True, + ) + + for form_id, result in zip(batch_ids, batch_results): + if isinstance(result, BaseException): + if not isinstance(result, ValueError) or not skip_failed: + archive_stream.close() + raise result + errors.append({"form_id": form_id, "error": str(result)}) + continue + + form_stream, filename, xlsx_elapsed = result + xlsx_finished_at = perf_counter() archive_name = self._make_unique_filename(filename, used_names) - archive.writestr(archive_name, form_stream.getvalue()) + try: + form_stream.seek(0) + with archive.open(archive_name, mode="w") as archive_entry: + shutil.copyfileobj(form_stream, archive_entry, length=1024 * 1024) + finally: + form_stream.close() + logger.info( + f"[export timing] bulk form_id={form_id}: " + f"xlsx={xlsx_elapsed:.3f}s, " + f"zip_write={perf_counter() - xlsx_finished_at:.3f}s", + ) exported_count += 1 - except ValueError as exc: - if not skip_failed: - raise - errors.append({"form_id": form_id, "error": str(exc)}) if errors: archive.writestr( @@ -217,12 +333,37 @@ class ExportService: ) if exported_count == 0: + archive_stream.close() raise ValueError("Не удалось экспортировать ни одной формы.") archive_stream.seek(0) filename = f"export_forms_{datetime.now(timezone.utc).strftime('%Y-%m-%d')}.zip" + logger.info( + f"[export timing] bulk total: forms={exported_count}, " + f"elapsed={perf_counter() - bulk_started_at:.3f}s", + ) return archive_stream, filename, ZIP_MEDIA_TYPE + async def _export_bulk_form( + self, + form_id: int, + current_user: AppUser, + direction: str | None, + sections: list[str] | None, + ignore_unsupported_sheet_params: bool, + ) -> tuple[BinaryIO, str, float]: + started_at = perf_counter() + async with db_session() as db: + export_service = ExportService(db) + stream, filename = await export_service.export_form_to_xlsx( + form_id=form_id, + current_user=current_user, + direction=direction, + sections=sections, + ignore_unsupported_sheet_params=ignore_unsupported_sheet_params, + ) + return stream, filename, perf_counter() - started_at + async def export_form_to_xlsx( self, form_id: int, @@ -230,9 +371,17 @@ class ExportService: sheet: str | None = None, direction: str | None = None, sections: list[str] | None = None, - ignore_non_applicable_query_params: bool = False, - ) -> tuple[io.BytesIO, str]: + ignore_unsupported_sheet_params: bool = False, + ) -> tuple[BinaryIO, str]: + export_started_at = perf_counter() + + stage_started_at = perf_counter() form = await self._get_form_for_export(form_id=form_id, current_user=current_user) + logger.info( + f"[export timing] form_id={form_id}: load_form={perf_counter() - stage_started_at:.3f}s", + ) + + stage_started_at = perf_counter() sheets = self._resolve_requested_sheets(form=form, requested_sheet=sheet) self._validate_sections_for_form(form=form, sections=sections) @@ -242,7 +391,7 @@ class ExportService: sheets=sheets, direction=direction, sections=sections, - ignore_non_applicable_query_params=ignore_non_applicable_query_params, + ignore_unsupported_sheet_params=ignore_unsupported_sheet_params, ) else: sheets_by_directions = self._resolve_sheets_by_directions(form=form, requested_sheet=sheet) @@ -253,12 +402,24 @@ class ExportService: sheets=shts, direction=dr, sections=sections, - ignore_non_applicable_query_params=ignore_non_applicable_query_params, + ignore_unsupported_sheet_params=ignore_unsupported_sheet_params, ) + logger.info( + f"[export timing] form_id={form_id}: build_plan={perf_counter() - stage_started_at:.3f}s, " + f"sheets={len(plans)}", + ) - - - workbook = self._create_workbook_with_metadata(form=form) + # XlsxWriter хранит XML-части и итоговую книгу во временных файлах вместо RAM. + stream = tempfile.TemporaryFile(mode="w+b", dir=EXPORT_TMPDIR) + workbook = XlsxWorkbook( + stream, + { + "tmpdir": EXPORT_TMPDIR, + "strings_to_urls": False, + }, + ) + self._write_xlsxwriter_metadata(workbook.add_worksheet("metadata"), form) + sheet_writers = self._create_xlsxwriter_sheet_writers(workbook, form.form_type.code) exported_sheets = 0 for plan in plans: if not plan.validation_direction and plan.sheet_name in SKIP_SHEETS: @@ -270,27 +431,44 @@ class ExportService: direction=plan.validation_direction, sections=plan.validation_sections, ) + + # Этап 3: отдельно измеряем запрос данных каждого листа к БД. + sheet_load_started_at = perf_counter() rows = await self.sheet_service.get( form_id=form_id, sheet=plan.sheet_name, direction=plan.effective_direction, sections=plan.effective_sections, ) - worksheet = workbook.create_sheet(title=self._safe_sheet_name(plan.sheet_name, plan.effective_direction)) - self._write_sheet_rows( - worksheet=worksheet, - rows=rows, - sheet_name=plan.sheet_name, + sheet_loaded_at = perf_counter() + + # Этап 4: рендер листа уводим с event loop, чтобы экспорт не блокировал API. + worksheet = workbook.add_worksheet( + self._safe_sheet_name(plan.sheet_name, plan.effective_direction) + ) + writer = sheet_writers.get(plan.sheet_name, sheet_writers["__default__"]) + await asyncio.to_thread(writer.write, worksheet, rows) + logger.info( + f"[export timing] form_id={form_id} sheet={plan.sheet_name} " + f"direction={plan.effective_direction or '-'} rows={len(rows)}: " + f"load_rows={sheet_loaded_at - sheet_load_started_at:.3f}s, " + f"write_sheet={perf_counter() - sheet_loaded_at:.3f}s", ) exported_sheets += 1 if exported_sheets == 0: raise ValueError("Не удалось экспортировать листы формы.") - stream = io.BytesIO() - workbook.save(stream) - workbook.close() + # Этап 5: сериализация всей книги в XLSX обычно заметна на больших формах. + save_started_at = perf_counter() + # Формирование XML и ZIP-контейнера XLSX также не должно блокировать event loop. + await asyncio.to_thread(workbook.close) + xlsx_size = stream.seek(0, io.SEEK_END) stream.seek(0) + logger.info( + f"[export timing] form_id={form_id}: save_xlsx={perf_counter() - save_started_at:.3f}s, " + f"total={perf_counter() - export_started_at:.3f}s, size={xlsx_size} bytes", + ) filename = self._build_filename(form_id=form.id, form_type_code=form.form_type.code, year=form.year, sheet=sheet) return stream, filename @@ -482,20 +660,303 @@ class ExportService: self._fill_metadata(meta_sheet, form) return workbook + def _write_xlsxwriter_metadata(self, sheet: Any, form: Any) -> None: + metadata_rows = [ + ("Форма ID", form.id), + ("Тип формы", form.form_type_code), + ("Год", form.year), + ("ID ССП.РФ", form.org_unit_id), + ("ССП/РФ", form.org_unit.title if form.org_unit else None), + ("Экспортировано", datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")), + ] + for row, (key, value) in enumerate(metadata_rows): + sheet.write(row, 0, key) + sheet.write(row, 1, value) + + async def _export_summary_rows( + self, + rows: list[tuple], + columns: dict, + metadata: tuple[tuple[str, Any], ...], + worksheet_name: str, + summary_number: int, + year: int, + sheet: str, + ) -> BinaryIO: + workbook_started_at = perf_counter() + stream = tempfile.TemporaryFile(mode="w+b", dir=EXPORT_TMPDIR) + workbook = XlsxWorkbook( + stream, + {"tmpdir": EXPORT_TMPDIR, "strings_to_urls": False}, + ) + try: + metadata_sheet = workbook.add_worksheet("metadata") + metadata_rows = ( + *metadata, + ("Строк", len(rows)), + ( + "Экспортировано", + datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S"), + ), + ) + for row_number, (key, value) in enumerate(metadata_rows): + metadata_sheet.write(row_number, 0, key) + metadata_sheet.write(row_number, 1, value) + + worksheet = workbook.add_worksheet(self._safe_sheet_name(worksheet_name)) + ( + header_format_resolver, + row_format_resolver, + column_format_resolver, + ) = ( + self._summary_format_resolvers(columns, summary_number) + ) + writer = XlsxDepthSheetWriter( + workbook=workbook, + columns_dict=columns, + row_normalizer=( + self._identity_row + if summary_number == 2 and sheet == "GO" + else self.form1_row_normalizer.normalize + ), + value_serializer=self.value_serializer.to_excel_value, + header_format_resolver=header_format_resolver, + row_format_resolver=row_format_resolver, + column_format_resolver=column_format_resolver, + fixed_column_width=SUMMARY_EXPORT_COLUMN_WIDTH, + ) + logger.info( + f"[export timing] summary={summary_number} sheet={sheet} year={year}: " + f"build_workbook={perf_counter() - workbook_started_at:.3f}s", + ) + write_started_at = perf_counter() + await asyncio.to_thread(writer.write, worksheet, rows) + logger.info( + f"[export timing] summary={summary_number} sheet={sheet} year={year}: " + f"write_sheet={perf_counter() - write_started_at:.3f}s", + ) + save_started_at = perf_counter() + await asyncio.to_thread(workbook.close) + logger.info( + f"[export timing] summary={summary_number} sheet={sheet} year={year}: " + f"save_xlsx={perf_counter() - save_started_at:.3f}s", + ) + except BaseException: + stream.close() + raise + stream.seek(0) + return stream + + @staticmethod + def _identity_row(row: dict[str, Any]) -> dict[str, Any]: + return row + + @staticmethod + def _summary_format_resolvers( + columns: dict, + summary_number: int, + ) -> tuple[Any, Any, Any]: + if summary_number == 1: + return ( + ExportService._summary1_header_format, + ExportService._summary1_row_format, + None, + ) + + roots = list(columns) + band_by_root = {root: index % 2 for index, root in enumerate(roots)} + leaf_bands: list[int] = [] + group_end_columns: set[int] = set() + group_end_header_keys: set[str] = set() + + for root in roots: + ExportService._add_summary_leaf_bands( + config={root: columns[root]}, + band=band_by_root[root], + leaf_bands=leaf_bands, + ) + group_end_columns.add(len(leaf_bands) - 1) + group_end_header_keys.add( + ExportService._summary_last_leaf_key(root, columns[root]) + ) + + return ( + partial( + ExportService._summary2_header_format, + band_by_root=band_by_root, + roots=set(roots), + group_end_header_keys=group_end_header_keys, + ), + None, + partial( + ExportService._summary2_column_format, + leaf_bands=leaf_bands, + group_end_columns=group_end_columns, + ), + ) + + @staticmethod + def _summary1_header_format(_full_key: str) -> dict[str, Any]: + return { + "bg_color": "#F8F9FA", + "pattern": 1, + "font_color": "#4A5565", + "bold": True, + } + + @staticmethod + def _summary1_row_format( + row: tuple, + _column_key: str, + _column: int, + ) -> dict[str, Any]: + thin_border = {"right": 1, "right_color": "#EDF0F2"} + data = row[3] if len(row) > 3 and isinstance(row[3], dict) else {} + section_code = data.get("section_code") + if section_code in (None, "") and isinstance(data.get("header"), dict): + section_code = data["header"].get("section_code") + if section_code in (None, ""): + return thin_border + + palette_name = {"1": "green", "3": "orange", "4": "blue"}.get( + str(section_code)[:1], + "blue", + ) + item_type = row[0] if row else "INPUT" + background = PALETTE[palette_name].get(item_type, "FFFFFF") + red, green, blue = ( + int(background[index:index + 2], 16) for index in (0, 2, 4) + ) + is_dark = 0.299 * red + 0.587 * green + 0.114 * blue < 128 + return { + "bg_color": f"#{background}", + "pattern": 1, + "font_color": "#FFFFFF" if is_dark else "#000000", + **thin_border, + } + + @staticmethod + def _add_summary_leaf_bands( + config: dict, + band: int, + leaf_bands: list[int], + ) -> None: + for child in config.values(): + if "children" in child: + ExportService._add_summary_leaf_bands( + child["children"], + band, + leaf_bands, + ) + else: + leaf_bands.append(band) + + @staticmethod + def _summary_last_leaf_key(key: str, config: dict) -> str: + if "children" not in config: + return key + child_key, child_config = next(reversed(config["children"].items())) + return ExportService._summary_last_leaf_key( + f"{key}.{child_key}", + child_config, + ) + + @staticmethod + def _summary2_header_format( + full_key: str, + *, + band_by_root: dict[str, int], + roots: set[str], + group_end_header_keys: set[str], + ) -> dict[str, Any]: + band = band_by_root.get(full_key.split(".", 1)[0], 0) + result = { + "bg_color": ("#F8F9FA", "#F1F7F3")[band], + "pattern": 1, + "font_color": "#4A5565", + "bold": True, + } + if full_key in roots or full_key in group_end_header_keys: + result.update({"right": 2, "right_color": "#9EAFA3"}) + return result + + @staticmethod + def _summary2_column_format( + _column_key: str, + column: int, + *, + leaf_bands: list[int], + group_end_columns: set[int], + ) -> dict[str, Any]: + is_group_end = column in group_end_columns + return { + "bg_color": ("#FFFFFF", "#F8FBF9")[leaf_bands[column]], + "pattern": 1, + "right": 2 if is_group_end else 1, + "right_color": "#9EAFA3" if is_group_end else "#EDF0F2", + } + + def _create_xlsxwriter_sheet_writers( + self, + workbook: XlsxWorkbook, + form_type_code: str | None = None, + ) -> dict[str, Any]: + default_writer = XlsxDepthSheetWriter( + workbook=workbook, + columns_dict=FORM1_DEPTH_COLUMNS, + row_normalizer=self.form1_row_normalizer.normalize, + value_serializer=self.value_serializer.to_excel_value, + ) + writers: dict[str, Any] = { + "__default__": default_writer, + "SMETA": XlsxTabularSheetWriter( + workbook=workbook, + columns=SMETA_COLUMNS, + row_normalizer=self.smeta_row_normalizer.normalize, + value_serializer=self.value_serializer.to_excel_value, + ), + } + if form_type_code == FormTypeEnum.FORM_4.value: + writer_configs = { + "OPER": (FORM4_OPER_DEPTH_COLUMNS, FORM4_OPER_COLORS), + "AHR": (FORM4_DEPTH_COLUMNS, FORM4_COLORS["AHR"]), + "CAP": (FORM4_DEPTH_COLUMNS, FORM4_COLORS["CAP"]), + "STRUCTURE": (FORM4_STRUCTURE_DEPTH_COLUMNS, FORM4_STRUCTURE_COLORS), + } + else: + writer_configs = { + "OPER": (FORM1_DEPTH_COLUMNS, FORM1_COLORS["OPER"]), + "AHR": (FORM1_DEPTH_COLUMNS, FORM1_COLORS["AHR"]), + "CAP": (FORM1_DEPTH_COLUMNS, FORM1_COLORS["CAP"]), + "AHR_RENT": (FORM2_DEPTH_AHR_SUB_COLUMNS, None), + "AHR_UTILITY": (FORM2_DEPTH_AHR_SUB_COLUMNS, None), + "AHR_SECURITY": (FORM2_DEPTH_AHR_SUB_COLUMNS, None), + "STRUCTURE": (FORM4_STRUCTURE_DEPTH_COLUMNS, FORM4_STRUCTURE_COLORS), + } + for sheet_name, (columns, colors) in writer_configs.items(): + writers[sheet_name] = XlsxDepthSheetWriter( + workbook=workbook, + columns_dict=columns, + row_normalizer=self.form1_row_normalizer.normalize, + value_serializer=self.value_serializer.to_excel_value, + colors_config=colors, + ) + return writers + def _build_sheet_plans( self, form: Any, sheets: list[str], direction: str | None, sections: list[str] | None, - ignore_non_applicable_query_params: bool, + ignore_unsupported_sheet_params: bool, ) -> list[ExportSheetPlan]: plans: list[ExportSheetPlan] = [] for sheet_name in sheets: effective_direction = direction if self._requires_direction(form.form_type.code, sheet_name) else None effective_sections = sections if sheet_name in SHEETS_WITH_SECTIONS else None - validation_direction = effective_direction if ignore_non_applicable_query_params else direction - validation_sections = effective_sections if ignore_non_applicable_query_params else sections + validation_direction = effective_direction if ignore_unsupported_sheet_params else direction + validation_sections = effective_sections if ignore_unsupported_sheet_params else sections plans.append( ExportSheetPlan( sheet_name=sheet_name, @@ -549,6 +1010,11 @@ class ExportService: return f"{year}_{form_type_code}_{form_id}_{sheet}_{date_str}.xlsx" return f"{year}_{form_type_code}_{form_id}_{date_str}.xlsx" + @staticmethod + def _build_summary_filename(summary: int, sheet: str, year: int) -> str: + date_str = datetime.now(timezone.utc).strftime("%Y-%m-%d") + return f"{year}_SVOD_{summary}_{sheet}_{date_str}.xlsx" + @staticmethod def _deduplicate_ids(form_ids: list[int]) -> list[int]: seen: set[int] = set() diff --git a/api/src/services/export_service/export_writers.py b/api/src/services/export_service/export_writers.py index 5591a45..7b7f1c8 100644 --- a/api/src/services/export_service/export_writers.py +++ b/api/src/services/export_service/export_writers.py @@ -7,6 +7,8 @@ from typing import Any, Callable from openpyxl.styles import Alignment, Border, Color, Font, PatternFill, Side from openpyxl.worksheet.worksheet import Worksheet from openpyxl.utils import get_column_letter +from xlsxwriter import Workbook as XlsxWorkbook +from xlsxwriter.worksheet import Worksheet as XlsxWorksheet from src.services.export_service.export_mappings import PALETTE @@ -14,23 +16,28 @@ from src.services.export_service.export_mappings import PALETTE RowNormalizer = Callable[[dict[str, Any]], dict[str, Any]] ValueSerializer = Callable[[Any], Any] +HeaderFormatResolver = Callable[[str], dict[str, Any] | None] +RowFormatResolver = Callable[[tuple, str, int], dict[str, Any] | None] +ColumnFormatResolver = Callable[[str, int], dict[str, Any] | None] class AbstractWriter(abc.ABC): - def _set_widths(self, worksheet: Worksheet): - for col in worksheet.columns: - max_len = 0 - col_letter = get_column_letter(col[0].column) # Get alphabetical column name (e.g., 'A') - - for cell in col: - if cell.value is not None: - cell_len = len(str(cell.value)) - if cell_len > max_len: - max_len = cell_len - - adjusted_width = max(max_len + 3, 10) - worksheet.column_dimensions[col_letter].width = adjusted_width + @staticmethod + def _track_width(column_widths: dict[int, int], column: int, value: Any) -> None: + if value is None: + return + column_widths[column] = max(column_widths.get(column, 0), len(str(value))) + + def _track_row_widths(self, column_widths: dict[int, int], values: list[Any]) -> None: + for column, value in enumerate(values, start=1): + self._track_width(column_widths, column, value) + + @staticmethod + def _apply_widths(worksheet: Worksheet, column_widths: dict[int, int]) -> None: + for column, max_len in column_widths.items(): + column_letter = get_column_letter(column) + worksheet.column_dimensions[column_letter].width = max(max_len + 3, 10) @abc.abstractmethod def write(self, *args, **kwargs): @@ -49,14 +56,17 @@ class TabularSheetWriter(AbstractWriter): self.value_serializer = value_serializer def write(self, worksheet: Worksheet, rows: list[tuple]) -> None: - worksheet.append([title for _, title in self.columns]) + column_widths: dict[int, int] = {} + headers = [title for _, title in self.columns] + worksheet.append(headers) + self._track_row_widths(column_widths, headers) for row in rows: raw_data = row[3] if len(row) > 3 and isinstance(row[3], dict) else {} normalized = self.row_normalizer(raw_data) - worksheet.append( - [self.value_serializer(normalized.get(key)) for key, _ in self.columns] - ) - self._set_widths(worksheet) + values = [self.value_serializer(normalized.get(key)) for key, _ in self.columns] + worksheet.append(values) + self._track_row_widths(column_widths, values) + self._apply_widths(worksheet, column_widths) class DepthSheetWriter(AbstractWriter): @@ -82,6 +92,7 @@ class DepthSheetWriter(AbstractWriter): self.value_serializer = value_serializer self.colors_config = colors_config self.palette = None + self._row_fill_cache: dict[tuple[str, str], PatternFill] = {} if colors_config: self._set_palette() @@ -138,17 +149,21 @@ class DepthSheetWriter(AbstractWriter): current_column: int = 1, current_row: int = 1, max_row: int | None = None, - full_key: str | None = None + full_key: str | None = None, + column_widths: dict[int, int] | None = None, ) -> int: if columns_dict is None: columns_dict = self.columns_dict max_row = self._get_header_max_row(self.columns_dict) + if column_widths is None: + column_widths = {} for key, sub_dict in columns_dict.items(): if full_key: key = f"{full_key}.{key}" if "children" not in sub_dict: cell = worksheet.cell(row=current_row, column=current_column, value=sub_dict["name"]) + self._track_width(column_widths, current_column, sub_dict["name"]) cell.border = self.thin_border if max_row != current_row: worksheet.merge_cells( @@ -166,8 +181,10 @@ class DepthSheetWriter(AbstractWriter): current_row=current_row+1, max_row=max_row, full_key=key, + column_widths=column_widths, ) cell = worksheet.cell(row=current_row, column=current_column, value=sub_dict["name"]) + self._track_width(column_widths, current_column, sub_dict["name"]) if new_column - current_column > 1: worksheet.merge_cells( start_row=current_row, @@ -189,24 +206,27 @@ class DepthSheetWriter(AbstractWriter): cell.font = Font(color=color["color"]) return current_column - def _set_color_to_row(self, ws: Worksheet, item_type: str) -> None: + def _set_color_to_row(self, ws: Worksheet, row: int, item_type: str) -> None: if not self.palette or item_type in ("SUB_ITEM", "INPUT"): return - row = ws.max_row - for i, key in enumerate(self.columns): if key not in self.palette: continue cell = ws.cell(column=i+1, row=row) if (palette := self.palette.get(key)): - cell.fill = PatternFill( - start_color=Color(rgb=PALETTE[palette][item_type]), - fill_type="solid", - ) + cache_key = (palette, item_type) + if cache_key not in self._row_fill_cache: + self._row_fill_cache[cache_key] = PatternFill( + start_color=Color(rgb=PALETTE[palette][item_type]), + fill_type="solid", + ) + cell.fill = self._row_fill_cache[cache_key] def write(self, worksheet: Worksheet, rows: list[tuple]) -> None: - self._write_header(worksheet) + column_widths: dict[int, int] = {} + self._write_header(worksheet, column_widths=column_widths) + next_row = worksheet.max_row + 1 for row in rows: if len(row) > 3 and isinstance(row[3], dict): if row[0] == 'META': @@ -215,9 +235,308 @@ class DepthSheetWriter(AbstractWriter): else: raw_data = {} normalized = self.row_normalizer(raw_data) - worksheet.append( - [self.value_serializer(normalized.get(key)) for key in self.columns] - ) + values = [self.value_serializer(normalized.get(key)) for key in self.columns] + worksheet.append(values) + self._track_row_widths(column_widths, values) if len(row) > 3: - self._set_color_to_row(ws=worksheet, item_type=row[0]) - self._set_widths(worksheet) + self._set_color_to_row(ws=worksheet, row=next_row, item_type=row[0]) + next_row += 1 + self._apply_widths(worksheet, column_widths) + + +class XlsxTabularSheetWriter(AbstractWriter): + def __init__( + self, + workbook: XlsxWorkbook, + columns: list[tuple[str, str]], + row_normalizer: RowNormalizer, + value_serializer: ValueSerializer, + ): + self.workbook = workbook + self.columns = columns + self.row_normalizer = row_normalizer + self.value_serializer = value_serializer + + @staticmethod + def _apply_widths( + worksheet: XlsxWorksheet, + column_widths: dict[int, int], + ) -> None: + for column, max_len in column_widths.items(): + zero_based_column = column - 1 + worksheet.set_column( + zero_based_column, + zero_based_column, + max(max_len + 3, 10), + ) + + def write(self, worksheet: XlsxWorksheet, rows: list[tuple]) -> None: + column_widths: dict[int, int] = {} + headers = [title for _, title in self.columns] + worksheet.write_row(0, 0, headers) + self._track_row_widths(column_widths, headers) + + output_row = 1 + for row in rows: + raw_data = row[3] if len(row) > 3 and isinstance(row[3], dict) else {} + normalized = self.row_normalizer(raw_data) + values = [self.value_serializer(normalized.get(key)) for key, _ in self.columns] + worksheet.write_row(output_row, 0, values) + self._track_row_widths(column_widths, values) + output_row += 1 + + self._apply_widths(worksheet, column_widths) + + +class XlsxDepthSheetWriter(AbstractWriter): + def __init__( + self, + workbook: XlsxWorkbook, + columns_dict: dict, + row_normalizer: RowNormalizer, + value_serializer: ValueSerializer, + colors_config: dict | None = None, + header_format_resolver: HeaderFormatResolver | None = None, + row_format_resolver: RowFormatResolver | None = None, + column_format_resolver: ColumnFormatResolver | None = None, + fixed_column_width: int | None = None, + ): + self.workbook = workbook + self.columns_dict = columns_dict + self._flat_columns_to_depth: dict[str, str] = {} + self.columns = self._columns_dict_to_flat(columns_dict) + self.row_normalizer = row_normalizer + self.value_serializer = value_serializer + self.colors_config = colors_config + self.header_format_resolver = header_format_resolver + self.row_format_resolver = row_format_resolver + self.column_format_resolver = column_format_resolver + self.fixed_column_width = fixed_column_width + self.palette: dict[str, str] = {} + self._format_cache: dict[tuple[Any, ...], Any] = {} + if colors_config: + self._set_palette() + + def _apply_widths( + self, + worksheet: XlsxWorksheet, + column_widths: dict[int, int], + column_formats: list[Any] | None = None, + ) -> None: + if self.fixed_column_width is not None: + for column in range(len(self.columns)): + worksheet.set_column( + column, + column, + self.fixed_column_width, + column_formats[column] if column_formats else None, + ) + return + + for column, max_len in column_widths.items(): + zero_based_column = column - 1 + worksheet.set_column( + zero_based_column, + zero_based_column, + max(max_len + 3, 10), + column_formats[zero_based_column] if column_formats else None, + ) + + def _resolved_format(self, properties: dict[str, Any]) -> Any: + cache_key = ("resolved", tuple(sorted(properties.items()))) + if cache_key not in self._format_cache: + self._format_cache[cache_key] = self.workbook.add_format(properties) + return self._format_cache[cache_key] + + def _columns_dict_to_flat( + self, + columns_dict: dict | None = None, + prefix: str = "", + depth_prefix: str = "", + ) -> list[str]: + if columns_dict is None: + columns_dict = self.columns_dict + result: list[str] = [] + for key, sub_dict in columns_dict.items(): + new_key = sub_dict.get("key", key) + if "children" not in sub_dict: + if new_key: + flat_key = f"{prefix}{new_key}" + result.append(flat_key) + self._flat_columns_to_depth[flat_key] = f"{depth_prefix}{key}" + else: + result += self._columns_dict_to_flat( + sub_dict["children"], + prefix=f"{prefix}{new_key}." if new_key else prefix, + depth_prefix=f"{depth_prefix}{key}.", + ) + return result + + def _set_palette(self) -> None: + for key in self.columns: + full_key = None + for sub_key in self._flat_columns_to_depth[key].split("."): + full_key = sub_key if full_key is None else f"{full_key}.{sub_key}" + color = self.colors_config.get(full_key) + if color and "palette" in color: + self.palette[key] = color["palette"] + + def _get_header_max_row(self, column_dict: dict) -> int: + max_depth = 0 + for sub_dict in column_dict.values(): + if "children" in sub_dict: + max_depth = max(max_depth, self._get_header_max_row(sub_dict["children"])) + return max_depth + 1 + + def _header_format(self, full_key: str): + properties: dict[str, Any] = { + "border": 1, + "align": "center", + "valign": "vcenter", + } + if self.colors_config and (color := self.colors_config.get(full_key)): + if "background" in color: + properties["bg_color"] = f"#{color['background']}" + properties["pattern"] = 1 + if "color" in color: + properties["font_color"] = f"#{color['color']}" + if self.header_format_resolver: + properties.update(self.header_format_resolver(full_key) or {}) + return self.workbook.add_format(properties) + + def _write_header( + self, + worksheet: XlsxWorksheet, + column_widths: dict[int, int], + columns_dict: dict | None = None, + current_column: int = 0, + current_row: int = 0, + max_row: int | None = None, + full_key: str | None = None, + ) -> int: + if columns_dict is None: + columns_dict = self.columns_dict + max_row = self._get_header_max_row(columns_dict) - 1 + + for key, sub_dict in columns_dict.items(): + current_key = f"{full_key}.{key}" if full_key else key + title = sub_dict["name"] + start_column = current_column + + if "children" in sub_dict: + current_column = self._write_header( + worksheet=worksheet, + column_widths=column_widths, + columns_dict=sub_dict["children"], + current_column=current_column, + current_row=current_row + 1, + max_row=max_row, + full_key=current_key, + ) + end_column = current_column - 1 + if end_column > start_column: + worksheet.merge_range( + current_row, + start_column, + current_row, + end_column, + title, + self._header_format(current_key), + ) + else: + worksheet.write( + current_row, + start_column, + title, + self._header_format(current_key), + ) + else: + if current_row != max_row: + worksheet.merge_range( + current_row, + current_column, + max_row, + current_column, + title, + self._header_format(current_key), + ) + else: + worksheet.write( + current_row, + current_column, + title, + self._header_format(current_key), + ) + current_column += 1 + + if self.fixed_column_width is None: + self._track_width(column_widths, start_column + 1, title) + + return current_column + + def _row_format(self, palette: str, item_type: str): + cache_key = (palette, item_type) + if cache_key not in self._format_cache: + self._format_cache[cache_key] = self.workbook.add_format( + { + "bg_color": f"#{PALETTE[palette][item_type]}", + "pattern": 1, + } + ) + return self._format_cache[cache_key] + + def write(self, worksheet: XlsxWorksheet, rows: list[tuple]) -> None: + column_widths: dict[int, int] = {} + header_rows = self._get_header_max_row(self.columns_dict) + self._write_header(worksheet, column_widths=column_widths) + output_row = header_rows + column_formats = None + if self.column_format_resolver: + column_formats = [ + self._resolved_format(properties) + if (properties := self.column_format_resolver(column_key, column)) + else None + for column, column_key in enumerate(self.columns) + ] + + for row in rows: + if len(row) > 3 and isinstance(row[3], dict): + if row[0] == "META": + continue + raw_data = row[3] + else: + raw_data = {} + + normalized = self.row_normalizer(raw_data) + values = [self.value_serializer(normalized.get(key)) for key in self.columns] + if column_formats is not None: + worksheet.write_row(output_row, 0, values) + if self.fixed_column_width is None: + self._track_row_widths(column_widths, values) + output_row += 1 + continue + + item_type = row[0] if len(row) > 3 else None + for column, value in enumerate(values): + cell_format = None + column_key = self.columns[column] + resolved_format = ( + self.row_format_resolver(row, column_key, column) + if self.row_format_resolver + else None + ) + if resolved_format: + cell_format = self._resolved_format(resolved_format) + palette = self.palette.get(column_key) + if ( + cell_format is None + and palette + and item_type not in (None, "SUB_ITEM", "INPUT") + ): + cell_format = self._row_format(palette, item_type) + worksheet.write(output_row, column, value, cell_format) + if self.fixed_column_width is None: + self._track_row_widths(column_widths, values) + output_row += 1 + + self._apply_widths(worksheet, column_widths, column_formats) diff --git a/api/src/services/summary_service.py b/api/src/services/summary_service.py index eb5550f..1408e04 100644 --- a/api/src/services/summary_service.py +++ b/api/src/services/summary_service.py @@ -52,6 +52,24 @@ class SummaryService: sort_direction=sort_direction, ) + async def get_svod2_go_export_rows( + self, + year: int, + user: AppUser, + org_ids: list[int] | None = None, + sort_by: str = "org_name", + sort_direction: str = "asc", + ) -> list[tuple]: + org_ids = await self._allowed_org_ids(user, org_ids) + if org_ids == []: + return [] + return await self.summary_repo.get_svod2_go_export_rows( + year=year, + org_ids=org_ids, + sort_by=sort_by, + sort_direction=sort_direction, + ) + async def _allowed_org_ids( self, user: AppUser, diff --git a/api/tests/integration/test_export_projects_integration.py b/api/tests/integration/test_export_projects_integration.py deleted file mode 100644 index 77fa3c6..0000000 --- a/api/tests/integration/test_export_projects_integration.py +++ /dev/null @@ -1,38 +0,0 @@ -import pytest - - -def _get_any_project_report(client, admin_tokens, auth_headers) -> tuple[int, int, str]: - list_response = client.get( - "/api/v1/projects?limit=50", - headers=auth_headers(admin_tokens), - ) - assert list_response.status_code == 200 - projects = list_response.json().get("result") or [] - for project in projects: - project_id = int(project["id"]) - details_response = client.get( - f"/api/v1/projects/{project_id}", - headers=auth_headers(admin_tokens), - ) - if details_response.status_code != 200: - continue - reports = (details_response.json().get("result") or {}).get("reports") or [] - if reports: - report = reports[0] - return project_id, int(report["year"]), str(report["report_type"]) - pytest.skip("Не найден проект FORM_3 с отчетами для smoke-экспорта") - - -def test_export_project_report_smoke(client, admin_tokens, auth_headers): - project_id, report_year, report_type = _get_any_project_report(client, admin_tokens, auth_headers) - - response = client.get( - f"/api/v1/export/project/{project_id}/report/{report_year}/{report_type}", - headers=auth_headers(admin_tokens), - ) - assert response.status_code == 200 - assert response.headers["content-type"] == ( - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" - ) - assert "attachment; filename*=UTF-8''" in response.headers["content-disposition"] - assert len(response.content) > 0 diff --git a/web/src/App.jsx b/web/src/App.jsx index bd5ed24..33d47bd 100644 --- a/web/src/App.jsx +++ b/web/src/App.jsx @@ -2,6 +2,7 @@ import { useEffect } from 'react'; import { AppRoutes } from './app/Routes'; import './App.css'; import { isTestMode } from './conf/config'; +import ExportJobs from './components/ExportJobs'; const BaseTag = () => { useEffect(() => { @@ -27,6 +28,7 @@ function App() {
+
); } diff --git a/web/src/api/exports.js b/web/src/api/exports.js new file mode 100644 index 0000000..050097c --- /dev/null +++ b/web/src/api/exports.js @@ -0,0 +1,101 @@ +import api from './client'; + +let jobs = []; +const listeners = new Set(); +let creating = 0; +const filenames = new Map(); + +export const getExportFilename = (job) => { + let saved = filenames.get(job.id); + try { saved ||= localStorage.getItem(`export-filename:${job.id}`); } catch { /* Memory fallback. */ } + return saved || job.filename?.replace(/\.[^.]+$/, '').replace(/_\d{4}-\d{2}-\d{2}$/, '') || 'Экспорт'; +}; + +export const createNamedExport = async (createJob, filename) => { + creating++; + try { + const job = await createJob(); + if (filename) { + filenames.set(job.id, filename); + try { localStorage.setItem(`export-filename:${job.id}`, filename); } catch { /* Memory fallback. */ } + } + registerExport(job); + } finally { + creating--; + if (!creating) receiveJobs(jobs); + } +}; + +export const registerExport = (job) => { + // The WebSocket can deliver completion before the creation response arrives. + if (!jobs.some((item) => item.id === job.id)) receiveJobs([job, ...jobs]); +}; + +export const subscribeExports = (listener) => { + listeners.add(listener); + listener(jobs); + return () => listeners.delete(listener); +}; + +function receiveJobs(next) { + jobs = next; + if (creating) return; + for (const listener of listeners) listener(jobs); +} + +export const downloadExport = (id) => api.get(`/export/jobs/${id}/download`, { responseType: 'blob' }); + +export function startExportUpdates() { + let stopped = false; + let socket; + let retry; + let watchdog; + let attempts = 0; + const connect = () => { + if (stopped) return; + const url = new URL(`${api.defaults.baseURL}/ws/export`, window.location.origin); + url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:'; + socket = new WebSocket(url); + const armWatchdog = () => { + clearTimeout(watchdog); + watchdog = setTimeout(() => socket.close(), 45000); + }; + socket.onopen = () => { + socket.send(JSON.stringify({ token: localStorage.getItem('access_token') })); + armWatchdog(); + }; + socket.onmessage = (event) => { + if (stopped) return; + try { + const message = JSON.parse(event.data); + if (message.event === 'exports') { + attempts = 0; + receiveJobs(message.jobs); + armWatchdog(); + } + } catch (error) { console.error('Ошибка статуса экспорта', error); } + }; + socket.onclose = (event) => { + clearTimeout(watchdog); + if (stopped || event.code === 4401) return; + retry = setTimeout(connect, Math.min(30000, 2000 * 2 ** attempts++)); + }; + }; + // HTTP fallback only when the WebSocket is unavailable. + const fallback = setInterval(async () => { + if (socket?.readyState === WebSocket.OPEN || stopped) return; + try { + const response = await api.get('/export/jobs'); + if (!stopped) receiveJobs(response.data); + } catch { /* Retry on the next fallback interval. */ } + }, 30000); + connect(); + return () => { + stopped = true; + clearTimeout(retry); + clearTimeout(watchdog); + clearInterval(fallback); + socket?.close(); + receiveJobs([]); + }; +} diff --git a/web/src/api/form_sheet.js b/web/src/api/form_sheet.js index f31596f..7ac4ecd 100644 --- a/web/src/api/form_sheet.js +++ b/web/src/api/form_sheet.js @@ -7,8 +7,7 @@ export const FormsSheetApi = { export(formId, sheetName, params = {}) { return api.get(`export/form/${formId}/sheet/${sheetName}`, { params, - responseType: 'blob', - }); + }).then((r) => r.data); }, getExpenseItem(itemId) { return api.get(`/expense-item/${itemId}`).then((r) => r.data); diff --git a/web/src/api/projects.js b/web/src/api/projects.js index 9d92132..7a5f74f 100644 --- a/web/src/api/projects.js +++ b/web/src/api/projects.js @@ -39,13 +39,9 @@ export const ProjectsApi = { return api.get(`/projects/${id}/summary`).then((r) => r.data); }, export: (id, year, sheetName) => { - return api.get(`/export/project/${id}/report/${year}/${sheetName}`, { - responseType: 'blob', - }); + return api.get(`/export/project/${id}/report/${year}/${sheetName}`).then((r) => r.data); }, exportProject: (id) => { - return api.get(`/export/project/${id}`, { - responseType: 'blob', - }); + return api.get(`/export/project/${id}`).then((r) => r.data); }, }; diff --git a/web/src/api/tasks.js b/web/src/api/tasks.js index d9b2f74..6c02944 100644 --- a/web/src/api/tasks.js +++ b/web/src/api/tasks.js @@ -28,8 +28,7 @@ export const TasksApi = { exportTask(taskId, params = {}) { return api.get(`export/form/${taskId}`, { params, - responseType: 'blob', - }); + }).then((r) => r.data); }, markForDeletion(taskId) { return api.post(`/form/${taskId}/mark-delete`).then((r) => r.data); @@ -43,7 +42,6 @@ export const TasksApi = { exportTasks(payload, params = {}) { return api.post(`export/bulk`, payload, { params, - responseType: 'blob', - }); + }).then((r) => r.data); }, }; diff --git a/web/src/components/ExportJobs.jsx b/web/src/components/ExportJobs.jsx new file mode 100644 index 0000000..19645db --- /dev/null +++ b/web/src/components/ExportJobs.jsx @@ -0,0 +1,63 @@ +import { useEffect } from 'react'; +import { toast } from 'react-toastify'; +import { useAuth } from '../app/context/AuthProvider'; +import { downloadExport, getExportFilename, startExportUpdates, subscribeExports } from '../api/exports'; +import { downloadFile } from '../utils/downloadFile'; +import { getApiErrorMessage } from '../utils/getApiErrorMessage'; + +export default function ExportJobs() { + const { user, isAuthenticated } = useAuth(); + useEffect(() => { + if (!isAuthenticated || !user?.id) return; + let stopped = false; + const processing = new Set(); + const handled = new Set(); + const toastIds = new Set(); + const key = (id) => `export-handled:${user.id}:${id}`; + const wasHandled = (id) => { + try { return handled.has(id) || Boolean(localStorage.getItem(key(id))); } + catch { return handled.has(id); } + }; + const markHandled = (id) => { + handled.add(id); + try { localStorage.setItem(key(id), String(Date.now())); } catch { /* Memory fallback. */ } + }; + const showLoading = (id, message) => { + toastIds.add(id); + if (!toast.isActive(id)) toast.loading(message, { toastId: id, autoClose: false }); + else toast.update(id, { render: message, isLoading: true, autoClose: false }); + }; + const observe = async (job) => { + if (stopped || wasHandled(job.id) || processing.has(job.id)) return; + const toastId = `export:${user.id}:${job.id}`; + if (['queued', 'processing'].includes(job.status)) { + showLoading(toastId, 'Идёт формирование Excel…'); + return; + } + processing.add(job.id); + showLoading(toastId, 'Скачивание Excel…'); + try { + if (job.status === 'failed') throw new Error(job.error || 'Ошибка экспорта'); + const response = await downloadExport(job.id); + if (stopped) return; + await downloadFile(response, getExportFilename(job), 'xlsx'); + markHandled(job.id); + toast.update(toastId, { render: 'Excel сформирован, скачивание началось', type: 'success', isLoading: false, autoClose: 3000 }); + } catch (error) { + if (stopped) return; + if (job.status === 'failed' || error?.response?.status === 410) markHandled(job.id); + const message = await getApiErrorMessage(error, 'Не удалось скачать файл'); + if (!stopped) toast.update(toastId, { render: message, type: 'error', isLoading: false, autoClose: 5000 }); + } finally { processing.delete(job.id); } + }; + const unsubscribe = subscribeExports((jobs) => { for (const job of jobs) void observe(job); }); + const stop = startExportUpdates(); + return () => { + stopped = true; + unsubscribe(); + stop(); + for (const id of toastIds) toast.dismiss(id); + }; + }, [isAuthenticated, user?.id]); + return null; +} diff --git a/web/src/utils/exportFile.js b/web/src/utils/exportFile.js index 5a3ebd3..4b8d59b 100644 --- a/web/src/utils/exportFile.js +++ b/web/src/utils/exportFile.js @@ -2,58 +2,33 @@ import { toast } from 'react-toastify'; import { FormsSheetApi } from '../api/form_sheet'; import { ProjectsApi } from '../api/projects'; import { TasksApi } from '../api/tasks'; -import { downloadFile } from './downloadFile'; +import { createNamedExport } from '../api/exports'; import { getApiErrorMessage } from './getApiErrorMessage'; -export const exportSingleForm = async (taskId, taskTitle) => { +const runExport = async (createJob, defaultFilename, rethrow = false) => { try { - const response = await TasksApi.exportTask(taskId); - await downloadFile(response, taskTitle, 'xlsx'); + await createNamedExport(createJob, defaultFilename); } catch (error) { const message = await getApiErrorMessage(error, 'Ошибка при скачивании файла'); toast.error(message); + if (rethrow) throw error; } }; -export const exportSheet = async (formId, sheetName, direction) => { - try { - const normalizedDirection = direction && direction !== 'null' ? direction : undefined; - const params = normalizedDirection ? { direction: normalizedDirection } : {}; - const response = await FormsSheetApi.export(formId, sheetName, params); - await downloadFile(response, sheetName, 'xlsx'); - } catch (error) { - const message = await getApiErrorMessage(error, 'Ошибка при скачивании файла'); - toast.error(message); - } +export const exportSingleForm = (taskId, taskTitle) => + runExport(() => TasksApi.exportTask(taskId), taskTitle); + +export const exportSheet = (formId, sheetName, direction) => { + const normalizedDirection = direction && direction !== 'null' ? direction : undefined; + const params = normalizedDirection ? { direction: normalizedDirection } : {}; + return runExport(() => FormsSheetApi.export(formId, sheetName, params), sheetName); }; -export const exportSheetProject = async (formId, sheetName, year) => { - try { - const response = await ProjectsApi.export(formId, year, sheetName); - await downloadFile(response, sheetName, 'xlsx'); - } catch (error) { - const message = await getApiErrorMessage(error, 'Ошибка при скачивании файла'); - toast.error(message); - } -}; +export const exportSheetProject = (formId, sheetName, year) => + runExport(() => ProjectsApi.export(formId, year, sheetName), sheetName); -export const exportProject = async (projectId, projectTitle) => { - try { - const response = await ProjectsApi.exportProject(projectId); - await downloadFile(response, projectTitle, 'xlsx'); - } catch (error) { - const message = await getApiErrorMessage(error, 'Ошибка при скачивании файла'); - toast.error(message); - } -}; +export const exportProject = (projectId, projectTitle) => + runExport(() => ProjectsApi.exportProject(projectId), projectTitle); -export const exportBulkForms = async (formIds, fileName = 'Задачи') => { - try { - const response = await TasksApi.exportTasks({ form_ids: formIds }); - await downloadFile(response, fileName, 'xlsx'); - } catch (error) { - const message = await getApiErrorMessage(error, 'Ошибка при скачивании файла'); - toast.error(message); - throw error; - } -}; +export const exportBulkForms = (formIds, fileName = 'Задачи') => + runExport(() => TasksApi.exportTasks({ form_ids: formIds }), fileName, true);