perf(api): отчёты и сводные проектов отдают готовый JSON из Postgres

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Andrey Epifancev 2026-10-28 21:24:14 +04:00
parent ca7160124a
commit 604e277371
4 changed files with 352 additions and 78 deletions

View File

@ -9,6 +9,7 @@ from src.api.v1.deps import (
get_current_active_user_with_set_db,
require_executor,
)
from src.api.v1.json_response import list_json_response
from src.db.models.app_user import AppUser
from src.db.session import get_db
from src.domain.schemas import (
@ -114,18 +115,17 @@ async def get_projects_with_reports(
return BaseListResponse(result=projects, count=count)
async def _get_projects_summary_response(
def _projects_summary_response(
*,
year: int,
response: Response,
branch_id: Optional[str],
project_type: Optional[str],
handler_name: str,
rows,
rows: list[str],
t0: float,
current_user: AppUser,
) -> BaseListResponse[SheetResponse]:
db_ms = (time.perf_counter() - rows["t0"]) * 1000
response.headers["X-DB-Time-Ms"] = f"{db_ms:.2f}"
) -> Response:
db_ms = (time.perf_counter() - t0) * 1000
extra = {
"type": "db_time",
"time_ms": f"{db_ms:.2f}",
@ -136,92 +136,89 @@ async def _get_projects_summary_response(
"user": current_user.id,
}
logger.info(f"db_time: {extra}", extra=extra)
return BaseListResponse(count=len(rows["data"]), result=_rows_to_payload(rows["data"]))
return list_json_response(rows, headers={"X-DB-Time-Ms": f"{db_ms:.2f}"})
@router.get("/projects/plan-summary")
@router.get("/projects/plan-summary", response_model=BaseListResponse[SheetResponse])
async def get_projects_plan_summary(
year: int,
response: Response,
branch_id: Optional[str] = None,
project_type: Optional[str] = None,
db: AsyncSession = Depends(get_db),
current_user: AppUser = Depends(get_current_active_user_with_set_db),
) -> BaseListResponse[SheetResponse]:
) -> Response:
branch_ids = _parse_org_units_csv(branch_id)
project_service = ProjectService(db)
t0 = time.perf_counter()
rows = await project_service.get_plan_summary_rows(
rows = await project_service.get_plan_summary_json(
year=year,
branch_ids=branch_ids,
user=current_user,
project_type=project_type,
)
return await _get_projects_summary_response(
return _projects_summary_response(
year=year,
response=response,
branch_id=branch_id,
project_type=project_type,
handler_name="get_projects_plan_summary",
rows={"t0": t0, "data": rows},
rows=rows,
t0=t0,
current_user=current_user,
)
@router.get("/projects/fact-summary")
@router.get("/projects/fact-summary", response_model=BaseListResponse[SheetResponse])
async def get_projects_fact_summary(
year: int,
response: Response,
branch_id: Optional[str] = None,
project_type: Optional[str] = None,
db: AsyncSession = Depends(get_db),
current_user: AppUser = Depends(get_current_active_user_with_set_db),
) -> BaseListResponse[SheetResponse]:
) -> Response:
branch_ids = _parse_org_units_csv(branch_id)
project_service = ProjectService(db)
t0 = time.perf_counter()
rows = await project_service.get_fact_summary_rows(
rows = await project_service.get_fact_summary_json(
year=year,
branch_ids=branch_ids,
user=current_user,
project_type=project_type,
)
return await _get_projects_summary_response(
return _projects_summary_response(
year=year,
response=response,
branch_id=branch_id,
project_type=project_type,
handler_name="get_projects_fact_summary",
rows={"t0": t0, "data": rows},
rows=rows,
t0=t0,
current_user=current_user,
)
@router.get("/projects/detail-summary")
@router.get("/projects/detail-summary", response_model=BaseListResponse[SheetResponse])
async def get_projects_detail_summary(
year: int,
response: Response,
branch_id: Optional[str] = None,
project_type: Optional[str] = None,
db: AsyncSession = Depends(get_db),
current_user: AppUser = Depends(get_current_active_user_with_set_db),
) -> BaseListResponse[SheetResponse]:
) -> Response:
branch_ids = _parse_org_units_csv(branch_id)
project_service = ProjectService(db)
t0 = time.perf_counter()
rows = await project_service.get_detail_summary_rows(
rows = await project_service.get_detail_summary_json(
year=year,
branch_ids=branch_ids,
user=current_user,
project_type=project_type,
)
return await _get_projects_summary_response(
return _projects_summary_response(
year=year,
response=response,
branch_id=branch_id,
project_type=project_type,
handler_name="get_projects_detail_summary",
rows={"t0": t0, "data": rows},
rows=rows,
t0=t0,
current_user=current_user,
)
@ -239,19 +236,18 @@ async def get_project(
return BaseSingleResponse(result=project)
@router.get("/projects/{project_id}/report/{year}/{report_type}")
@router.get("/projects/{project_id}/report/{year}/{report_type}", response_model=BaseListResponse[SheetResponse])
async def get_project_report(
project_id: int,
year: int,
report_type: str,
response: Response,
sections: Optional[str] = None,
db: AsyncSession = Depends(get_db),
current_user: AppUser = Depends(get_current_active_user_with_set_db),
) -> BaseListResponse[SheetResponse]:
) -> Response:
project_service = ProjectService(db)
t0 = time.perf_counter()
rows = await project_service.get_report_rows(
rows = await project_service.get_report_json(
project_id=project_id,
year=year,
report_type=report_type,
@ -259,7 +255,6 @@ async def get_project_report(
user=current_user,
)
db_ms = (time.perf_counter() - t0) * 1000
response.headers["X-DB-Time-Ms"] = f"{db_ms:.2f}"
extra={
"type": "db_time",
"time_ms": f"{db_ms:.2f}",
@ -274,28 +269,26 @@ async def get_project_report(
f"db_time: {extra}",
extra=extra,
)
return BaseListResponse(count=len(rows), result=_rows_to_payload(rows))
return list_json_response(rows, headers={"X-DB-Time-Ms": f"{db_ms:.2f}"})
@router.get("/rf-rollup/{branch_id}/{year}")
@router.get("/rf-rollup/{branch_id}/{year}", response_model=BaseListResponse[SheetResponse])
async def get_rf_rollup(
branch_id: int,
year: int,
response: Response,
sections: Optional[str] = None,
db: AsyncSession = Depends(get_db),
current_user: AppUser = Depends(get_current_active_user_with_set_db),
) -> BaseListResponse[SheetResponse]:
) -> Response:
project_service = ProjectService(db)
t0 = time.perf_counter()
rows = await project_service.get_rf_rollup_rows(
rows = await project_service.get_rf_rollup_json(
branch_id=branch_id,
year=year,
sections=_parse_sections(sections),
user=current_user,
)
db_ms = (time.perf_counter() - t0) * 1000
response.headers["X-DB-Time-Ms"] = f"{db_ms:.2f}"
extra={
"type": "db_time",
"time_ms": f"{db_ms:.2f}",
@ -309,24 +302,22 @@ async def get_rf_rollup(
f"db_time: {extra}",
extra=extra,
)
return BaseListResponse(count=len(rows), result=_rows_to_payload(rows))
return list_json_response(rows, headers={"X-DB-Time-Ms": f"{db_ms:.2f}"})
@router.get("/projects/{project_id}/summary")
@router.get("/projects/{project_id}/summary", response_model=BaseListResponse[SheetResponse])
async def get_project_summary(
project_id: int,
response: Response,
db: AsyncSession = Depends(get_db),
current_user: AppUser = Depends(get_current_active_user_with_set_db),
) -> BaseListResponse[SheetResponse]:
) -> Response:
project_service = ProjectService(db)
t0 = time.perf_counter()
rows = await project_service.get_project_summary_rows(
rows = await project_service.get_project_summary_json(
project_id=project_id,
user=current_user,
)
db_ms = (time.perf_counter() - t0) * 1000
response.headers["X-DB-Time-Ms"] = f"{db_ms:.2f}"
extra = {
"type": "db_time",
"time_ms": f"{db_ms:.2f}",
@ -335,7 +326,7 @@ async def get_project_summary(
"user": current_user.id,
}
logger.info(f"db_time: {extra}", extra=extra)
return BaseListResponse(count=len(rows), result=_rows_to_payload(rows))
return list_json_response(rows, headers={"X-DB-Time-Ms": f"{db_ms:.2f}"})
@router.patch("/projects/{project_id}/report/{year}/{report_type}/cell")

View File

@ -10,6 +10,7 @@ from src.db.models.rf_project_report_line import RfProjectReportLine
from src.db.models.rf_project_year import RfProjectYear
from src.db.models.smeta import Smeta
from src.domain.schemas import placement_type_from_db
from src.repository.row_json import ROW_COLUMNS, fetch_row_texts, row_json_select
class ProjectRepository:
@ -355,22 +356,22 @@ class ProjectRepository:
query = query.where(Project.org_unit_id.in_(org_unit_ids))
return (await self.db.execute(query)).scalar_one_or_none()
@staticmethod
def _report_source() -> str:
return (
"FROM v3.v_form3_report_jsonb("
"CAST(:report_id AS INT), "
"CAST(:sections AS TEXT[]), "
"CAST(:user_id AS INT))"
)
async def get_report_rows(
self,
report_id: int,
sections: list[str] | None = None,
user_id: int | None = None,
) -> list[tuple]:
query = text(
"""
SELECT row_type, depth, sort_order, data
FROM v3.v_form3_report_jsonb(
CAST(:report_id AS INT),
CAST(:sections AS TEXT[]),
CAST(:user_id AS INT)
)
"""
)
query = text(f"SELECT {ROW_COLUMNS} {self._report_source()}")
return (
await self.db.execute(
query,
@ -378,18 +379,32 @@ class ProjectRepository:
)
).all()
async def get_report_json(
self,
report_id: int,
sections: list[str] | None = None,
user_id: int | None = None,
) -> list[str]:
return await fetch_row_texts(
self.db,
f"{row_json_select()} {self._report_source()}",
{"report_id": report_id, "sections": sections, "user_id": user_id},
)
@staticmethod
def _rf_rollup_source() -> str:
return (
"FROM v3.v_form3_rf_rollup_jsonb("
"CAST(:branch_id AS INT), CAST(:year AS INT), CAST(:sections AS TEXT[]))"
)
async def get_rf_rollup_rows(
self,
branch_id: int,
year: int,
sections: list[str] | None = None,
) -> list[tuple]:
query = text(
"""
SELECT row_type, depth, sort_order, data
FROM v3.v_form3_rf_rollup_jsonb(CAST(:branch_id AS INT), CAST(:year AS INT), CAST(:sections AS TEXT[]))
"""
)
query = text(f"SELECT {ROW_COLUMNS} {self._rf_rollup_source()}")
return (
await self.db.execute(
query,
@ -397,17 +412,44 @@ class ProjectRepository:
)
).all()
async def get_project_summary_rows(self, project_id: int) -> list[tuple]:
query = text(
"""
SELECT row_type, depth, sort_order, data
FROM v3.v_project_summary_jsonb(CAST(:project_id AS INT))
"""
async def get_rf_rollup_json(
self,
branch_id: int,
year: int,
sections: list[str] | None = None,
) -> list[str]:
return await fetch_row_texts(
self.db,
f"{row_json_select()} {self._rf_rollup_source()}",
{"branch_id": branch_id, "year": year, "sections": sections},
)
@staticmethod
def _project_summary_one_source() -> str:
return "FROM v3.v_project_summary_jsonb(CAST(:project_id AS INT))"
async def get_project_summary_rows(self, project_id: int) -> list[tuple]:
query = text(f"SELECT {ROW_COLUMNS} {self._project_summary_one_source()}")
return (
await self.db.execute(query, {"project_id": project_id})
).all()
async def get_project_summary_json(self, project_id: int) -> list[str]:
return await fetch_row_texts(
self.db,
f"{row_json_select()} {self._project_summary_one_source()}",
{"project_id": project_id},
)
@staticmethod
def _project_summary_source(function_name: str) -> str:
return (
f"FROM {function_name}("
"CAST(:year AS INT), "
"CAST(:branch_ids AS INT[]), "
"CAST(:project_type AS VARCHAR))"
)
async def _get_project_summary_rows(
self,
function_name: str,
@ -415,16 +457,7 @@ class ProjectRepository:
branch_ids: list[int] | None,
project_type: str | None = None,
) -> list[tuple]:
query = text(
f"""
SELECT row_type, depth, sort_order, data
FROM {function_name}(
CAST(:year AS INT),
CAST(:branch_ids AS INT[]),
CAST(:project_type AS VARCHAR)
)
"""
)
query = text(f"SELECT {ROW_COLUMNS} {self._project_summary_source(function_name)}")
return (
await self.db.execute(
query,
@ -432,6 +465,19 @@ class ProjectRepository:
)
).all()
async def _get_project_summary_json(
self,
function_name: str,
year: int,
branch_ids: list[int] | None,
project_type: str | None = None,
) -> list[str]:
return await fetch_row_texts(
self.db,
f"{row_json_select()} {self._project_summary_source(function_name)}",
{"year": year, "branch_ids": branch_ids, "project_type": project_type},
)
async def get_plan_summary_rows(
self,
year: int,
@ -471,6 +517,45 @@ class ProjectRepository:
project_type=project_type,
)
async def get_plan_summary_json(
self,
year: int,
branch_ids: list[int] | None,
project_type: str | None = None,
) -> list[str]:
return await self._get_project_summary_json(
function_name="v3.v_project_plan_summary_jsonb",
year=year,
branch_ids=branch_ids,
project_type=project_type,
)
async def get_fact_summary_json(
self,
year: int,
branch_ids: list[int] | None,
project_type: str | None = None,
) -> list[str]:
return await self._get_project_summary_json(
function_name="v3.v_project_fact_summary_jsonb",
year=year,
branch_ids=branch_ids,
project_type=project_type,
)
async def get_detail_summary_json(
self,
year: int,
branch_ids: list[int] | None,
project_type: str | None = None,
) -> list[str]:
return await self._get_project_summary_json(
function_name="v3.v_project_detail_summary_jsonb",
year=year,
branch_ids=branch_ids,
project_type=project_type,
)
async def upd_form3_cell(
self,
report_id: int,

View File

@ -195,6 +195,93 @@ class ProjectService:
project_type=project_type,
)
async def get_report_json(
self,
project_id: int,
year: int,
report_type: str,
sections: list[str] | None,
user: AppUser,
) -> list[str]:
report_id = await self.resolve_report_id(project_id, year, report_type, user)
if not report_id:
raise ValidationException("Отчёт не найден")
return await self.project_repo.get_report_json(report_id=report_id, sections=sections, user_id=user.id)
async def get_rf_rollup_json(
self,
branch_id: int,
year: int,
sections: list[str] | None,
user: AppUser,
) -> list[str]:
org_unit_ids = await self._allowed_org_unit_ids(user)
if org_unit_ids is not None and branch_id not in org_unit_ids:
raise AccessDeniedException()
return await self.project_repo.get_rf_rollup_json(
branch_id=branch_id,
year=year,
sections=sections,
)
async def get_project_summary_json(
self,
project_id: int,
user: AppUser,
) -> list[str]:
project = await self.get_instance(project_id, user)
if not project:
raise ValidationException("Проект не найден")
return await self.project_repo.get_project_summary_json(project_id=project_id)
async def get_plan_summary_json(
self,
year: int,
branch_ids: list[int] | None,
user: AppUser,
project_type: str | None = None,
) -> list[str]:
branch_ids = await self._allowed_summary_branch_ids(branch_ids=branch_ids, user=user)
if branch_ids == []:
return []
return await self.project_repo.get_plan_summary_json(
year=year,
branch_ids=branch_ids,
project_type=project_type,
)
async def get_fact_summary_json(
self,
year: int,
branch_ids: list[int] | None,
user: AppUser,
project_type: str | None = None,
) -> list[str]:
branch_ids = await self._allowed_summary_branch_ids(branch_ids=branch_ids, user=user)
if branch_ids == []:
return []
return await self.project_repo.get_fact_summary_json(
year=year,
branch_ids=branch_ids,
project_type=project_type,
)
async def get_detail_summary_json(
self,
year: int,
branch_ids: list[int] | None,
user: AppUser,
project_type: str | None = None,
) -> list[str]:
branch_ids = await self._allowed_summary_branch_ids(branch_ids=branch_ids, user=user)
if branch_ids == []:
return []
return await self.project_repo.get_detail_summary_json(
year=year,
branch_ids=branch_ids,
project_type=project_type,
)
async def upd_form3_cell(
self,
project_id: int,

View File

@ -4,20 +4,27 @@
в той же сессии теста. Спека: openspec/changes/aurora-1848-sheet-json/specs/sheet-read-api/spec.md
"""
import json
import uuid
from types import SimpleNamespace
import pytest
from sqlalchemy import text
from src.main import app
from src.repository.project_repository import ProjectRepository
from src.repository.sheet_repository import SheetRepository
from src.repository.summary_repository import SummaryRepository
from src.services.project_service import ProjectService
from src.services.summary_service import SummaryService
ADMIN = SimpleNamespace(id=1, role_id=1) # fixture.sql: admin
ISP = SimpleNamespace(id=2, role_id=2) # fixture.sql: isp1, EXECUTOR_DFIP
YEAR = 2026
REPORT_NOT_FOUND = (400, {"code": 400, "field": "", "message": "Отчёт не найден"})
RF_ROLLUP_SQL_ERROR = (400, {"code": 400, "field": "", "message": "<class 'asyncpg.exceptions.UndefinedColumnError'>: column r.year does not exist"})
RF_FOREIGN = (403, {"code": 403, "field": "", "message": "Доступ запрещен"})
HEAVY_GET_PATHS = [
"/api/v1/form/{form_id}/sheet/{sheet}",
"/api/v1/svod/1/{sheet}/{year}",
@ -206,3 +213,107 @@ def test_svod_get_does_not_use_old_path(client, admin_tokens, auth_headers, monk
raise AssertionError(f"GET свода не должен вызывать SummaryRepository.{method}")
monkeypatch.setattr(SummaryRepository, method, boom)
get_ok(client, url, auth_headers(admin_tokens))
def _create_project(client, headers, year=YEAR, branch_id=1) -> int:
r = client.post(
"/api/v1/projects",
json={
"name": f"PT_{uuid.uuid4().hex[:10]}",
"year": year,
"branch_id": branch_id,
"project_type": "Открытие ВСП",
"vsp_format": "Типовой",
"placement_type": "Собственность",
"object_address": "Тестовая улица 1",
},
headers=headers,
)
assert r.status_code == 200, r.text
return r.json()["project_id"]
@pytest.fixture
def project_id(client, admin_tokens, auth_headers) -> int:
"""Новый проект сразу получает отчёт LIMIT со всеми строками статей (POST строки на нём даёт 400/409
и ломает транзакцию теста), так что отдельная строка не нужна."""
return _create_project(client, auth_headers(admin_tokens))
@pytest.mark.parametrize("sections", [None, "q1,year"])
def test_project_report_matches_old_path(client, db_session, admin_tokens, auth_headers, project_id, sections):
pid = project_id
result = get_ok(client, f"/api/v1/projects/{pid}/report/{YEAR}/LIMIT", auth_headers(admin_tokens),
{"sections": sections} if sections else None)
old = old_rows(client, lambda: ProjectService(db_session).get_report_rows(
project_id=pid, year=YEAR, report_type="LIMIT",
sections=sections.split(",") if sections else None, user=ADMIN))
assert result == old
assert any(row["row_type"] == "INPUT" for row in result)
def test_rf_rollup_error_same_as_old_path(client, db_session, admin_tokens, auth_headers, project_id):
"""v3.v_form3_rf_rollup_jsonb в миграции 0002_functions.sql ссылается на несуществующий r.year
(дефект вне AURORA-1848): на любой БД ответ - 400. Фиксируем, что новый путь отдаёт то же самое,
что и старый, а успешное сравнение с эталоном для rf-rollup невозможно."""
r = client.get(f"/api/v1/rf-rollup/1/{YEAR}", headers=auth_headers(admin_tokens))
assert (r.status_code, r.json()) == RF_ROLLUP_SQL_ERROR
def test_project_summary_matches_old_path(client, db_session, admin_tokens, auth_headers, project_id):
pid = project_id
result = get_ok(client, f"/api/v1/projects/{pid}/summary", auth_headers(admin_tokens))
old = old_rows(client, lambda: ProjectService(db_session).get_project_summary_rows(project_id=pid, user=ADMIN))
assert result == old
@pytest.mark.parametrize("kind", ["plan", "fact", "detail"])
def test_projects_summaries_match_old_path(client, db_session, admin_tokens, auth_headers, project_id, kind):
result = get_ok(client, f"/api/v1/projects/{kind}-summary", auth_headers(admin_tokens), {"year": YEAR})
method = getattr(ProjectService(db_session), f"get_{kind}_summary_rows")
old = old_rows(client, lambda: method(year=YEAR, branch_ids=None, user=ADMIN))
assert result == old
def test_projects_summaries_not_all_empty(client, admin_tokens, auth_headers, project_id):
pid = project_id
got = [get_ok(client, f"/api/v1/projects/{k}-summary", auth_headers(admin_tokens), {"year": YEAR})
for k in ("plan", "fact", "detail")]
got.append(get_ok(client, f"/api/v1/projects/{pid}/summary", auth_headers(admin_tokens)))
assert all(got), [bool(g) for g in got]
GET_METHODS = [
("/api/v1/projects/{pid}/report/%d/LIMIT" % YEAR, "get_report_rows"),
("/api/v1/rf-rollup/1/%d" % YEAR, "get_rf_rollup_rows"),
("/api/v1/projects/{pid}/summary", "get_project_summary_rows"),
("/api/v1/projects/plan-summary?year=%d" % YEAR, "get_plan_summary_rows"),
("/api/v1/projects/fact-summary?year=%d" % YEAR, "get_fact_summary_rows"),
("/api/v1/projects/detail-summary?year=%d" % YEAR, "get_detail_summary_rows"),
]
@pytest.mark.parametrize("url,method", GET_METHODS)
def test_projects_get_does_not_use_old_path(client, admin_tokens, auth_headers, project_id, monkeypatch, url, method):
called = []
async def boom(*args, **kwargs):
called.append(method)
raise AssertionError(f"GET не должен вызывать ProjectRepository.{method}")
monkeypatch.setattr(ProjectRepository, method, boom)
r = client.get(url.format(pid=project_id), headers=auth_headers(admin_tokens))
assert not called
if method != "get_rf_rollup_rows": # rf-rollup: см. test_rf_rollup_error_same_as_old_path
assert r.status_code == 200, r.text
# Характеризующие тесты ошибок: литералы сняты со старого кода (до правки роутеров).
def test_project_report_not_found_unchanged(client, admin_tokens, auth_headers, project_id):
pid = project_id
r = client.get(f"/api/v1/projects/{pid}/report/2099/LIMIT", headers=auth_headers(admin_tokens))
assert (r.status_code, r.json()) == REPORT_NOT_FOUND
def test_rf_rollup_foreign_branch_unchanged(client, isp_tokens, auth_headers):
r = client.get(f"/api/v1/rf-rollup/1/{YEAR}", headers=auth_headers(isp_tokens))
assert (r.status_code, r.json()) == RF_FOREIGN