137 lines
6.1 KiB
Python
137 lines
6.1 KiB
Python
"""Сверка серверной постраничной выдачи свода 2 с прежним полным ответом на данных стенда (только чтение).
|
||
|
||
cd perf && python analysis/svod2_paging_check.py --scale 1 [--api http://localhost:8000]
|
||
|
||
Пользователь dfip и год берутся из perf/results/manifest-s<N>.json. Проверки:
|
||
- склейка `view=full&limit=300` равна прежнему ответу (`result` целиком);
|
||
- `total` один на всех страницах; страница за концом пуста;
|
||
- `view=compact` - проекция `view=full` (compact_projection из api/tests/svod2_oracle.py).
|
||
Код выхода 1 при расхождении. Поиск и варианты сверяет svod2_search_oracle.mjs --check.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import importlib.util
|
||
import json
|
||
import sys
|
||
from pathlib import Path
|
||
from typing import Callable
|
||
|
||
PERF = Path(__file__).resolve().parents[1]
|
||
REPO = PERF.parent
|
||
PAGE = 300
|
||
|
||
|
||
def _load_compact_projection():
|
||
path = REPO / "api" / "tests" / "svod2_oracle.py"
|
||
spec = importlib.util.spec_from_file_location("svod2_oracle", path)
|
||
mod = importlib.util.module_from_spec(spec)
|
||
sys.modules[spec.name] = mod
|
||
spec.loader.exec_module(mod)
|
||
return mod.compact_projection
|
||
|
||
|
||
compact_projection = _load_compact_projection()
|
||
|
||
|
||
def stitch_pages(fetch_page: Callable[[int, int], dict], limit: int) -> tuple[list[dict], set[int]]:
|
||
"""Страницы подряд до короткой (или пустой): строки и множество значений `total`."""
|
||
rows: list[dict] = []
|
||
totals: set[int] = set()
|
||
offset = 0
|
||
while True:
|
||
page = fetch_page(offset, limit)
|
||
chunk = page.get("result") or []
|
||
rows.extend(chunk)
|
||
if page.get("total") is not None:
|
||
totals.add(page["total"])
|
||
if len(chunk) < limit:
|
||
return rows, totals
|
||
offset += limit
|
||
|
||
|
||
def _line_id(row: dict):
|
||
return (row.get("data") or {}).get("line_id")
|
||
|
||
|
||
def compare_stitched(full_rows: list[dict], stitched: list[dict]) -> list[str]:
|
||
problems: list[str] = []
|
||
if len(full_rows) != len(stitched):
|
||
problems.append(f"число строк: прежний ответ {len(full_rows)}, склейка {len(stitched)}")
|
||
ids = [_line_id(r) for r in stitched]
|
||
dup = sorted({i for i in ids if ids.count(i) > 1}, key=str)
|
||
if dup:
|
||
problems.append(f"повторы line_id в склейке: {dup[:10]}")
|
||
missing = sorted({_line_id(r) for r in full_rows} - set(ids), key=str)
|
||
if missing:
|
||
problems.append(f"пропущены line_id: {missing[:10]}")
|
||
for i, (a, b) in enumerate(zip(full_rows, stitched)):
|
||
if a != b:
|
||
problems.append(f"первое расхождение строки на позиции {i} (line_id {_line_id(a)} против {_line_id(b)})")
|
||
break
|
||
return problems
|
||
|
||
|
||
def check_compact(full_rows: list[dict], compact_rows: list[dict]) -> list[str]:
|
||
problems: list[str] = []
|
||
if len(full_rows) != len(compact_rows):
|
||
problems.append(f"число строк: full {len(full_rows)}, compact {len(compact_rows)}")
|
||
for i, (f, c) in enumerate(zip(full_rows, compact_rows)):
|
||
head_f = (f.get("row_type"), f.get("depth"), f.get("sort_order"))
|
||
head_c = (c.get("row_type"), c.get("depth"), c.get("sort_order"))
|
||
if head_f != head_c:
|
||
problems.append(f"позиция {i}: row_type/depth/sort_order {head_f} против {head_c}")
|
||
elif c.get("data") != compact_projection(f.get("data") or {}):
|
||
problems.append(f"позиция {i} (line_id {_line_id(f)}): compact не равен проекции full")
|
||
if len(problems) >= 20:
|
||
problems.append("... остановлено после 20 расхождений")
|
||
break
|
||
return problems
|
||
|
||
|
||
def main(argv=None) -> int:
|
||
import httpx
|
||
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--api", default="http://localhost:8000")
|
||
ap.add_argument("--scale", type=int, required=True)
|
||
a = ap.parse_args(argv)
|
||
man = json.loads((PERF / "results" / f"manifest-s{a.scale}.json").read_text())
|
||
user = man["users"]["dfip"]
|
||
year = man["sheets"][0]["year"]
|
||
problems: list[str] = []
|
||
with httpx.Client(base_url=a.api, timeout=300) as c:
|
||
r = c.post("/api/v1/auth/login", json={"username": user["username"], "password": user["password"]})
|
||
r.raise_for_status()
|
||
headers = {"Authorization": f"Bearer {r.json()['access_token']}"}
|
||
|
||
def get(sheet: str, **params) -> dict:
|
||
resp = c.get(f"/api/v1/svod/2/{sheet}/{year}", params=params, headers=headers)
|
||
resp.raise_for_status()
|
||
return resp.json()
|
||
|
||
for sheet in ("GO", "RF"):
|
||
full_rows = get(sheet)["result"]
|
||
for view in ("full", "compact"):
|
||
stitched, totals = stitch_pages(lambda o, l, v=view: get(sheet, view=v, offset=o, limit=l), PAGE)
|
||
if view == "full":
|
||
found = compare_stitched(full_rows, stitched)
|
||
else:
|
||
full_stitched, _ = stitch_pages(lambda o, l: get(sheet, view="full", offset=o, limit=l), PAGE)
|
||
found = check_compact(full_stitched, stitched)
|
||
if totals != {len(full_rows)}:
|
||
found.append(f"total на страницах {sorted(totals)}, ожидалось {{{len(full_rows)}}}")
|
||
tail = get(sheet, view=view, offset=len(full_rows), limit=PAGE)
|
||
if tail["result"]:
|
||
found.append(f"страница за концом не пуста: {len(tail['result'])} строк")
|
||
problems += [f"{sheet} {view}: {p}" for p in found]
|
||
print(f"{sheet} {view}: строк {len(stitched)}, расхождений {len(found)}")
|
||
for p in problems:
|
||
print(p)
|
||
print("CHECK_OK" if not problems else f"CHECK_FAILED: {len(problems)}")
|
||
return 1 if problems else 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|