620 lines
31 KiB
Python
620 lines
31 KiB
Python
"""Генератор данных стенда через HTTP-API (и WebSocket там, где у API нет HTTP-ручки).
|
||
|
||
cd perf && .venv/bin/python -m perfkit.gen.run --scale 1
|
||
|
||
Порядок: оргединицы -> пользователи -> ВСП -> формы -> программы/проекты FORM_4 -> строки ->
|
||
значения ячеек -> этапы -> проекты FORM_3 (годы, отчёты, значения, этапы) -> манифест.
|
||
Всё делается под admin (у admin на ячейки и строки нет ограничений по этапам), этапы для
|
||
EXECUTOR_RF и DFIP с открытым окном создаются после заполнения; в конце проверяется, что
|
||
`rf`, `rf_coauthor` и `dfip` действительно могут читать и править.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import asyncio
|
||
import dataclasses
|
||
import json
|
||
import os
|
||
import random
|
||
import re
|
||
import sys
|
||
import time
|
||
from collections import Counter
|
||
from datetime import datetime, timedelta, timezone
|
||
from itertools import cycle
|
||
|
||
import asyncpg
|
||
import websockets
|
||
|
||
from .. import RESULTS_DIR
|
||
from ..manifest import Manifest, Page, ProjectReport, Sheet, User, dump, manifest_path
|
||
from .client import API, Api, ApiError
|
||
from .plan import FORM4_LINES_DIVISOR, PASSWORD, PROGRAMS_PER_SHEET, PROJECTS_PER_PROGRAM, YEAR, GenPlan, plan_for_scale
|
||
|
||
DSN = os.environ.get("PERF_DB_DSN", "postgresql://perf:perf@localhost:55432/perf")
|
||
CONCURRENCY = int(os.environ.get("PERF_GEN_CONCURRENCY", "8"))
|
||
CELLS_BATCH = 200
|
||
EMAIL_DOMAIN = "perf-stand.com" # .local отклоняется email-validator (см. design.md)
|
||
|
||
# листы, в которые добавляются строки: (лист, direction)
|
||
LINE_SHEETS = {
|
||
"FORM_1": [("AHR", "Support"), ("AHR", "Development"), ("CAP", "Support"), ("CAP", "Development"), ("OPER", None)],
|
||
"FORM_2": [("AHR", None), ("CAP", None), ("OPER", None)],
|
||
"FORM_4": [("AHR", None), ("CAP", None), ("OPER", None)],
|
||
}
|
||
SATELLITES = ["AHR_RENT", "AHR_UTILITY", "AHR_SECURITY"] # FORM_2: по строке на ВСП
|
||
|
||
# колонки-кандидаты для заполнения; реально записываемые определяются пробой (computed_field отсекаются)
|
||
NUM_CANDIDATES = (
|
||
[f"plan.q{q}" for q in range(1, 5)]
|
||
+ [f"q{q}.actual_m{m}" for q in (1, 2, 3) for m in (1, 2, 3)]
|
||
+ [f"reserve.q{q}" for q in range(1, 5)]
|
||
+ [f"sequestration.q{q}" for q in range(1, 5)]
|
||
+ [f"seq_ssp.q{q}" for q in range(1, 5)]
|
||
+ [f"seq_dfip.q{q}" for q in range(1, 5)]
|
||
)
|
||
TEXT_CANDIDATES = ["plan.comment"]
|
||
REPORT_NUM_CANDIDATES = (
|
||
[f"q{q}.base_plan" for q in range(1, 5)]
|
||
+ [f"q{q}.m{m}" for q in (1, 2, 3) for m in (1, 2, 3)]
|
||
+ [f"q{q}.adj_by_items" for q in range(1, 5)]
|
||
+ [f"q{q}.adj_increase" for q in range(1, 5)]
|
||
)
|
||
|
||
PROJECT_TYPES = ["Открытие ВСП", "Закрытие ВСП", "Переезд ВСП", "Реновация РФ", "Открытие УРМ"]
|
||
VSP_FORMATS = ["Флагманский", "Типовой", "Розничный", "МСБ", "Лёгкий", "Мини", "Другое"]
|
||
PLACEMENTS = ["Собственность", "Аренда", "Субаренда"]
|
||
REPORT_TYPES = ["LIMIT", "CURRENT_EXPENSES"]
|
||
|
||
|
||
def log(msg: str) -> None:
|
||
print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)
|
||
|
||
|
||
@dataclasses.dataclass
|
||
class FormInfo:
|
||
id: int
|
||
type: str
|
||
org_id: int
|
||
org_kind: str # "ssp" | "rf"
|
||
org_index: int
|
||
vsp_ids: list[int]
|
||
cols: dict = dataclasses.field(default_factory=dict) # (sheet, direction) -> list[str]
|
||
|
||
@property
|
||
def has_satellites(self) -> bool:
|
||
return self.type == "FORM_2" and self.org_kind == "rf" and bool(self.vsp_ids)
|
||
|
||
|
||
def sheet_params(direction: str | None) -> dict:
|
||
return {"direction": direction} if direction else {}
|
||
|
||
|
||
def now_window() -> tuple[str, str]:
|
||
"""Окно этапа now-1d .. now+30d. API кладёт значение в TIMESTAMP WITHOUT TIME ZONE (CAST) и
|
||
отвергает aware-даты, поэтому отдаём наивное UTC-время; сдвиг на utc_offset оргединицы (+03)
|
||
много меньше запаса в сутки."""
|
||
now = datetime.now(timezone.utc).replace(tzinfo=None, microsecond=0)
|
||
return (now - timedelta(days=1)).isoformat(), (now + timedelta(days=30)).isoformat()
|
||
|
||
|
||
def rand_value(col: str, rng: random.Random):
|
||
if col.endswith(".comment") or col.endswith("justification"):
|
||
return f"Комментарий {rng.randint(1, 99999)}"
|
||
if col.startswith("plan."):
|
||
return round(rng.uniform(5e4, 5e6), 2)
|
||
if re.search(r"\.(actual_)?m\d$", col):
|
||
return round(rng.uniform(0, 3e5), 2)
|
||
return round(rng.uniform(0, 2e5), 2)
|
||
|
||
|
||
# --------------------------------------------------------------------------- шаги
|
||
|
||
|
||
async def ensure_empty() -> None:
|
||
conn = await asyncpg.connect(DSN)
|
||
try:
|
||
n_org = await conn.fetchval("SELECT count(*) FROM v3.org_unit")
|
||
n_users = await conn.fetchval("SELECT count(*) FROM v3.app_user WHERE username <> 'admin'")
|
||
finally:
|
||
await conn.close()
|
||
if n_org or n_users:
|
||
sys.exit(f"БД не пуста (оргединиц: {n_org}, пользователей кроме admin: {n_users}): сначала perf/stand.sh reset")
|
||
|
||
|
||
async def create_orgs(api: Api, plan: GenPlan) -> tuple[list[int], list[int]]:
|
||
ssp, rf = [], []
|
||
for i in range(1, plan.org_units_ssp + 1):
|
||
r = await api.post("/org-unit/", json={"title": f"ССП-{i:02d} Головной офис", "is_ssp": True})
|
||
ssp.append(r["result"]["id"])
|
||
for i in range(1, plan.org_units_rf + 1):
|
||
r = await api.post("/org-unit/", json={"title": f"РФ-{i:02d} Региональный филиал", "is_ssp": False})
|
||
rf.append(r["result"]["id"])
|
||
return ssp, rf
|
||
|
||
|
||
async def create_users(api: Api, rf_org: int, all_orgs: list[int]) -> dict[str, User]:
|
||
spec = [
|
||
("rf", 3, "Исполнитель РФ"),
|
||
("rf_coauthor", 3, "Соавтор РФ"),
|
||
("dfip", 2, "Исполнитель ДФиП"),
|
||
] # ключ манифеста; логин = "perf_" + ключ (в API username не короче 3 символов, «rf» не проходит)
|
||
users: dict[str, User] = {"admin": User("admin", "admin", 1)}
|
||
for key, role, full_name in spec:
|
||
username = f"perf_{key}"
|
||
r = await api.post(
|
||
"/users/",
|
||
json={
|
||
"email": f"{username.replace('_', '-')}@{EMAIL_DOMAIN}",
|
||
"username": username,
|
||
"full_name": full_name,
|
||
"role_id": role,
|
||
"password": PASSWORD,
|
||
},
|
||
)
|
||
uid = r["result"]["id"]
|
||
orgs = [rf_org] if role == 3 else all_orgs
|
||
await api.put("/users/dfip-many-ssp/", params={"user_id": uid}, json={"ssp_ids": orgs})
|
||
users[key] = User(username, PASSWORD, role)
|
||
return users
|
||
|
||
|
||
async def create_vsp(api: Api, plan: GenPlan, rf_orgs: list[int]) -> dict[int, list[int]]:
|
||
out: dict[int, list[int]] = {o: [] for o in rf_orgs}
|
||
|
||
async def one(r_idx: int, org: int, i: int) -> None:
|
||
opened = datetime(2012 + (i % 12), 1 + (i % 12), 1 + (i % 27)).date()
|
||
body = {
|
||
"registration_number": f"RF{r_idx:02d}-{i:04d}",
|
||
"address": f"г. Город-{r_idx:02d}, ул. Улица-{i:03d}, д. {1 + i % 90}",
|
||
"ssp_id": org,
|
||
"vsp_type": "ВСП",
|
||
"approved_format": VSP_FORMATS[i % len(VSP_FORMATS)],
|
||
"placement_type": PLACEMENTS[i % len(PLACEMENTS)],
|
||
"open_date": opened.isoformat(),
|
||
"staff_count": 3 + i % 17,
|
||
"area": round(40 + (i * 7.3) % 260, 1),
|
||
"rent_contract_num": f"Д-{r_idx:02d}/{i:04d}",
|
||
"rent_end_date": (opened + timedelta(days=365 * 15)).isoformat(),
|
||
}
|
||
r = await api.post("/dict/info", json=body)
|
||
out[org].append(r["result"]["id"])
|
||
|
||
await asyncio.gather(*(one(ri, org, i) for ri, org in enumerate(rf_orgs, 1) for i in range(1, plan.vsp_per_rf + 1)))
|
||
for v in out.values():
|
||
v.sort()
|
||
return out
|
||
|
||
|
||
async def create_forms(api: Api, ssp: list[int], rf: list[int], vsp: dict[int, list[int]]) -> list[FormInfo]:
|
||
kinds = {o: ("ssp", i) for i, o in enumerate(ssp, 1)} | {o: ("rf", i) for i, o in enumerate(rf, 1)}
|
||
spec = {"FORM_1": ssp, "FORM_2": ssp + rf, "FORM_4": rf}
|
||
forms: list[FormInfo] = []
|
||
for ftype, orgs in spec.items():
|
||
r = await api.post("/form/", json={"year": YEAR, "form_type_code": ftype, "org_unit_ids": orgs})
|
||
for f in r["result"]:
|
||
kind, idx = kinds[f["org_unit_id"]]
|
||
forms.append(FormInfo(f["id"], ftype, f["org_unit_id"], kind, idx, vsp.get(f["org_unit_id"], [])))
|
||
forms.sort(key=lambda f: f.id)
|
||
return forms
|
||
|
||
|
||
async def leaf_items(api: Api, ftype: str, sheet: str) -> list[int]:
|
||
r = await api.get("/expense-item/", params={"form_type": ftype, "sheet": sheet})
|
||
items = r["result"]
|
||
parents = {i["parent_id"] for i in items if i.get("parent_id") is not None}
|
||
return sorted(i["id"] for i in items if i["id"] not in parents and (i.get("depth") or 0) >= 2)
|
||
|
||
|
||
async def ws_events(api: Api, form_id: int, sheet: str, events: list[tuple[str, dict]]) -> list[dict]:
|
||
"""Отправляет события по очереди по одному соединению, ждёт ответ на каждое."""
|
||
url = API.replace("http", "ws", 1) + f"/api/v1/ws/form/{form_id}/sheet/{sheet}"
|
||
results = []
|
||
async with websockets.connect(url, open_timeout=30, max_size=None) as ws:
|
||
await ws.send(json.dumps({"event": "user_login", "data": {"token": api.token}}))
|
||
for ev, data in events:
|
||
await ws.send(json.dumps({"event": ev, "data": data}))
|
||
while True:
|
||
msg = json.loads(await asyncio.wait_for(ws.recv(), 120))
|
||
if msg.get("event") == ev and ("result" in msg or "error" in msg):
|
||
results.append(msg)
|
||
break
|
||
return results
|
||
|
||
|
||
async def make_structure(api: Api, form: FormInfo, sheet: str) -> list[int]:
|
||
"""FORM_4: программы и проекты на листе. Возвращает id проектов (для привязки строк)."""
|
||
res = await ws_events(api, form.id, sheet, [("program_added", {"name": f"Программа{sheet}{p}"}) for p in range(1, PROGRAMS_PER_SHEET + 1)])
|
||
prog_ids = []
|
||
for m in res:
|
||
if "error" in m:
|
||
raise RuntimeError(f"program_added form={form.id} {sheet}: {m['error']}")
|
||
prog_ids.append(m["result"][0])
|
||
events = [
|
||
("project_added", {"name": f"Проект{sheet}{pi}{j}", "program_id": pid})
|
||
for pi, pid in enumerate(prog_ids, 1)
|
||
for j in range(1, PROJECTS_PER_PROGRAM + 1)
|
||
]
|
||
res = await ws_events(api, form.id, sheet, events)
|
||
out = []
|
||
for m in res:
|
||
if "error" in m:
|
||
raise RuntimeError(f"project_added form={form.id} {sheet}: {m['error']}")
|
||
out.append(m["result"][0])
|
||
return out
|
||
|
||
|
||
def new_line_id(resp: dict) -> int | None:
|
||
ids = [x["data"]["line_id"] for x in resp.get("result", []) if x.get("data", {}).get("line_id")]
|
||
return ids[-1] if ids else None
|
||
|
||
|
||
async def add_lines(api: Api, form: FormInfo, sheet: str, direction: str | None, count: int, leaves: list[int],
|
||
rng: random.Random, stats: Counter, project_ids: list[int] | None) -> list[int]:
|
||
"""Добавляет `count` строк, обходя листья дерева статей по кругу (дерево покрыто целиком)."""
|
||
pool = leaves[:]
|
||
rng.shuffle(pool)
|
||
ids: list[int] = []
|
||
bad: set[int] = set()
|
||
vsp_cycle = cycle(form.vsp_ids) if (form.type == "FORM_4" and form.vsp_ids) else None
|
||
proj_cycle = cycle(project_ids) if project_ids else None
|
||
i = 0
|
||
while len(ids) < count:
|
||
usable = [x for x in pool if x not in bad]
|
||
if not usable:
|
||
stats["pool_exhausted"] += 1
|
||
break
|
||
eid = usable[i % len(usable)]
|
||
i += 1
|
||
body: dict = {"expense_item_id": eid}
|
||
if direction:
|
||
body["direction"] = direction
|
||
if vsp_cycle:
|
||
body["vsp_id"] = next(vsp_cycle)
|
||
if proj_cycle:
|
||
body["project_id"] = next(proj_cycle)
|
||
try:
|
||
resp = await api.post(f"/form/{form.id}/sheet/{sheet}/line", json=body)
|
||
except ApiError as e:
|
||
if e.status in (400, 404, 409, 422):
|
||
bad.add(eid)
|
||
stats[f"line_rejected_{e.status}"] += 1
|
||
stats.setdefault("line_reject_sample", e.text[:160]) # type: ignore[arg-type]
|
||
continue
|
||
raise
|
||
lid = new_line_id(resp)
|
||
if lid is None:
|
||
stats["line_id_missing"] += 1
|
||
else:
|
||
ids.append(lid)
|
||
stats["bad_leaves"] += len(bad)
|
||
return ids
|
||
|
||
|
||
async def add_satellite_lines(api: Api, form: FormInfo, sheet: str, rng: random.Random) -> list[int]:
|
||
ids = []
|
||
for n, vsp in enumerate(form.vsp_ids, 1):
|
||
body = {
|
||
"vsp_id": vsp,
|
||
"contract_number": f"ДГ-{form.org_index:02d}/{sheet[-4:]}/{n:04d}",
|
||
"contract_end_date": datetime(YEAR + 1 + n % 3, 1 + n % 12, 1 + n % 27, tzinfo=timezone.utc).isoformat(),
|
||
}
|
||
resp = await api.post(f"/form/{form.id}/sheet/{sheet}/line", json=body)
|
||
res = resp.get("result", [])
|
||
sid = next((x["data"].get("id") or x["data"].get("line_id") for x in res if x.get("row_type") == "INPUT"), None)
|
||
if sid:
|
||
ids.append(sid)
|
||
return ids
|
||
|
||
|
||
async def probe_cols(api: Api, base: str, params: dict, line_id: int, candidates: list[str], value_fn) -> list[str]:
|
||
"""Определяет записываемые колонки. Сначала одним батчем: ошибка называет отвергнутую колонку
|
||
(computed_field: X), её убираем и повторяем. Если ошибка колонку не называет, добираем по одной.
|
||
Батч нужен потому, что на FORM_4 каждая одиночная правка стоит секунды (перерисовка листа)."""
|
||
cols = list(candidates)
|
||
for _ in range(len(candidates) + 1):
|
||
if not cols:
|
||
return []
|
||
changes = [{"line_id": line_id, "column": c, "value": value_fn(c)} for c in cols]
|
||
try:
|
||
await api.patch(base + "/cells", params=params, json={"changes": changes})
|
||
return cols
|
||
except ApiError as e:
|
||
if e.status not in (400, 403, 404, 409, 422):
|
||
raise
|
||
named = [c for c in cols if re.search(re.escape(c) + r"(?![\w.])", e.text)]
|
||
if not named:
|
||
break
|
||
cols = [c for c in cols if c not in named]
|
||
ok = []
|
||
for col in cols:
|
||
try:
|
||
await api.patch(base + "/cell", params=params, json={"line_id": line_id, "column": col, "value": value_fn(col)})
|
||
ok.append(col)
|
||
except ApiError as e:
|
||
if e.status not in (400, 403, 404, 409, 422):
|
||
raise
|
||
return ok
|
||
|
||
|
||
async def fill_cells(api: Api, path: str, params: dict, line_ids: list[int], cols: list[str], plan: GenPlan,
|
||
rng: random.Random, stats: Counter) -> int:
|
||
num_cols = [c for c in cols if c not in TEXT_CANDIDATES]
|
||
txt_cols = [c for c in cols if c in TEXT_CANDIDATES]
|
||
chosen = [i for i in line_ids if rng.random() < plan.fill_ratio]
|
||
changes = []
|
||
for lid in chosen:
|
||
for c in num_cols:
|
||
# плановые кварталы заполняются всегда, остальное с вероятностью 0.5
|
||
if c.startswith("plan.q") or rng.random() < 0.5:
|
||
changes.append({"line_id": lid, "column": c, "value": rand_value(c, rng)})
|
||
for c in txt_cols:
|
||
if rng.random() < 0.3:
|
||
changes.append({"line_id": lid, "column": c, "value": rand_value(c, rng)})
|
||
for k in range(0, len(changes), CELLS_BATCH):
|
||
batch = changes[k:k + CELLS_BATCH]
|
||
try:
|
||
await api.patch(path + "/cells", params=params, json={"changes": batch})
|
||
except ApiError as e:
|
||
if e.status != 400:
|
||
raise
|
||
stats["cells_batch_fallback"] += 1 # computed_field и т.п.: пробуем по одной ячейке
|
||
for ch in batch:
|
||
try:
|
||
await api.patch(path + "/cell", params=params, json=ch)
|
||
except ApiError as e2:
|
||
if e2.status not in (400, 403, 409, 422):
|
||
raise
|
||
stats["cell_rejected"] += 1
|
||
stats["cells_written"] += len(changes)
|
||
return len(changes)
|
||
|
||
|
||
async def process_form(api: Api, plan: GenPlan, form: FormInfo, leaves_cache: dict, stats: Counter, scale: int) -> None:
|
||
t0 = time.time()
|
||
rng = random.Random(f"{scale}-{form.id}")
|
||
sheets = LINE_SHEETS[form.type]
|
||
for sheet, direction in sheets:
|
||
key = (form.type, sheet)
|
||
if key not in leaves_cache:
|
||
leaves_cache[key] = asyncio.ensure_future(leaf_items(api, form.type, sheet))
|
||
leaves = await leaves_cache[key]
|
||
projects = None
|
||
if form.type == "FORM_4":
|
||
projects = await make_structure(api, form, sheet)
|
||
stats["form4_projects"] += len(projects)
|
||
ids = await add_lines(api, form, sheet, direction, plan.lines_per_sheet(form.type), leaves, rng, stats, projects)
|
||
stats["lines_added"] += len(ids)
|
||
if ids:
|
||
path = f"/form/{form.id}/sheet/{sheet}"
|
||
params = sheet_params(direction)
|
||
# записываемость колонок зависит от типа формы и листа, не от конкретной формы:
|
||
# пробуем один раз на (тип, лист, direction) — проба стоит ~35 запросов с перерисовкой листа
|
||
ckey = ("cols", form.type, sheet, direction)
|
||
if ckey not in leaves_cache:
|
||
leaves_cache[ckey] = asyncio.ensure_future(probe_cols(
|
||
api, path, params, ids[0], NUM_CANDIDATES + TEXT_CANDIDATES,
|
||
lambda c: rand_value(c, rng)))
|
||
form.cols[(sheet, direction)] = await leaves_cache[ckey]
|
||
await fill_cells(api, path, params, ids, form.cols[(sheet, direction)], plan, rng, stats)
|
||
if form.has_satellites:
|
||
for sheet in SATELLITES:
|
||
ids = await add_satellite_lines(api, form, sheet, rng)
|
||
stats["satellite_lines"] += len(ids)
|
||
log(f"форма {form.id} {form.type} {form.org_kind}-{form.org_index:02d}: готово за {time.time() - t0:.0f} c")
|
||
|
||
|
||
async def post_phase(api: Api, path: str, body: dict, stats: Counter, label: str) -> bool:
|
||
"""Создаёт этап. Колонки, чей раздел (секция) не входит в form_type, API отвергает целиком
|
||
(«Колонка X некорректна») — выкидываем их и повторяем."""
|
||
for _ in range(2):
|
||
try:
|
||
await api.post(path, json=body)
|
||
return True
|
||
except ApiError as e:
|
||
bad = set(re.findall(r"Колонка (\S+) некорректна", e.text))
|
||
keep = [c for c in body["column_keys"] if c not in bad]
|
||
if bad and keep and len(keep) < len(body["column_keys"]):
|
||
body = {**body, "column_keys": keep}
|
||
continue
|
||
stats[f"{label}_errors"] += 1
|
||
stats.setdefault(f"{label}_error_sample", e.text[:200]) # type: ignore[arg-type]
|
||
return False
|
||
return False
|
||
|
||
|
||
async def create_phases_for_form(api: Api, form: FormInfo, stats: Counter) -> None:
|
||
opens, closes = now_window()
|
||
sheets = (await api.get(f"/form/{form.id}/sheets"))["result"]["all_sheets"]
|
||
default_cols = [f"plan.q{q}" for q in range(1, 5)]
|
||
for s in sheets:
|
||
sheet, direction = s["sheet_name"], s["direction"]
|
||
cols = form.cols.get((sheet, direction)) or default_cols
|
||
for code, role in (("PERF_RF", "EXECUTOR_RF"), ("PERF_DFIP", "DFIP")):
|
||
body = {"sheet": sheet, "phase_code": code, "role": role, "column_keys": cols,
|
||
"opens_at": opens, "closes_at": closes, "direction": direction}
|
||
if await post_phase(api, f"/stages/form/{form.id}", body, stats, "phase"):
|
||
stats["phases_created"] += 1
|
||
|
||
|
||
async def create_projects(api: Api, plan: GenPlan, rf_orgs: list[int], stats: Counter, scale: int) -> list[dict]:
|
||
"""FORM_3: проекты РФ с годами и отчётами. Возвращает записи {project_id, org_index, year, report_type, rows}."""
|
||
out: list[dict] = []
|
||
cols_cache: dict[str, list[str]] = {}
|
||
|
||
async def create_one(r_idx: int, org: int, j: int) -> tuple[int, int, list[int]]:
|
||
body = {
|
||
"name": f"Пр{r_idx:02d}П{j:02d}", "year": YEAR, "branch_id": org, "level": "project",
|
||
"project_type": PROJECT_TYPES[j % len(PROJECT_TYPES)], "vsp_format": VSP_FORMATS[j % len(VSP_FORMATS)],
|
||
"placement_type": PLACEMENTS[j % len(PLACEMENTS)], "object_address": f"г. Город-{r_idx:02d}, ул. Проектная {j}",
|
||
"staff_count": 4 + j, "total_area": 80.0 + 12 * j,
|
||
}
|
||
r = await api.post("/projects", json=body) # technical_number выдаётся с гонкой при параллельных запросах
|
||
pid = r["project_id"]
|
||
years = [YEAR]
|
||
if j % 2 == 0:
|
||
await api.post(f"/projects/{pid}/year")
|
||
years.append(YEAR + 1)
|
||
stats["projects"] += 1
|
||
return pid, r_idx, years
|
||
|
||
async def fill_one(pid: int, r_idx: int, years: list[int], j: int) -> None:
|
||
rng = random.Random(f"{scale}-p-{r_idx}-{j}")
|
||
for year in years:
|
||
for rt in REPORT_TYPES:
|
||
path = f"/projects/{pid}/report/{year}/{rt}"
|
||
rows = (await api.get(path))["result"]
|
||
ids = [x["data"]["line_id"] for x in rows if x["row_type"] == "INPUT" and x["data"].get("line_id")]
|
||
if ids:
|
||
if rt not in cols_cache:
|
||
cols_cache[rt] = await probe_cols(api, path, {}, ids[0], REPORT_NUM_CANDIDATES,
|
||
lambda c: rand_value(c, rng))
|
||
cols = cols_cache[rt] or REPORT_NUM_CANDIDATES[:4]
|
||
await fill_cells(api, path, {}, ids, cols, plan, rng, stats)
|
||
opens, closes = now_window()
|
||
for code, role in (("PERF_RF", "EXECUTOR_RF"), ("PERF_DFIP", "DFIP")):
|
||
body = {"year": year, "report_type": rt, "phase_code": code, "role": role,
|
||
"column_keys": cols_cache.get(rt) or REPORT_NUM_CANDIDATES[:4], "opens_at": opens, "closes_at": closes}
|
||
if await post_phase(api, f"/stages/project/{pid}", body, stats, "project_phase"):
|
||
stats["project_phases_created"] += 1
|
||
out.append({"project_id": pid, "org_index": r_idx, "year": year, "report_type": rt, "rows": len(rows)})
|
||
|
||
created = []
|
||
for ri, org in enumerate(rf_orgs, 1):
|
||
for j in range(1, plan.projects_per_rf + 1):
|
||
pid, r_idx, years = await create_one(ri, org, j)
|
||
created.append((pid, r_idx, years, j))
|
||
# первая проба колонок отчёта делается до параллельного запуска, чтобы не пробовать дважды
|
||
first = created[0]
|
||
await fill_one(first[0], first[1], first[2], first[3])
|
||
await asyncio.gather(*(fill_one(*c) for c in created[1:]))
|
||
return out
|
||
|
||
|
||
async def self_check(plan: GenPlan, users: dict[str, User], forms: list[FormInfo], stats: Counter) -> None:
|
||
"""Проверка прав: rf/rf_coauthor/dfip читают листы своих форм и правят ячейку в открытом окне."""
|
||
sem = asyncio.Semaphore(4)
|
||
rf_form4 = next(f for f in forms if f.type == "FORM_4" and f.org_kind == "rf" and f.org_index == 1)
|
||
rf_form2 = next(f for f in forms if f.type == "FORM_2" and f.org_kind == "rf" and f.org_index == 1)
|
||
for uname in ("rf", "rf_coauthor", "dfip"):
|
||
u = users[uname]
|
||
api = Api(u.username, u.password, sem)
|
||
try:
|
||
for form in (rf_form4, rf_form2):
|
||
rows = (await api.get(f"/form/{form.id}/sheet/AHR"))["result"]
|
||
lines = [x["data"]["line_id"] for x in rows if x["row_type"] == "INPUT" and x["data"].get("line_id")
|
||
and (form.type == "FORM_4" or not x["data"].get("header", {}).get("vsp_id"))]
|
||
if not lines:
|
||
raise RuntimeError(f"self_check: у {uname} нет строк на {form.type}/AHR")
|
||
await api.patch(f"/form/{form.id}/sheet/AHR/cell",
|
||
json={"line_id": lines[0], "column": "plan.q1", "value": 1000.0})
|
||
finally:
|
||
await api.aclose()
|
||
log("самопроверка прав rf/rf_coauthor/dfip: OK")
|
||
stats["self_check"] = 1
|
||
|
||
|
||
async def count_rows(api: Api, path: str, params: dict | None = None) -> int:
|
||
return len((await api.get(path, params=params or {}))["result"])
|
||
|
||
|
||
async def build_manifest(api: Api, scale: int, users: dict[str, User], forms: list[FormInfo], projects: list[dict]) -> Manifest:
|
||
m = Manifest(scale=scale, users=users)
|
||
primary = [f for f in forms if f.org_index == 1] # ССП-01 (FORM_1, FORM_2) и РФ-01 (FORM_2, FORM_4)
|
||
for f in primary:
|
||
sheets = (await api.get(f"/form/{f.id}/sheets"))["result"]["all_sheets"]
|
||
for s in sheets:
|
||
sheet, direction = s["sheet_name"], s["direction"]
|
||
rows = await count_rows(api, f"/form/{f.id}/sheet/{sheet}", sheet_params(direction))
|
||
m.sheets.append(Sheet(f.id, f.type, sheet, direction, YEAR,
|
||
Sheet.build_url(f.id, f.type, sheet, direction, YEAR), rows, owner=f.org_kind))
|
||
for p in projects:
|
||
if p["org_index"] != 1:
|
||
continue
|
||
path = f"/projects/{p['project_id']}/report/{p['year']}/{p['report_type']}"
|
||
rows = p["rows"]
|
||
url = f"/table/form/{p['project_id']}/form-type/PROJECT/{p['report_type']}/null/{p['year']}"
|
||
m.project_reports.append(ProjectReport(p["project_id"], p["year"], p["report_type"], url, rows))
|
||
# своды: у страниц одинаковый путь, лист задаётся состоянием вкладки; после '#' — имя вкладки
|
||
for sheet in ("MAIN", "FORM_1", "FORM_2", "FORM_3", "FORM_4"):
|
||
m.svods.append(Page(f"svod1_{sheet}", f"/svod#{sheet}"))
|
||
for sheet in ("GO", "RF"):
|
||
m.svods.append(Page(f"svod2_{sheet}", f"/svod/2#{sheet}"))
|
||
m.svods.append(Page("svods_navigator", "/svods-navigator"))
|
||
m.svods.append(Page("projects_summaries", "/projects/summaries"))
|
||
# страницы, с которых открываются модальные окна (этапы, год, экспорт, создание форм/проектов, админка, справочники)
|
||
m.modal_pages += [Page("tasks", "/tasks"), Page("projects", "/projects")]
|
||
for f in primary:
|
||
m.modal_pages.append(Page(f"task_{f.org_kind}_{f.type.lower()}", f"/task/{f.id}"))
|
||
first_project = next((p["project_id"] for p in projects if p["org_index"] == 1), None)
|
||
if first_project:
|
||
m.modal_pages.append(Page("project_card", f"/project/{first_project}"))
|
||
m.modal_pages += [
|
||
Page("admin_users", "/admin_panel/users"), Page("admin_ssps", "/admin_panel/ssps"),
|
||
Page("admin_default_stage", "/admin_panel/default-stage"), Page("admin_audit_log", "/admin_panel/audit-log"),
|
||
Page("dict_vsp", "/dicts/dict-vsp"), Page("dicts_navigator", "/dicts-navigator"),
|
||
]
|
||
return m
|
||
|
||
|
||
async def main(scale: int, overrides: dict) -> None:
|
||
plan = plan_for_scale(scale)
|
||
if overrides:
|
||
plan = dataclasses.replace(plan, **overrides)
|
||
log(f"ВНИМАНИЕ: план переопределён вручную (только отладка): {plan}")
|
||
log(f"масштаб {scale}: {plan}")
|
||
await ensure_empty()
|
||
t_start = time.time()
|
||
sem = asyncio.Semaphore(CONCURRENCY)
|
||
admin = Api("admin", "admin", sem)
|
||
stats: Counter = Counter()
|
||
try:
|
||
await admin.login()
|
||
ssp, rf = await create_orgs(admin, plan)
|
||
log(f"оргединицы: ССП {len(ssp)}, РФ {len(rf)}")
|
||
users = await create_users(admin, rf[0], ssp + rf)
|
||
log("пользователи rf, rf_coauthor, dfip созданы")
|
||
vsp = await create_vsp(admin, plan, rf)
|
||
log(f"ВСП: {sum(len(v) for v in vsp.values())}")
|
||
forms = await create_forms(admin, ssp, rf, vsp)
|
||
log(f"формы: {Counter(f.type for f in forms)}")
|
||
leaves_cache: dict = {}
|
||
await asyncio.gather(*(process_form(admin, plan, f, leaves_cache, stats, scale) for f in forms))
|
||
log(f"строки и значения: {dict(stats)}")
|
||
await asyncio.gather(*(create_phases_for_form(admin, f, stats) for f in forms))
|
||
log(f"этапы форм: создано {stats['phases_created']}, ошибок {stats['phase_errors']}")
|
||
projects = await create_projects(admin, plan, rf, stats, scale)
|
||
log(f"проекты FORM_3: {stats['projects']}, отчётов {len(projects)}")
|
||
await self_check(plan, users, forms, stats)
|
||
manifest = await build_manifest(admin, scale, users, forms, projects)
|
||
path = manifest_path(scale, RESULTS_DIR)
|
||
dump(manifest, path)
|
||
biggest = max((s for s in manifest.sheets if s.owner == "rf"), key=lambda s: s.rows)
|
||
log(f"манифест: {path}; листов {len(manifest.sheets)}, отчётов проектов {len(manifest.project_reports)}; "
|
||
f"самый большой лист rf: form {biggest.form_id} {biggest.form_type}/{biggest.sheet} = {biggest.rows} строк")
|
||
top = sorted(manifest.sheets, key=lambda s: -s.rows)[:8]
|
||
log("топ листов: " + "; ".join(f"{s.form_type}/{s.sheet}/{s.direction}[{s.owner}]={s.rows}" for s in top))
|
||
elapsed = time.time() - t_start
|
||
log(f"ГЕНЕРАЦИЯ ЗАВЕРШЕНА за {elapsed / 60:.1f} мин; запросов {admin.requests}; статистика {dict(stats)}")
|
||
(RESULTS_DIR / f"gen-stats-s{scale}.json").write_text(
|
||
json.dumps({"scale": scale, "plan": dataclasses.asdict(plan), "elapsed_s": round(elapsed, 1),
|
||
"requests": admin.requests, "stats": dict(stats)}, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
finally:
|
||
await admin.aclose()
|
||
|
||
|
||
if __name__ == "__main__":
|
||
ap = argparse.ArgumentParser(description=__doc__)
|
||
ap.add_argument("--scale", type=int, required=True)
|
||
ap.add_argument("--extra-lines", type=int, help="ТОЛЬКО ОТЛАДКА: переопределить extra_lines_per_sheet")
|
||
ap.add_argument("--orgs-ssp", type=int, help="ТОЛЬКО ОТЛАДКА")
|
||
ap.add_argument("--orgs-rf", type=int, help="ТОЛЬКО ОТЛАДКА")
|
||
ap.add_argument("--vsp", type=int, help="ТОЛЬКО ОТЛАДКА")
|
||
ap.add_argument("--projects", type=int, help="ТОЛЬКО ОТЛАДКА")
|
||
a = ap.parse_args()
|
||
ov = {k: v for k, v in {
|
||
"extra_lines_per_sheet": a.extra_lines, "org_units_ssp": a.orgs_ssp, "org_units_rf": a.orgs_rf,
|
||
"vsp_per_rf": a.vsp, "projects_per_rf": a.projects}.items() if v is not None}
|
||
RESULTS_DIR.mkdir(parents=True, exist_ok=True)
|
||
asyncio.run(main(a.scale, ov))
|