del profiling
This commit is contained in:
parent
91ad3f8003
commit
197d26f050
@ -1,8 +1,5 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import cProfile
|
|
||||||
import io
|
|
||||||
import pstats
|
|
||||||
import tempfile
|
import tempfile
|
||||||
import time
|
import time
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
@ -31,7 +28,6 @@ from .protocol import ProtocolRow, write_protocol
|
|||||||
from .report import ReportRow, StreamingReportWriter
|
from .report import ReportRow, StreamingReportWriter
|
||||||
|
|
||||||
EMPTY_BATCH_STATUS = "info"
|
EMPTY_BATCH_STATUS = "info"
|
||||||
CPROFILE_TOP_LIMIT = 30
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@ -45,7 +41,6 @@ class BatchResult:
|
|||||||
log_path: str
|
log_path: str
|
||||||
messages: list[str] = field(default_factory=list)
|
messages: list[str] = field(default_factory=list)
|
||||||
duration_seconds: float = 0.0
|
duration_seconds: float = 0.0
|
||||||
profile_path: str = ""
|
|
||||||
|
|
||||||
|
|
||||||
def run_batch(
|
def run_batch(
|
||||||
@ -143,17 +138,11 @@ def run_batch(
|
|||||||
report_path = join_path(run_context.run_dir, report_file_name)
|
report_path = join_path(run_context.run_dir, report_file_name)
|
||||||
protocol_path = join_path(run_context.run_dir, protocol_file_name)
|
protocol_path = join_path(run_context.run_dir, protocol_file_name)
|
||||||
log_path = join_path(run_context.run_dir, log_file_name)
|
log_path = join_path(run_context.run_dir, log_file_name)
|
||||||
profile_path = join_path(
|
|
||||||
run_context.run_dir,
|
|
||||||
f"profile_{run_context.date_key}_{run_context.run_number}.txt",
|
|
||||||
)
|
|
||||||
|
|
||||||
with tempfile.TemporaryDirectory() as temp_dir:
|
with tempfile.TemporaryDirectory() as temp_dir:
|
||||||
temp_dir_path = Path(temp_dir)
|
temp_dir_path = Path(temp_dir)
|
||||||
local_report = temp_dir_path / "report.xlsx"
|
local_report = temp_dir_path / "report.xlsx"
|
||||||
local_protocol = temp_dir_path / "protocol.xlsx"
|
local_protocol = temp_dir_path / "protocol.xlsx"
|
||||||
local_log = temp_dir_path / "process.log"
|
local_log = temp_dir_path / "process.log"
|
||||||
local_profile = temp_dir_path / "profile.txt"
|
|
||||||
report_writer = StreamingReportWriter()
|
report_writer = StreamingReportWriter()
|
||||||
stage_timings: dict[str, float] = {
|
stage_timings: dict[str, float] = {
|
||||||
"smb_read_seconds": 0.0,
|
"smb_read_seconds": 0.0,
|
||||||
@ -161,22 +150,16 @@ def run_batch(
|
|||||||
"report_append_seconds": 0.0,
|
"report_append_seconds": 0.0,
|
||||||
"move_seconds": 0.0,
|
"move_seconds": 0.0,
|
||||||
}
|
}
|
||||||
profiler = cProfile.Profile()
|
_process_files(
|
||||||
|
scan_result=scan_result,
|
||||||
profiler.enable()
|
run_context=run_context,
|
||||||
try:
|
samba_conn=samba_conn,
|
||||||
_process_files(
|
launcher=launcher,
|
||||||
scan_result=scan_result,
|
report_writer=report_writer,
|
||||||
run_context=run_context,
|
protocol_rows=protocol_rows,
|
||||||
samba_conn=samba_conn,
|
log_lines=log_lines,
|
||||||
launcher=launcher,
|
stage_timings=stage_timings,
|
||||||
report_writer=report_writer,
|
)
|
||||||
protocol_rows=protocol_rows,
|
|
||||||
log_lines=log_lines,
|
|
||||||
stage_timings=stage_timings,
|
|
||||||
)
|
|
||||||
finally:
|
|
||||||
profiler.disable()
|
|
||||||
|
|
||||||
if not protocol_rows:
|
if not protocol_rows:
|
||||||
protocol_rows.append(
|
protocol_rows.append(
|
||||||
@ -202,12 +185,6 @@ def run_batch(
|
|||||||
|
|
||||||
report_writer.save(local_report)
|
report_writer.save(local_report)
|
||||||
write_protocol(protocol_rows, local_protocol)
|
write_protocol(protocol_rows, local_protocol)
|
||||||
profile_warning = ""
|
|
||||||
try:
|
|
||||||
_write_profile_report(profiler, local_profile)
|
|
||||||
except Exception as exc: # noqa: BLE001
|
|
||||||
profile_warning = f"Не удалось сформировать cProfile-отчет: {exc}"
|
|
||||||
log_lines.append(f"{_now()} | profile report failed: {exc}")
|
|
||||||
# Use CRLF to keep one-record-per-line in Windows viewers.
|
# Use CRLF to keep one-record-per-line in Windows viewers.
|
||||||
local_log.write_text("\r\n".join(log_lines) + "\r\n", encoding="utf-8")
|
local_log.write_text("\r\n".join(log_lines) + "\r\n", encoding="utf-8")
|
||||||
|
|
||||||
@ -220,15 +197,6 @@ def run_batch(
|
|||||||
log_path=log_path,
|
log_path=log_path,
|
||||||
samba_conn=samba_conn,
|
samba_conn=samba_conn,
|
||||||
)
|
)
|
||||||
if not profile_warning:
|
|
||||||
try:
|
|
||||||
upload_local_file(local_profile, profile_path, samba_conn=samba_conn)
|
|
||||||
except Exception as exc: # noqa: BLE001
|
|
||||||
profile_warning = f"Не удалось выгрузить cProfile-отчет: {exc}"
|
|
||||||
log_lines.append(f"{_now()} | profile upload failed: {exc}")
|
|
||||||
|
|
||||||
if profile_warning:
|
|
||||||
messages.append(profile_warning)
|
|
||||||
|
|
||||||
processed = sum(1 for row in protocol_rows if row.status == "success")
|
processed = sum(1 for row in protocol_rows if row.status == "success")
|
||||||
partial = sum(1 for row in protocol_rows if row.status == "partial")
|
partial = sum(1 for row in protocol_rows if row.status == "partial")
|
||||||
@ -256,7 +224,6 @@ def run_batch(
|
|||||||
log_path=log_path,
|
log_path=log_path,
|
||||||
messages=messages,
|
messages=messages,
|
||||||
duration_seconds=duration_seconds,
|
duration_seconds=duration_seconds,
|
||||||
profile_path=profile_path,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -445,21 +412,3 @@ def _format_duration(duration_seconds: float) -> str:
|
|||||||
if minutes:
|
if minutes:
|
||||||
return f"{minutes} мин {seconds} сек"
|
return f"{minutes} мин {seconds} сек"
|
||||||
return f"{seconds} сек"
|
return f"{seconds} сек"
|
||||||
|
|
||||||
|
|
||||||
def _write_profile_report(profiler: cProfile.Profile, destination: Path) -> None:
|
|
||||||
stream = io.StringIO()
|
|
||||||
cumulative_stats = pstats.Stats(profiler, stream=stream)
|
|
||||||
cumulative_stats.sort_stats("cumulative")
|
|
||||||
stream.write(
|
|
||||||
f"=== TOP {CPROFILE_TOP_LIMIT} cumulative functions "
|
|
||||||
"(sorted by cumulative) ===\n"
|
|
||||||
)
|
|
||||||
cumulative_stats.print_stats(CPROFILE_TOP_LIMIT)
|
|
||||||
tottime_stats = pstats.Stats(profiler, stream=stream)
|
|
||||||
stream.write(
|
|
||||||
f"\n=== TOP {CPROFILE_TOP_LIMIT} tottime functions (sorted by tottime) ===\n"
|
|
||||||
)
|
|
||||||
tottime_stats.sort_stats("tottime")
|
|
||||||
tottime_stats.print_stats(CPROFILE_TOP_LIMIT)
|
|
||||||
destination.write_text(stream.getvalue(), encoding="utf-8")
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user