from __future__ import annotations import csv import json from collections import Counter from pathlib import Path from trace_analyzer.helpers import safe_float, safe_int, series_stats from trace_analyzer.layout import resolve_details_summary_path from trace_analyzer.report import build_markdown_report def _iter_feature_rows(features_path: str | Path): with Path(features_path).open("r", encoding="utf-8") as handle: for row in csv.DictReader(handle): row["message_count"] = safe_int(row.get("message_count")) row["conversation_depth"] = safe_int(row.get("conversation_depth")) row["declared_tool_count"] = safe_int(row.get("declared_tool_count")) row["assistant_msg_count"] = safe_int(row.get("assistant_msg_count")) row["tool_msg_count"] = safe_int(row.get("tool_msg_count")) row["user_msg_count"] = safe_int(row.get("user_msg_count")) row["system_msg_count"] = safe_int(row.get("system_msg_count")) row["assistant_to_tool_count"] = safe_int(row.get("assistant_to_tool_count")) row["tool_to_assistant_count"] = safe_int(row.get("tool_to_assistant_count")) row["tool_to_tool_count"] = safe_int(row.get("tool_to_tool_count")) row["assistant_to_user_count"] = safe_int(row.get("assistant_to_user_count")) row["user_to_assistant_count"] = safe_int(row.get("user_to_assistant_count")) row["max_consecutive_tool_msgs"] = safe_int(row.get("max_consecutive_tool_msgs")) row["avg_tool_burst_len"] = safe_float(row.get("avg_tool_burst_len")) row["has_tool_loop"] = safe_int(row.get("has_tool_loop")) row["input_tokens"] = safe_int(row.get("input_tokens")) row["output_tokens"] = safe_int(row.get("output_tokens")) row["total_tokens"] = safe_int(row.get("total_tokens")) row["reasoning_tokens"] = safe_int(row.get("reasoning_tokens")) row["cached_tokens"] = safe_int(row.get("cached_tokens")) row["cache_hit_ratio"] = safe_float(row.get("cache_hit_ratio")) row["uncached_prompt_tokens"] = safe_int(row.get("uncached_prompt_tokens")) row["output_input_ratio"] = safe_float(row.get("output_input_ratio")) row["latency_ms"] = safe_int(row.get("latency_ms")) row["ms_per_input_token"] = safe_float(row.get("ms_per_input_token")) row["ms_per_output_token"] = safe_float(row.get("ms_per_output_token")) row["long_context"] = safe_int(row.get("long_context")) row["high_cache"] = safe_int(row.get("high_cache")) row["tool_burst_alert"] = safe_int(row.get("tool_burst_alert")) row["tool_loop_alert"] = safe_int(row.get("tool_loop_alert")) row["slow_request"] = safe_int(row.get("slow_request")) row["pattern_labels"] = [label for label in str(row.get("pattern_labels", "")).split(";") if label] yield row def build_summary_from_features(features_path: str | Path) -> dict: model_counts = Counter() status_code_counts = Counter() role_transition_counts = Counter() session_ids: set[str] = set() latencies: list[int] = [] cache_ratios: list[float] = [] cached_tokens_list: list[int] = [] declared_tool_counts: list[int] = [] burst_values: list[int] = [] record_count = 0 success_count = 0 high_burst_requests: list[dict] = [] slow_despite_cache: list[dict] = [] long_context_no_cache: list[dict] = [] tool_burst_alert_count = 0 tool_loop_alert_count = 0 cache_bucket_input = { "lt_0_2": {"latencies": [], "ratios": [], "count": 0}, "0_2_to_0_8": {"latencies": [], "ratios": [], "count": 0}, "ge_0_8": {"latencies": [], "ratios": [], "count": 0}, } for row in _iter_feature_rows(features_path): record_count += 1 model_counts[row.get("model") or "unknown"] += 1 status_code_counts[row.get("status_code") or "unknown"] += 1 if row.get("session_id"): session_ids.add(row["session_id"]) if row.get("status_code") in {"1000", "200"}: success_count += 1 role_transition_counts["assistant->tool"] += row["assistant_to_tool_count"] role_transition_counts["tool->assistant"] += row["tool_to_assistant_count"] role_transition_counts["tool->tool"] += row["tool_to_tool_count"] role_transition_counts["assistant->user"] += row["assistant_to_user_count"] role_transition_counts["user->assistant"] += row["user_to_assistant_count"] latencies.append(row["latency_ms"]) cache_ratios.append(row["cache_hit_ratio"]) cached_tokens_list.append(row["cached_tokens"]) declared_tool_counts.append(row["declared_tool_count"]) burst_values.append(row["max_consecutive_tool_msgs"]) tool_burst_alert_count += row["tool_burst_alert"] tool_loop_alert_count += row["tool_loop_alert"] if row["tool_burst_alert"]: high_burst_requests.append( { "request_id": row["request_id"], "session_id": row["session_id"], "max_consecutive_tool_msgs": row["max_consecutive_tool_msgs"], "tool_to_tool_count": row["tool_to_tool_count"], } ) high_burst_requests.sort( key=lambda item: (item["max_consecutive_tool_msgs"], item["tool_to_tool_count"]), reverse=True, ) del high_burst_requests[10:] if "slow-despite-cache" in row["pattern_labels"]: slow_despite_cache.append( { "request_id": row["request_id"], "session_id": row["session_id"], "latency_ms": row["latency_ms"], "cache_hit_ratio": row["cache_hit_ratio"], } ) slow_despite_cache.sort(key=lambda item: item["latency_ms"], reverse=True) del slow_despite_cache[10:] if "long-context-no-cache" in row["pattern_labels"]: long_context_no_cache.append( { "request_id": row["request_id"], "session_id": row["session_id"], "input_tokens": row["input_tokens"], "cache_hit_ratio": row["cache_hit_ratio"], } ) long_context_no_cache.sort(key=lambda item: item["input_tokens"], reverse=True) del long_context_no_cache[10:] ratio = row["cache_hit_ratio"] if ratio < 0.2: bucket_name = "lt_0_2" elif ratio < 0.8: bucket_name = "0_2_to_0_8" else: bucket_name = "ge_0_8" cache_bucket_input[bucket_name]["count"] += 1 cache_bucket_input[bucket_name]["latencies"].append(row["latency_ms"]) cache_bucket_input[bucket_name]["ratios"].append(row["cache_hit_ratio"]) latency_stats = series_stats(latencies) cache_ratio_stats = series_stats(cache_ratios) cached_token_stats = series_stats(cached_tokens_list) declared_tool_stats = series_stats(declared_tool_counts) burst_stats = series_stats(burst_values) cache_buckets = [] for label in ["lt_0_2", "0_2_to_0_8", "ge_0_8"]: bucket = cache_bucket_input[label] cache_buckets.append( { "bucket": label, "count": bucket["count"], "avg_latency_ms": series_stats(bucket["latencies"])["mean"], "avg_cache_hit_ratio": series_stats(bucket["ratios"])["mean"], } ) return { "record_count": record_count, "success_count": success_count, "session_count": len(session_ids), "model_counts": dict(model_counts), "status_code_counts": dict(status_code_counts), "thresholds": { "long_context": 32000, "high_cache": 0.8, "tool_burst_alert": 4, "tool_loop_alert": 3, "slow_request_p90_latency_ms": latency_stats["p90"], }, "tool_patterns": { "role_transitions": dict(role_transition_counts), "declared_tool_count": declared_tool_stats, "max_consecutive_tool_msgs": burst_stats, "tool_burst_alert_count": tool_burst_alert_count, "tool_loop_alert_count": tool_loop_alert_count, "high_burst_requests": high_burst_requests, }, "cache_patterns": { "cached_tokens": cached_token_stats, "cache_hit_ratio": cache_ratio_stats, "latency_ms": latency_stats, "cache_buckets": cache_buckets, }, "anomalies": { "slow_despite_cache": slow_despite_cache, "long_context_no_cache": long_context_no_cache, }, } def write_reports( *, features_path: str | Path, output_dir: str | Path, pipeline_summary: dict | None = None, ) -> dict: output_root = Path(output_dir) output_root.mkdir(parents=True, exist_ok=True) summary = build_summary_from_features(features_path) summary_path = output_root / "summary.json" summary_path.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8") report_path = output_root / "report.md" report_path.write_text(build_markdown_report(summary), encoding="utf-8") combined = { "summary": summary, "pipeline": pipeline_summary or {}, } details_summary_path = resolve_details_summary_path(output_root) if details_summary_path is not None: combined["details_summary"] = json.loads(details_summary_path.read_text(encoding="utf-8")) combined_path = output_root / "analysis_snapshot.json" combined_path.write_text(json.dumps(combined, ensure_ascii=False, indent=2), encoding="utf-8") return { "summary_path": str(summary_path), "report_path": str(report_path), "analysis_snapshot_path": str(combined_path), }