diff --git a/runs/frontier-code-trace-v0/README.md b/runs/frontier-code-trace-v0/README.md index 4abc76b..fe6d48a 100644 --- a/runs/frontier-code-trace-v0/README.md +++ b/runs/frontier-code-trace-v0/README.md @@ -34,6 +34,9 @@ selected.selected_window_stats.max_model_len_coverage[推荐值].coverage = 1.0 ``` 旧记录预计 source block size 为 512、推荐 max model len 为 131072，但禁止把这两个值写死为实验事实。 +审计会单独记录并排除 `input_length<=0` 或 `output_length<=0` 的 source +行；这些行只有在 raw trace 同样显示 zero usage/empty response 时才按 +“未发生模型执行”处理，不能无记录过滤。 ## 2. 物化稳定窗口 diff --git a/runs/frontier-code-trace-v0/audit_code_trace.py b/runs/frontier-code-trace-v0/audit_code_trace.py index 250b47f..99462ac 100644 --- a/runs/frontier-code-trace-v0/audit_code_trace.py +++ b/runs/frontier-code-trace-v0/audit_code_trace.py @@ -131,6 +131,9 @@ def choose_window( def scan_source(path: Path, args: argparse.Namespace) -> dict[str, Any]: rows = 0 + source_rows = 0 + invalid_zero_token_rows = 0 + invalid_zero_token_examples: list[dict[str, Any]] = [] first_timestamp = None last_timestamp = None previous_timestamp = None @@ -145,6 +148,7 @@ def scan_source(path: Path, args: argparse.Namespace) -> dict[str, Any]: sampling_rows = 0 schema_keys: Counter[str] = Counter() for line_number, row in iter_jsonl(path): + source_rows += 1 missing = [ key for key in ("timestamp", "input_length", "output_length") @@ -153,6 +157,22 @@ def scan_source(path: Path, args: argparse.Namespace) -> dict[str, Any]: if missing: raise ValueError(f"{path}:{line_number}: missing required fields {missing}") timestamp = float(row["timestamp"]) + input_tokens = int(row["input_length"]) + output_tokens = int(row["output_length"]) + schema_keys.update(row.keys()) + if input_tokens <= 0 or output_tokens <= 0: + invalid_zero_token_rows += 1 + if len(invalid_zero_token_examples) < 20: + invalid_zero_token_examples.append( + { + "line_number": line_number, + "chat_id": row.get("chat_id"), + "timestamp": timestamp, + "input_length": input_tokens, + "output_length": output_tokens, + } + ) + continue if first_timestamp is None: first_timestamp = timestamp if previous_timestamp is not None and timestamp < previous_timestamp: @@ -169,10 +189,6 @@ def scan_source(path: Path, args: argparse.Namespace) -> dict[str, Any]: max_gaps.get(previous_bin, 0.0), timestamp - previous_timestamp, ) - input_tokens = int(row["input_length"]) - output_tokens = max(1, int(row["output_length"])) - if input_tokens <= 0: - raise ValueError(f"{path}:{line_number}: input_length must be positive") input_lengths.append(input_tokens) output_lengths.append(output_tokens) total_lengths.append(input_tokens + output_tokens) @@ -186,7 +202,6 @@ def scan_source(path: Path, args: argparse.Namespace) -> dict[str, Any]: isinstance(row.get("prompt"), (str, list)) and bool(row.get("prompt")) ) sampling_rows += int("sampling_u" in row) - schema_keys.update(row.keys()) rows += 1 previous_timestamp = timestamp last_timestamp = timestamp @@ -205,6 +220,10 @@ def scan_source(path: Path, args: argparse.Namespace) -> dict[str, Any]: return { "source": str(path.resolve()), "rows": rows, + "source_rows": source_rows, + "invalid_zero_token_rows": invalid_zero_token_rows, + "invalid_zero_token_fraction": invalid_zero_token_rows / source_rows, + "invalid_zero_token_examples": invalid_zero_token_examples, "first_timestamp": first_timestamp, "last_timestamp": last_timestamp, "span_s": last_timestamp - first_timestamp, @@ -253,7 +272,9 @@ def scan_window(source: Path, window: dict[str, Any]) -> dict[str, Any]: if timestamp >= end: break input_tokens = int(row["input_length"]) - output_tokens = max(1, int(row["output_length"])) + output_tokens = int(row["output_length"]) + if input_tokens <= 0 or output_tokens <= 0: + continue inputs.append(input_tokens) outputs.append(output_tokens) totals.append(input_tokens + output_tokens) diff --git a/runs/frontier-code-trace-v0/experiment-card.md b/runs/frontier-code-trace-v0/experiment-card.md index ec77a14..e43f658 100644 --- a/runs/frontier-code-trace-v0/experiment-card.md +++ b/runs/frontier-code-trace-v0/experiment-card.md @@ -58,6 +58,7 @@ strict decode-only 必须同时具备： - timestamp 单调，存在 60–75min 连续稳定窗口； - `timestamp/input_length/output_length` 全行存在； +- `input_length<=0` 或 `output_length<=0` 的行不进入 replay，但必须计数并保留样例。已抽查的两条 0→0 行在 raw trace 中同时满足 `usage.total_tokens=0`、`response_message={}`，属于未发生模型执行的 source request，不是 full-cache decode； - `hash_ids` 数量与某个 source block size 在全行严格满足 `ceil(ISL/source_block_size)`；预计值 512，但以审计结果为准； - 记录 ISL/OSL/ISL+OSL 的 p50/p90/p95/p99/max、gap、request rate、prompt/sampling 字段覆盖。 diff --git a/runs/frontier-code-trace-v0/prepare_code_window.py b/runs/frontier-code-trace-v0/prepare_code_window.py index 28065d9..0ee7872 100644 --- a/runs/frontier-code-trace-v0/prepare_code_window.py +++ b/runs/frontier-code-trace-v0/prepare_code_window.py @@ -63,6 +63,8 @@ def main() -> None: continue if timestamp >= end: break + if int(row["input_length"]) <= 0 or int(row["output_length"]) <= 0: + continue chat = row.get("chat_id", source_index) parent = row.get("parent_chat_id") has_parent = parent not in (None, "", -1, "-1") diff --git a/runs/frontier-code-trace-v0/test_code_trace_preflight.py b/runs/frontier-code-trace-v0/test_code_trace_preflight.py index 0f5a19f..7557ba3 100644 --- a/runs/frontier-code-trace-v0/test_code_trace_preflight.py +++ b/runs/frontier-code-trace-v0/test_code_trace_preflight.py @@ -20,6 +20,21 @@ class CodeTracePreflightTest(unittest.TestCase): source = root / "051315-051317.jsonl" with source.open("w") as stream: for index in range(4501): + if index == 100: + stream.write( + json.dumps( + { + "chat_id": index, + "parent_chat_id": -1, + "timestamp": float(index), + "input_length": 0, + "output_length": 0, + "hash_ids": [], + } + ) + + "\n" + ) + continue input_tokens = 513 if index % 2 else 512 stream.write( json.dumps( @@ -53,6 +68,7 @@ class CodeTracePreflightTest(unittest.TestCase): self.assertEqual( payload["selected"]["hash_contract"]["exact_source_block_size"], 512 ) + self.assertEqual(payload["selected"]["invalid_zero_token_rows"], 1) self.assertEqual(payload["max_model_len_recommendation"], 40960) output = root / "window" subprocess.run( @@ -74,6 +90,7 @@ class CodeTracePreflightTest(unittest.TestCase): self.assertEqual(manifest["requests"], len(rows)) self.assertGreaterEqual(manifest["duration_s"], 3600) self.assertEqual(rows[0]["sampling_u"], rows[1]["sampling_u"]) + self.assertNotIn(100, {row["source_index"] for row in rows}) if __name__ == "__main__":