feat: new router and benchmark setup

2026-04-16 14:23:53 +08:00
parent c86d931d8f
commit 996511f300
35 changed files with 1480 additions and 76 deletions
--- a/configs/glm5-nvfp4-8xb300-128k-plus-n1.yaml
+++ b/configs/glm5-nvfp4-8xb300-128k-plus-n1.yaml
@@ -0,0 +1,37 @@
+# GLM-5-NVFP4 on 8 x B300 for the 128k++ bucket.
+# A single instance already keeps mean TTFT below 5s, and routing is
+# effectively irrelevant at N=1 because every request lands on the same node.
+
+model:
+  config_json: ../models/GLM-5-NVFP4/config.json
+  name: glm-5-nvfp4
+  compute_dtype: fp8
+  weight_dtype: fp4
+  dtype_bytes: 1
+  block_size_tokens: 512
+
+hardware:
+  type: 8xb300
+  hbm_bytes: 1900.0e9
+  dram_bytes: 1.5e12
+  max_batch_slots: 256
+
+cluster:
+  num_instances: 1
+  meta_store:
+    ttl_seconds: 300.0
+  router:
+    mode: cache_affinity
+    precise_probe_latency_us: 50.0
+    precise_probe_topk: 1
+    load_alpha: 1.0
+    prefix_k: 8
+
+sim:
+  trace_path: bailian-traces/glm_coder_blksz_512_040915-040917.jsonl
+  max_requests: null
+  output_dir: runs/glm5_nvfp4_8xb300_ablation_131073_plus_n1
+  sample_interval_s: 1.0
+  seed: 42
+  input_length_min: 131073
+  input_length_max: 4294967295