From 506e63a63319a133ea511c46ee43a852acc8d4d6 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sun, 19 Jul 2026 16:45:35 +0800 Subject: [PATCH] Add measured collective profile gate for Qwen235 --- ...p8-collective-profile-ablation-20260719.md | 64 ++++++ .research/ongoing.md | 72 ++++++ .../analyze_qwen235_v020_campaign.py | 15 +- .../jobs_qwen235_collective_profile.toml | 45 ++++ .../materialize_qwen235_v020_allreduce.py | 213 ++++++++++++++++++ .../plot_qwen235_collective_ablation_mock.py | 74 ++++++ .../qwen235-collective-ablation-mock.png | Bin 0 -> 105057 bytes .../run_frontier_qwen235_v020_surface.py | 26 ++- ...run_frontier_qwen30_exact_trace_surface.py | 141 +++++++++++- ...run_qwen235_collective_profile_ablation.sh | 51 +++++ .../test_fidelity_envelope.py | 119 ++++++++++ .../profile_vllm020_allreduce.py | 198 +++++++++++++--- .../run_allreduce_profile.sh | 29 ++- 13 files changed, 1002 insertions(+), 45 deletions(-) create mode 100644 .research/experiments/qwen235-tp8-collective-profile-ablation-20260719.md create mode 100644 .research/ongoing.md create mode 100644 runs/frontier-fidelity-envelope-v1/jobs_qwen235_collective_profile.toml create mode 100644 runs/frontier-fidelity-envelope-v1/materialize_qwen235_v020_allreduce.py create mode 100644 runs/frontier-fidelity-envelope-v1/plot_qwen235_collective_ablation_mock.py create mode 100644 runs/frontier-fidelity-envelope-v1/qwen235-collective-ablation-mock.png create mode 100644 runs/frontier-fidelity-envelope-v1/run_qwen235_collective_profile_ablation.sh diff --git a/.research/experiments/qwen235-tp8-collective-profile-ablation-20260719.md b/.research/experiments/qwen235-tp8-collective-profile-ablation-20260719.md new file mode 100644 index 0000000..4b9365f --- /dev/null +++ b/.research/experiments/qwen235-tp8-collective-profile-ablation-20260719.md @@ -0,0 +1,64 @@ +# 实验 EXP-SIMFID-Q235-CC-TP8：真实 TP4/TP8 collective profile 消融 + +> **状态：** review 通过，执行中 +> +> 本 card 是 SHA、command、config、log 等 provenance 的唯一归宿；本轮只重跑 simulator，不重跑已经冻结的 48 个真机 trial。 + +## Claim 与决策 + +- **Parent claim：** Qwen235 Fixed-PD 的 30%+ TPOT/E2E selection regret，是否主要由 TP8 collective profile 缺失及 TP4 profile 与真实 serving backend 不匹配造成。 +- **目的：** 支持或反驳 mechanism hypothesis；不是用同一 workload 的 E2E calibration 修正 simulator。 +- **Competing hypotheses：** + - H1：collective profile coverage/backend mismatch 是排序反转的必要主因。换成与真机 serving 一致的 TP4/TP8 实测 profile 后，Frontier 的 Fixed-PD TPOT winner 从 TP8 翻到 TP4，mean/p90 TPOT 与 E2E selection regret 降到 10% 以内。 + - H2：collective mismatch 只解释部分误差。换 profile 后 TP8 仍是 Frontier winner，Fixed-PD TPOT/E2E regret 仍超过 10%；下一主因应定位 decode batch/state-conditioned MoE composition。 +- **事前预测：** 当前 Frontier 在 Fixed-PD 上预测 TP4/TP8 mean TPOT 为 87.77/61.59 ms，TP8 有 26.19 ms 优势；真实 TP4/TP8 为 21.04/27.99 ms。若新的 TP4/TP8 collective profile 使这个 26.19 ms 的 simulator margin 反转，则支持 H1；若不能，则支持 H2。 +- **判定规则：** 只以 frozen simulator rerun 的 winner 与真实 frozen surface 计算 selection regret。绝对 latency ratio 作为 secondary metric，不用它替代 selection verdict。 + +## Setup + +- **自变量：** + - A0：当前 `measured-allreduce.csv`；TP4 是 Qwen30 hidden=2048 的旧实测，且 profiler 只检查 FlashInfer 可用、没有证明每个 payload 的实际 dispatch；TP8 无行并静默 analytical fallback。 + - A1：Qwen235 serving-matched piecewise collective profile；TP4/TP8 都在 dash0 H20、vLLM 0.20.0 commit `88d34c640...` 上实测。Frozen server logs 证明真机同时使用 `disable_custom_all_reduce=true` 与 FlashInfer-TRTLLM `allreduce_rms` fusion；profile 对 fusion-eligible payload 测同一 FlashInfer communicator，对阈值外 payload 测真实 PyNCCL/symmetric fallback。 +- **控制变量：** Frontier commit、Qwen235 operator profiles、runtime contract、四类 frozen traces、候选配置、MNS/MBT、prefix policy、real results 与分析脚本全部不变。 +- **System context：** Qwen3-235B-A22B-FP8；vLLM 0.20.0+cu129；dash0 8×H20；`{TP4/EP1, TP8/EP8} × MNS{64,128}`；MBT=8192；Frontier piecewise graph path。 +- **Workload 或 trace：** 重跑四类 simulator surface：Fixed-PD 4096→256 @ 0.2 req/s/GPU、Fixed-PO 4096→1、Trace-PD、Trace-PO；每 cell 沿用原 129-request trace。Fixed-PD 是 primary，另外三类检查 profile 替换是否引入新的 selection regression。 +- **Profile protocol：** payload 覆盖所有 Qwen235 decode graph buckets（1--256，含真实 capture sizes）、fusion 阈值两侧 `{63,64,65}` / `{255,256,257}`，以及 512--8192 prefill sizes；每个 TP 与 payload 先 warmup，再保留 3×20 个 per-rank CUDA-event samples。raw JSON 记录实际 backend dispatch、fusion byte limit、world size、dtype、payload bytes、GPU/runtime/commit 与 source hashes。TP4/TP8 使用相同 payload grid；不把 microbenchmark 直接当作 E2E 结论。 +- **Profile contract：** H20/SM90 上 vLLM 0.20 的 fusion limit 是 TP4 2 MiB、TP8 0.5 MiB，即 Q235 BF16 hidden=4096 时分别为 256/64 tokens。simulator runner 启动前解析 CSV，要求所选 configs 的每个 `TP>1` 都有有限、正值的 measured rows；缺覆盖立即失败。结果 manifest 写入 CSV SHA-256、TP coverage、row counts 与 piecewise backend 集合。决策实验禁止 analytical fallback。 +- **Baselines：** A0 current Frontier、A1 measured-profile Frontier、frozen real hardware surface。 +- **Metrics：** profile latency median/p90 与跨 rank spread；simulated mean/p90 TTFT/TPOT/E2E；winner、selection regret、tau-b（可定义时）；每个 TP 的 measured-profile hit/fallback counters。 + +## 预期产物与 review + +- **预期数据：** TP4/TP8 raw collective JSON；materialized Frontier CSV + manifest；四类 A1 simulator surface；A0/A1/real comparison JSON/Markdown；profile cost ledger。 +- **Figure prototype：** `../../runs/frontier-fidelity-envelope-v1/qwen235-collective-ablation-mock.png`；左图固定真实与 A0 TPOT，并为 A1 留待测 series；右图明确“winner flip→0% regret / unchanged→33% regret”的判定。它回答 profile 修复是否足以改变配置选择。 +- **人工 review：** 通过（2026-07-19，用户明确要求“推进实验”）。 +- **Review 意见：** 保留 frozen real surface，只补真实 TP4/TP8 profile 后重跑 simulator；每个 simulator 实验必须使用真实 profile，缺失 coverage 或运行时 analytical fallback 立即失败。 + +## Benchmark design audit（experiment-design-review） + +| Crime | Verdict | Severity | Evidence | Fix / gate | +|---|---|---|---|---| +| 用 microbenchmark 代替 E2E | PASS | — | collective profile 只作为自变量；结论来自完整 simulator surface 对 frozen real surface 的 selection regret | 保留 A0/A1/real 三方结果 | +| calibration set 等于 evaluation set | PASS | — | A1 只测 collective operator，不使用 real E2E latency 拟合参数 | 禁止 E2E scale/calibration | +| selective benchmarking | PASS | — | primary Fixed-PD 外，同时重跑另外三类 workload | 报告所有 16 个 simulator cells | +| 缺失平台/版本 | PASS | — | raw/manifest 绑定 H20、vLLM commit、model、backend 与 hashes | 任一 provenance 缺失则 profile 不可采纳 | +| 缺失方差 | NEEDS EVIDENCE | Major | 尚未执行 profile repeats | raw artifact 必须保留 per-rank repeated samples并报告 spread | +| profile 覆盖静默降级 | FAIL（A0） | Blocking | TP8 无 measured rows，Frontier 使用 analytical fallback | A1 runner fail-fast；fallback count 必须为 0 | +| backend/fusion 阈值未对齐 | FAIL（A0） | Blocking | 真机日志启用 FlashInfer `allreduce_rms`；vLLM 源码规定 H20 TP4/TP8 fusion limit 为 2/0.5 MiB，旧 CSV 未记录这条 piecewise contract | A1 在阈值两侧实测并记录每行 dispatch | + +**总体建议：** 已批准执行；coverage gate、backend match 与 provenance gate 任一不通过则 Block。 + +## 复现信息 + +- **Code：** AITuner branch `feature/sim`；本 card 创建时 HEAD `f4a75aa8e400ead4eb6d305178192e85940c6de6`，后续运行 commit 待填。vLLM source commit `88d34c6409e9fb3c7b8ca0c04756f061d2099eb1`。 +- **Environment：** dash0 8×H20；`/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1`；model `/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8`。 +- **产物路径：** 待 review 后冻结；不得覆盖旧 campaign `/home/admin/cpfs/wjh/aituner/qwen235-v020-fourcase-20260719-r1`。 +- **已知 deviation：** `--disable-custom-all-reduce` 只关闭 vLLM custom AR，不关闭编译器的 FlashInfer `allreduce_rms` fusion。真机 TP4/TP8 日志都显示自动选择 `trtllm` workspace；旧 TP4 profiler 没有记录/执行真实 fusion-limit piecewise dispatch。A1 因此必须同时重测 TP4 与 TP8，不能只追加 TP8 行。 + +## 结果 + +- **观察事实：** 待运行。 +- **异常：** 待运行。 +- **含义：** 待运行。 +- **Claim update：** unchanged +- **下一步：** 依次完成 collective profiling、profile materialization、CPU simulator rerun 与 analysis。 diff --git a/.research/ongoing.md b/.research/ongoing.md new file mode 100644 index 0000000..7475282 --- /dev/null +++ b/.research/ongoing.md @@ -0,0 +1,72 @@ +# AITuner 研究当前状态 + +> 2026-07-17｜写给未参与项目的读者，可直接作为 presentation 讲稿。历史过程与复现信息见 `../runs/*/` 各 experiment card、`../docs/` 各 campaign 文档。 +> +> **2026-07-19 update：** Qwen235 四类 workload matrix 已完成；Fixed-PD 出现 33.0--37.2% TPOT 与 30.7--34.6% E2E selection regret。当前发现 Q235 simulator 输入的 collective CSV 缺 TP8，并且旧 TP4 profile 未绑定真实 `allreduce_rms` fusion-limit dispatch。active ablation 见 [`experiments/qwen235-tp8-collective-profile-ablation-20260719.md`](experiments/qwen235-tp8-collective-profile-ablation-20260719.md)。 + +## 一眼看懂 + +- **Topic / problem：** LLM serving 的自动、低成本配置调优（AITuner）。当前主线问题：用 simulator 给部署配置（并行度、批量上限等）排序，什么时候可信？需要补多少真机证据？算上这些成本还划算吗？ +- **Central claim：** simulator 要能帮助配置调优，必须先满足 scheduler transition 的 liveness/coverage，再满足「配置相关残差小于真机 decision margin」；前者决定 capacity 是否有定义，后者决定排序是否正确。(ID: C0) +- **当前结论：** 早先 35 个 trace stall 不是 Frontier scheduler liveness failure：adapter 为不满 16-token 的 prefix block 错误生成了 cache identity，Frontier 又没有 fail-fast。修正为完整 block、使用真实 graph buckets/KV blocks 和 `piecewise`/`KERNEL_ONLY` profile 后，Qwen30 Trace-PD 的全部 12 个 cell 完成 129/129 request，Frontier 对 TTFT/TPOT/E2E 的 6 个 argmin 均与三次 fresh-server 真机一致；但绝对 latency 仍高估 4--511×。这只证明一个 MoE Trace-PD surface 的 selection fidelity，不能外推到 prefill-only、fixed workload 或 235B。 +- **最大 uncertainty / risk：** 这个正确选择是否依赖 Trace-PD 的 queue/margin 偶然性；Fixed-PD、Trace-P、Fixed-P 与大型 FP8 MoE 是否仍保留正确 config ranking。Q235 还缺少同栈 profile/runtime contract，不能复用旧 vLLM 0.10.2 数据。 +- **下一项 critical action：** 完成 Qwen30 其余三项 fixed/trace × PD/P surface。Fixed case 已先启动无请求 vLLM runtime-state preflight；Trace-P 的 graph-aligned Frontier 12-cell CPU surface 正在运行。Q235 先做 vLLM 0.20 TP4/TP8 compatibility gate，随后才允许新 profile。 +- **停止条件：** T1 出 verdict 且成本账本建立后：pass 且摊销论证成立 → 转向「sim 剪枝 + 真机终选」的 hybrid 机制设计；fail → 转入失败机制归因；两条路都无 insight 增量 → 收敛写作。 + +## 核心概念 + +- **Frontier：** 本项目使用的 simulator，属 Vidur 系（直接使用 vidur backend），加自研 FP8/MoE/EP/decode-profile 兼容补丁。 +- **Regret：** 按 simulator 排序选配置，相对真机最优配置的性能损失百分比（以每 GPU capacity 计）。primary metric：排序选对则 regret=0。 +- **τ-b（Kendall tau-b）：** simulator 排序与真机排序的秩相关，1 = 完全一致，−1 = 完全反序，0 = 无关；tie-aware。 +- **Decision margin：** 真机上头部配置之间的性能差距，即 simulator 误差的容忍带。 +- **Action-differential residual：** simulator 误差中随配置（action）不同而不同的部分。Why needed：所有配置统一偏移不影响排序，只有差异化残差才可能穿过 margin 改变选择——这解释了「绝对误差 33%」与「排序全对」为何可以同时成立。 +- **Capacity bracket：** 真机 anchor 为候选配置的 capacity 划出的上下界；「bracket 不反转」指未测的负载点不可能推翻 top 选择。 +- **Decision-valid coverage：** simulator 能从初始状态推进到所有请求完成，并为 config×workload cell 产生合法 SLO metric 的比例。若 reachable nonterminal state 没有 enabled transition/future event，capacity 与 rank 都没有定义，不能把该 cell 当作 infeasible。 +- **Workload realism 阶梯：** prefill-only（无 decode）→ fixed-shape mixed（固定输入输出长度的混合负载）→ trace-faithful mixed（生产 trace 忠实回放）。fidelity 结论不能向更高一级外推。 + +## Claim 层级 + +- **Central claim：** 见「一眼看懂」。(ID: C0) + - **Subclaim：** zero-shot 排序失败是真实现象。(ID: C1；supported) + - 30B 纯 profile 驱动的 regret 为 25.63%（τ-b=0）；另一 throughput-proxy 评测口径下为 30.46%。Boundary：均发生在 capacity-point + SLO-gated selection——恰是 Vidur 论文自己声明预测误差会爆炸、评测刻意回避的 regime（见 claim map）。 + - **Subclaim：** 少量结构化的真机证据可以恢复低 regret 排序。(ID: C2) + - **Hypothesis（decision-bearing）：** trace-faithful 回放下，同栈 profile + 真机 KV capacity + 兼容补丁、且不做逐案例端到端校准的 Frontier，能满足 gate：regret ≤5% ∧ τ-b ≥0.8 ∧ bracket 不反转。(ID: H1；weakened) + - **Supporting：** 235B prefill-only regret=0；235B fixed-shape mixed 的 top set 全中；30B 加 per-TP 校准后 regret 0.76%（但这是外部端到端 scale 给出的上界，不是原生 profile 保真度）。 + - **Counterevidence：** 修正 prefix trace contract 后的 TP2/MNS16 `none`-graph run 完成但 p50 TPOT 约 96 ms，真机为约 14 ms；然而该比较尚未对齐 real vLLM 的 `FULL_AND_PIECEWISE` graph path。 + - **下一项 discriminative experiment：** 补齐 `KERNEL_ONLY` graph family并以 `piecewise` 重跑相同 trace；若 full surface 仍错，graph omission 不再是可用解释。 + - **Hypothesis（机制，active）：** 误差机制是 action-conditioned residual——执行状态的转移（并行拓扑、kernel family、graph mode、batch 组成）使按算子 profile 的组合预测跨配置不可复合；残差大于 margin 时排序失败。(ID: H2；supported) + - **Supporting：** 三个 TP 档的端到端校准系数为 0.72/0.47/0.35，残差确实随配置剧烈变化；235B 的批量上限交互预测错误但被 2× margin 容忍；30B prefill-only 在低负载近似对齐、饱和后按 TP 反向放大，最终 τ-b=−1。 + - **下一步：** 按 collective 通信 → batch 组成 → 调度器逐步轨迹的单变量顺序定位组合性失效点（对应 experiment card 中 A1/A2/A3 消融阶梯）。 + - **Subclaim：** 成本论证只有在摊销前提下成立。(ID: C3) + - **Hypothesis（active）：** 每个 model×硬件×runtime 的一次性对齐成本，摊销到大配置面、频繁重调（引擎版本 churn 的频率证据见 claim map）或禁止在线实验的场景后，低于重复真机调优。(ID: H3；untested——分母已实测，分子未入账) + - **下一步：** 建 cost ledger（见「下一步」）。 + +## 当前 critical experiment + +- **Question：** 生产 trace 忠实回放（prefix 打开、原始到达时间与会话结构）下，best-effort Frontier 能否满足 low-regret gate？ +- **为什么现在做：** 这是 H1 的判决实验；所有已完成的机制分解都在人工 workload 上，不能替代这个 verdict。 +- **当前状态：** Trace-PD 的 graph-aligned surface 已通过 selection gate，但绝对 latency 不通过 calibration。现在以不共享结果的 Fixed-PD、Trace-P、Fixed-P surface 检验它的泛化边界。 +- **Result → decision：** 若其它 surface 排序失败，保留 Trace-PD success 为条件化 envelope，并按 fixed/trace/prefill/decode 的差异定位 state composition；若都通过，才扩大到 Q235 或寻找 simulator 已解决范围之外的新问题。 +- **Experiment card：** [`../runs/frontier-fidelity-envelope-v1/experiment-card.md`](../runs/frontier-fidelity-envelope-v1/experiment-card.md) + +## Key evidence（最多 3 条） + +- **E1（否证「prefill-only 是充分 easy condition」，支持 H2）：** 30B BF16、去掉 decode/prefix/混合 batch 后，真机最优是 TP4（8 vs 7 req/s/GPU），simulator 却把 TP4 排最差（6 vs 8）：top set 无交集，regret 12.5%，τ-b=−1。产物：`../runs/frontier-phase-factorial-v0/results/final/`（dash0，12.07 H20-GPUh）。 +- **E2（margin 可掩盖残差，支持 C0/H2）：** 235B FP8 的 prefill-only 与 fixed-shape mixed 都选对 top set，但 simulator 漏掉并发上限（max-num-seqs）×批 token 上限（max-num-batched-tokens）的交互棋盘格，并把 10/34 个真机 anchor 误判为不可行；真机头部 margin 有 2×，残差被容忍。产物：`../runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/`、`../runs/frontier-multicase-sufficiency-v1/results/t0-final/`。 +- **E3（selection 与 calibration 分开）：** 用完整 block projection、`piecewise` 和 graph-compatible KERNEL_ONLY profile 后，Qwen30 Trace-PD 的 12/12 sim cells 完成，6 个 mean/p90 latency objective 的 argmin 都与真机一致；但 sim/real latency ratio 仍为 4--511×。产物：[`../runs/frontier-fidelity-envelope-v1/graph-piecewise-experiment-card.md`](../runs/frontier-fidelity-envelope-v1/graph-piecewise-experiment-card.md)。 + +## 下一步（最多 3 项） + +- [ ] **完成 Qwen30 remaining matrix（direct progress）：** Fixed-PD、Trace-P、Fixed-P 分别冻结 Frontier/真机 12-cell surface；P-only 的 TPOT 一律为 N/A。 +- [ ] **Q235 portability gate：** 先验证 vLLM0.20 TP4/TP8 FP8 runtime 和 deadc4a profile provenance，再决定是否允许其 Fixed-P sweep。 +- [ ] **建 cost ledger：** parent H3；完成标准 = 每 case 一行（profiling GPU-h、补丁工时、校准探测、sim CPU-h），与已实测的真机调优成本同表，随每个 case 更新。 + +## Blocker 或 anomaly + +- **下一启动已准备：** dash0 8×H20 当前空闲；graph-compatible attention/linear/MoE/router kernel-only profile 会先在一 GPU smoke，成功后以 3 张 GPU 并行完成 attention shards。完整 replay 仍为 CPU-only。 +- **Anomaly（保留）：** 235B pilot 中 simulator 把 10/34 个 anchor 误判为不可行——false-infeasible 是 H1 的主要威胁模式，T1 分析时须单独报告。 +- **勘误（已固定）：** 统一平台为 dash0-only，早期 dash1 出处是文档错误；fixed-shape pilot 的主 SLO（TPOT 40ms）无判别力，150ms 是事后明示的敏感性分析，不得写成盲选的 primary。 + +## Related work + +- Claim map：[`../docs/simulator-claim-map-20260716.md`](../docs/simulator-claim-map-20260716.md)。核心缺口：capacity-point + SLO-gated selection 的 regret 无人用真机 ground-truth 面验证过；alignment 成本无人与真机调优成本放进同一张表比较。 diff --git a/runs/frontier-fidelity-envelope-v1/analyze_qwen235_v020_campaign.py b/runs/frontier-fidelity-envelope-v1/analyze_qwen235_v020_campaign.py index 25853e6..4f4c739 100644 --- a/runs/frontier-fidelity-envelope-v1/analyze_qwen235_v020_campaign.py +++ b/runs/frontier-fidelity-envelope-v1/analyze_qwen235_v020_campaign.py @@ -97,14 +97,18 @@ def compare_metric(real: dict, sim: dict, metric: str): def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--campaign-root", type=Path, required=True) + parser.add_argument("--real-root", type=Path) + parser.add_argument("--sim-root", type=Path) parser.add_argument("--json-output", type=Path, required=True) parser.add_argument("--markdown-output", type=Path, required=True) args = parser.parse_args() + real_root = args.real_root or args.campaign_root + sim_root = args.sim_root or args.campaign_root cases = {} lines = ["# Qwen235 vLLM 0.20 Frontier vs real", "", "| case | metric | Frontier winner | real winner | match | regret | pair agreement |", "|---|---|---|---|---:|---:|---:|"] for case in CASES: - real = real_surface(args.campaign_root, case) - sim = sim_surface(args.campaign_root, case) + real = real_surface(real_root, case) + sim = sim_surface(sim_root, case) metrics = ["ttft_mean_ms", "ttft_p90_ms", "e2e_mean_ms", "e2e_p90_ms"] if case.endswith("pd"): metrics[2:2] = ["tpot_mean_ms", "tpot_p90_ms"] @@ -118,7 +122,12 @@ def main() -> None: f"{item['pair_direction_agreement']:.1%} |" ) cases[case] = {"real": real, "sim": sim, "comparison": comparisons} - payload = {"schema": "qwen235-v020-simulator-real-comparison-v1", "cases": cases} + payload = { + "schema": "qwen235-v020-simulator-real-comparison-v1", + "real_root": str(real_root.resolve()), + "sim_root": str(sim_root.resolve()), + "cases": cases, + } args.json_output.parent.mkdir(parents=True, exist_ok=True) args.json_output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") args.markdown_output.write_text("\n".join(lines) + "\n") diff --git a/runs/frontier-fidelity-envelope-v1/jobs_qwen235_collective_profile.toml b/runs/frontier-fidelity-envelope-v1/jobs_qwen235_collective_profile.toml new file mode 100644 index 0000000..be41524 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/jobs_qwen235_collective_profile.toml @@ -0,0 +1,45 @@ +version = 1 + +[[jobs]] +name = "qwen235-vllm020-serving-allreduce-tp4-20260719-v1" +gpus = 4 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner/runs/frontier-qwen30-vllm020-profile-v1 && timeout --signal=TERM --kill-after=30s 1020 bash run_allreduce_profile.sh" +artifacts = ["artifacts/qwen235-serving-allreduce-tp4-20260719-v1"] + +[jobs.env] +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "4" +NUM_TOKENS = "1 2 4 8 16 24 32 40 48 56 63 64 65 72 80 88 96 104 112 120 128 136 144 152 160 168 176 184 192 200 208 216 224 232 240 248 255 256 257 512 1024 2048 4096 8192" +HIDDEN_DIM = "4096" +COLLECTIVE_CONTRACT = "qwen235-serving-projected" +TRIALS = "3" +REPEATS = "20" +MODEL = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-fidelity-envelope-v1/artifacts/qwen235-serving-allreduce-tp4-20260719-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +VLLM_SOURCE = "/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build" + +[[jobs]] +name = "qwen235-vllm020-serving-allreduce-tp8-20260719-v1" +gpus = 8 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner/runs/frontier-qwen30-vllm020-profile-v1 && timeout --signal=TERM --kill-after=30s 1020 bash run_allreduce_profile.sh" +artifacts = ["artifacts/qwen235-serving-allreduce-tp8-20260719-v1"] + +[jobs.env] +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "8" +NUM_TOKENS = "1 2 4 8 16 24 32 40 48 56 63 64 65 72 80 88 96 104 112 120 128 136 144 152 160 168 176 184 192 200 208 216 224 232 240 248 255 256 257 512 1024 2048 4096 8192" +HIDDEN_DIM = "4096" +COLLECTIVE_CONTRACT = "qwen235-serving-projected" +TRIALS = "3" +REPEATS = "20" +MODEL = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-fidelity-envelope-v1/artifacts/qwen235-serving-allreduce-tp8-20260719-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +VLLM_SOURCE = "/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build" diff --git a/runs/frontier-fidelity-envelope-v1/materialize_qwen235_v020_allreduce.py b/runs/frontier-fidelity-envelope-v1/materialize_qwen235_v020_allreduce.py new file mode 100644 index 0000000..e1ca264 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/materialize_qwen235_v020_allreduce.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +"""Validate Qwen235 serving collectives and materialize Frontier's CC CSV.""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import math +from pathlib import Path +from typing import Any + + +VLLM_COMMIT = "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1" +TOKEN_POINTS = ( + 1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 63, 64, 65, 72, 80, 88, 96, + 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, + 208, 216, 224, 232, 240, 248, 255, 256, 257, 512, 1024, 2048, 4096, + 8192, +) +TP_SIZES = (4, 8) +HIDDEN_DIM = 4096 +ALLOWED_BACKENDS = { + "flashinfer_trtllm_fused_projection", + "pynccl_symmetric_with_copy", + "torch_symmetric_memory", + "pynccl", +} +FIELDS = ( + "time_stats.all_reduce.min", + "time_stats.all_reduce.max", + "time_stats.all_reduce.mean", + "time_stats.all_reduce.median", + "time_stats.all_reduce.std", + "rank", + "num_workers", + "size", + "collective", + "devices_per_node", + "max_devices_per_node", +) + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def materialize( + input_paths: list[Path], output_path: Path, manifest_path: Path +) -> dict[str, Any]: + if len(input_paths) != 2: + raise ValueError("exactly two raw inputs are required for TP4 and TP8") + rows = [] + seen = set() + backends_by_tp: dict[int, set[str]] = {} + environments = [] + for path in input_paths: + payload = json.loads(path.read_text()) + if payload.get("schema_version") != "vllm020_allreduce_raw.v2": + raise ValueError(f"unexpected raw collective schema in {path}") + environment = payload["environment"] + if environment.get("vllm_version") != "0.20.0": + raise ValueError(f"vLLM version mismatch in {path}") + if environment.get("vllm_source_commit") != VLLM_COMMIT: + raise ValueError(f"vLLM commit mismatch in {path}") + if environment.get("gpu") != "NVIDIA H20": + raise ValueError(f"GPU mismatch in {path}: {environment.get('gpu')!r}") + if Path(environment.get("model", "")).name != "Qwen3-235B-A22B-FP8": + raise ValueError(f"model mismatch in {path}") + if environment.get("collective_contract") != "qwen235-serving-projected": + raise ValueError(f"collective contract mismatch in {path}") + if environment.get("disable_custom_all_reduce") is not True: + raise ValueError(f"custom all-reduce was not disabled in {path}") + backend_env = environment.get("backend_env", {}) + if backend_env.get("VLLM_ALLREDUCE_USE_FLASHINFER") != "1": + raise ValueError(f"FlashInfer projection was not enabled in {path}") + if backend_env.get("VLLM_ALLREDUCE_USE_SYMM_MEM") != "1": + raise ValueError(f"symmetric-memory contract mismatch in {path}") + environments.append(environment) + + for source in payload["rows"]: + tp = int(source["tensor_parallel_size"]) + tokens = int(source["num_tokens"]) + key = (tp, tokens) + if key in seen: + raise ValueError(f"duplicate collective row: {key}") + seen.add(key) + if tp not in TP_SIZES: + raise ValueError(f"unexpected TP: {tp}") + if int(source["hidden_dim"]) != HIDDEN_DIM: + raise ValueError(f"hidden-dim mismatch for {key}") + expected_bytes = tokens * HIDDEN_DIM * 2 + if int(source["payload_bytes"]) != expected_bytes: + raise ValueError(f"payload mismatch for {key}") + backend = source["selected_backend"] + if backend not in ALLOWED_BACKENDS: + raise ValueError(f"unexpected backend for {key}: {backend!r}") + fusion_limit = {4: 2 * 1024 * 1024, 8: 512 * 1024}[tp] + if int(source["real_fusion_limit_bytes"]) != fusion_limit: + raise ValueError(f"fusion limit mismatch for {key}") + expected_backend = ( + "flashinfer_trtllm_fused_projection" + if expected_bytes <= fusion_limit + else None + ) + if expected_backend is not None and backend != expected_backend: + raise ValueError(f"missing fused projection for {key}") + if expected_backend is None and backend == "flashinfer_trtllm_fused_projection": + raise ValueError(f"fused projection exceeds runtime limit for {key}") + backends_by_tp.setdefault(tp, set()).add(backend) + median = float(source["critical_path_median_ms"]) + if not math.isfinite(median) or median <= 0: + raise ValueError(f"invalid critical-path median for {key}") + if int(source.get("trials", 0)) != 3 or int( + source.get("repeats_per_trial", 0) + ) != 20: + raise ValueError(f"repeat contract mismatch for {key}") + samples = source.get("per_trial_rank_samples_ms", []) + if ( + len(samples) != 3 + or any(len(trial) != tp for trial in samples) + or any( + len(rank_samples) != 20 + or any( + not math.isfinite(float(value)) or float(value) <= 0 + for value in rank_samples + ) + for trial in samples + for rank_samples in trial + ) + ): + raise ValueError(f"raw sample coverage mismatch for {key}") + critical_samples = [ + max(float(trial[rank][repeat]) for rank in range(tp)) + for trial in samples + for repeat in range(20) + ] + ordered = sorted(critical_samples) + measured_median = (ordered[29] + ordered[30]) / 2 + if not math.isclose(median, measured_median, rel_tol=1e-6, abs_tol=1e-9): + raise ValueError(f"critical-path median/sample mismatch for {key}") + rows.append( + { + "time_stats.all_reduce.min": median, + "time_stats.all_reduce.max": median, + "time_stats.all_reduce.mean": median, + "time_stats.all_reduce.median": median, + "time_stats.all_reduce.std": 0.0, + "rank": 0, + "num_workers": tp, + "size": expected_bytes, + "collective": "all_reduce", + "devices_per_node": tp, + "max_devices_per_node": 8, + } + ) + + expected = {(tp, tokens) for tp in TP_SIZES for tokens in TOKEN_POINTS} + if seen != expected: + raise ValueError( + f"collective coverage mismatch: missing={expected - seen}, extra={seen - expected}" + ) + if output_path.exists() or manifest_path.exists(): + raise FileExistsError("refusing to overwrite immutable collective profile") + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("w", newline="") as output: + writer = csv.DictWriter(output, fieldnames=FIELDS, lineterminator="\n") + writer.writeheader() + writer.writerows(sorted(rows, key=lambda row: (row["num_workers"], row["size"]))) + + manifest = { + "schema": "qwen235-v020-serving-allreduce-profile-v1", + "inputs": {str(path.resolve()): sha256(path) for path in input_paths}, + "output": str(output_path.resolve()), + "output_sha256": sha256(output_path), + "vllm_source_commit": VLLM_COMMIT, + "hardware": "NVIDIA H20", + "model": "Qwen3-235B-A22B-FP8", + "dtype": "bfloat16", + "hidden_dim": HIDDEN_DIM, + "collective_contract": "qwen235-serving-projected", + "tp_coverage": list(TP_SIZES), + "token_points": list(TOKEN_POINTS), + "rows": len(rows), + "observed_backends_by_tp": { + str(tp): sorted(backends) for tp, backends in sorted(backends_by_tp.items()) + }, + "frontier_target": "time_stats.all_reduce.median", + "unused_stat_policy": "repeat critical_path_median; std=0", + "raw_environments": environments, + } + manifest_path.parent.mkdir(parents=True, exist_ok=True) + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + return manifest + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--input", type=Path, action="append", required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + args = parser.parse_args() + manifest = materialize(args.input, args.output, args.manifest) + print(json.dumps(manifest, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-fidelity-envelope-v1/plot_qwen235_collective_ablation_mock.py b/runs/frontier-fidelity-envelope-v1/plot_qwen235_collective_ablation_mock.py new file mode 100644 index 0000000..6b1a208 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/plot_qwen235_collective_ablation_mock.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Render the preregistered Qwen235 collective-profile ablation layout.""" + +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np + + +OUT = Path(__file__).with_name("qwen235-collective-ablation-mock.png") + + +def main() -> None: + configs = ["TP4 / EP1", "TP8 / EP8"] + real_tpot = np.array([21.0442, 27.9942]) + current_frontier_tpot = np.array([87.7735, 61.5878]) + + fig, (ax0, ax1) = plt.subplots(1, 2, figsize=(10.8, 4.4)) + x = np.arange(len(configs)) + width = 0.34 + ax0.bar( + x - width / 2, + real_tpot, + width, + color="#4c78a8", + label="Frozen real", + ) + ax0.bar( + x + width / 2, + current_frontier_tpot, + width, + color="#e45756", + label="A0 current Frontier", + ) + for index in x: + ax0.text( + index, + max(real_tpot[index], current_frontier_tpot[index]) + 4, + "A1 measured-profile\nresult: TBD", + ha="center", + va="bottom", + fontsize=8, + color="#222222", + ) + ax0.set_xticks(x, configs) + ax0.set_ylim(0, 112) + ax0.set_ylabel("Fixed-PD mean TPOT (ms)") + ax0.set_title("A. Does A1 reverse the simulated TP margin?") + ax0.legend(frameon=False, fontsize=8) + ax0.grid(axis="y", alpha=0.25) + + outcomes = ["A0 current", "A1 flips\nto TP4", "A1 stays\nat TP8"] + regret = [33.0, 0.0, 33.0] + colors = ["#e45756", "#54a24b", "#f2cf5b"] + bars = ax1.bar(outcomes, regret, color=colors, width=0.68) + ax1.axhline(10.0, color="#222222", linestyle="--", linewidth=1.2, label="10% gate") + ax1.bar_label(bars, labels=["33%", "0%", "33%"], padding=3, fontsize=9) + ax1.set_ylim(0, 42) + ax1.set_ylabel("Real TPOT selection regret (%)") + ax1.set_title("B. Preregistered decision outcomes") + ax1.legend(frameon=False, fontsize=8) + ax1.grid(axis="y", alpha=0.25) + + fig.suptitle( + "SCHEMATIC / preregistered layout — A1 values are not yet measured", + fontsize=11, + ) + fig.tight_layout() + fig.savefig(OUT, dpi=180, bbox_inches="tight") + print(OUT) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-fidelity-envelope-v1/qwen235-collective-ablation-mock.png b/runs/frontier-fidelity-envelope-v1/qwen235-collective-ablation-mock.png new file mode 100644 index 0000000000000000000000000000000000000000..e6fb39b0504495bd4717be121312c550c045a4af GIT binary patch literal 105057 zcmeFZ_dnO|-#`A6l(bZ&kXb?@LM1CBG9z0EEh|~sqbQ@KvLaF0gp`q;24$5kLMc0u zk?imNIM4Uz`d-)V^9Ou>xw@V2+j+jl%jRy@^k5(d2N=!$^_SHH+w;a{` zd9CSZ*_j+aKkcM1gPY?Y_6pikZ$CNd^YF#s{XWoOTA&oQ(7_WJt8 z-f}5cR@O5X7OR|`oDNG!JW+ojeqZDq-IbpmS6{w7{BUypx^JOf%#fTgb z6T2Q7x-mmH^TuQOK#QgM8Ml??vKsgP&@7P{wbXNb;%1#lr?Kl~0kH6q=eormwKBu6lNcG9i_`J)H_gmEB zC3nt@wkxG;CzJPf^ytx+Y!eo4rQ2^C9v=Pu?akVU?S|oafA{a-$7laGJ>2Z^Av`EZ zWZ1DmYtNoNG@P8A2F1>NH#W219%;>d^Xe5vBxVC6zs~!LRok|ZKaEO(ROVb@Y2LySkZOnnPNe&GpxpO8m{&{{HKZ{Wae{%B(0HKYpAlE8S<; z5y#azre#gBm)_jiSX8ruclFZk_mk<_ik0P?Nl8hk`y<)8xVamjpHp=nZlbA;5_r>J z8~uBvbzf3SO5a#l;Y*(-iN(dm17}`Qbka^zm6w;hn+{a^ue}=^8`AggW}R;U@-+ukdj`%h4?TxkP*W1iuDxLQF*{8Ki_A4nVJ*=gn zEc_YsFDWT$sV(tYn%^OMkv>-3E@bN-6%I~Ls@d`0>Z{aipSg~kO8PFdHm9i9I(;7~ z_gA3rvapM0`dX}%C7tzI1Oq@%TA*~rlFu~y>omoB5*@s#h{_wV`VKt2iE z_U|!zJlvehEg-NF5Bae35WR%+&>ex$2M<^V>SFv^4qZyy$S;lGROT^r?^>nPV8c7t zUhlP*mfvly0ynbU7#g~WD#?UL-8@cbWo~5^Ea5Wp<4A&OnTI((N5@Nt#45iDY^OU> zQ8z@+)jU#(&1EX$mXgrcAkyt@oqL%A}(98(zb!q;FvM{YlOy z_BFStC_|xruO3T8bbRR*K$ipFyomFwPhK{PvVrX zgtt32+~Jg`dvgDux34h0?(5Nj;GiIBkuWx?d*%A++I!mPPM3QbGS?Jhb^gwe=D5Tc z`mQXilo`k{Vto~|jPlVj4_(rU8Td2S_44w!Rg_oF%?Fj{YN zA#&QGnKGZpk00+4wph)tlSVmp>QtnBLWSG! zRWHuh%Z2l(pDcBAxE~w)`oo8HIBZpTsI7vktw(1``M%U${Fqd1QBj57a_<-QOk1{G z{V2V#p*2%qDc9oN8EiOIAm_i{P~mAPUq3KjlN4W(c0xs19ex`c;0*n z-%E9Nlar8H`QoX^ax{^(AvbTn$*EWl!2?M=`{CXJVc}yjBIiD3>I?ns$d&(oRHNVh zW6Z&|l*`jYS{rs8p!C;7`s4MPMK9j5i!X5<@2eH6P{1iQx3r{2qteOHS%+rOGJN$; zUY_WF)6yO4v7%QaBN_3AK0Q%f|MYax*5Tn{qrvt-R4vQT&(G~RaE8LKpS@xI`t|o? zVjg3|>^OY+(e%+eekrM4=W3&_tMN3auU$bwO|~m_8P$ul(ug~J)%?v3@~sCB9`t|s za96CD%?+CM+fnFE?Rq9Hc}CM{4AYNBDcxpWhjtvenSB)n75e!uC&e+7HuljYO24Yv zeFM~Z{I6c6Vw3TCD(3xA!i6LJ!}Bs5X>M+6r7*U4`exqOal&8s^srS`Rq>jZ9*z~Y zy2kGF_n5~^fkSu;oV&PVZ+!gt;i#_bSm$-Lp%WF<#>U2?R!yr>P!w%#pH{is_ml>s z)v%JILLRKObqKo8V~6im_YXU8T^a9rIO-~8R^&*7T`51<@bKHv5bd^Y+YC_LJZ46A z$oQ02wd+t;W{-xq_i83cze58jN0?!s-Ujsa;fHOMJ8^Nhut<+Bt8ep~6pN~R>@mpWtHNzL@ zPt3B$7DP&PHau8qwIKz+hWZ%kkdYG;m#Jha^xpH+A6%r(yk{aKJ{G%-PM#Lj*VpHh zF=3VQIjth&`NuZW>PNGQiHVa*HmalQoTSg9!6GWkD!(+H^m{eaGRuDxDI6m$>D&Q- zD`qlI7d@})FZG<&z=Eq$=zDz@vZhynO6Vw0P>Lj?4STzuqY^buKYGORu=eq0mywn& zl(*v*EAijmy}i7W90RXk4@Tus_PwyQf%Wi70EUdZ>d&7IDfFhMrWW{Iuiw8{wlZa5 zVM(4w1!IVcin5NTk-fF^;PL)RHc7Wf(49%2|A6FR`GA-q8H5-vegJr~jzyCc?d)GUm?!K`6m?OX5 zQ{H%K?+o#rPoGYb^AvjS;Q{k&%BrfTiqxldXNNL1wsXz4AME_I^U-pr$*y;nd%3ve z#tI4w3z^U*8Rckm&EKra9~{E@4*vVsEx*?D@5JLYvx@R6mo)9<5CEE}nzsU4=zP&v9R43nA?EE{1@rUgdAjc}EG!K2CjdvP9{PdGUguOK9qBiL zXPor43;X;LM=9Qc)e)exyUP_mOCG~3?b#;H>2>oxQF~OQ7$^#; zR0EokUMwi9Y$LCeuJl+uqwRN&A*EsU*7J!vQ&83ZQ>E| z3DeOlKk2(mT%)6A`=iay8%J4ucSbv*p){6zd*-6vZ`r<`2N+{NAcRoz%Hq_=$lFfr z?CcJ}!VceF)8u;0SZihK30TbogT4?DI&#F&ie-Ad*X(C^iGBdiADB(BxLqeLE7aLMKX)E(kK%MFDfV~Xn}UlwST|b&(3^R zv~Bg_j(XMasVRP!t*tW-95_Ia4z-oE^21(#pMfvZA3Ju;qBTReser=a^Y>ez?G!$P zaqKdVYMOvNx{vd(7*Q+jg5i+r6KFpHg@uK*8#gkdC2sj%6&miN4=B_3`+N20fsZz_ ze*p`~=X_%O`T45%UK3oS!mcaJ-l6WkD&#CCee`=qFXwS*ugzD^On?O=O3blmPuq6` z>TE^xVKnpleSKnfHhLrh+Cm7{Y#rbbm23pBmYm(0T#i)$K_5uPT;&JI!$3+O`W@Hq z-3oEi-bQr^GG=s-IzN97PD|rWo=cGN72Lmn|LQwCMdJ_I#~*h5LD(@p7Jo}mPtWT& zZ{$&o_v=%A1Ih?O30>#v>Pq$i*$?1IoSRq=b8O#lzya=@WK;VlaC2szJX_jx96$0- z_4M@mQWJc(rfNLg3jD#8sh71C43R;O)?u(=7cdOBU)|cA^YyWN0EYLF=L9bOb}+$L zQ0h@@sk;*hZXjz;c%Ss$_RqVHUSa4eu-o?S+qb^2mHtoAptWPMfzs;+f%d?s^sr37 z*KOko!buE4xyZdTW~7y@!pOJzozp}5`5ez_-MxeT9s_`0eV-o7H%-q?4PG!#jq5r3 zKJ4MWJ*pugjQa>h3hMP;F*d?BC%G=KtZj6Ta4zb=R&LZJu`==i82R>n6C2wpnekp!@B+ zac9Q06B*LuJ!Q2~N^N9Yfy!u5=pmKpsCW+4?MxRw8L!vwcR5GSjTYpE!$7i?5OWM$@Yn*zf(32){`KxUHC|3Y|V`Jly8vRE6 zbdqU2 z-%Fq$&VpJOfWw+ROL; zUz_+7aBFha8QlaR<6v@P(YGR}s^7gkVZ|bDn-(p?w?^1$zHvK!+e~7H`7vEtx3SLr zRqua(d(v^3%RUWSop-@Z69c`^pPwuJdu3~dR%k>n$2|%6* z=8JUqN)Nq?;5+3gR$=iq6-36m&;+tgghuOg2I|)t9@lG^(B+zZR8E}GdLSq!rVH^e z!&X7ZRyO~|i{=hm-p$jS0I2MkIU@p1>X}z(L0zD5fP_9OcsvdY`TEP3)0Xt=WTz^> zJo{m5XupnPyWg`?{lx3nue(hRgjVHE_B0s!^T{omJv0Y{hai+B9)_B=p5ujnEKhE3 zuC~ylrewD9Z%h_qBL!a5V@n#G=mAQ$6l9H-R3#!Qr_1USZ@ESL?;7E5-`<~O3 z7^%?b^FLpFOq%27;%bSKH{A;)aGs?&lFGb$cp)xP8S7ph#;!^wqbt>NoxTs!V3V{i z>oLx~Kcre(X+BLJ<%qj;hlA!TgJ*a8bBlL$RI)o*SoTT1FE~@VitoLy0fWCpj#)*6 zxQTFDA->}`t|NjuebNrj&N`mI^nQ*N+q4TNTcIvT`e>GU z&gzvs#dGD`tb6qW*o}Ku_>8wy;<(?l;`TGVxA${{{?VlOdwF{&J2MHH*6CfETgFao zG2oedzZ$)xzE#31!&Sod*M7f6WgW7#MJ%Q7K-&%w1xgvE1LlpH){@s<|f z8rjTz=+Z3^pH&pVj12Lx$^Kd*{8^`te}1u+@~Ont9M1*lR;|-uZ*G1SkJ0Azjq>Ew)bEh*Ub;{5WSf?SWa{Vi0mU|5DRUmy zE^r5*S`ETP@yp4vFBm>#-}4r2lZt{mUL$U@Tw?R-5f#L$v;tcKb82{deIlILr;hB+ zB*Yttp5lkvpQ-l3s+ou4S5$Na{32ECBLwR&6~62M!r`|aY;0^OuYkPCBawmc>6G_|8a-u){lF}$+%J0J!Ua@pD#}QEP8hVJOtrnen`Qqj z75T3Cpu4=OsZklt4_oZ;{PS}sIo`r{8l$`B#^+knPkpX2__SQxPI`F6!QRvr8U&oo3 zas<+RXQ6%0DWB}@Y!ADZg52C=(9@scM{K{m+=6pgwY;>jg@t83)us!L-w$Nq>c?6b! z9jJextf{GQ-4cbZ5#KonO~C<@1LYOAd#=}<2_cIPG$E<*8%I7(1RaEE;MG(aiz22Hf0n&Xp`IxG3GX>leYyuYv@;g8D)pwT6b? z191rniLRBp`g&pKp_7Z_zAJC=0dg-kazP$cY3`?52bvAJgunj5y?fW7sI8(v&PvIR z!#<5JH$!V4^gu#IIs#4;X{d8} za?D|@Q|#gBGz^-zZ{Pm#=uKbGw(6vW{yA@D?cb*}RCitJuVMUB>LIkQC3!#1y>PGxm#N-(_d{WCntB?=O8FJk(VNuqoX}8ybao7 zP~juhQ|j&yvn>&*X{a-QALwV|@4|5QWt)HZwdYSxsNvJ=;CQ?Ows>~=n_S#shgEo` zS5FOJBt7atdHFUzE|gqYP!Q#@ZNSxkwpyiUy&Dz^Zbme7hv)1s0jb|PH^KJ4VOLR> zbM*D}paV8-`@I50Tt4$$M=`hX+_7Ia?)LtICdEU^l{&`Mxn9MkS{jV%SEEs8PtT~` zJLK2z>|WH(6jFOLG&Hij){{OHpj=Up=u9ot!_XfLo}XPeKRxUZnEwb+;O|VkS!B&U zmZHyWu|7(3Wn-2W7D?m9Y!Bx4967QRG`afyU2cc@X{*r6GqOupSPq`w;8*wE8rojd z3XVbYeC~yh@t<$+dFP#DXQCR(zsFnu%MJEK#(jE^AUe;@f2fydW?wj>6iM{TK`M9q zGezK0U$NCFBGlCxKYJ_KQLU;#1;y5!`$1>vJ)@P=(bZ-7F#M0ReiTMt;Bj?P`A76>J zICHXx#XxZHj5|aepx?ju+o$5ii?YI}zIz-`zzGr4h#udkJ}f$>>AlO-Jsz&5-_Yd|f$^S}$EPseiJ-_<%=g;O4G%0>igemWB z8U9R8-fJQ|8AQ!(ewq(1&P+ndvGKMn7WBBQ=pbiMoey9BN}-$`wR#*{s84ucZJYslQZvG*^UBjk#P+9lJUVRtdbQW@ zcT5uf>3khoqmymfCYdcx+S}C2tS4_mGzZZO%qgFzLm`tpd6FK5_YOS5n@mD|@Cgm_ zE^b-1YSq`izI&PH-WMw22u?s-|4~#|d}$%UciAwN)B0+dtJ@OV-FLJmib&v>cu}jZ zXm&wO$*OPa>eS}Tpj@zl=;$52CQm$w98)P+Cp)1?Xv_Y_&T&IKptQ<|^Ec)1<>>FJ z_G!?1*U+#Dddr)5DbHt6TlO2}DN6|UQ(3^+bsq1L>MDhXeKjyJP_*wY)Mr8kiNxR5 z)~3Cy^isD3pI=HX@1FS-gPhaF&L;+^Y>OQGBbF~Ur)&g!xrtJ8W@UNlu*(QTyo~Sd zjt&C|3^Wj4uEM(n(h_TP%DA;dcoir()c7~|g)QOKz&x~mguSIb-V9ithOaMjpZlDd;G1T+Cpw&zk^~;4y|$FC5gei@$vClh2etYFwP%^P67P9 zBX0Pw?xS&%?jpT`n>TLWTs<}Tk$81z$3uHQqMlo^N}ud`Q~Bx@hrax?^SkC>g2^9g z;NHvNc?r(yfVF!_rMdg&qb@(*Pz@+T{Yg4AhV>xi{O{Zx@gafbHv$a>;Hx%cONk+{ z@$eMk-bphI6&w;5ZFNXjVxH2Ei=R1^x%GO^;*q#r_lx!J*NZu+YVhn_*!} za0ne|=#!*JAL5``qTJq>_LczDg0x6_r6stuvYHyvNr=mzJbSvf_GuPVg>*yU|B+m8 z%OC$|V=%DfKNk<;{j=S)z&57sc;FM|o(_I?i`MRL6OWJ21%bLlAS+S7_Zn|QlNa;( z@x~Ld(7@)aLc;OetbE>Pa0SbSfReo_BE8#Ea^eR=#xLmX0cE6|#7AbGja?<;Sw#{R zl%(TRi!G0iQS)7n?&^oor#dg_`%-?FU9(e7Q`1%{kLgz^C%<8=P`$V1m2n!lr7dCN zc?&y`o^lza3|_-X{!>HU44o6qzeEeW%5jdgzZ$n&9+W|YR;CJp;hxdm15NfdFh}0N z;J0s7Q0)L7ElG4_;ZK&R&M?Z1Zd({!8c5T0esxQ!!F|(DLmNWIkLn-#d6beRZ@azU zv?iej4Fv#1b$l)Q@VoZb)-iAw^Y$&d`iZTb4U5_*@tx%+K={MO3(;P>bScfaP>^y4y%>3k zn+;R^n=~`_xyu?Pyiv68cy?@TA`<%KOUHgX%0=6k%_*+qJrqB}Z8~x-L7tcZgcNRW z7x>4@R&aK9zC8uvbZ?adB*s%L%HB)!V(@$;7G7IFep&zGkB<1kgVYqnQ!duYmyZEK z=0eGriar?w)Y+2B55ZV}V+M@&%PuaFlPBBdk7BK`s2{z}^G|*mjO|%@Y2C{IaB?3+ z9?{cD2?^{4wx7AA4kTJJ1fvX5Q3Op(g8+Wg)e9k7QeqZl%a?vYV&L%BpQXrKTk~J} z^(Eb5m$TG>v?6)@jKKE89i2r^d$qN-lX@C%-@Z+J@|w9H)ozZC&pX$)o0fZ}Pv35h zYwGWy=AgN(Gu)CMKE0_6pZZ7bvsN(4V`#!u7ugxP_L+=?K=c|u6stceVM2Qq--3d< z*aSad_?U?MxUsIBynOPkYi4oV`>^7@g)YyfvoHJ@rS>wKu|g$CI`ap*U3jJ>-IJc- zPmlelrAEgMP518Ez56R_GXG#$aBwBG4yQ7&q9@M(NSW@{|4B>$QIcRnLA;w`nfC%I zk9U`UHK%G8h(gSg`*3d$6{XjEb{9@|YObrPn%V|LuWnFLuleCARDR7702tXMZs0V0 zl4kdEJYY@^4vOq~`Dgk0`EBm7%>BSHsNS1A5-U6_vU721DLHjk>DG>`Vm56TZJ+r} z{#Yj|DS5zs@&~KHD7rlMD&}bfSMa+aC^M|e7-;t@vOA*Oi1A&@WGO$C=06c5UKl_4|355bR@4l4hm4Z27m;(-d z3P_>!XWSAN7Vao<&8f0f{JqQDkX`a_nD*s=^nW*Z_evxfD5Mr!XY~@21h!!qV{Po!V?h~*

w=gpr2*HXj_z71Z_C zO{1tfx|EoTgSurF(8l3o#f(a9de8bxXZce^SXfw%srymaUvCTSx>^RzZEW_HWzK*u z-#_A%QYA5te)#!XPT;0pO$KI?o`2LvitguUw;nIwMV02AIu6jARLJ()q}cgxb~lOn($yV42+kvj5 zn#2HG{_&=T;K@YnH`G=!Mr!Q$_b{0d>b0AKo;^F@0A)TGl8LZ&3omg=lEG(mYpPpY zbz@U*LY7h;2Jgrc3H9;-?0gH0FV6=7pm>3>|FYRrpQjt;fLH8W>V;cT)85r(Jh*ti z?mlNd;uB(_wJn1M401tG3+NC8;j_atePzjG7p2yc2dvU- zxpxUEj}#7s;P{h+LqclwHq8kSj*MhDdJ~74tP8X>LDC>m>=b2X|54u~?ETklL~ij$ z)8gcJD!=>UcCobU*Kb3%aSQ(ULGSk0uV>6roGG;2+}y%me=p+zZ6f|D0)ZSzn+W*K z5AW3~yktTpyB}eZE{~zfqj!isnLJJ0Km@|JLy2NQPwm5Lv_KBeV)Dm_f7O39E|mgF z1S|#n8&lL-{&{6$fByW;?J+&{*!kxwdWgI2MuNJcyWYUfiI}u+dm2MJA+a6Fc1!U7 z6LSE93?Rwf5qG-^e-wGGTnj}$O^O8&m{ILtfSD7+HZ5tKSY-uB;vk!8#1TO^yHhX& zSxjd@7|A)lFhm1YT$+ie1p^mWafa?bzIrH>&-;aK+V)$`!=vO!2jku41pTtHn2kMi z@$ev`g{S}i{mbk9VFFI+n(7Gp1>EB*u^w5a<~YDT6wBH%UB=kC+Q(U@`0OzK^WHqh9sX$%jiXcQG|jQ znOznoWG0BZ2~C_!;pE9XApF5Y&8cAsT5W;Gs2AH?TdNAWfEJOMTmaxL3=F&Rt#9#S z@17{$O!A=kbrd;W&=41hf*Ha7HirN0XcH^Dn@k|X@}6;&=|^G3+gRI+Lb(o#yJyA>K5ieei!BRjGcJK$e` z7F@H4_=G?buW?FsvW#?N%}kC=W&ZjAy460qlQZBVO0W*QZD-zz(-l4@MyW$XL#$bW z+10lN+ig4lcse`h<(X0xO0JCkV>@r!w4HdQ0F!;-Tnxm>M)9+NGNUpzn%ILx%64Q! zELzgEUgH2~tOd^5UA{bTgWx3}g;rHnHFEGRmi~l$*sh~`&~fC5tq+kc2-{zUY|^dnE5YJjwq@s8{1-~H8eC{_x73%x@?mR zlJ=gbrC1<)d?Z$r@txmgY*>=x=zRWXv|SS-{;_k?N7m7<=X?ullXToxot>JR8c#P# z)Lw)7HB?E9Utfwy{%3R&$xuZL3yT6t*r%xsyzl3KPfs&IM2TGXfv#B~nMV_l@13ES z#gD&_7U%_!O+Ie;E-d5tHyn6kF*S=UA>LM>5MHIp833Oxze#@Uh%^E(eSkcl2Yi9% zt9nlXXum@dORkvW=3s@hLi0h_8x?tP_UWQ~iM^G0Hv8-C&!3MPqvFtQYHE7>_xE?P zOVyxEWc%d1leRbZd)3)B77U{_x>qY8i>ECuWvpN0LpS+7F467Rr~bsSss-5peEo)r z$2Vq8J%thK@kqAy0_vqv*m^svZ?H7J4Gg4Q*$J(<54yF{=qy4ojTOGW4+}1U(nbzM zuB{XGCJrDfQ2wQZy71sdO=XH-z6`337P2{A9k)6F-Vsn5F}@}2!oEC>E?@OpjuvTK zexR%5+*@czOz4UHgC9Q@?>*%5oZh`8ev4cX8t{o@{ZH9VKE80I*-5{hYwBt=0K$;- z&roJD{;oyHO`;~zA7r3uTA)_**ZQI_X+2R`m%lKIBcWAXt+gk0VdPZNzn;$<&cuIG zs;C5S(XKE5oiO)W7%Lq5^q4v>F7C|v^J`FpJ3fDYj6RO`%u1*sDwxU}Ih&d(#03t! z{$fR$zY6e{2-bZ-T>J)7l6c4=B)tzU9yn?p30+@tt6sU5s+rJ-$3gJ}I)PqpI_QpV zx(U&0ieE)VMW<_ce>4<1bgtE?w6Bv+gcIopoqQ`O09xfSbU;lk>UmAh!*IQc%ku8d%{0VG;7LyLy0Dd-Nu&=mZ(dt$SXX1{#*`x z{440DjEoGa$;^L@k&1}CJU4l+E>>*4n01R^Ow3L=y0`IR$s7Zm0qU(=wF^T;DBj|%8B+rWNd!CzH31yjR`xv31N1BhUJ=!QK?5q;b zv|!Tif~h1JA&N^dblj*{Srt;J% zpAM}E_(ur}8zwKf?84lq2EO4ST!f?1DOnT2=TFuJ>IQJ12pZT)>|HXk0QB+>8#q0m z6|9WJqk&NT2@)$?J9TP?FTv?KGKMGTjia6FKoKtYyimOy&m?JoPxq?DA1et?%fMy z@CXlas6B@rerYfQk0X``&lwMDF8!MYJkAqvgQ=Gu{b|dRFTfj#`I=w4bQ>cKk(IS^ z6EidK5Zk4kE&rZ{IIah8zi%`2hHp>_Hv+Zl0dh?|*Xk020axZrsx7 z%QAWv2!@NF_CM|-*Ssi^-$6IdTb`prsQwzzQH}Y27#6~ypr!6pYbetqbs~&*ga@KN zA)(GbXXYbC*`j4``S72)2Ab<+D2R)z=>y8(`V!ie6X=iED=K7&0fs}6h=>Z@wklAM zXGntu1O`%5;7W!;>}Pbe8n&Fq(86x?Lv&CuIv%j!Gj?{N!Q1w3ztR{KwCrTCmk=i{ z^E0yd$p;?dW@2Q#Zss$mI5rKwY6%erlUp4~h19L6bqg)A)1j)@a3%*c7-F+S*(UGnOLzDB zlEVrLD^^d#iB^@y@gacz$+Pt&<^13N%wIU!LBm1T~mUsXAmH%3g z|6Ps$S_=Hf|AW=g+A=UYx;{p24gN7R>wPV#ag`a9ka@oV8m^nfd65)5GAwZR9zVM| zsdZ<391%>6Q4Vbrf$uQfft-PYIa44Gg|tQKvY_O32m7vUlcQEK>?Mmm(kTCTCKC=` zxZN|g7l|AwdRm)EjK27<@PPxbaN6jSStnWUJY`37Ax#{0S`IcGfd;lnBRDuXT-R>U z>_#<-D6rQMN<)Tx3oGj_$Rid%-rr3-mAdKHty>*KYVMP8$)TmJ#XLho{-K zTEa>YUX2`V{o=*&kuhYWvY`w_h0D&)3eeXlW_x|P#l;II#3_tvK)X+k3pY`(*&qva zg6J_78JbDgrh$s^4!@1ABnYS9Ft*`0DAKcn0@MYuufxN^XfKQxKQs6X8_Ez6_2Pv* zT+9QPz8pqWqgb}~_&fxR6#|O>!tkdn;Z@LiPZF%%ffbk_dUjJ@LD5gkoXKzK)bHgb zPeNMfjcKXO%*-(7R!Fa!jJCZI8H}LYx2v#Lmmy_P{LseMY-YO!7xfWbyX&WGGy}uJ zGz%)=vZYTgq5}s72RAyU4^{kvhoW8f9pfM1<=Zb^9g?|P{cw5DzjKw<%ncX6%;>bpGJCV~|Q-T<__*ASYma{P-G#5O2aeBFSZgk}CphHte7PQz9q1-TE`{h2^2|J*v5!%=U{JP|oxw*L$Z#Ha?g69B8 zd2IeDrbl7eyhHkeNIZ~Hsgd#|jTOBFU0p)_$0!FsKfm$j9<=)J087AM&o$iPr0iQV zDfe;*6{JG);FG>^urf7v{>qmFpCgU}Khx6Eo;)hW%S#K`NrP}yFB{RQ9u$x)=ZXxh zE`kr{iEuOF8Uc~Jx8ao-og4Uu0QQEPHCxT@%_1F5>Xq#J@nAQTwPXGM84fKlF z@cHg|N%w0I8T!ECkeI0^Gfi+b9z(06(H6w?jNoYgF|v+GQy>{<+f%v|N$tb#leuF4!+A?!{gv?Bm)GH5#b@Y0&N5oq7ok$GhHh*iZi;R)59!B9M9gC5dNV9 zuFVnfzHl)(h<62(iAmh<_F6c3BnC$$;=dD|7$p2e5W_xwIJctZB?FKvh$jT8$`3v< z`A$$7IdH=D*>@5+uvLfUTyYeeda+Ru*ooJ0$4{&!BXbCP5rRREHNp%L@xQn+J?xLL zV^v@>L*O94S-Dp>+fzpe2d%kF-|fri))3%-`SJm2ufKlK7)cQPqr>vF6%b`rw3$yZ z`4$io?btsMq$-=6w}ES|p)CHcW+%O!k;`Rp0qV^6%$#yLJc2tI&I7a|5paarJg-J( z!MoG1T;O-_!6k$;DsVGS7-9RotB{s^% z;9D3%YIc_q@gVLsg#-k^ATc%l_>StI-&^IBj~N3(>^Ck@V_sGF@#AI)2@?I^2L}TY zY-U4JK@pgEGOHBnm*MC6`MM(?d|(E`Red=@r!ZH8SQ0#@n(Z6bP=Mr<0~>%J=$>Ch z6pR|fOO=3z$rX^y41+`}g%P+>B!G-^VG}XC6Ujm6^=X9@d0S0t6L^S-L0xZN3l_M35jEu-zntRshV& zJlZF?orEQVC6Uqz@f;bQoKt=-z_dFg-40+nef7Unf|ZSpTQH_n{pgtgGXzDEp&UKU zqmCeo;700x%z8pI!pzZX%EaH_maIpws0r0J7Sb{DV<7#6R29G{;bSjfzI;T8_kjZ^ zKOYsq)E_Y38%&~}gd_?2L8d)HMPI>*a6oh^vbMx)P6VzE{z092bZTexenYN$ZgwBt zNQf|Mafi16xL?c45<|JMAymW=Gd4PncM`&hC(d`PMGHK8L271!oouyv%fYHQV3}tOD!h76cfbVngYIU5 zaVW^xM{$Pk*!A4yROFkt=4%76qn9DJ}meqb3JO@Sk74(!(82&&I>J28g z#l^)j?;s9ui#ViK0=_7ANt3m;KSoDyLT-D5y$OkcUrE7zZgLONF(N0d&lA{R+^blY=LtXOX|j64w}Kj4=EgU(n93vo17WJ`P=ua7eflX)i_J_5122wh=*m*oSn;a&tR@2xJ^eE?_aT%^?stH; zLh?QW#szP&3YbYxa~|LGaTY&A5y|n^HF3)H+@oPSuBh96bnHvfHuyKkVK+ijd2+>I zy7WuMK9DeW%(`9wH=;ytdLZNR$R4{O>VgbM7)TZkKEyQ~30qrRqFdF)OR*qz^ufUg zy7X(rK0aaiBz;#F;lW>u4q}QbeKKB`k#jW=&UL(#L^6X~P@rcWz5wu`^Gj2AmO6vv z{SfU`nY{s2bsU;ANk1>n4JIH0mR>XGTqdKA$OleXD5z~BUYgvWf$fCg=LC!p>#T#i ztxPDp6B84D(AFaoaCra>%QWGfxEdSJG(neug1pJz#6DVoD1dc(>=&82B+<&D8z{k& zN){KK^ZtvVfFBaUHv7Ba<_h~Py6NPY?t;EZ)IUa_A*R0yeTAQ*67_iV!9zmYderSzj zj`~0dCK5>#Q&Vs7y)J^=8kLUG-#re=`w3>IVV>pZGotd!(Q=ZJKnhfBNF6jrJUEzE zpEt8fesVAW$KcRS*H*MEMpIdb*fw8Qh9(}ZB0Q3+{^XRD5WZ6>v}hus?qW#)6Cv*$ z8@BZm9CqfxXkRio^sw+c58j7%Jk}Av3~|n+m5qqAx4UE5^TZsPkQ3^U3}1ar#YxY*rJCUJiM{>>llu=v-3uNyks^@M~3)QgW%fUser9}%z(z56H^{7&bzCn@b|UCC7eCZ-pZI!Jqr(5`%r*Q-lLRJ9X^{lUpYp(*ssK_&VNf!g z3~p#As{}(h>9;oNN8)W?hEDpEW(kauo_}EWdp0E_yUyb&F%!{I((253!h>^Rs7@ z96e)zBM1xP3_!q@2|-f*5<56;kDr%EWuAO|O{2ORH(;br6@=Wry&bRqdGftS+*%r% zbr3M}Yr$Bu-KK}`w)K9%T8jPJ`>ywDbaXT^_8=)*M|YudZ`rjg%fY+V5avUKCSn+l zk!!bX>5sSY&aS^kvpy-WVsTQnJX-yC=gF@81{~VtJZxs&SiK#yp>n(l_Wswdkph!v zdTJz5{n#ZGb>=DJZ<=OT2*JEWHF7SN)-DayvmI=l)+!B)Y_2X#>nm$UI!qBdsm?&6 zGW+bf9PQnxsFXY}-Rd^`ilwMAk3&LG__7=xKo-(~amsI!Xg!Hg<;BvEQ7hXKBuExc zK>8^JEX%4ph1^0To>MRu9v?4Qb$eH4vx7K5k?qu|hY$-Z>wZX+nZmy4q7Fl!IP=}l zmPun)vAoLTD+UUiUYfWhhy6Ke=Mo@jCAP%IAd~S+9og+YkaLro7c-J@UeMyL9=al5 z&Lu3YHCFtg8QH=R5|v#nL;X`Zd-g0X%$p*|PuPl2Gf5;-=4+Kp{ zATU-;7}l3EPbLtnraL|Ej0o34jK?5VE*Y+*$igwzLjV3!?#+VVh7g(na_KYA)x1?t zoHzymK|;ZbsPJB*YmW)E&c z7`vxr9$7n5x>i&p> zlucB}vuBAY{Rk0JavcxMdJ>F94I55(&_!2l#4mV=Ly{VTZh`s#=3ch8$=JXZVN9 zn8k(#&LJko=;Pyq2!g=zFt#j_Fv#+jIC97&mXO&-B3u9Y^M~Y1(ei>&PQN1mGSrg3 z4PgW=a30pIeRm84$yG!sx-=Bgi=QY6iYS0Iow2pO1^PxI30G$AhPDwt z=-97|8;k>JVE($prA3-y7Tl_0b@BkVkx%UV<1G&YisriHZkzdm;PvfLKx1;_Y zC+UbssKGl#&eD>_V`QX(WB_;P_?YlQx*Lw#w285hwjCBS5L=T_mK{g0?1zErpOm!w zCm;@a?kXyDH#Y3N1}gljws!s1t5;!&ixK#W?fw?29|M*Qw)36TP7&$BZ3dRS~yn(Brtk z1XZ;{v}O|=xAa5g$|~fTf|2d~h9}ff?6Qw+L~Pih{IzqyB8@LDXd>?U3eE&JJGoru z^3uG^ezS5`s6E6aK@Yuw!#v14j+g+6#!^I-Tw{*9X7|(}*DxO$7qLu=rm-l>ip*V%SYov{-t)H&rc$ zYbF5UP4q1$Ks<19Ke%ONOoxm^cg}hY>VT-?)(Ah?+6Z;J^Nok@zK;KYf;!tme z4A5!b0J)?%{Qeq1v|!ZW*EqE}=|LeOukqSIF~3;{ z1(nH}h|ME>O?W1eWC(Lb&}M9t3$~7L3<>81hR+%bR2R$ z8>^%nJ<8H``c+6t9z1Pg(g>1&I@-i+?D718b2SXOEs6;V`MZC}=;HHkC)8Kd;c-kn zC8C591DHZe3fYnH@yzY)1iCZHr;u2F!7S}{7%9?K6mlsKmKio24IVzi8ssh-Oh0xM z*s=8W^$`+rWtsvBE))?8a$Lw^#9-wu%ajn80uf{1un93g>-mA_|W>9DaTsU{Y zWBOuCmN65KJLt_?jQacG){$GxVszLd4Q(m3Lpi3H?22pgF~pcX1}7PSYyDwSL!EE{ z`^LN|9r~1hNsj`1*6W*!+q{{v*gR4*;rX*V*lEdDFrav!iS!o)VlW+G-6Yl(BY`T- zc-Il}XP3YW5VDv#k;|hVn|8VZ3&@>Jq8i9vevIPb=9c%uys>sM^rPce9!5s7uoUmc zM|sXoCU?%Dwndu$xoYt-4(rILj>-@(Y6lWfR8zs>L0c;WSw{N-x#Y!Tqy>e5NVMmQ zF^8(Gd5r}J4TX?j-51nlI^sMYw^-CTE_#TlDZuWsZDvg9h(}COSzWz8Y5;rR0_qI& ztbgCBOxg4H_1IYH0?~cp!+7kv4+I4S)KLt@yH>t`|9%}>2oC%fT(1{UGY2h#J5(IE zPw}C#W?eCiJv)}3654m_we0ziV!9Q{I14IP)@@JIMabnR!zb!d@q#e>o}pHVo7>E7 zZTI`trOr6v-Y^~hum=wwkV{b>`~ipW!#4c9oDk0TnnHMwZg#lEIGN?ZdsVoD~mOUj=u@xQEv64`O=_XP^&uK65mg)*N<~M1mpG%F*M?@TkEFytagW`iRTD zd-u2?Q$NiUlF4U#4Y?fy3_3bG@inxvqBd=6D1K*EQ{6U^u|Hgtu{6_eJB%nYADSdL zg%;cM_s`DR>TOGfFkMY1eD zu2QcEkDI2CYM+_hqQKBy;cGUcbp~_Poyu3dt@R`NY<870SRo;p8pFh?TXpdV9#-7hKrf>w%V?*A_ zK2%>{uXyt0$pTSS?I#!$Q|Yg*pP5CEN}hx9k#?m!;QdcUKK82`?Tas(2FotCu>~V3 zxRsBO&pJ9QXJHm9BcF*S6w!04PE!NwTE)&*$&>xbffG<%le~%%jGFAwmRxRWT_<<; z4{xOAIKtUA3)9_kByCuzjlwpz?E*amVy!;L=}BzhcoD!{5g%N^%wX;vD1`QRBEDlY`hzxC}B3H#C8~%ZFll; zPgzN)OVjZLRG=Zw{i#{%-22|$O~C9|+Le|k2nj|?85bg*UT=o$b|USpuq#x`yrTt- zQyh1OEPTY=6D#0>aV+#|Rqog$M~}iF@FXd>NN#$hZj5TiA!(c{QuA(6#pO5=H7yt> zjjVpySMa{R?5LKxh3tK0f4B%L^7VR84YZic?;;mh9O*faHyv>!^AN^qU=m2wD*?fz8uZ#MV4%hr`6OQbS^A>KHnMK0 zpepj2ztFd=Jj}Zf@yFXkJrt8ZoSY+0!x_O&F@1UYan)okb#+nM@-jv;ljriwEL0!H zo$ei-GS&Irla6FHvp>gtsqZ+VoO!3!+hw8fpPFCl7w1@<>{m_y3HZ!xHbtma^0^Ey zdln~0$EKI{$SaeHry0|>1-o|(%1lg5S=FWdovU6?a%Lf>^YDA=^-n&&#h$a5tUS;V zMB73`a&FXiit`0HdYjwZ=S&{c=Zr9$xtS{nSX9Zx>o8_=kz;qt~F??3|;(}%1rgixxa7+u%b zI8lYvBs8r$3b9!;ORI5*dYXS-M{$QC@PK166xw6!AE zCR)ds;LxnBnR4NP?Sk1CW>3w%N+H<3QQ~E@x}SfhfBd+Y0pkUa%Q|6(uhJG`tu(;4 z0=yWrzH=7y3eyH24z-%q=JTcY+t)B_8_#_+n$~~dz;U!)K9OCgoRA9G)ojtig+BSp zGuG!!rhif_Z(Majr|0^0pNi%>H~FjX$X1i)sM0o1aT}mDIJ#FK+>T_p>?2wUVKl>D zYc4}O1w6M840@jwU8SB-2Cs8Xq(5|hw%+ZAd&HrYWd+J3{f)0vp7<0b8#6YUWcISM zt&#-dp~nmdOVP|dd@(J_$m#q>3L>*lYk&;k@zw(>-Ky0C#70~E2j|YsT>a@)*jCI! z60AhZ7zF!U#%)|g85k-R7FY2;QfJl6Yrewwzc z9UX{+VXs2H;6Ep*3~p+Q}yYU|ucL9h@1H%?a>H zD){7LlJcJJeqiH{Kk@l0wa7dq8f_NuP$8((p?LE-7>A{+lMibi7rfCgL_iYR|%BG;T#+>wKxfJ zc4LCw#q$+C?Osk+UgXqVb%4M#yXY_V)3Qcsj|24==GLHfZyl&R#pi<#M|s};HZE!E zHKvE?G#@=Ne?m7%rZ-9xmcR6!+#+Y&s|W8Sk)<&$4Z^;}(MBu6Kbx?ZP=Tq}T0z3) z`M{x4U@x>n%deRw)<&7$s_03h=*kfofw-C>0qq?=I0iDccB9gAQyxrG&*~{1(T&ajVb3Q|nq*l26PLW5)Yni7Q6v?-+mf~d=<=P*`E`XVPA4S|j4;LG$2Xr+ zJwl7-Lt+4eSuu-#KgB9YQ z*nP{WnJvB@o!RiM*1Bcwi$+{7EWbyB%MIzCy7Z&zBA$+=u-0g?pJ2R;&&apJ!qG4s zHl8}(J2St*9DgQt30rK&9KT^X@20WwQ2SrQ<)SjQ7TXSbi`A`_WZ2Dx%HBG!eE96l z?9O1*%+wbz(GpSun5!Lk-U>!Oqbvm61 zJmSUc(k{LTzc6VA!`K1SnlxzO60v4j?`z>MJ^k?k_3Ep*6wr3o3Ye657AO?bP=hW! z+tvZzg0@P#DK12F4WW?=I)>%}?L$ih9fyEY&e?7;-+p^@UMq1P>5C_^dvcx4JO7Sc zZ{(A=Ffee?9znE{+7@o{U7f&%1#q3PtkqO-#99z%I@FS9OXw10#4&U?n>_r7mf2{# zxm3CwuPEiI)K_YQ=cQxP+>swkwlB{2&MIwD_;Y}BWSzO&FCBAD-^|edQGvzzPdp@y zgX(2!{8fAj{91amb#HI&B&{0rt-yL*lGDtlE#j<=G+nTa)Q_jj%k!$hLVwW%Nr)3}|C@+CX~lgv9F%{&1x}{MeGhIf}F+m>c2T1S|GF{mO5h zH>9X!Xcpk{3k)-0%MY^GwLzVhtF~@^!Sdepi*x%*;=L8!+STAh(3o$$Mpp)&!XAqETdyCit-dYOJ6_8Y9I> zj2f=%eppC5{jNdTU(CqHLG+ywy4jq2tc+qgA!n4zB9+Vnaps+JqK^=%hPqBYo-ujB zyIy*|6=w(mtr9W!V_rQwr@X6np`nwhD;_Bz5{iclSDKFqgygui1dHmtYhrQCOGZ&i zZ-8YBqf?pQwb!kv73k&_J$m#w+G9>*T1j}5`x4Hno>wWh^VT%a-fDMUD>Kt=RMG%9 znY1!5?nR$BC(OR8t&p_9_TFb^?q)D@mhz%HX*n<4sHvAfZqrr!!i`mzZ`oVkK9&?y zvVGbyEtf~)m%cE}dPbIIbRHnq+cvIaEQaBbIx*=X*i};b^lj@Zak>-xc|p`UMjxQ* zB&?dtff!FYMPaaS;g&mxZ#Kf6z7@m1R*9|D)z5&?-NHEnm4}oi;=(lLLR{|pS;cSK z&OedaB$5NX>Py+>L+K$#6d-H1Mh|eiu`A;1RMkX3AD`v!et}yb0WDKHMB!&5FCi>4 zQ085Pv-vmH)}xfP1ROVL65m|4i|(tDK6{x^6)&2Vzw|cDrb#Vgg#`}iP#C_}=B)MK zJ?CBZDPB~tf5M`Ds?K_g+)B*!7q#9q%UMsOyLjzz0Z5zw9A4KYjR`H&hM!=00v7{^ zUo3dEauF!f^pG7nFE1QV?z->NrAzS`X)8_|v~H)fp}j%u(+07JIiwElwmQwW9{JAzCXwy~ya|KgLA$8o240 zx*hB=AQEtZ=CT$l(-#Z+sCT=SAr4Angm*2j^}yA;(V4Z9dqR5k$(S6p&$V~V@f#r( zOlX1pU%&FbJIOO}AUhil>cC3#*jC3YFoiARxq@$Xt-Pt5GNyscPo2kkeUB#Q9d1{V zzsxLnK~{yZ%E zoMNNZVDMBLCBGtu+U=(UmC7wrbzk~)+pPu#e)`SJ_E^5^TBV|0U-(hvn~&$iHZqRh zjT(hpyoW)^DXM)bks*Ren5J&SKPjCc_TyJY{-)~ zL+R<|?Jdqd{B#ESh2iNqDrsu#Iy}3A+2)VAr#2|M!>>&pyLYcoN@N;ya}#bK*TdVY z?9-0zVb`_?H+js$g{P7~>{HFz`c<=%e(R9sDNNGln>U$bm*3i^<~V}BA!F*=d-Pr{ z8x!>5!?M=qyB!>^vCd&Ud@LDVa>6=x>vnEpcpY86udO!)e+j#$)vflx`h{Dsik%Ym zssuLSGSV$ccIv$(qG5+xJir!T-wxs9XK%gl9TnE%Wm08m`cRb@zv8p|sTS3*Uq7L| zdvmMICdU+ym2!WI4Y{yq0b%#KjH(zl-J$>jp!V<%yL9p5H;}A0H?$F^V16er*r?V#*v_2{GTESP)}{HXRKw1G z^ML9MCeRbiS+f{>!DFW*x-YnKk;M98kEP9=Gne6WJ&&;4CekDZJxB>vw^!s1*0Qc( zx-oggj7^#_-3YEvXPu;ycA|XDUY#06!Q63Cx12Rq>aAKuF)tnYaeCjvQiuE0D=9Ys z@XpP;fX#DO%s)*>2ks1b%l}oQbnQAKs~rNt;zJ#6b~JhzxvX-T(d*pR>(`6kH60LZ z%*2UxQK# zicTSSYC!Mo-U*%zdMA!)KFBO#Y{rI|^5GGRtlUH3?Q5oBt9Rk%2d@%wg9E_8nmws} z+8C7NI$Dj_2k5An&p6z!)G;I>sbu-I%PlRZx+G@FOoN)^0k?VsLsZMpFY19g-)Lw5 zvu~13i;tBjBfh4omOAxLDlEU18uzH*o~Esw>e=PhB%9~V{|3x;`?H7pO!^lr6FcKl zeu&U4^Bq!~%hx$c7w7FlM~@)Nrga8ybFFsgJxyHrg*O&rgH7)>N(d?p9v+KdzS&Tk7991 z1jGchd3eUpIz3@t$Lg=qWZ%x@&4UCghkVEiGAfjVJ5fVbk6%PT0{6H0L&nmFn!)YIXS+h(;mU@%1OzN6PdUch&sw(efLO!f#c)`VNqO3}1! zTly-W(Y;+a7oNSX$aCB)eAmRQD(%)*8{A)`Xi3l+(KW_H*GaXktTPk>Fy-ebX8&qm zU-cj8#%#?{dRS55I+INf&Jpb(?JS#b9y8T=Bj##U6f&v%LaQ$%^Rc(j{1{mn^Q-*8 z+lNjS;}?C({dI-T&G2xie*Fw{4||o_y_=2+1B95Gu-Fwx^Lu4wCcR6(e7ix$VxwtG zD*SoC4cmVsJkH{u56Qcle7v^j46j25KxTG>3ibKuZ3&ZJx?L-|i3d zGKs;+&M*kN^+WydC|HSxHJcm`O|u_$0kF{Wq9-s_t$eiJOxhaP;(pE5i7Z#=hoHk9B&zje;JL5xaWQ>$*6CHk!$vrG2&uZ%dfov?kxbS?olbGmy|V$I<}=ep5ntvWnRKbHptj`H3U+` zwcf;~u|!xaw?OWDwQbRRZYpm4aq$})q}%zpDk1Jjd4{Wnq zoZ-Gg*F9N$Ismjlf5ebk78n(s9vi+XyRk4mDXY#+^`yr`?ree)U#nQ}gVg68vDH}C zDthEhIH#_D#FBdOJW!c5Xd&ymo?92GwP>*qgmRUg$!rfuACvFtFAB2`8o8|=J`G$+ z|IpB3)db8+npJw|hQj)bi7hNG6;9uS#gUK<6C4^M>?!Y-y1?5JDm5rBVYi~&cirRL zs(y{w+gQUbyMuRDq(%Fpy$fe8i{X;>1(v*IlT~~xF)2upUN{Aa87br9^j5kLh9@Iz z9=0I7DrsK{Eag0YLcInJ#*7>}^2Px0&3h2gg<~GxJCxeiP+PoeO@x1w`)+A7z+-oSrKQO^6 zhtg}-q23T4cJZ*U{_@oSAeZJnb^6N_aWePI{duwUn8K&DGhE&b3jbEe=R7a<+Q>x0 z6G2e1os8bLtxScPpVDp1?OAA1#b9!-(Mvu2WV{U9rx@c8IxhtlVO3Po>Ll*IT z=<0V&Es31_Z{ldtCdym>m!^aAdY?j6n~Q-6gBv5 z2@xJlz6X3BWt=c49L{_PRu0Y0aA*cbX4+6Ser5`{%AS7+D-N<4azEnM2A& zm_N@!Tah3u#CbM&WfP2#PqZ6nv=JG9j+gSowYD-F*;0`l&Dk8ZJ~J@Y7Nz%ej%AW!uY^)N`p zIMys3CNEGiX7KRoZ2qDyFc5#6!65K>p#zJV#7q+byBC1+D!?}pyHhwZ)0W2-J4_eL zof#iK?4%FLUR9y>b$zsdj-~j%C10YyV5b}{I52Yk%s?bxF+s_xlh zqYJ$weW`9#dEoCO<=Ft)R=DD#ZKKtYtMNzrbojK9Z9%%SxOozXI~b&-bCAW|Lv^kQSV>dmn@~ye>5a z!Qp4clLq7Qj)YcX;C*_mrfO=n6`~I!gbs9v}Dt0tdzNLF(JAt?xufD}zl5$^b!9U=AK}&A1V%!CX^S zSN*S`n*e zR(Ayeyo9;wX**9YDa?(S(Ly8$JK$F=1?D}`e`RC18=@`|qK?K;OL2Xmm95WH7G_)4 z3>@}ITb;P4R+N>su$-+gj--H&fUvK^{&}NO@Lh1n*iY4W2I3N2pLm_!8#o2k#uso!9+3Y; zHXo0)Aj4FH0tZ;!}Hkc>$cF`eCDualGEk2DrvM{rc1ErC=X*AgKHO(bG2uvmj4$njd?L=B;ji4O|F z>mE%ppL!3f4|-)W?2yWu(d?VH#e~5=IN@I56d@~U!{ZP|c`mVvZkiQ2GA~HDHyyXN z8AEEUFvb_H7zb04R!OfVv`{jHH^_6sdRfLmnseFO-oSlOuO-Q84uy{GHTCh}QnXh% z0Ltz!E-r5KG}5NX)a)1WuG`d27$*9a{rHg?aEkiGaB+w&zQlWAyK@5ePb_nrwrR zwa0`aJ)6R@te=7;^6c3(ECYK?C1Gwp^u4*FfRon7b(HhZTN^fSer0o-XKg zX2?M>P%hcrfI(pv!GYV)Da|!Hee|db6~hvtGI=~60P}0Djp!=eblG-xDfXgp`(roa zu!QJ)b zVotPq9v_4Ac#tnVWxYXdHC!S)O-&huMKDrhz+jIMuey(reF8}ty4{P{(jdBpDX#O?O> zmKAOQh_D?dcr7gfwC2opkcc8w>@_<1y49EtDgH{i^7Hgs^_S z=zPdC6bY-Qech5-Y*E#FcT;{wxE`ys`!n^8gbaV^@|Nqj+3xk=#tu1mq}6;5x+hF*M>8CL<6T93~e(keck@Urp&?|1aj0;_f>r}%BOK|*6f z6Fxs1y#CCC&2w#rnrHX0ON;qJ+aJ`S9~Eu-sb zE`Lo_VL(|K%sNEgEy9R=lk=F%*P=Rs`sNJ+!!hf?P=7DL+T<~IpJh9e05?#qc9N?z zh$y7(eI6bfT;pBvKxO%O)s zeze(krk%WB!hBO$IMMSUaE9XKm~dt+y(Sb5 zW2rLTnb_8F1|@ML+UyjQ)oqH5$$1CyS{6~XV9ns682TwtmdX^0f@~bLu;+|D#5sA* zX?l_v+(djLv8nVgzO0Y9Y|0hXH&ftE6RsrG3Y<-)A&xF!BIj7K0TKEA$B%x+otCeI zoA-gssbg0@fOIP6oA;xmQ7kAZ`^Pc}muge0@Ql9T7gjJ5meGEQ@D|Mg6S)R5?INKQ zHpUae;q=y_%v`W_z0o=$=L-cNdvRunju2fgXiV#_C%R0eUU$ea;tU7zL5>)riG8v} z+9=>}NF>!P4=CQ3w@w!LlZov;sq?6_M3E1fZY=!_0!6`eM7|C*K92sY(70Do5-gR5 z0NXubHj6Zb{TWhpW7Z=L4xIlPx!3s5ip1}km9$uHiY>@Xxq|~sT zZuOl0W)_6Mwo=_;jIKYsKnbV8;!e=<-bD{nd-u7|Qjv}Zf$3?I7wA;(MoaCLYxE;DqicS#^ zFlG9)VfWZ&J9Be$>d{8>wYiA1Qf3=5QAU9Bo(a=v;Aq_&H5rk=CtR zi8q!yx$+mQsu@!^;DM60+;Wj+OE=Vm7rD7fG!1*DE#A{^m~cQHD$Y%6A{Mu) z%Pt+{wwmA=JC~V-Oyv0aNqJL&V?+zfubM4sJ+g+P`DVFF?takG;8VQ9+Gu+7>%wR4~*&U8o+XKud*u>Bt2~F(DgY5)_ z?vs9K_TsPn-4TY3tTCZE?r@CBj)Sf#Gxvm0tI05Lj>n&?et|AxRKTD3uGz4*TJ1An z3&0S@Em*)0Y+|GF>yZ53R9T0?eSA3$f52X%UZHNQsX!6*G|3r9sTt0{J??Nra+wJM zfJ()iL5Bo(*+5LWBRmvzT0CP%6F8yAtM|57oul;^eG??-AtI{|6O72c}i+d5=W~<@rdUsCh>PWV<=xJh6UdJ5v@DE zvZQI0tp)GDd|9aQt-dsK|JT>fr{v2qyG`o4iy@=b1XEEv$-(0IVTH1n?ZEO)B@)_K{la@~AFIfXTgXl7wxrLWh1=CPlK(Lp+frq- zl3f1(`P0q*fB&g}zvch@-v3|!q^0U+rHPUF3K+N8S^XJ4d3V^|?ThC0-rrJ>{r(44 znI);mM#z5c|NJ@7?R(SSUO!ZZ%aQr3@_5ibv-msP7RV*lEZ0s<3h%Y;_i?KJ_wqlg zGyYuZ|NBq=UsoeXTv|v#^wbUT;Q8~XnRirggv**Bhhlnm}-(-fLUJPChlLb=2 zw^A@LX~&VAi)`y0CDjdEShxpyP{jXMx`xF`v)aL7P$&Q?y1&@?sWo7?{P8@&@a4i! zdph>|H`}b@cUtT!mZJ5o>+~JZ7AiQ9wxmQ$}WMkwL+A^|3!=lFEV$LKzN^&wx9YQ~g zsM!jpEjSZR3yt&WNu@mfSXI46Qu!z&%L%mXc^i{YVB7I9)s;X_~-A?gKP5*wCx`S(-Qp%frF6w)-S{ zkuz|@41Z9J4qfa&>GQ!<>xueNAzLhfDwS}Yy+bgk;ApH6!6Q9ywQR09(Lk|-I4R1y z*Vrc@Z|Rv;2_l5;$&5)8l=tP!q8q~2XfH1>Aca0$0a2tv&K1)PkP$I)RGd^gTUD8F zzbg5f7GOthDpwg*65hl?+HgD7+%7}~6=q47Jq*jyQ3>3GJ?0Kta=bg9BT)ps>A4{F zzzRn!`#5!QJC%SRLMs+^vF0Cnk>51=HIwM zYs>afwNmY4lBwq;y6+J_}_OGa)S71H5Iah zx3UU=OtC{eg{xfbj~mq7Atp(bl+-_qPA?wDgtmIs3~Pt5X;MemMxN<%eMz_SZy#!M z$M#an2whz)Q5J{kNIs#L_Pr(^lps-*h6T`ueQ!8j49pH;i=B*f{&+c(bA;YRJ>{BP z@_2pqo+@%jEo2x?sJ|y*B7-&RP7oSupPCAoyoV?u#j1mHYtNJ}j2;wB7dJ6`tc{bB zp)T;#6SZRH9b86O9TUwy`IDHXxR-{A6xny}>pt)nKLDkkU>6OJ;!fB_ERW6Q63EVX z4(o3IR?trLVdIz{$o%w@fp(7r`}dEc@2%!G!Vv~P>Ne}Vief4ACUw2-!>3H08c$OK z`uTxdCjMK}V8O^eiem8b8nQO9IF#!PFM9MtAI>XDA1a}||Zit23*X4D$A zPs&ntgNDeij?#EKo`m7pt0@#zx{8zG4*m#T#q5cT9B1`rdsXEYBEn!IkVq9de;d$o zOKAu8{U#yi{(a%h3+67mMC8fo93BZhc%ARWz-KAl2K*ci*sE)+Wv)9ImFbsti}1`Jy(Zrs{B;!dkLafH)yu6c}@KOUga5Wcld2GhXj$U+Jt@dFS0ELTFXz@Q9T8 zug5)Kl^-~!_=wZb`35hC?e4+Cjgo7Z<_2jh-iPK#+~f0m9bLz4PmYl6ed37_09>&H zEa)DxQ@%BOm}7+Fe=LKkfSCBg$0r{Mt&z@m9%Jzq3=@^b91S$Bh63%KXljK{NDDTa z`>z-oa1pd;#{58v_p=-d+0lrk&) ztwXwUgvpUp1=mEtW1o`4Mc=-Bv(G6in*<<6ZrAt$MS7kPtEXI-Wt%;&3H) z@=%|jk_bYN1hoUOA^-o`pm1iELOw!6eTeeDhN2vd6Y;vZ6o6B~-x`C1mZ+rhk&~Y2 z7QRnxc&oK(Q_(`#N^T@}{zI`PAoD8mP%BQBHiV0Pj|1xynI#hpSOP!<)3gGS!^w$4 zopX<;)Ag|t4u^`9=4$hzA3X5lbPzFRgZ*^C2(x5nO`=%?nP|i--=05VRzDxTk_@qJ)s;K}nnVokPG;uEhOOv?(`SaM{ zcR?s6vI^&oT4I5g!)owmX}T#u0!o8U(W|jSypC^`6^@HcsbavvudQvgf&l)5V!-w9 zy9jt@0~*nz1R3^GS4f7XJ{JrE{`>Uqb6)W&B!^-gt;3c>g1ONNK>l3>t+O0b^}Z&% zUlog%aj#9_O#eg4mW5_?A0wws`Dc6m$#Ub26l&Uw{~>$k7MDTd3x0~8AT4)z6@%d5 zr+Rc%ZQ^D&MTV;wxx#5dbQP|Kv&->xX$Qk@53qp={o|^n%Nn4|^+|L>FyXPoeSR?6 zP*f7)E5_H6pW)75o_Sl`fk}Ea%Jb^iu6a@^k?(vy%$||&Q?-W20`DRE7T*Xo3LeLr zC}ey`wb2(HrB(zgIap)CJ@{*QRzd z4<>9Ex+D%RHw4IMFSahS1DyclIv|gh6NUIFYCGR8^;fJw2SJ0oyh7?JqtshNh79R| z8~wZP=jHq`-C14{Ioy-_?R?mqZLraJfGuEN<<$E`;1`={)~nqm@!N9pKH6zD_! zr%s#pY_qxBsZ*!CSFG@;BDW1W-8C@tZYrVnx{bfTf6a$&*3Fyc#+y@d7)R$cQ0sXI z$CslC44>-&uMCk}#FHj-aT!Wc3d|rF$q)?$e-vdY47&lxM$oY-rZcu!n|1ZtHG)Y; zlI&6Q`Y0?U3@>HX5#-)Xd($@=|M9beDbmha^|en#lH04KxiD;Hc!n0@_w zw}G11Z#EmY`Q_<)`)AY|^>BEkUSVw6(5Is|963BZ|K@^qQ!kfn505k3HY~{|JTQ0Y zJ&{uJL<5VPhLz?~5t=@FcgrTycUv0ras~O-G`nLA#-qCfQWLf`f23c}Sn6$O70gH0 zs>L$xTEDefX(WbcQk2VL$gmJ_ALxzu==e|SSZJCjpIE=k4q&7G=Bq7kDqreMJp5-4 z-k_#I=1^6hG`mu-;9|w0gX?*XxV$P$TMXDl;N>%NXARa_onq2*^2irZnhwK<|MWF! zuHPF4NhTjzLa0i&>Fr0^onG9ls(9XqsuIh0iFWcRjm^xe@*^sGbF5;hdl(c=JEe0M_-66yb@ruS7b)}0byKZM-0G0f5w&vGirpDrf&ODUOxXrsE%GdzmZn_|4c%tA zTlQ^e20VxK{fPgTJ+350J)tUJa=Qv$%97jp7Y?|$9;Tx+yeN1x(AW^h@6|VO5Hl!B z&OKDQ@3XR2SggFN15@T+Ow2y;w8PpB77oE`MhS3bTRubaEik=ONb1a?e-G4VW zAY;cGI@50eci*F$$#{vSnhOUCfPKyEU;K1`yK+lL8!;3DCKhis#~mP+$adYSDHFo= zspM+i0cULX;um>Af%?5?s?B{|65iuh`Kr)MS6+Pg*)!QewSj#a)+Kxc)8-)z=aj0p=uSUHVR;-D#elwJRD;}TAvf?W zQp1gM)*Ocm>Df1Ni7Ss)-t38|tzSsC$MMSlJVYdpx7-E_8H$HQWu|Cp`6ntj;Yr37&H^1k{;(uSViPUJq zejwd>(+o2Ksj=(s9n$7)suW^J-(ffXsoG$Oh&RZ=1;%7^Cyr80wu)~MOEyu)keJ7C ziyxls-&HU&;7aCspX zD&C7RsvL9x1_2kMw^O+(OycsNo#^YW>fNIHEUpo-oQ}2DduSR8Cd6?AjMWV5V>%31sux&1a@7Fq{b@WvVn@NI zj>mP{*@~u=(_z8@Txz{`?L4iM0+4Wl*ZAPuD&<_AI(6i=;O;#-s?{8MmBcxSGW+2v z%H?HKE~NMs0J@C!vVCXkc2({CCm5WBD7A}kMA6$uSm%lZn748=GDc*zE*<-YCa~wB zS4Gkp46-E2)P}AAR(B#aM`s5)C;;NMvtV)<{** zmBU}qEQBqgGhZ)y z&9VWlrW(9Vx-dsOV2nz2_^1VwX>M520qsR0_hE%A9=Z<>oHtti47@IA0MctP1H4Q9 z0)Dcb^-?m1*mZYl!F*Dq5BM@Y{Am!^`xjuR%J>}Jz5~7dbh@eSI7uXMThCyr607~l zwTIAfOu2bF!-ma_+-@hXom{Pb2M->cR{@}Xh3P5>M^@MbUBB+j0tBc<9*898O{v?Y z^`L)vHyNr^0gbV=96*BcqKg~3HtGk>vTf-MoY$iPq?=)~% z{Hg(6wlp;vVNvzhXLHr+FaBc6Chr!gf>d~XYxj(t1}Ddl?n%92ZLakMCIL@BV_IRl z^{@#E1xJFE)j>Ix)xd@G?)r@yb>@VEnz#|^I&aaLz2ees=Vwi2Y|kHOYcPhfEFkRL z$`Ny}19ZGfLD?p~mPDK)ebyAL=0cZU)d^&5Pln8Dz*D9x$YjXj50m)5rEbgbJahSa zUGtW|rwE#w1-R|)rRkUnX9VC$9L~w4zyJ>{yUZq{GcTOcZ$kIqVMq4mQp4B64zB~u zJRXKCo3f`7S0wx9P#ACA_v3-#=(&=?_|ucP;F)9FlSmGY_#Y-Kkb<-CHPQ1S=MyZu zZaGo?x<~3W=5iLFAzqE!SaT$F(K4KXWvT_F^r+1T_sJ{k`u7>SQvdfWZ}PtBk*fFm zqIh)ZS^X;i>wmP*?7s3JSlhv2VtwELGyX}Z*DT@aqu!9`u;cA9zd!z8&b21Gy!|~e z2)q3LhWU|UyR#qPULNo=1_H-$ddixjWLKfGx}7I}2oRiNC@c3LI^;*_w=I0s73$oh z&q2P+8Aq8QeUZaWBgp12QoqJb4?5isM_+Yf)P(Ze-=?pS6o zp5l}2pQ*h|$vgfbZ(0ThefrHWx@ZsrRliF0B!t6Rn4%wp08w(7_y{rr9EWd^%a7iMT`{<#p>Ta`9HvGlAV0edYxw7a z>bL)MBnpq6q|;G=srmCX0gbv-jU}3-kke2@Fa@`JGW(3_!^1myryAn+8u6lpVek|Q z(MLKr0Q1Xy=HU(kEBh8_x?|X6+(%tJ$D{i;tUV?y&)HMgU5Z9d?_WRwGv8{TuY<@* zC};8nGjhww%pA)nE`$A_9xdcR$wHYbL}>ew!%QKx%T5)4{=ux#LI$c_QWdGWN9C zvZX8a06@echO%zg17~>GrrlDKk-llE`j*OUj}Eq^a8AS&p1h!&H@gylpr;=aD&d+G zq2<-C2_B!0yJfn^mQ=9k0>{^b7W{q=J9Un)|GO8u`H-86>*qiy@?IntwAJ6FQiY&R z^t3-#2q#Nt@PbHgu*OY1Xo(K98#sCKa{t z2$2wjZyp;d%wFlPTVYCBnC|Ebdo97=z<_T7;_5T`_}AM>vIrXLdqV-tV4WsftYl)4(Wf$;zFrDm3`Wde zt>fvW01@j&CgJE0f@X$5nU%W-C4V9sT9#ZNm={T{(Fa%(%-*a!{fGu8#Ih#dw4U(} zMhk+`1t(?Nz-@?Snbbr|A{iW$-TS}v(6RWNCm{aPg29vv%>wgzc_9}OiVdD$|NY>M z&CK%owEG*&x3SNU?@_8}`=bwCHJ5mp0w^5QCg$B)|P#QIF8alc(^3l>AJTKudDu>HqIDN9=_SuzO+aEm9 zT5~R+n%7UgJyAu#2dRz7>M@h+VvG0JW-Gp+Z5 zcoj3b_Lwhf?}<{VXWft7RnS-&!;pha;|RzfJ-Kf7Q}^SPpC?^k1}^agIiH)=wnfjm zccAC|z_~U^ccQaqLBy42l5>JG8PBHkWDzIQdUlX?H0TH(0@25XK;P@Xi@s zr|<|(U0I;(? zy&0WUMK>|>OKw%#?r#TE*Z)b8HY2o#cKAtlXS0gQ>_N*P9UJC}%V5}LsKK9Y0{gfn|G9ENSDSel+ z8FQUsE&DCEu52x(tFwwZ8dXKX>|p-2Gr{gE+ZqsYkD~&fYK6nFc%>UPxKt2 zA7t(kGD(`}N6_ zERufG<#|?=Lr>LOyhP({HPof4dE^!kz>irTBeor*WmZU2aL^pw)|yU6>2;V*@Vs|i z+^I{BjJRMQVO!R1{tY zg7-a14x0!wxKArbwV3k2TN)dx;hwKB7wd(j#9aj;;+zA&lakY|OV;*fSuGget|;&axKMW-FMt0zNd!G@+Ej;%XO{^Yz1e-C5MTUy z2hLK(iZ$3_RO!|w2FJ;ijR7$kIZR<|Yrt?Y88MsqF!=OHqS9F|J6qcvAjQp}V#?B; z_@48Abx*(X?$EX!0|M*_bi60+iq@0uCm7T!w7Se9sLRUPe|{Gyrr%;llx< zm{MrLV>|+-qmbN$5M>f|?GUn>w3tbbliT1Bl4deo`+U0*TL&;~FvSExIu1I z^nLKu_fO8gUw*Xd4T4V{pwcbJk`t{U&i1j}|5sAgxa=#@FZJxCR{YDkYCbmC3Mx4J zb)lp$A*OEr^7<}ePc~aNb$=M^(zOZI>N2h=Cy0eK_kpJMsn(ZtKSpFZLKf&trXEBq zLaE90Te=AbRVS`{FuDhse~^@PoUC-X|H^4>;+94RF!1H%MT0vaMx6&PD8m)4_wW{VOPm8c`wh9Ulmy6lt$V+FKrb%u(bx zz&K%>*ZzC>3mcC2jz95sR3~5B9aO&$Vb-@~2SW599@O`2Q>tO&dBVWu!Nyb?lapfQ z;IJsmo!U|XU@ObQ9>Lgo)BIai32T|CzH6~5r(TmL_KuS($5it=xqTQ~npXi_O+4*>4#Pen(71erj?Ryh zDPGET>ePIno^G=F(H91wt$DE?VVhYu3P5*xwi5blNFQdE|8tC ziTNO@X8>5$Qm9C4>;Tjqanl~?*)t^);2=;nFl@*_<2%wf-doy-JR|Q6W_HV%GnJOox~Qcfh;#r$Q<~0Vm*5Z{Ik(x$dKc`Wg79r`yk3&J7)V=_cV>Vt0Yml+ z^NKxhJmm{=oZpk~32^&hk#@GYcEs&nm`(E?Jks|s#L}t@`2OZiXRH71LM-j0{^OIk zlY({Y_2``ol92~F!bUS$vt}}J_33A6SZK`@CqSoRPdFv%ty^)9K{$=Wm9LE^UY*Cl zTtTfXx+TH8XeADsQ#uzwV|e;B?|NWG@sJzbT2B5T36P8`iYP~$vj5-?1#S*^3l+gx z{Np*v_r9EZ?GGH`gi3e@7lPn89;P;Pxm`J<9Q;REf7>Bd79J0H{%9Hc)TI#hr}Stu zWaItxAA%**`dV+>HWN?|ZOBAE_z%GOrzDc3#!Z?ywMZ>ZlxRjU5RDb{1zCa%baVoK zfm`yzFl#|;Po`fYK~s=sQ#Gmus)7^{L1Z&7%YMZapYHCRNlJ_T!@NrY`Xy2l_6>PY zK2=L9TEd<~Qr8Zp*H*^072tq4riH62AL1j;z>8FlDe!AbE6Eu-R!kaObd zF^vLTI!MF}KD-Y&bGhr{c7JM8`Ag7gmBgw63)iNC4~UdTdU%k0 ze@uGRc^$q|KXI{hcY1a^(?x;yr$mTc1tE97yKRnm!RyryXI{33Zm z1@w@Xnn+5g*BERAfxu`yubPIwETT$ZS`b1-vXa9a#`sIn2@cK&`e7R7sr5bsm7bWL z^C%^bkuQ5W50LD8>q+34@kEw*n78Ciib& zY)AfwFXSou3ek+rEaSvrhhYj00v_fIqqePDP2!45TM4Xw7d@m*qu|g@re%7XF6Wp0 zu$-4O-Ji%H1ea^yav6SYf`m_Ix*2jMnsfRCm(~P7*w$%#nLhpLxnTgJo|-k zm=n0FI3KLO5n_cxg62|s?0AahnPr6BPcJVd^@w2mKR#8NBW_ZSY-tE%4w#)*+?$UXhOhIr3^Mjs>VgEgf$bbbmds8`X&)O!NGnhVP zg6o{2Qukxe_y(|;N8kizudjUQ(b@gKS+v9~_ER=3NIFw`R$T6YL?Y-D-XbD8VFb1u zOuS(Bw-J|+$^ene=);A9yxd(nMwD~C$>HS1I=k*3-o2&M{^(T$t|GyL+-IF1UOFX< zuDRdK&H^dA0YA{F`1N|!K(+hwV-0Ih33}kMQlm5!CHki9HI+jF zKhEv-Cx%KWrI?PsfS%Mmtl#8}u=9Y(Zk?woQ9IRZ)F>-BPPacu$J%C*L?1cWpxiZs z^z(>qF2lC0OPpP3fxqmU?^`1;Qih%sP-A-|2(7JKyIZF$)yqFkYss*q7LCI{H^aY^ zb9t6c2s4x(ksB`_lN(6UCF=U%<$ZR*_oX&?uxh~S+*SV*Po0+^O`lVD!O*JG70aKx zxss3ilac3q&j0Y?E*1cbd(^fm;u9SwTz#YaKSD!4(}wtZf{`0!kZOVSvuC|$WLR%? z&(I+Q+8K*2#MBjpQDu`jC=Z{zl(RkE_>7l6^b@3N*8uD^Y zVKK|FR461=0|97001=lnZN6Jtt_jXw4QJTAt@O~fOHP+88G9lUPG>-#!OliOBA%1-3KCrL?z7azGr0w`z}71z4Ij?M{c zg%h23ds-P-^qqeoE2j{w;$TT&_pjH7Q`e&Xvo{UBm1Ov*uaUnTIn8t?FbDim#Qi&% zZYScCa4-)OoFkv=8Lx4e+=jvIDgJ%eb8)iJj=qp%bD!ny7r#=pA2{Ad`5anCNU@!I z_WXx1At??3S2oehHZZec*|%z~!a_xBr$*P_f31ELd}EnYXe4F8yoKf!7rzr>Zp^jk z>RPWrqq_YAyc4x<)8C(5_G|F|Dz9y#C1A8L>6>+6k9A>OqZ4dOeSCh|!9UX_`HR)3 zhQ`iG>|caRSc_T)2v7IQ51E{h{bl8h%dodCx`PTg4OO-r4P+1I-cq||8zL&yXU~+u zWG^8w@X80EH8PXMDAF?^72;l>v{yrA(TB@>^^E)P(&z!3zladZEye>UK7IORU2^Qm zyLu}E@_`gY&kxGeZ-K_C!|J^pTeNERnDUDQ>#!x_Tkh6uk1gP``rp=6xAcEw{pMF2SdSy}dM5G^9fsb`5bg<-F1nPa z`EQE}F=CZ1TEShr?)8XSy&py#Ly!zcg&&xmTs_gJNyCP=J=*R@S@!QA{eWvLu034y znywMkV?Pixzx%$O+u43oVq07TVWk02`(1LvM1ASd;KA$0eI>fjgA$_P^!TzvXaBb5 zYC`2eO|cVlPaj|`Af-J_#^9RD^z;xXaX5GBTn162o0u z@cyms(TNB{if?_S$DH2aR$_igv(ORTm2&X@iyH*qC#<1@G_k3?*o+NLEH${p5QZWx zh;I3K$W62f;vG~3*dkgQC?liUvP#6^VfWwx-UwTczTNzDe`XA+_EmPmPau&=9nBp@ zZTu#^vhz_m>7biuc}=Jd!6=jYovHAWVWm#^J4Tof4Lxv_x!c z`{8P0>r9@MJ{yIy@Mxr?efQ1-hUdnD^l{MGp7Adjzishaxa2&q8;tjN(zrWG+E0ReT*^qKgVjEAUciyqheAWuJYvtUXF+&8G+YVtCEoIvexx0_U{ zMDXLJLT>bUDLJUZg&6N?Td4o{P0#J~s1xuWIzfwbg$sZL8Hr~tMirWr*bO0@(4TdA zeEq)#PxK!^#GwR^Nc+v9i~q9ePxVn3+QiSFKOge3H|`T`_lTz1y5h}=6~8-elkoo# zL;gHT`e%>+OBMP5_$QWoH`MpubnN%N9x!-o${+g4U-2!Y{!?P}w}mw()%|aPb#U;R z&Odp$Kt#ZRA)U6U|MwTxyAF7yvBTlvg=C&&@GogeY`?!*88~n@%@6U{K0udpQ8@2n zfW|_x;GtN8lU(H%Qt9|`gbT{62H!P@V?YlkkqD@Qv5F@+QVTY@&Z z=||B7y=)ReTbRqh(ghEO<8m#GJsxgv0402O4YY-UVO*iE)VqU2k&!YcRPiG^~WQAsLHCI+HIv#Zz}g->B-#u5y`9 zj2#&J`G^(-%C>sjr*PSU6qZ5^TK??RNgCHgCa`q5>n;9)@VbH6vGP|YgBi2+qNCx@ z;sb3nRQ0o9<&Fm8dgQ66$3vY>eqy|;6tw>pDH)@;@qA684aiXzim)3wTSb@#$UY;HrwWswa<2^HO5yaT%R81e~SGjyB4s z0|Td)t1J8=ljgzPciAcEx7_>`bG8g%#h15oe^cbNI-EM(%cFS;)iot)9hP{W;yN;O z+FW0)=R?Yo#b!@Hq@78XM948b$&?j0Odrb8I`VjgUFLpY4cp(>JTuRK_mzp=v%h|w z28}Vp7014IJL)y>a-#DtC5HGB2`+7>Pq z;#%1^Z$md8ri=POCRUs(=dK2A+h}O$AMdTfcl0H=VqQMHXseJRC}1Aky}OeOEYu+P zkA_vM&6|$_qj+>&yB&OiS0O71NBW(MhfGKvKc8Um_;zUMf}WA!N6-U*LZOMvYX`%d z23pj8Y@hnTOv0GQN+>crYADwigKut$Y2Nd+RUH1zI8RR*`BMK#8V2B6Qf`HFWHz0H z0?v2ezN3W>>1>4d;Tq8)rbj7x8M2gGI~Cb5+%7^6vz?*fO9V$ZK^7B3=2AKOq+twCyfmR={Qw8?da7{^AC5*yjCv*fiB# z8DLQmNDgJ(RcvUd(WVU$PgHE|K~hjbWL2dTAQzr%jee$`jKY7SE7g*zT@`$fT1#SH zy;9pjk9GWjc0lx{Z+35bc|(ZDhp*Mh^9zitiL9?miw9Xsw1ku-P?^!W={te9`f_@L zNkBK92JeN;K!y|>5y;q%a(bw|=_*ajPkA)x(zz^r%#jAcqU;i+Vl|Bs}OX7C8p$UV^~G-6lWCp zCVd@Cm`$d1IK!9$~5;;jnXukL6M?RA8Of6+*arCAewGAeN9gavY{Wg z*}_0;$k3r~k5g^3H59CqSpABAHuwoC?QY+Y+qacA5DQn{sMoMzCcq#7a2M-^Yw$~! zgQ0+%*!ONFBb=vVKLf7Ci1!RA#eP*UX_-5n%?yXqAT55n|trpB0mZ#(^r^|eYsOL z;!J}pEQI7iiV=U*;$qv9*3$La)V6!(JlJ(MrR6vf5itUiWKRg0iT4c;^EjrS@B~EG zt*x!i@7Iy_T~b-lDs+V70=MiCPj*Ij_CC-Zkkx^X*z7N2_7WpU9uf=WgmhjIEW?#M z0v2D+yqg&tAMXJ-9lbAt={D5cGCWV5{fgqVla7wMo?hhGw2e04Mf3$V@g#c0Py@r; z282+l18DH(eHm=^*{J>ZF{3F-=!vr#OcDQOi4}wNi$&G+>(tQ^6z~&xpLF3N7bZT1&Pk7f zU*5kLt*O+7++op@t8_PVpz`3C+}`iLLhSv2czg4>8vE|;f18QxGG>g+utPG-n0emF z5K2kLkYt`R#%0RT#SW#<?8?ny8Q=Qj{c-WJr?CLm8g;I(OXn_qp%i^E`k3yk7Tp z8`^uH`#e9N^*PqDj&&?LJaQzN!zw{MP>A`a(tQlco>@%2N@c-TMiv3Jsjmq= zv+TLkO>u#(67}7d@DjhXNjd(7LCCj*yCh0aZ`OjMp%nEMmuo4-3_S97Z$^_x z#CH(lY0#^P+JdLWUnBt-2H*6gexTMPyQk?y3i0tLjr>Nic?l{rSU%E$Ba(LvNFQ(b z&eysOgsx*Uvx*i|aMV+xqP3z}u^+ff^l#2U`F0}bA^unBXiVp}~yhBuven{GNz zw3z2wv!cV+UifnO?-fr2tGUlhF0MiQSuIG|PE@>%??JdLsJ;B6Pq-4_Y7((JlD6OSKxe~h*la-jmmaU z`q#5}Ux6Qz#uvQUq?D@dwPtln-U0?yz>mfeu9l$-qssCWsOY=!8cDwdU+F_3%c9q+ zqB8|3JVvF1F&^r%giKoelD)N!ZJzhfp$jv1Ie@mYTy|r`>L!vg`u;c8`ozA|<0KOu z2&ctD9ylPn+M*tWq*-^4Uto>!k(G#GGE_~Z1e`Yu(Xg?AHUVBqnJ*Y#&&y6e*ZUXNPRbhe)048G30xxk|nPl=oOPY7fC1-<1$9F zgHzh?=Ccln1lxC5ozdx>sRGAx=fZ|lgim7oxoq23dWu^X{v|R2E2ze!_cP~s1(UUo zB65>yrW1@igpFBg&32Xm&+Xb)-*gY(_WUV9_f*H4!aWmMnaMa95GELbnyGYO$(SuM zNb#S$sx?>d_fOB9&7oEfbEZw*K8Y_Vzsx&0H2b^J665EVW@Ssksr?P(`kwUMoQ#-U zm8y;*N-kE(En=k*_9T<2QN^0?DUz5Yu!4lc@_ZjivQ2a@%U)m!V4U1?l=cGBfa|-r zF3of*=giroKK>8U5}Ie3Ur(}k8DK(|qR)#PE_pVbWsg9w?<+ zOF3P4_Zu7S0Mk%m^MwU%^RP4T!s*?%>fvfG`uh5IpGG)*u0Wj;C@!UPu6OTs%t5Il zwRgyS^{SHSX{#B?4!#!hVxj9fmovZmcGPvS_1lVM-Vl06^O}G!cFN9(1)3cn0ht9y zKXl#!#EIDAT!x9CY2gj+NGOFHIqvrg45A8;ifYzeQGU+2onzoM+UQr0jbWTrS4CmC0XNEktt}{hPfUT)2Bwt^)b4$6f z@pZNZt%Oq^S;c+mEE0M=El3)o-41hy1+pDHV&ACl?FlS6nW{hw0!U~P_u$Rck- zlQqM;$j~U3vsDs0_Tcu?6^4hg)ae6FX`DJcY+*e7~-jH~KlIkDM==q@N zWgn@v36nWtkYy(5mrQhx1>Ig98f79}O^@7T3h zBo)em=;@d`m(CSf!m8N!li+joQ+@Mb^E2;u+l5Kupwp(iY0Hkc9#|+B7Pow16GQHW znWyeSYM0LWfRFNRx+*DA4yOFYd0v)qQ+g7Y$e=-YzN+U(?tL@v_ffq1%1jwS|`)YaKpJMl7S6kQjZ4@f{tHeB=Ln`Dv>4qh!! zN4sc?bZv=MovJr))Uu|1vhFUwd!DdeQgSj|p&sdZ$>Ll(L>VT&Zc0X$nbS|@%k9r{WXBQ!L?FXaB{S<69{&-T8EXJ?WBXf5;gIRXU8j>t}mD(r+~ckaWpO1CXc@ zIrWh9A5#DG-pn^5P}%gG6RYf*am&X_xl*;$^MR1V!cYUOLL9)X)U7EX3_TSZT1J}a zIn&ocAEF_XD9UAQMXpCout~$+gNIIwqQ7(G$yM*m}wOUK<>I)k+;p}Fd zHk#p!i-nCF0^!SfGfe7!<@^$tAo3R&Hbbj~xhFb1&8hd`tl5H14YdIaT3dNOXauNh zylh@TKl!Uy`-cAXe$XgxYGA~ux(=nAG=A6gb}bWajSq(Mq2DY~z7TFZql}O=n&-3d z*>(M`-)Ry9@r!Xe%d)eP!u<&Q6W27NRB@$GQ$d>gsC}V+>1144PfyHN!5xnyzVjZJ zAyxN!G}3*|!7Y4#H*D%-mLzEevIOxh2bKw=2P`-E`$BJi=Vsc!c=2qgw)n;ih_%zm z(j@5uEl3bV5!{HgQD|epps1UmAa;37);;;n>PQGmY0Z;%GWagDX9JJyloV?6#r4y+ zDeW{YHe>&0>lu+O%{X9o@RS(4j-K#f?bZ3^8eWa}>}X8%;JY;#@}~|IUmI^kY9>Jv zlpy_sq~K;Ga3W6ul`VT*+{%xr`@*DcH#;oz2J9yTyh#i{;U;mIbZqw60`>4~qy8q}A(Qorr>xE7H1FCqN zE^UpND~mXdJ{e3m9D$Npz#`i4G4uMgcNGYaL_~^x(km{~CXRugU$ROGJ;l{KZm&{& z78P%P^ZNek^}Nd#APZPgAXp^4_?qJ7Cpi3!y3eVIH6La|p!)QR_0#r!suR1@gGMkBhtH)G}}IEzFC* zs}sB)wgi(>+v%=O{qgjWzW4{{T0Fd;KB?`S3+4V2@umOvV!B;B4$r+|YfF|9FKV=Q znGV}S8OlzI-j@sQy6nxt#U+()cUJn17CO5V?)076aG4aB-U+Y8S`k5;G`o|m6kUu) zFfaTiXq%ZGeF)@Mv~`K@j|9QxuXDO(=`+)!Yro)MunB276<=Jc$KOUK==#!2RA?!JJTeanh%_vJ?Y|tOl|S|S8>@>h3=;;&_2~z zH*gSJRP7-QpgGf+ENfeV)z(HP`lE` zWvj>mRS@M;vzC;E%Fg)N#>K~Ti)yVJb;pkcudShUZZZ48)oKEJjzLpl_Sq|teLOfH@U*WA=4P7v`Ti&aOrJu)#Mo zPMvP|SI>qGodM-LNl3wvb6b#E^Z-Q@5Y)e)kM2u1_8ZQ(Mm2hk<0WEgfw$i)5OB$Z zXd>R;A2eV9b%3_e$^3}Mgq{MQ%0 z#;}dBiS|N{b*W!4su0u1Y?#H)4pC48Vp-w&Utc?itxuNS6!waOiBRZxJKW%cw=fgi zDucCbZPT?F;1z$r#zC#kru^9x#$Q-&2%Px~7arj;;)E0%U`C2UOPL4`#AmPFrT_69 zs%pko>)5E0jQ0o&``jUq$p(ALUZ!k9u=a1iv@g~7e@AG)bWUvY3a51$MTLbdnRWfQ z>5QBEn*-R2BVDyVoK3m3!K}-u42w!))a7kSr9Y79V01fd+Ye4@(V>$|h*45#XtVI zoSE4(;*#!=-C3h4s|=IVKZt@QvN%qTLVXIwXJTJ%)u>?I!@f3NaT!5DNL@$0v^c(M zFRHDVETO;(^Auh9(wOD!HqO2H<~cQyT~BG;Yxf2$P!OKa_ykiUB^D|X(8b7SmQ2}H z95?-jvHmEoZ2{yWpm14SLH9O0JIkne{7%~%%PDF3sU_fLI@F29Kfh32SpT8ri7|VL zMOxu+BE{R9U)Smp0qRa)!y!=@{WJZQUlMwv-9&RL%?{)>RDK*A5BhZlW?X^)SpO@` zqv#_gOBs%RLoIXI+tONtMh3I5rf^<_=h37X;srz~K9K$yenpzjaY0>28C0r-G&iHs zZUxmSx4J?>@%F4Xlo#B5zG;4i$4DNVjvof0JehK`ocX}7`M^;6Ik3AvL?Xya+{`*f zi*{v(>wMgC=io4A+SH?GCc->%NKfk)c4E87zV5>*9hAg3(Z zyj|;^FNm`o3D5ES>#&>ZKuM`Bj83PK)uX%9f^$C*5piGthLcgTa0bDcPy$v_N7~3~ zEnOY{mkl~s9pMu;_Gw@h9US!QooG1J&9pSn%E`6!XAi|0mwuxxM4M(>`?k$>|TLmO?0z#wp@owY&u`im-bcU6z5gxf%dyDN)H-F0^KU z#v+!B1wq>1?cAcp7B00KO$`)DDA818#V{;p8)=i|j$qE*TYY}I>CP983u<_;SEkd) zQZNE0oSyrrr?hmF!HI~MwIItL#(g2m4C^y9FThgWEX<5qbAX;E@sJWr0Dj7eZVNsq z8>bA(Hi=`mW7hn98qrm^?ln95k8f0okv}&PZq?~L$7bEx9qnsZ@;$cgQE7_a)1W_) zVJx7Qn(Wi=(PUF7PSTxuxygq2&pO^<2~R#j15_t6no^TcLcR4q^8=J9CWa~^^HCz7 zh09m7)myC9elJ{BY4n!2Ab>o*`Kjs6?R3d`(8`5MxpCXJI|w}ssb`Otc4RBs>Dd6> zxwe|A88ne|%gf2YDlyCOjRc=`pq59$xhr%8JHKHO^eNXWN9Tyc6I9(ky*#nwEt z5W21><)cWgr^J$c|P3VN*zS_qytDd6Jziy%XS* zQfrWIX|GrUKvN2kpiEype_`seoS+f^YT=Cu9W+1(27StM%6U`1fLe;Xa%CeVq*x_| zFk*;F(LWLn_;`Pb9t_}934UsI(#93DE3lJ_4uZrn4QkfBB61GJr|R?{WlKTdsiqvb zrc1GR?s`TwP`EaBg*m;I!ZtyFW`}mDU5TOW`TzsUK5iyg2$g$Re}40(?f2CT9zdZ| zvil0#Jft76h!tnr>d?}=rdwlPr4IS=s#YiC=!l6(}YnuH~7gPtE7YeXN zuPV<94oI3XX{5lBw@Q#Bz)T7wz?7jM!Y^LBB&`Ti6``|J#d1|DMqW(k@L0X(_PI`T zp6Sxk&FA*Yp#{UiJosFLhhd|5->-a(z$AqEv>0YFmNu3xCZ{%_nyP=|E29G5dhhC_ zy&ZBoTwN3Sn%twpkn+u*ww&&a*r(a?{ey?p@H8Asit6Y5_m5t3kJ^Z@6Zb|;K00+u zdUVhqcdv_^-J& z`^RehO$QyGFn1+p*w7>f#=Zm=GqMSvH)TT8QCJ*H*M`UpIMftJ29!T4QcpWc4{XAB zP}Px|3)SNN{r`!dY0@*S=W8Q}rAt5e|7XBcGyNY0G4b`+`IIdD8NezyHyA-LVMlXo z);Ja@fbpg_^lr^6RVEdFrwnL^UPTTkQksPmXo6e%>DF$?sY}F(%FZUdEl~V02`nTrmGt36 zwRDIJ*sW}k|MVO+YGSX|l`B?U@%+n(v>p$OA#vtvh)0o@uc^MHclTv)8giZ?Lg`hZ z@z#f_4)hDpeh^+26Fhb+oj-wow(oUx-OgQ2fJB07pCgHa-fD;P6#feFtfhC+dtYgk z11hVs*j|QGM!HX0KtbX!6$TuRo=M!3W6`upvE@$nqb?*pDJiu8Wed-_BVn&0W z?z`Lw^w{Atx!T)=CCIK#zON?Dr>5DLA|w+>8-d8#Yf3l&suBCTl!)_wvJ2p9Jg~TH zK0cq8ld15nkmn`HPUx7g9&{5vOk6S3PfadG32%DT2*cgbxFuzF&uA-z#KU5+S6fG_ zZT+fV=b%<(sf`B~5%N4#I00;doS%p4-f?a)_z-Q#*WIf#YoM#Z(x$4I zhg;c1a8fzmH#a5-iq3dU8ZI{~g1T!=%$jr2K)KFlk-Z9EG{H4f_g3)*^dn7TcL5>A z^_vS=#}{B1W1+ze((=@M_jaC(0_MZ1m!Y^?OJ&kAgq=k%oa)=7nrT&{{|xdqbe}V20%uO5SLXub@8^uTNNEXGffWxCqD19PiB$X#;T6T; zhdM?Xk-Ia@%u}2lT;Lt2nAEk$f}H%llq6&;d)P9^YyE%N? zWAR0qaiH|;Sq3$brZ_e(LZ_~y8q|np)*~eMV}Rom!qW8k$GqYlqOlaFg#jvI5~S^J z`uCrV6B3IL5QOxtv!ly4Y}|Mq?4Q_wOEY#Kka0kR7~NSiZWHD2s~bk2COoDvqc0@? zG>3o+Nm%FItKd(FOt)GBf&gZKWU~?G#}ILNc+}>$m+VavFLiw!Lrz&IU_X|%VZ_?4 zKGXgSiM6cy^MM@RrL)9~>gLT>@j-v_j$|!nYceu21Qu|x;IpyQkC}nr1jSQokE{2D za;Jb%UaC5sF+@B(si`(x;!icn((C#4yYHHwTa$EHVwz_~m(4!ID!VQotcz&5kK9bx zFAM0{dB_=RziXl4FyNShv`(Y}kji^)-yjdA@77rGJjvV>)Q%!W;fdCvL5@b0SQ@Nn z%-BQ?;L)nj^{JYK&1zpv?qnLk*d7GI#qYV3fck-jDz$^7yfN{yI?aK^XJkOKT13)L zwq-r6Y$@WHOeJIbX>a-pT()gfqahX*33ZrYz=40WVjIA?$bjI!%eY$M(eG})Dy4CB z#Zp%BclAm>A3tK$R<(YQ(`__;MqDN-JC~lhK4KIyK*;WA#N+symoi?OE&J&`;6E_l zabT`E6VMj(KD1OM>xyf*0QpRkHLi(nicHm=>YUtM8QquQYM@Jot7>LhXv*X9>MG@G zWR-5{wPkkIeEKpKqOft*MTItE#yi z5Nu}>3wth_PR^M2ttQeg(F4xns8qN~D#1S4NLZ*J;m{>e35k(2VuFTCB0{L!Na2~K z-)++i{d4gdvs~2rj*pva`egV39(DHx|6|rm4v-7?kH3UoNT1@AajrtaqmY@=jr{gM z?+fMRc6M7$P5StO$4(DytD@$MJy&HoE4a!;mIbQLMy$eVM75el-RrNE{_pgVW(+=! zW%7vis&@zoyXf`iM~|(~>uK-Xw^ldSwzuDKYO_W8m}<1u3LQ!}MYX(Y zx`5YsZU9W2Ve=Bjq>a`CK=|wTo{R>-`>?nsaN%F&0o^yLx-;(3Z|c7or2aSe#$ub~ z{{5@9%#m}`(o1p$%hUvnhdG&u$G#3E;!$gIoBCocpH}n|^5EVW# zg+moJf{d|xuW`lq^z^@KQKh9|Pf{igKj_g#i$(4)bvG5b5>C>MMFdAb7|uf=p*X>k zr%5NcB*h3@#I7`9X)$dx*ZAMA2${+>zo{T(J?OIF{nd^By{j+!R5nw2Q{$8jA#8h!Dry>shv*BO$v{LQ zjMJkPqIL5P#m~iPM{2c;l=2Req1~-pT=jSwcoy>`uwg$E)QR`bqU#aqlr)81Co$a$ zcZCGQrH-%eLHW$H05ro=Qpp(S&_#C}cZS`EB}H`r4F0jf5#6WIg3ztL?J0(%NTK)J zETL^NAHN~xH|gfm5CU5GC8BN2seF=Cn9K(D$sWDJ>N7pa-``=%hGL}R_sdz5Vu}o< zRY~L12FFcI*TbLnV~KF69?&9qyW{OC#eDrP+M9~6u>UQFBXjaeYYJKt=>*4~eDo?d zJ6lN<2;xWxE5@^&apDn0C>1J}Vejr0U-Yw6_kU|L-mQU-NQ4GR?1`)Y=-e%wFY8XrmKo>Zm{lm4zce1pFS$Qnsc0d=MyIY%D%qer32Pwbe z^F(XH@gP-5XlL0;1;ZxOGcKJYB!n|9et;^P*&WyYytc1!NVn^j(rs$n75k`M8TWTv ze))X4|6VF`dP0sSUOb|zL$<&F!Q*qgI91<6Crb0a*yC`upzKyOsa{M=tA(Tp7N|^b zh&ICGF(NL4PNTg|+-Epd&k=FQ6Z&Wnl~i^>C_c>~<#xY^QN2|~g>%>+s! zoA@f0U6A@%Rk*dErrAJD><#NK%5+SJnh@eS2)agG($+YRB7ew_;)0_#WLDoU6zotb z-1O`xe|c^Eb^6AU=aXSVZ9^Z48}wGkzt^o3`IYJ#a&+)Y7H6cU&Pkss!s?R$i+~mp zW?aA={9~O6^djA}K=XUGIO6Txh12u|T9##=vG{DkUeWKp{_@?UwvIbp7Z}{Xp;Ba2L}-ZywQD8rQqb7#K4$7XkJxpZy$0U;S#y^ z$xL8H)6>$`sUWilte8wGZH@|~SO~~&GPcb?f!>yKk<4T!9|yM|LKP!k9R${K20uy6 zWbitPgmfNpObOg2f*oOv<=R&zX#MDO zcH#n#*x=#A=Wj%AcPRA-xQcK$d z>mQ`R^t^Zw(Lvw?n^V-k*je12eGXwE#6oS6i~?(}P-=X&c-z(jtDkeerLjx)Vk9Fy zn|$IfdZ)oRAxLQ~m#=!7zZAuv+{hIL=dV)VxK2DtKRtDmRXucaJ-@x_hnM&kvXytG zbp}R_jrOD_>H7I12pmN4COUxz^_w^lk{I^zNg8c!(2&?J`LWc{wKftWnw!YtYR65X z8$~p@GV4KzQ{LtYwK^>R8r1XpxnfRx03kQIYKEV07bC^oN=~={A3@e#9A8P)q5$Z&+ZU4!OP{O+AI@IU z`H+dS!EU^Yw!MqKqU*dY6(hnABQ=r;PGgC*enOLM{QK{I>k90_PZBZN^kV%JXpTyyh%X`A(K_tbAVl95lq)-X>)6>&$hdNkN#SpE7h6m(kG^KcZx)fs# z|Mzd-G9yEa9I3>}@OhJ^07*j)^3jGQy)F$d@ubIv&pTvv8XJ&)#2bjA|3BPDj4%nb zI)Fym|6EP-h&D9z3IIljpEzbULyB>K{)fb9&9}GZgu6)HkNF@gzXK67BsEZKw=4ze zeic`h>tVCg`+}5gyIz5Al8Fg8%vXqueoguyF1{!h|0Z}fec{avzB-5}fn8VA($Z!R z_z395ki4oipko#$Y+6#!qDshg#88H{Xa6wrqw%9la6m#@%YZ9FsLAe4n>M9{%orgK zL?}ik8l|_KdyITa#=hcryOnjzM@UNd7N!WwMAW=I!De+D7nh-T4I4jD(RGZivwa{P z7+a{1Bl+|8JBR4gH_}HGp&?`G^XVk*@Xk;`dr1OE2cQ7%w0l4GV(>rX6DEnpNPHYJ zIN?iLEX6^>y;9k0-aSlVlBEvFu4-i~SoXe>FE&D>Remsd0{dxMFGA6L}%oq|B0D3sYDp|pTNabE=`Ib z9I_=f$6(NaW32!7C%KNUPrzp%H!Mb{-+6ca9?NTN?osPY?tA+uae-Oq=RIC$^_9%d z1M5hlTO)XE2%3;Wgb_PvXKJ`)Y!(1}CSx0EUhxIPk{|En(U*Rdx0SCk(r56_WzXh+ zUP^^!>*u9}l7^qQJr)@ja7KkiBFXOdv}n{PZxefcWDD{d+bGKAmyL^p5d7}IeU38@ zs7iO`vkjjYk$S6ISe`TpN=!}N&CBKk@mbn6<_`FjCo{g$kAWAr2HA<-P?7bJm$KB5 z^K-^v_o?f7Mo(g!T}~NehaP#$*yWcBk<@5~>I*;5F$%JB7 z<6wKYx>s49=|NJ&F&f*l(YJsVTejXdZ#lw!)5eX~0c*z24|)bI(slK>FC8*2pWmbH zMr)vSqrNSnH3Yt&V$IT2VAkAqglf1_@ z639HbS`^K2fD(l}`%v)c(Ud4wv3hymdv&v`T_wyMgQ~6Y@;$=`U}COv08%qC@h@V? zK(^ta67<+#e64%`g1$oiiXY zdsfDbP-c*wy~@HTX70l__e!oVE{~zelJdP;Pw0kQ#YIU=2`nr^j4hJJ6uf+?=P}ozKAFuA z&Ae^%vtJhg+A7TY%l&va)T6BTw=Y{6MpeIV{s>ehWAjJ|?9O)U(x{zv-8LpByLgf~ zg;gn7`8e%kqGu+2^kZ8z%sjL^j?@?7<3|RPuriQ;U12o>kGVB zQ5^8RPGl$(H=?yTu~NQ07q|cNtcQ^uu?3Ju+Has8Wn>QxYcaV6oFf8tPTHQoPQXdH zKPNr-S!lJ3w6AqGb?5?}3lj8%2bDxCYd$sRwK(fg0u>Q?z#PIYuCO~OQa{PcdUO4> zbv6|f?$Q??#1~M1%ccoTLVdKQT%)+Q!Tio4NP5!_+E`%H@2v^g)U|cMvbnpL=l~xO6SuWo;SpB}@w$ zFOG1JQJ+zc*vQ@_@J#JiDh&R=5;j8w^W=klN|DouGfxB>e<@G`U8g_!r%s2BkevDB9&fYlK!29TfKMSVd)?Et{w06LMV z?oQ@t7MnW6p0qZ_gPIAFUHAtXwBoX0mD781CIS|{gB=#NC2%5^xK={s#~@)k9aBGy zjHmSnFC7U2^o#6=(~(>NHc1M*Bw&m+c#?BAiJoxkVZJP5$-yMl#jIJA@=_s0oTB0a zt-sTo=bh1-Z-Lun2$Dx|=Rm@H2_(2j()C6;O9}hqP}lYIpv(j^uwcawM%180bMo13 zv=E6R6M;{s(77lTaF@joxH8&BxUiv8x}$?3E0-rlnr|J3x`UA$ZO$==%i&E1AP6!V}UvmWa$ILCf`XXwdU~GzREy z#2$hYBTe|;J4kJ4+D{Z|U{?OIiF@!1B?A?^NVx$(5SZ>hLb`IJ0h}Ei8@2SiO1>;h z9uXM!y|p&?C{@I)gA!AgR$=DAA3Vt!(BJWOPUxPU&qxXJXRh={sF4GI=ff6Bm=Ih( zP^H>?LDRp_80zFl3zy_3Zy@_ffxqI@rfi>Q-shHFZ1+*Jq;kKtouZjx2s7rIIYAje$B^ttEi;0e? zrWW?s9d+8e>Ej#@a__NQYCa+I!_d#I=#Xd4ki=U3l2Y!Rwi};6nX$pN(FL1yuWan+ z!AJL^{|t;1v*0KHtxo@R;5;2B?XTU9k&GA?KtcVQxwCUj!Yt$U4<9}hN>MD>bEEn1 zbA%PlxQVycNh7a~mw691=a!AiDK~Tr=S5Ed3N5k2UrR?Ezts96)_ubX3o3*ukB=6e zWFjs?by_C_^pV!euff5yZXcF5T+;2H|NiL}9#aqJ)RF=b0(H&27}ix=C)?_F+(JCPP1F^Th+Puy&q=R7lT;W^-c|gNlC4@b zdPRwiW;OArAs!#W1C2Yri9Fk!PVfTP`XIvhpg_A#pp&zG<0_(v2q)RgFD|6N7hdE7e_s=vzM0$n! zvtngTEI=o=V$NTGU-&%?YTRzZq1bZg5VT>4J(!OmYp}fxj>jPiqbC(=6mV0?&HJll z5HOKDk@Qcxse9f}^&`T=p^9UcJ(G@Q=gW~&{QHhO+R)MIV#ly3mzkrlCP8RiJ7k%8Oa4X6VGF>K#H`HoPi9G2MZ)|k7EK!{%c0!!~e zy`MCQ&wLc!ql(v=k0PV$UVqI3f6PHuG)S~bO`=~93P((NnDLt^0pV9Ql1UWtFOAn9 zuwdE?D&-v>?n)u*v?p-o77r3(^$3C?_0|fkvfi@~!vXnWRWgd%LsCbEZV_X2S`@M2 zGobd>uv47}ypCN9KxxWREohBmkTmuJ(OmcM-=CUIU}&u(MUuvETy{KPdS_mbAcw(j z9t(q7bGPRBoT6t%#i8~!d6nbr*YuO#mn~{g2*2xeGJzj*tq2Z-c_*ZC`IwP1MISzEd~#14lsg?8o~YuSfvwR`ZEIS z7$?))>$D$Njs>J9(6dWYhQ;_-#tB^CRmIf}wsG0ih{#CmcG>PB;ANUH#nt@q#2aF* zxfkDi9z_!Zb&!cK&6~*XA26{5K8!+MH3U3c&_~^3R+ea7P!Brad%vAD@G2-qls6C& z)4*Ugui~;7me@}x7j)Wc)v-}`3`!{5{Na3irZU@7#v2Hq55|)O*TGh?=!~1i=9o4A zs0fXU9gR3~8riP;?P%mqr{pcL6nVlRn6;U=4wlHw%oGxYK2x9Y?K6g4&ufF95s0tE z=^&ZV#43M+#Bl~VM_Nj-opo=}d5-s?{KEsPJ?BwN>-pR3bmv^9fFTG)s8SkOOyS*b z{asVHsFbQ9Xi7$YAGy7m*bFQd-V@m|wu-}$F3{^FX#gjfbzZRYDhotkb54I_0n(8s z-A+Bj)}R)pmU0k?ON-}v=Jaj=LVmSP1E(Qn z?p3*I%taPst5Q-iiz`1?efjjRN~^7@u!v=Z^F7D+yNiwSG)@6~s{Eh=ike3Z!HI+7 zY(DxSMuJ4AGTg6eAib5}Kc?M{6oK64&8giTQvHBpSkjkms5c7rmcE@XW=R{5n3VRY+;kr?GAbdF4f6GYqVyEQ} zMHt*n+ol5+{j1%$QE>ZHCBedUB|(6QftGYtbbhpwk(1NF%XcGvbJCc<4lsrmbeewKx+3r8zF9EA^>!3c%K^vC-$FUWHU$I$nCag9Mf5hP~-IT0I zY2RmsHM7{<*zY>2uQ(d3WK16U$k}_Xkr7gPf*LI=ldus~$=qD2|MJm7KbzRwzRugQ z-n&$T&p&T=*32Z^sK0J-@PcKfen!=(x-+fTziUMe{0-M%x_FU-U`B^#Vxmz_At8P} zAUrFci_#1*DMci(%nk%EU`gVgjQ1^;B&uX`5?sH;bBY&DxqHIzlDncqqjxLN{*_oq z07F@xm)IWS)5QXx3ee$D+(d!AS@SCtMI}3dd*4K6jMop+BboZSlTjfmsRUu+&FSS5 z{p{T@zczzcP?^C(XjB3L!W7pojevmo#WKe~a>N}F@;u?_) z*|+GDze;{R-IStt@vEJuuYczX{LvTVwT`<`PI2+?GUBjP#YU$JAvU$2Y%q^ z_uOf57M4>DHSa!t9Lo?ID&>JV;fz2ZG?JKI|Fccmd8@pgp4UNSHB1Wyp}so1jQcD? zq-&eGEG0j>7_1`}9Br^Ht3)fS;Qj#9yPPJ0e0JAp0J~w#7Bp;^=!Y$9qMhRn>}jVjjiJk6wGxbzvjbn_=-W9~Uew9h<+- zVa@B;<-2ytdvMLqvOJx4)=MAEzuNKh{L3zFlm~lK;5!-9kR?$)CN_KX^?cat(B){R z>R*;kIB{XSaa#N#iIwFqU%l#pFaGc$<>iI*IvuOI9^dkV;5OT{BbF~;?%H$X1q5Oa_+cSwH+SZx>e1@_dgEI74e*U|t&KfO#U%tHkQ0*1h`u?O?;fCepHL;=l zmejnDF{NIl{89C74UCjl{cvGY4`d)jFGiQznt(}5j;@3i=E~@km7NU3AJW%YW6!k%fgm}*nm+r4jJ z053Hyq+{F?v*)z~f^?r)Y54y3-N64={-xwEvNnFAGjkRSZ_i$9P_Mpx_AWmo|I_Rw zN!-(a7#SJ4=CE(?K=2+%`u+l8YYHjVVp@Z~78UtZi<_0~<3gA^2CHbj=P%_w4Tc++ zSKi(Er}&&RLD_~O?PsV990R(CFDl|pSUT+EO3@G?C~HW4?pnN`a(yQU_NlG>rsBD` zSL#K6|CI3&5OV?(mRa}D_z~WHm@an1bp;4w!mcXvlOt=7tWs8f6I&7~&yLBpwdxd; zkL*f&k}*qzLAv+L8dhVy^2ks1<|HyGGg)4tdT`mV%wwVq6$-1gP*=`0Q=>nWe|asH z)1`xy-^WIjdFjeUx*Q2Dogak--hdimzQ8mp`kE0B1N07#Y-8JBH4>jx%rdrqI7TC@9FQy7CU)ck^0+UgX#3Ui*K& zb|#Yo{cKm{cVwI8+;32|>LGMfen>G|4_Cf=^Gf@A+L9*kM=IrcjsjT(*bZ;t&<^i$0TY9C8yk=QLuPEhlsj^%++m6v(9B*Q_I5&9! z8q#Ax%>bVKOsF=B5vNcy93_N~^tGRbo}&BNB&d(xqE0 ztO+|R$v=ejhZy+#}0_9s8z0AullhGjb7zywOSLq>ow#+pbRw1++0iT>?CKxSdNi9MC670 z+naXnY9-(vQO#l)RwvF9Kp5f}FYo0qjlLEZM*g_W1s=S$Iqz=j!>S|8@7h~*G^$CB z!K62LNR~{X_P>7pn!9sQgZR)#mpV1x9EA?w~0xYzpR-~5$ccF(_5^S*1>vt3OeIVev-KD6rw|m%pdr9Z@8#Gvb*~Tg(T#M#%<;POpN13}+ zd+qAnH};?JEuHI?zB(YhJX9Yp2?qM^lKq@^&R+7Ku5O-GWe_Swil}ZK09#1hoSP#3ty@JiS-E z?y50gupoHqY(^@1XM~^0Pj08&)OtgKUF;yL!r9Hs&uA4@cjM&Z^Q_BmyE2dYPA7=< z-IjkiJ}f06&Ewmrg&P8l4up7PP@urtsf=oPW`YdrJ1D!5|9!kWer_(s%ROOXp|vJV z%4fdChK| zaOWf)X=AgGJ=Bpwn-BSBEha>sp*$RL-)A&~_e{MXEzb)~-*Ulfb|)|Svc#crXhf4C zH%?G@v#aN|XV0FTPOTlI7AfI7>q<3PkRUtdBf>pzp3q-|6G{6@{iw00K{zf-G3)%W z?d3^EMc-G!vU<*lp{Qu;{a5#8BelV+nslf9n{?X!?1XO0d8fRKYTt$6b#DM8;Ku!V zyF)v#d-LYa>;aF@*>u~m@a~vN$Sd4#E_oaHkbJ{?YRG*&?qosqn6K=pG6^VNS{E5U zUSrC=rFc4Ra$|Yh-jknWOun`CD=ct=;a!m8msIXKw?q7Hr!OMq@%&8_mmy zQDGrRlr632JaoqH;KRL?kG7m*)#nSSgOLC`Y?nPQhSW3;Vn?u6D0B5xk5CA~ckkr)2XPWMd? z5_lip3Yo3@Q_~xXCdOm}bgZ2q%SS4^-*)v^JBHDZgha4#Os)nQeTV4wfwb|3Tn#W4 zeT3l1K?lfxO;46j#9lH`-T<&CAssSSzjp2BQ4de@Q5r2N^AVDU@DCrAZ>s0b)_4h7 z(PGr$^-)K3H-4L4naPkUSKmYPIgn{v8W7E%=sj4+?8Cuw7pa^1a^gEc%ncbn+ybWL z4i)4cAMcC?#%fMZg5zdc-%;V1{{ucP{lzVfXD7>d$sCMBQr zJx6_fUTp82d2Qh~d(R<*2j7B9YQjl!d{mRIqZL@ESMSo97i9;y4?nPP;>O9%J9J1{ zXR=`vQ7gQlhF)0soCy!>m)OvIZ^Xn%(~_#E6^TrJ`*d#|`Q(8lNp-s}n0D52Mvs_k zDJIVf3WCV0PLSbPTshaM#?yu4|Bq+tPCLu;K~^DGPK?zZ09n-9Y{4aqzMY%%u-n%! zUycL!oLo<*&(^C@VHDB2hQ2=P;uaB&TgW{*!mc@X7q{mHenXvOJCS4OQbn%GL|AK2bb&u7ak(htYxVDV2!eSYOR-+!&QiI z0JM+vi|TGXv~61@|0c>OPl}M|j2Ua)c2nZ?i^Kp%mn9P#&vvP!e7Clzts>sRpXTnJ z;eDGeXy^G{|F0!c53hHTe^!Mjkl-0=D);zsOTP%w_joWb@Xk0w3^-3udFV)L1l z_R3oMOq)N55bTv*lWQ+$0Q^V5O7n&dvwn7(72-$j0hjY+fYxH7WiB1?DO_~m7UjV@ z50zt)&Gz;6Uo~p@5Jbj%D7pB^+Ex}-GS51Csr@kMN#FCcs6`eHN*tU$ffk3yYPkZ_I-jfL~T^cp&g8)sAys3Mc^fLA3i=iFoJQxQS z5_0JG#mJTk&dA5xEnUBe`7deL6A}{AJ|zF^ONvBya_#gz`mL2M? zcJ&e6AHTk*Tyo{9oC>_5)&~Wv=^lzd@o)2A@cTRvy&xh2HODx;WV3$X&9-evQV6zS zw0Vuc{tDvKPk@FwRz~*j?F=fAEo{`Yr!9%1b>t#HB#;k&Wm0^n8TxC8y#?bwA$k&A z-ECf$^N;rjeqMX@Xd=5bnWA4K*zoi3t{YoiR#trMGtvovK%=l|?>>FnWAB;X06ZdR zMKSV^GHos2CbpE*_|WXA*O5bsiSeEOSVUF*4D)n8=B5m)*J<{=mTfFg?nCV(6JxCO z74k(D)_nZg0G$2i#yUAu8(D0Ko0!LN5lkBCQ$9Q;kL(m8L2hp&C))Jv#r zZgWKF*R3lzFurZ2rEzb%SQQ-c`eC)UWTTVgEd|nyoc`MVwW~_?_2T@hCr4KKXDdFV zPoF$FLPWTl1ZVy3-8*x(=Ts{|ZYU;0vnRet6b<{YG^q9PzxP9Q!kG+Jjv7Q`m;rR{ zxU|oY&r@TzTs+zE_4^X4v^HPEZ26e*3G?!k9@cMEZpQw4R-NoV>}E!R1+=k+sOQTn z8*0&&{$$h(L1&A3x1+(LIJv$BufxijYI z(WCar#Pt6B^Dd9>j(@Gc8c5LT-RJq);&gdKA8Tt*F@<9(YLMoQWIu(3;>Y@O&Cj9( z(O*@TZ2 zE<&9?yVK(Q`(tSva8c(XqZu`9*tV1qgjy{Lu$*fZM^Nu%pxnPzbH9-GP!8cFMeU~|IU zG0Kx4(6UwesG_)9@(0z@t^|J$03X@6{B+%3tIr|9Sn#1krGEc(f46y9${phL1kc;d z`+M7+ll5))R;|n^%D69j+u%|}yGs3_cXz01^|^j=m-6;`lR-%XyS!E6LTn2v7bm3= zG}Jz2AT2q`#GPZ#&dxSPOSrY#Gc^2K4!lF&TtA8JlDjPW-%}fd1F58a?e;TT(Q}9H z&9fa-x3uZ;b^<&x;26s8*wd#^4OW@*4cPG!--^GMzf{2hz|`Y+-(2#wU@9gn1K6%Y z&TLKA5PFw`kPS7&7)#uV3>?wEE$%MEh`jU2ACS7~Rj!X(>j98>bkiVCXm)%H1CqD)I>Ey?NerNH3E$tta1G>evtRE@KTtQ8U*^b&LrDjDCY z=0-;Gg_E?emP9pJ{bllZur3Ev8s=EL0>3A>Sy--Y+2;}7Lo*f>l~j($*zYZ)`hKw< zz7iLd-(jDW&uPVqoV;1%QcQMhwTH>pYN_4+2cyn~u;R_5u6rglve{9*=1y6I*>>&O<6He}^M+{otO$~p z(7P>@IzokB)xRJQ6AHoo$OxF8Cr;MZ*4}xMztgRCmx9;L>(it7$?KEj8frYLYU&q( zM$9)(7Ne~_(}m*h$WR7NGCW#G)Py!ghc<>uwkuupWW*7^pYT>-)Cqg37Qbe^gz@gEuh-Z0%s-f0L#zWn|YGM{>6Jf5s z6TNChGhby>ba5m1T(>B@qOzn-yCWX-?6&v>+wKf*o2Vs*``W*xbb5zxv&-B2B!#S- z`?`*n*&gakjDHC+tG?>!lzDs5%xjWKDu_A02KfZnG?EEJ4K&^oa|Q4?ht%=4_q3QZ z37^Qi-4-1`QoE%FeC!YcButf7uJ|KvwKq1>1b;c8-~3i~EsB_TZmQ=C(vX*s zc{e}bBSOQnSQ7q5vaD* zog9lf-!YrX^$ZLGP;|{AAsIecStav2>RnVm%jQ_;%w?DC^Oh6R_McTShN@+o@&ym0 zEGbNg>Axg;_|Ok)riEp^?`XC= zWVpAo59S@Ecrb6cvLu`D03x9nv7)5ys+RL(R*LU-BtAxS=T7%i@{DDBNRuR+l3x+Q z0o_dDXkko|Y|SK>u6SW-`@{ND|5S|}JZ?$u99TD5KNO(+J1JkF_u4%>cOIgk z^}^R{j=A8Sixt~QewSW|PFqM<0!M#sq>jtDG!TWr9jP6ppn@yQ=7?(i(Dphs_{a6yUzs(RGh)E27`3m8hUG0sw%X1Au z)7MYjR6cTKLolCAHzozzhmB#IjL-1G!syS5{yP14fuNJWg+aY~dBv&i$a{B)A=)!& zdW=7Pe-)CX)_?@hnkdWU;2ZL1gPJvcDKJnqw+dAjh)OR2^*>C@`A_#9NL%7V9w8X` zKpC6tX25^}5p5gE^&D(_Dxl#5DxQo+XwkZL;;$7j>Enyr{p1^ivOga9`HeIB!_MTC z6hjRTDEsmIIkDn(#2_hM@|TNAbL5~{v~ZTZr}Hr`=v(s8Ht|VGBP1Y>^^(6<9gO86 zNxDLL>;vMum*m8w`u1I?_9HYce|7rm8Jamb-r>|OaD<&dcapryc~EdbLmTGaGgoG? z09v7m=-O!Bv196qo66H3nT3_0JxCNafCbS264yAz1hD{Y;vG~~a&ADPjU)$oP+viO zS!>S)TAy}zcYpGgd4^}e{Kl@JIFf3M0m_1meFm@+kkaGZn@ePw-RLN2-=V`;G_p=B z#xJq{Q@J#C#flew?KxZJPpUbCspgVu{URp%vwZ%I7nyhI#0a)a3oZi z*7jcoHde_|qUcKw$@D!3*!Dq2BtT9dtqQuj1NmOOcdPOO$v@-d@Nzl5U$D&;yIpCZrdN3@wt@Ml>Gz zzGvjqR3irbDlVhw>yJ`Lv>XsLM@dv<*GsuMm2Q*St1JV?HPXzwcj}PoVcn3&$K`Oy|`Ad3+_9#UXN6ID|^tPABpt+TFmDZxHSeAjl4kJ>K4|yma!5Yl{@9!p=+p~7x^&6xGxB~}nl+%Se5@n zd=po`-5|f0N(&RIx02~e0iQ~S1Ja(vZOZ*u-b}T@8=VnHE%mj10BsazA9%?Al%kwI zAsvsFvBf=#UWIU;2rfVimlvR;@|4$)e9<+q;W_LpNW+h_4=opm763EGAU_673+7AI zaZmsDY2uT>N4bsYiA?0(U$N6uEA^b~(?fEp(VTyInzT+!n4>bkub5?V2P^NPmQ?Us$uLyYp5zDU8yXrS zmHOa$l{U{7BxzQ^&PSjOa&+_MFAi;bP+y5Os&Z7M_aTLvIPsA~g94VsnKLFce(^&* z{-wG#6FlGg-&G7FPjzv;K2rlncL$;+D7W5ij_3dF)+0-x*xq7d^tX=NJDQE&IJrxQ zfB*WkhPNueFi3asD&Mzxmwgshm|e3m#6r2W*gtZKkI*oY&)4xP;-dWEhW$KM zfNP?)zVheJEvF|b550yOtx9CuW%OWv;K1s-I{JajYADO5=$Z%>FJV%I)gf>J;yVk8 zPe!|Vz%+Rm=dJX%*kh0U+V$}c(nYpqF4I0z<;0{(NW!nyhwtiu-=>pm{dQB_YcyzZ5)sSz({O?wkhbcv{c}F2$XQHFyqC|N5;v^j0$;Bk^%`v@5S~7QT zw1aX_%j~_m*EovcB{iH4n>P8gT1EfDap=otcc5U?`NJ72ibZ69DF_-lNT1yJbo1J^ zkucKw_3HW4rJYMtI6?RXITPN!3p)nc1Kc>C;2}P(yNJOR_2kIHWaVbk^EQ30lq`uqXcl+ zOGg(!xL|Kt9y7{`0x*W9j3H#pdJtbt)K(q@_*N+<9Vw2JJ0)6qE!qpK_+q!E?I>dA z2NXB9r`J%OS3+}yt=R+C3Wmgev?h4v=WJBgX4cR5@7^JaQ&x@_QQ{j~l%ZOvm>_P& zOm6pbT8lfPC;!eroERT=Qf*IFBJsY@4cap&`uX{JcA0+|)yL}eXyTJ~%(1s=Aa@4b&)hj zPx%Q!5kDtvs1~EKoikzr&hja_U|B)#mJJ9|scqnpCVp3q*foZ_Sd)WQ-c0RPF{`X3 zpt19BhkWOVq@cV}s)z^>=}S0HroBAdr5Rb~eh2HE?JMWpSgpUIe$h&Vg7!JB*R(Sj zS$p4SM2q{DY?!xldb87fX*YlLs&9OEs!gIS9d_o;Z~@&zH+=r3_#6 zn<|Oj&PA_y`gH5dIK-3ihug`zBm1V-+M23(C{r8cv zPI(eTpx*RB^Z*Z=)X2HN3vmBQTyY|-@RXCBvZ5uY=*I^OlOjx&9hFt~9cIvZp=q{^ z+TLmqreD~Z$5F1qvfgvgy;alz+3PLuqBtqKo0z-%qZ6#jeQ4>~ZQ4 z*{PBjf+UWE)g_(zG-p1HcnanPdxJ4tlZemb5lI*`89goX9291+s6Z^B+4 zl~t#*c}jx+G9xUARVPSCB*&gTS49f-%s#nBz64|xP2KD0qB$Z86S)QQwpLQ*U|_T) zjZqVIMb1lI^0g17uI(_k^BELY9UbqF5iEhx#~~8pXD_CD`ht3lG$JqX*X7wQ#XKJm z3}%omum(H0~t$6$jg+*z#;!s~()SFs)Xo__c7Fw&V(1cxV+!!istEZFot$9#OcTp_`$ zJPUokRGzq=E_;|d+J|E5tju!}T!C66w2P!D$8wbK(PdN0|7!2u!*a~uw(rZ#m_^2n zG1i2kNQtpbWHp9d)|DhmW4)-zBBJFijQI`1C1sI>2&D)qtqem5A(d2WBq5bhDJ`DQ zaTYVb``+&Rxu17?{&=@{du`jyHjS(6IT*GHgr~Qv!7P> z>_ol^j$iP`nD)0cXIGVK{Y6wxZ0*mi9=RER{`tL>M;fZf*WeM)I3?DsTOg~->s3^> z)?Z>@@_Wdf26{mutKACNf%}aRi;1;EH+LF73%ixrV?F+qUs5&RgLg>Y`19!xS(EIM zm2(c*Tal;A>nIq_$az3D0)P4CBRX$S$U22X^P2oyxE#gaO8FP{!Nu!E-J)8-(DsqA z2b0liY2>Xd=MuMXtfO;- zugwQL@5_v^8$H{qSw$t?I=||fy8wF*yM8;p|3tMpZ}ysT)q}JyE0226_N3rV zko!*&xpJS~P!?q_-XcCz*!8|Wtq)Lw>Cs9)ym)~t!o*|1F@oCRopPMA^W2bu@-F8` z^=aV$wtA8keTqmF%Hw9~gLGPGf~fqT0xtQUH41*Dd}!hFa^>E8tp|mXx)SWBuwJgF1;7B7#6nOJ&djyEk1{CL3BnxBuIb;mV z-Ql;)KbvVuSPOZmtUTpkBlhRfK;^_*UW0*m74cuvzVunH6VGO?7#}(8lP@~lq91dV z(gHNUdw|P@WA%?}dM{WUa>hJ|*`M-GH2k4DwES_m;&}H{>*y0Y$&gwu#&_yUm1peSnib_h0jxF49q9lH_eR z5=ewT^H|zD*V4N8>^UJku6(yNJ`D~kU+ahHB-^&i@#i&U`iy42L98n|$w@pz{!J2) z8g6IDA- z+G#R{HHpg0I=A<;`6d>}C+YY}?J6vSx6k{TW>Qg$K7e~l?HJvq{$WmH0-gc&@Hi&z z6niU6&+F$yjsN1jLzFccyx3P~*I}TM6)k>&0I(WP>===BEsZG}XwX2!YMavk-E}$i z+|#!TP)hVLt$NQ2B{35Ef&zy%Cbpp@7Shm(AAW4u7s@pU-U6~Vp(=FDiZWvnY?PBp zFgQry(d9*V4s8tDSTR$%IOX-KK^thC2!{-y6RwaYnPl(Gvz+_v5Gm#V)0@XKp^z_5 z8^i7-QjBo62iMi(KQGIaJ9k=i=+F~%#jPxX`aCwyaHEofd*usnZ6;h#k%;UkU>NsN zAkM%Or$Rqyk77gcEfQXR^Pns9-*z12G=d+bP<|Hy{9%eaI6NP)#pv^kYiBE8Wb69I ze-SUlU2#UdTQuXi)@8)w^|N2+D0T8X+RqAZUlozGumLj~#2znB58o$(Bk3M}i?HF-sWz zb3b}zFb5($0--tn`>ld3HC;o>X2`jCV^=>@Y0A&7M99-idOde8h5W1g#NG4ex2S(y z@^8IW4%5Q2;5G>-pCB73gksKMB8e9TkY$*Su~ZD82efok9u+w*c2siD`ghRM_M19; z&j9~W)Rk7u=Bvm}~ zE|Ll7%A+#B_4~cUrkCC__4#YO&Gh^A>zC{yn}`#wj3g{+>DieLK;HOlw^ujxp-jch zRX%a+W|riJj_bx!Zwap>71}9{!*5OZ3H6^&Wg4CIM{qmQ&b*^sDZi+t&4uTLK{Xh2dPmff4XEe89x`bGCBaZm(JBG>}SmCKg$9o6}1rcAig zEhkSs^-q2_zt{Q~U6#38`l_?r@aQ%26)EVg9F)*p|NdIKNlh*=4keCHI0O~!8U-xI zDrOMdS1~$55Go0#5K|fgqVj7>RPYC)ZavFoB#~Eq7pKyxuEE|x0Ai22nl++fk{eDn zD?DFr_^o4l^&cw#o}wr1QS@tG=8e;pC03SWVo^?;X`%JJTbk)r$& zV%5K|bvdojwM#6MNemQ_%fiI6Gn}<}w1HwRy(-BMj@7r6ray!$>~#O!EYBL};N!}{ zs^TfjDj?mrNU3JpaqXXr!xib)4?;Sn{O}16Ulw1*pJ(`w z-J+ekSh$Q{9~^o$UU|iHM3ojJxn{7A8Utc8 zP+E_wr`_dQNHoXPb9c!rPr2t_h1~Oa-|c@8j;*&UiJ8j&xNS^Y_E#@kfT{@Vy(7M( zq1213FD+*}$~g)}iN>H#mXfav;e{?YpCv4gQ~+5hIzNz&Wi$>~-p|R9jjcpwRfPmU zB(r-RGf;`|3_nSRpUy%Nw2B&IjC65Ul{U#0Y7TqiwLm<`Nfa5aOOmowyj%wz>Rb)n zKUM=_T@e^3g-Msx|Eu~G`n%+$q{7{aH$_xKC)mie&l__QtO!X50A&&R6a?-^1C}Wd z`Umac7Qf7dcZBEy=Q=5{GJI{DUi{W)+qSi{f2xlOgMHd zj02sOM53`z%Hd4|Ct4uRO^gV!t>2eA)XI(tp((Z*9uM~HJo3lz z6CT^UUE10#E+^JHa=rm19Md7IANT1J^Re!@MJ_*A zW%%Bxe7ns3PoU_44b;od+9F6 zjP#JX1D02nyKP=(6CCuQ-R_#W-O)~G545h?8(zA1Mo#GdEe`@5Gu|vtu&>?9Q)_iLVCKMw>1<`bsL6u%xo^z}t2Kr9=8Ud6{llHT8NM~KWjkl>dNH26TwXWp=BqE)lvMa%q76`=Y}KtK z9%L_1)J9}z0VN;sYCL?1$%MCOq=SSGQ00L=&|K2-rv%;}({+eW_wGXigv6Lpawif- zrd#WJL2y{!>7P!}u0}yfh3R+D5vh;#jX8|+LmMazfpkdmTHR;?R?y<>uMZOI<>Sad zy$xKcJ|0ld6_>>E{hQ@f z*JWmY?&@7S+U;dsML|N^lX;^9LkE50U-MM4yCKsa3sItr0T7~>$MekiV z-@UjRI*)P~axRx4n6%y|KD9hlN zCKnRyW(~?2@!w!%FY8kM3a2)kTwem8xJJi+UYjxAZ|~->F3V@P>#3_aMQt@j<4EF* zTmmvD{7|>F$!DLV-_9YmTlV}r9UoFGoiLBger1G0G$I7+NG7PXg z&Sia+z9JL`7`7awUkxq&^K05p0!EerlMf!+Pj0~ZwXu7nBS>Lr@fakRC!JBZ4h{nS zbXuSzf33=T7@4^=1@n~^J@SCi05W}IM_%1;FFw#}py)OFi*1OoMuFKzH^$6)9d^sy&I57Dk%pK{3E-Vkw zgXlrJaz*D;jPV~(t}VAt#(immga`@e`}+L1sGQ6-(y~ojsbo%Ct;61VQiu*g2ZM_z zzq>P_NO}e67H2ddrdQt6pRn1T0y;=F^)fYwP7`Jkt%8-!*<#@PRgb8XWg5kcT3qRd1wVv_&CZ*zoIpi zT5)e4Ze-)oVOVaaP*+CGe&=@41OvjcB8nmw2ufCz6`gW>HIpkuyv^*ysj!k(y?$aw zrEU$GV`)9(;l=mwIZ)j}K7JF1j<{_T1d<0}h-a$wIieTu)<<3pBGePL<$KcHyL;G0 z3L|ifFhb)s?u{Vfa3+qBONsJIq|?Pw%2)gH&z22=Ob->FNQ*g*6}|@vtKgFJ<^>Ui z0y)3em@2h`%eP_8;x~zK3EHY~Bc#WHY5}i>qs+lM2XgV6&a~}vQ7s`Q{6%PXusGO7 zQ+cYlaJ++ZTl|xi^4#@4rCTYSx}g{ZJiC7;gto)3<89oyO;R8)o$#q#=8NxbzHw_y z^HH{L_4AUF-h697$R53VO%mL@L7`-maNGSK0wwWjke<0{uh_8txqA(yaLe9GAu{Fs zyafxE2#ez&1GD0|O2p@S3c zEY#I0^!#>^viNE8&c`V2k358znvUqc=@1VIoNK?)#R#7v0T_QrXq|T-Hag@Sd zoEpTILt5;p8MYkEp`|a)+o3P7bQPdS=PBM&G!gb>wN&Cj0lQwYks zYN3WYht6&zF;IASd~|r-}(|dE6z`DjjZQ%N+Veq7n}8Ts_Bz~_`X3gTi00VP2fe(8)uMz?z%zIrCm0els_TIUHw${wA8$O720MAkm0dV@0 zb1UJ0^#9?!iky2QF45+6XK2j@2^+DGp{)486M>!GHse9^8k7uXC#5Wng+cfKri!H3 zJ2t$sY&P)ou6C7B>tCfN?-;lK+fl|gU5|dRxEADEsVYuIopiGKC4_$roNQtju~jrd zz;**nGws9U4x=(rYsA^+_xe!h_t*9-_CK7>EF(=QWF2h-R3AnS>QKaP_AY=_zBa7j zPUQG{Y<#H+u%|g@DKvQY94vyQzrz2+2dKfqyp;q?@6V|3-n1j&_mgd{Bm5)5(vl(_ z6)jU(XP75Wqy0l(PS#{&;WG9Ye{lfQ`$+4dM=PUNc&D`FpT^!BTcbF&Tm3i~b22d$ z!Cqbu$I}XBh6tF&zooL0KB!68Q1yJCg(lvqW)JF26pzJEPO8rV>)V@r>U?gUp}9S zM{I^e>zw-y$)RE2zAqzEepiCp($4{rpeb5~ZTY&+Q>!BOeq3zqqoNyo_bP9BbM(kT zo&L`iMo}9V?@q5#-r@YNH_u92;$+xAv;S?sn7(h!IY7i&NYoWLrl^IXpEit1=HdmS z5hIp@UzH~kl||p?U2~xWH{)`>8P|aT=oxNXHlvm%nHLV%5qQ%=K!@|p;ywc%O;)ew z?#a2P1fI(pMRBT1-(Qcc=>4{{@8B6-SK9_0TGa~Gtu!A9 zepb+`i9_N^B93?obv5CnmLk^y^iI_7U7yOxcf@j|<%P8;GK(?-@YSl0sIf3NU(wUR zE7~! z%pZV{VlYg2L~Bfyl=RCmD?R11FTjP!lB=g_$5vnO1)wUKU5_cS&)VU$nPwOPkLdgq-%99d`YBntJnYgGzh(|Zl*G)``MY!AYPcV%1R$1TucREd*ZxmQk6}FqMG@z8O z8iFbv`!jp$-A`7GYTz8(Y$roC4r`Ug439l2%FTWMcgj!im!;E0lGyhiep&v?!`%9= zTc3m_#*8bNf(O!F!SShBhG_lCMS*os<>=tZ zwNq;(=ixMp>eov?yYrXtGFFw=VL%>xdOI2&j6&hh8UQGvH(m4kDJ(J3UauLgB#}0a z8g|jvQGiipbX=DZVnvZ4ScLRMY0*ilz%bVX9yL`*Ij7!{bUts$+Bq2f?(%KvO_Dlc`>Yk1fH{}<&Byx+M0_ioM>=Ah_HmIWcmf|NnZ0{TNU zsoRXJ?iZ~_us4Gf!jDG1SUs>a3!;k=_5+?w;yBZwd5vecgnjlt{(2>fDME4Vt4Y$D zhnFGttE)>M97k$oP54M7FF#e*3BvO%_cBY207F11@ySWW$m1B*biuh=GGGfD^}9{h zMtpwJ>C*TQRJq3*Donn)OAu|u(JV}CHw1nm^%sM*q6cv^>G0p74-iU4d-ZmmsA)); z6q6CkE~WGm;tlCiJ1TnUkrBr|fnS94EdF1V0?}3ZjgaI1eSCC$iZ;n-@owQ$;{4Dd z35nN}ogc3Q@EMHviw-+rqM)qOy@aSYUL%YFshWP@vLzih%Bka?fH)%gLn)De!0IE_ z`P75;hcjR0#UPpHd}hfGouhCIjqlGqy=XsuAhGe_oZq2?VnVG!{(Gv-5b~Q&k3M~F zpL`CMc8fZrj7F76X*KCIqx${h_V^Psd+-j>R!GD>!rUo*x~cColSSt!&Vm}Bym7gq z`%Y@gn*P{U<@J-7K|Q+8rFTpO0@qL_2Q1u2M|Hf06Enh)sWp44!2Ax@L3rZ0 z8NQgx6%lKSH2+xuvbu#=qi7L@#6e3D0m6{(@$~(}wEwYfDl1mqioTkB`i`{}7yt_) z$m?a@=`2P-1e`(7+X9P#4u$@xQShR%qQPCVPKdhcoBz&~ItF9Ns0kv02KVfmJ`CtB zDXn6~thMmqO7^T(>=X|#7MuaKP^|cNQ6|YCMbRu!o$WPA z+{YaiTRkx>ICA7jhazyw2(Xja;&wwkk;77|R%+2>{5UPXze@FTbAu8RFmCQpYWNu^ z=d5Q51w%$sf|!9-8ZgfpY^~rDJvJ_fWVyL zUS3Ylu{5^d{V@=LN)VFpby;+@SH{)7I7G{PKTqDqp6lpIk@oJjJfQzg#(3fq#)U%I zPYt!Ukk&ZcL?EcozV4wgC!T31cj9qd^f zD=yh0)25va^lV0r?fSEVplto*-S1v*d7;(S|Dgt`fk?R3;N35qj=a|ejOIQwb9EM@ zg%j4b&eeLwPSvCBlQFDud$k!a&n(}s<(j=>ot7#cZKGjY7h3(R!HRN*KqI#w{8MA& zw?N6LI}aapR#J7Z_9!@;1YvMPBVVC~KfVv5EN7_(t*^e{{C|77DtW}ndmq5K(FlGo zRHI}MYe$kd$#!iu^}3b~%+H$5u9HXSb=m zwl&AYtGj+-u}ZH)ynjLQNA(^O%{#On+PolAwc--aq$-zC?B1 z*1UHzR25&2RzMAxkAV*V*m)Qy3Y7iMjTL0C*5-6 z6YZ=%o!NQNpnKv>V<;E(&c|>~Uc-fd$UX6S?=kXUy#l4##DSTY&aNhEuPsknvT@U< zsUX1PBb0J)YbXCqSrJ|o`3^q<7mDaWa~c55=IzeIM@c=#iKM9{!a<&&>d_~BWB%lV zBFxl**)9@xskeKWz9n>>@vQSuo3m?ecmMlMUh3B37enTw{BOwq$A}=so0PJ2MA*AW zzi;+#?pLz_{eL=Lq&O0<{-e{y<{pib=d}41AB^-yswwxD+kBM!xqs$?&VT*eeQi8@ zqewUZVb_O>hD<=b4d2zz#lOk7RN1tu|GG|j|DmXJ{U20($~*jD$*QY8K}98VlDG(P z09pFtx2Ii7=P1N*m_6`?x6OT?f1kC~TI<4yZ?=_fC_H1b;HG~MHQC;cGug1CSkJgG zpM*!-R2-p)q>~$9;kh{RoPSy3OB90Os{-4W^kM4n)e2Yhc(=+9d8cY~>ngg^yJ&&3_9FeT3Y7hly?(6J?Skhv#RW+@+ug-^6@5){tj>N zqZNMS>L5@kJtKx6379US5!bc>3`^vR&fM9~Z#x$zOlcIFbUK{YbU5EPJ)#cGT!`^v zs|2bmCdz=o9P)h8<>N&DWRK`mN-y{{`)OhxBXz@8W6iayM$>}>i><=axIGe&6I+{k z6-u;K*~lcU=w6f6YO^Ivp3Q&z(MKOWn%=5;^M}*PS;9CZgmWi(pePea>O;tftge?y zYZy%NYDvRx9D-;_3KrN2`rFT8D;-scEvhWO5s%l8hHh?LcF017rd#I3>2k^ z`=#Zgx#5_X#+Bk5j~3uGW4CGT2wE=+@uk>aBt>*xgBukbrl6pWc6Y@3&-KGbt)hKv z(&_5njdYxu)K|=i5Jqp)*{nTe8XY?Fi^`66kTx-99u0IbZS+S@Z5uC7SYaq%8=M-G z3)_@vlNZa#MO!o{oF|p_VFHRl`BRQ+?=tZ&@-C-=QwYCYti%Ktd-&&PGL|Cai`ZyP zZx4~Yq|#~$j!#4Cz{!YQgPdqq*&B%6J(4TB6rO|=W0GjZ6o;8PbR^)Olek$@dPyJ0 zml36NZRiIH$6iEDFuKk?%Tjq&udG#b_#esUnLP=@s{9v4;?;C4qz$roi93?9Fd1-Z>w@L40Eo7yB`X@P<$)R zXh_g-C^|g>;e!@igys>Qgv##_dgzn4^dvtn&>ScK6VkWksCXGO%rhvA{-wbN~E7N@2O7wd8Rmlqe9Zh8q!X2 z)DdWz=^}#-t$5CPYq2EpkDR4SGdS|LWnkyP&Yzz@DooDAS4p;pqOAa^AGNOZQtNQn zcGo?V&V^q%W7By4A!90c6^i92E-(;Q+-SS> z&%8NlZ?~trGRo)}S2F%we8tBqlaXnb-$E60b~`%u+@PTk^S?JWr96(#_qAD=0Lkbi zb+7Kzy7(h24aJ&Y(0{&~H24XwBIOacW`h5;$hQ5-AGIsqZsgclxmVDL*=>xqQtLV< zh}@~ow#?fTO~cZxhx(ul+;oDjsD$$KB3$+^?z<6ky1I%c2kIX^o+ zC4)0x#E8IulyaJ)A|5YyEO?uLok;SU|t`s`GQ|qb_PWR*4AV#uo`W*VlQz-2KI1 zXx~y12E}f?%qR?h{KCY1M(1kjj|<7&$OGuYJ&u~}zEp~5wK+YFm({A%Y!^)sLnYKa z?zm@eLaRkmh>`1{8eCjO$ch~)@%S6*bkL0zFwo5rl*06s+MGDfR3ICLvTh$h}yC zZc%7e82iYC8#!f($UF$k$Xc?UjN+l~AWd?f5AMj1>2JbRjT6hBCxkYK$SYJO0G*T9 zkLAR1S|5P{v<}Y*v1Fjolj2>v>J;!R3}I^;0Mube*!wN9P2+eIEJ9R4Fk}W9 zc6B<~kD89|7UlWEiWkF@DhMf`;(tx)lLP% znv`8jZ&rjTN9o~bgoWzsMLg&5Q$(%Z88aeJ`}xu-l#-Q^GDIOp#z~45anzwBYD>B-f@&P z7Jz~>?&TT0V{kyL25K3dy5X6h?7k`Zg!l^IM=S~H5`Rg}!58{!GC##J0p}omvgCKY zzZ_ptFXs9bvj!3-4$A2OoE`zz0d6nVKsHN|_<8pfV!Lcz z@+pau7((Cnq&=6hAo(7j47v2NOV1gnul%c|MQ+FRb9E8sV4*9ky{*e)j2M?B3x%G6 zIB&o}C?h@}W7|s(Fvuy88@Jo>y7VNaXTwSsKXwUftlK9PU2sfOz{A9eF?jw*OZ$^D zb+N6Ulg;!+q{aO)ppfMZ0Ld&2OXSkhx4~mEG3R^Yn%GEn=odAQG4t;8;)bc^F-V?F z`|}BRMf?t!HWp5$B@V%YDTpU88_yvcy#hkX-Xy8rGg~J52?#JZ?I$<`gMB?Y;DkRV z$_-FkA=PpVMW`>?5u5ZWcJ8YlXMWmUIUqSMp1agyO4i-=Sr zAAl+&rptVtLeiP1Y67vg~!mx1l?%IkQE(@*|O zbU48|-|3APcM%K#7Vc4ZDO~-bpGc-f=ioTdTSmWV`ZM5GU@3vlz`yR2(ohU1xjCWLu~J{Ogn-Q$Gb<5_FzNME5@I5Y55Yij{mC*Kem@l#bh+1Rhcq@ zi0De6e&SoK(D%h2fGdWxc*bF_glF{~vJ&wPA5?CNbpdp)2Gmvr&u&GRUPX~GnRb0qTGo&|xqERNiB5Fa=UY!y!` zXp-vIlLH}G6+6@*VkN6e% zqzD%kZYM_I`wEVKAzc$B-jAlwLj0N-_Y?N@9+yJBf};Xz{DK!@^dqj#bJwDMV+nLq z3(!d{(RO6F?O3(E-8xl9%SSci(n^Zy`|Q}Y>*>p)HE>w(Tr${EXK24#*%&YxGww@d zA&Iv!FAw?oP@U^U>wY(LFOL}6=KKKZVCHozmtF_!c3Aa`QD89QcS8T<#_Ezo+!^xr z$@EI4n~)dy*90>6R8AdBWF9tkM{b4-v-j}gS>@R4|7MaHj`dFIgEqhJmxTetNF2V% zAzu?4j!-|O`+upL^1`NQxtWF(=@1Q&C4cgArAAGXc_HW_hHg zjsX0)X*KKo*DW5^_=PYSPut4j6bz2Tn{QVA^pwmnh&F{i^h#r`K_g~cZmlpN=@o{N zn5}cC>`4*5d*3up;)n^8%8)H`5Khc$B<>scguJ9M#Oi9Yuk z2{Z-eyA<@u`sUfkkShTT5n7IM^gN@B6?H6QUT#MaeH$Q>W{DG@^URJDVxBQ(M!NTB z$nG3c9MyLOxe*~aJ1ZO;%{l7BL<5=c!Kl%v72V?7k*9iBbb~5MB`}4<7rXLrrTUj1 z^vr=OuK{7@Q7&YG;6m;K-w5FhNJbhUG602YnwLlfHy5v~unXvtkWywhcIU^`=j{)@zTA&rvNetON@y{8l_n$54F*0eu zk-ncD@a^Lly{g&X>CIiYeVXAv_GZbJ>YMJyny=NnAL%5%T5emB8dZ|)o>TI+*m+om zae@1=eH9PlB>5HeQOauUU^o+dl$HfP-`ecsDP72v`}}^-E@t&a&gm3Dz7QG}Sl=Xv zIkG5uh8R@^vytF!=g)NL*=l^e^JvR(WpTEvcuc-=&ZYDK^wZbSAX-g7rKOru z*|H#7Pt*caFd34AoBi6-S$@~HnD5?8=1l-$_$t2DjnjsOK}IYc=}E$z z@Z@4Y)zmTKZQAGBT)0@P`}Mi!EB<5V4}9KIADzssD{bez{>JGSZz3PwJ!7yqyXwx= zyX{_i-yb^cP#*SZ0fT4$nz|$vqSW}VtIi*)xl>zc-5La&c4T31&&rM53Efi}p@bG$ zux#V~-?t4=@7QnGmIy`UW#{VS411Ka{+#>UNNL5;A`M~RodYF5xy6-na$=(y9UVNl7`f5zfzOXVGm-?@`M!9kfiXU%twGM`iS)7$frsy^m_2%-%zGZf9fP zR;9n^;iNTh=DijCg5H}Zh{_=0i;7c@E7zOiW|dv-#IIEAgmf28!KKvC~R3O$A~6FCsg_ElU33yd2vyW~TM64)G1 zbz>^N*pium&&u3g&`W}1U0q&1o_H|e#KGHtc++khx6i$Nm-M*RuT5Jv{cN^sei#|r zUU{#1qU&YJG130K+-A$_T|G_|jmH z`5wLQ7jM?tkkDZ)%a42^5^z_^?9^d4-na9)EwEV+E0^Z?{kF^_f$kiO;mUd@Z67Ai$nw?$f6E)|Q`5P-{fm z6H{^-Vak&iE*(2|ynJMYsTyvk<*y!xPNawF&A5()Vj_{HFX`QOy4CMD_82O#X^N+Pq)XiAk*dcdFxzU7QrdyuPh+6%9tgv z?YuW^60}%iR!}sq@mQ%Ubkzdouz<-qd-PcL5utdlIIy-)z7?#=*226Rf7Ag^-gM<+s z88FZ*p-ChE(7vw?K!8g`Ax_0}BP`1dWLNHb^}WigD<)H%`RK^FAHs6uj+xWiS7L=A z@IdK@XGnqFG0&B+@kZ}k{$v1B*)-z=_XxW`Gx`n-7>C3#iu&tI-$knRWJ zpG{iO_{D^czx{S%TturwL!wqa)!bf9jdZBphN39CA(tMkA z?Ck901oU;!G(JA_&U*UwvJKUQBE{)(yV9{Dcky_2@mH$AMb}S-Vdg$O7@S-Fjed3N#~Xm@Va)!8=K zwTI#mzb!p_b z_S2`|v&j2cfA^i@1c&gw!G^kPd(ZG0cHl(%(%G2~HR-hSj+0&vn3V(cyt?K(N0)Z- zTNk@W1qBP&!~s$omY037!*c#pD~El@9XE2%N*`Q~1WLQQ+t+u&Sv#=nG61iB_ep~; zwrk#e4*qW^$-3_@GGe@H%-mf!7KCtaTWSVyzMR8DA|+P!_dMX2lgh)bW&R_zFIoN=8nz2jOb+ltEmcyKRHvO_C|sm`Ag5x8X2 zQgDN;C(Y|C=Q-a%C>Nx`2SKV@x&FD$z@)YP-6Q)sLY`9aq>I1-K(xn@Av5VqDqI@e z4Q26CqRl?K_iL5@hnadi**7>Cof`XAqB{iEvza;Fun!Z)L}Dk;P}HCdpS^nH#zX-o zkhMBvTm&)AXV3#Odp+h+F=57xl;Y9Tr$=e5@mEJ!>C*oEYR_iwpeTzR@AsFBhulE6UGUV2*hwr{4v0ltK7@l*Ch`81_nusDsVv2r$Q0a%hXQ>&a^V$)=s7W zqqKc>b)SNzpS8^%sZs@uHfasJ!|Qi(2}fZpSn%&dwTG*G4nY|dGsAPP6=4EO3G*i7 zoBcl90zR2?RA55d^AH(_*{%Cf6|h!$7ZSqd+;250*^aC8?jYOo*Z$&~S1`k!yzIKL zun0~9m!2BcNOk_HOD~0Ap8FAgV=4< zm>OpC9L>tI5$>UQaa~_<>eQ%FZGX6?h?Nx@MQ+;_WzQ^a6Fh9e@)Swf*I$q7dvAT8 z26npgBAXUn-xCPodyU@qPYpxs(f|HZ(w6ssoXm9I|Kg-GjI|Y^KId4z$5JDsv(0gE zMUQEVo6`9r&VT8s&=Qdog~L4f%5rf$g{U(q4J*`Zk3UxFML?(HEs|Y|T3d~Nx{THo z)5AWHX5xZV*7Wz}9oSpAvv$db&i;M+hToG?G88_JTvLD}AD9j;L01jhZJ53i!=aG# zWztQaJh}I2nV$9Pct4iRx{a3(`1rH{tDPWQ6@=vV7SHNYJgv1$Y^2@7Usaf9s9vjHvi)ihS?E*Pa56YnJV&Ju0D5#Oe$3Pf zo8I3`lT$n{>)gIT=)sYnHEEKxCYJKHP3HFP+hr&=E$|V1#VXt^WvXls1A~xTJEe~# zZa7?N&$7bCxh6M@mk~090E;PyXNm{5G_X-bQ<6+3>5(FOLbvUtxQ7{U;F~mbWM$_D zr2y-(?PvVLzS`H_We$X9RsOY>?qnQaWYRzyLDpyi#vnFpJFIr9Y|f`t?LqMJgs`s( zVm_pdcZ8C%O@}d-LK6{#JRM2dY)`$pK_ABY})syF0mZ%s|OhbI^|9oszKaP1%bVQ+Uy;yvt3g z2E-9ol&;dM4Dd74;jkG{8&6sKqodFUNXd@ELD)Uib@kfyADdXY7ndW1v@yr+IL-;T8QNz9+a=t5cq9R^U|VSgF9O*~a(f#< zWY%!f1_P7tnC+uC`R{c#Xf*Uhme%6C*O-j9F8tv&;kcY2Bw*4i9Wnir0Xg8fm<_|M+4ks&9=({w z)!9&b+%oG*>S`y`B@rWOnyEGc`x_3JvZ!^`EJiuA=)t^>$*Gw{dLTIO z_O6D;rZt<%k>q#)7uVw!fsFd`x%dys2y8*%*vrzSfk5?S(=D?xW8ZVw^kbzpY5rj9 zk;rYQCrf)t=PA??8P1txIM9?6lAnTki1mD(0gLZ^pHNe@tmwXcQI2M53gC|@v;$@%R~u6207<3_As zgZM=)#lWNDSud$# zb)K}`b=7P^PWGzQa1X`#o^s}j)%KNu*wP8(zAT)uwrSvVO43o#pHg{Xo1MFDR|Bw; z|82iC$Bae$CdJZ6_8S%d%`YdNvwDG-m#1YRvYpjdXoVz!}VfCzABXnCS2mw%z4}ie3x1P1HI*E+?i;CXk>@445B_IL+ z5i7}W+3s4K_Yc&|0(UlZ^9QTKBEO2C43}$feORHEg`^h7L8@MB5evLJ4Nz-`VkXS8 z%5P@ubuEZK?h4R39uqpirLT?HGvJ|S^uU^Ex3n(|5ieJ1;MmORH4!*q;FvL#Gs1+Y zf4oCS7-rBugh``Aq;DBAtdIV{f7}>lFaa|}z(vKCRA@$=jFQL4?EV8kaWiUE68ruz z>~JR3gIP*ud-=#;kUa_^39U>!q&jk5$f5`l$7)3BO7eQmVPF!1kosw7=`AcsL6#n_nZFQLoYb&ON;4EBoDt-%u0(x?t!E0flJd40 zY%>4ueR7yPm9Dsq;HWAu3MmM1oy2~yq>US(bXEH25^BrBWzrbycZg7(GBPrR`GCjeags9x9QXc;ryLS-Cq@znY1`?; z?tc=(`&}#59~bsNDEW@D;BJ1=^3atVGpr5l{#K!?w4|bZ|L30x?*HS=`2V}VSoh`P Y(Pm?(c8K{=?wES?r0>JN`{Aem1r5JoMgRZ+ literal 0 HcmV?d00001 diff --git a/runs/frontier-fidelity-envelope-v1/run_frontier_qwen235_v020_surface.py b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen235_v020_surface.py index e275421..2d85d9c 100644 --- a/runs/frontier-fidelity-envelope-v1/run_frontier_qwen235_v020_surface.py +++ b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen235_v020_surface.py @@ -158,6 +158,10 @@ def main() -> None: selected = [config for config in GRID if config.name in wanted] if {config.name for config in selected} != wanted: raise ValueError(f"unknown configs: {wanted - {config.name for config in selected}}") + allreduce_profile = Q30.validate_allreduce_profile( + args.allreduce_csv, {config.tp for config in selected if config.tp > 1} + ) + allreduce_csv_sha256 = BASE.sha256(args.allreduce_csv) builder = BASE.load_module( "qwen235_frontier_builder", args.replayserve_root / "tools/run_frontier_sweep.py" @@ -182,7 +186,11 @@ def main() -> None: result_path = run_dir / "result.json" if args.resume and result_path.is_file(): previous = json.loads(result_path.read_text()) - if previous.get("status") == "completed": + if ( + previous.get("status") == "completed" + and previous.get("allreduce_csv_sha256") + == allreduce_csv_sha256 + ): results.append(previous) continue run_dir.mkdir(parents=True, exist_ok=True) @@ -217,8 +225,16 @@ def main() -> None: "trace": Q30.trace_manifest_entry(trace), "elapsed_seconds": time.time() - started, "returncode": completed.returncode, + "allreduce_csv_sha256": allreduce_csv_sha256, } - if completed.returncode == 0: + fallback_evidence = Q30.collective_fallback_evidence(run_dir) + if completed.returncode == 0 and fallback_evidence: + result.update( + status="failed", + failure_class="collective_profile_fallback", + collective_fallback_evidence=fallback_evidence, + ) + elif completed.returncode == 0: metrics = Q30.find_request_metrics(run_dir) result.update(status="completed", metrics=Q30.score(metrics, trace["shapes"]), request_metrics_sha256=BASE.sha256(metrics)) else: @@ -233,6 +249,12 @@ def main() -> None: "frontier_commit": subprocess.check_output(["git", "-C", str(args.frontier_source), "rev-parse", "HEAD"], text=True).strip(), "profiles": {name: BASE.sha256(path) for name, path in paths.items()}, "runtime_contract_sha256": BASE.sha256(args.runtime_contract), + "collective": { + "backend": "vidur", + "allreduce_csv": str(args.allreduce_csv), + "allreduce_csv_sha256": allreduce_csv_sha256, + "measured_profile_validation": allreduce_profile, + }, "prefix_caching": args.prefix_caching, "results": results, } diff --git a/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py index a2c39a0..5590ed8 100644 --- a/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py +++ b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py @@ -121,6 +121,91 @@ def percentile(values: list[float], fraction: float) -> float | None: return ordered[math.ceil(fraction * len(ordered)) - 1] +def validate_allreduce_profile( + path: Path | None, required_tps: set[int] +) -> dict[str, Any] | None: + """Fail before simulation when a measured collective profile misses a TP.""" + if not required_tps and path is None: + return None + if path is None: + raise ValueError( + f"measured all-reduce profile is required for TP coverage {sorted(required_tps)}" + ) + if not path.is_file(): + raise FileNotFoundError(path) + + required_fields = { + "time_stats.all_reduce.median", + "num_workers", + "size", + "collective", + } + rows_by_tp: dict[int, int] = {} + sizes_by_tp: dict[int, set[int]] = {} + seen: set[tuple[int, int]] = set() + with path.open(newline="") as handle: + reader = csv.DictReader(handle) + missing_fields = required_fields - set(reader.fieldnames or ()) + if missing_fields: + raise ValueError( + f"all-reduce profile schema missing fields: {sorted(missing_fields)}" + ) + for line_number, row in enumerate(reader, start=2): + if row["collective"] != "all_reduce": + raise ValueError( + f"unexpected collective at {path}:{line_number}: {row['collective']!r}" + ) + tp = int(row["num_workers"]) + size = int(row["size"]) + median = float(row["time_stats.all_reduce.median"]) + if size <= 0 or not math.isfinite(median) or median <= 0: + raise ValueError( + f"invalid measured all-reduce row at {path}:{line_number}" + ) + key = (tp, size) + if key in seen: + raise ValueError(f"duplicate measured all-reduce row: TP{tp}, size={size}") + seen.add(key) + rows_by_tp[tp] = rows_by_tp.get(tp, 0) + 1 + sizes_by_tp.setdefault(tp, set()).add(size) + + missing_tps = required_tps - set(rows_by_tp) + if missing_tps: + raise ValueError( + "measured all-reduce profile coverage missing " + f"TPs {sorted(missing_tps)}; available={sorted(rows_by_tp)}; " + "analytical fallback is forbidden" + ) + return { + "required_tp_coverage": sorted(required_tps), + "available_tp_coverage": sorted(rows_by_tp), + "rows_by_tp": {str(tp): rows_by_tp[tp] for tp in sorted(rows_by_tp)}, + "size_range_by_tp": { + str(tp): [min(sizes), max(sizes)] + for tp, sizes in sorted(sizes_by_tp.items()) + }, + } + + +def collective_fallback_evidence(run_dir: Path) -> list[dict[str, Any]]: + patterns = ( + "all-reduce profiling data is empty after filtering", + "analytical fallback", + "fallback to analytical", + ) + evidence = [] + for name in ("stdout.log", "stderr.log"): + path = run_dir / name + for line_number, line in enumerate( + path.read_text(errors="replace").splitlines(), start=1 + ): + if any(pattern in line.lower() for pattern in patterns): + evidence.append( + {"log": name, "line_number": line_number, "text": line.strip()} + ) + return evidence + + def manifest_offered_rate(path: Path) -> tuple[float | None, str | None]: manifest_path = path.with_name("manifest.json") if not manifest_path.is_file(): @@ -415,6 +500,14 @@ def main() -> None: selected = [config for config in BASE.GRID if config.name in wanted] if {config.name for config in selected} != wanted: raise ValueError(f"unknown configs: {wanted - {config.name for config in selected}}") + allreduce_profile = None + if args.cc_backend == "vidur": + allreduce_profile = validate_allreduce_profile( + args.allreduce_csv, {config.tp for config in selected if config.tp > 1} + ) + allreduce_csv_sha256 = ( + BASE.sha256(args.allreduce_csv) if args.allreduce_csv else None + ) paths = BASE.profile_paths(args.profile_root) coverage = BASE.validate_profile(paths) kernel_paths = None @@ -477,8 +570,14 @@ def main() -> None: run_dir = args.output_root / "runs" / config.name / trace["label"] result_path = run_dir / "result.json" if args.resume and result_path.is_file(): - loads.append(json.loads(result_path.read_text())) - continue + previous = json.loads(result_path.read_text()) + if ( + previous.get("status") == "completed" + and previous.get("allreduce_csv_sha256") + == allreduce_csv_sha256 + ): + loads.append(previous) + continue run_dir.mkdir(parents=True, exist_ok=True) command = builder.build_frontier_command( python_bin="/usr/bin/python3", @@ -520,6 +619,39 @@ def main() -> None: timeout=args.timeout_seconds, check=False, ) + fallback_evidence = ( + collective_fallback_evidence(run_dir) + if args.cc_backend == "vidur" + else [] + ) + if completed.returncode == 0 and fallback_evidence: + result = { + "status": "frontier_failed", + "failure_kind": "collective_profile_fallback", + "returncode": completed.returncode, + "config": { + "tp": config.tp, + "mns": config.mns, + "name": config.name, + }, + "trace_label": trace["label"], + "offered_request_rate": trace["offered_request_rate"], + "offered_request_rate_per_gpu": ( + trace["offered_request_rate"] / config.tp + ), + "request_count": trace["requests"], + "elapsed_seconds": time.time() - started, + "trace_sha256": trace["sha256"], + "collective_fallback_evidence": fallback_evidence, + "allreduce_csv_sha256": allreduce_csv_sha256, + } + BASE.write_json(result_path, result) + loads.append(result) + if not args.continue_on_failure: + raise RuntimeError( + f"measured collective fallback for {config.name}/{trace['label']}" + ) + continue if completed.returncode != 0: stderr_path = run_dir / "stderr.log" result = { @@ -542,6 +674,7 @@ def main() -> None: "elapsed_seconds": time.time() - started, "trace_sha256": trace["sha256"], "stderr_sha256": BASE.sha256(stderr_path), + "allreduce_csv_sha256": allreduce_csv_sha256, } BASE.write_json(result_path, result) loads.append(result) @@ -575,6 +708,7 @@ def main() -> None: "trace_sha256": trace["sha256"], "request_metrics_sha256": BASE.sha256(metrics), "score": score(metrics, trace["shapes"]), + "allreduce_csv_sha256": allreduce_csv_sha256, } BASE.write_json(result_path, result) loads.append(result) @@ -720,7 +854,8 @@ def main() -> None: "collective": { "backend": args.cc_backend, "allreduce_csv": str(args.allreduce_csv) if args.allreduce_csv else None, - "allreduce_csv_sha256": BASE.sha256(args.allreduce_csv) if args.allreduce_csv else None, + "allreduce_csv_sha256": allreduce_csv_sha256, + "measured_profile_validation": allreduce_profile, }, "traces": [trace_manifest_entry(trace) for trace in traces], "config_results": config_results, diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen235_collective_profile_ablation.sh b/runs/frontier-fidelity-envelope-v1/run_qwen235_collective_profile_ablation.sh new file mode 100644 index 0000000..f38c5c7 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/run_qwen235_collective_profile_ablation.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +set -euo pipefail + +BASE_CAMPAIGN_ROOT="${BASE_CAMPAIGN_ROOT:?BASE_CAMPAIGN_ROOT is required}" +OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT is required}" +ALLREDUCE_CSV="${ALLREDUCE_CSV:?ALLREDUCE_CSV is required}" +RUNNER_DIR="${RUNNER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}" +PROFILE_ROOT="${PROFILE_ROOT:-/home/admin/cpfs/wjh/aituner/qwen235-v020-profiles-20260719-r1/frozen}" +FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-q235-v020-5b953f5}" +REPLAYSERVE_ROOT="${REPLAYSERVE_ROOT:-/home/admin/cpfs/wjh/replayserve}" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +PYTHON_DEPS="${PYTHON_DEPS:-${VENV_ROOT}/lib/python3.12/site-packages}" + +mkdir -p "${OUTPUT_ROOT}"/{provenance,sim,analysis} +exec > >(tee -a "${OUTPUT_ROOT}/controller.log") 2>&1 +echo "Q235_COLLECTIVE_ABLATION_LAUNCH_ECHO host=$(hostname) base=${BASE_CAMPAIGN_ROOT} simulator=Frontier-6e8e0d8 configs={TP4/EP1,TP8/EP8}xMNS{64,128} cases={fixed-pd,fixed-po,trace-pd,trace-po} real_rerun=false measured_collective=${ALLREDUCE_CSV} analytical_fallback=forbidden expected_wall=1-6h output=${OUTPUT_ROOT}" +date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ +test "$(git -C "${FRONTIER_SOURCE}" rev-parse HEAD)" = "6e8e0d845bceff11b0b62cb29df3a1a93411fdd4" +sha256sum "${BASH_SOURCE[0]}" "${ALLREDUCE_CSV}" \ + "${PROFILE_ROOT}/manifest.json" \ + "${BASE_CAMPAIGN_ROOT}/provenance/runtime-contract.json" \ + > "${OUTPUT_ROOT}/provenance/input.sha256" +git -C "${RUNNER_DIR}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/aituner.commit" +git -C "${FRONTIER_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/frontier.commit" + +run_sim() { + local case_name="$1" prefix_flag="$2" + "${VENV_ROOT}/bin/python" "${RUNNER_DIR}/run_frontier_qwen235_v020_surface.py" \ + --frontier-source "${FRONTIER_SOURCE}" --replayserve-root "${REPLAYSERVE_ROOT}" \ + --profile-root "${PROFILE_ROOT}" --python-deps "${PYTHON_DEPS}" \ + --output-root "${OUTPUT_ROOT}/sim/${case_name}" \ + --runtime-contract "${BASE_CAMPAIGN_ROOT}/provenance/runtime-contract.json" \ + --trace-tp "4=${BASE_CAMPAIGN_ROOT}/traces/${case_name}/tp4/public/frontier.csv" \ + --trace-tp "8=${BASE_CAMPAIGN_ROOT}/traces/${case_name}/tp8/public/frontier.csv" \ + "${prefix_flag}" --allreduce-csv "${ALLREDUCE_CSV}" --resume +} + +run_sim fixed-pd --no-prefix-caching +run_sim fixed-po --no-prefix-caching +run_sim trace-pd --prefix-caching +run_sim trace-po --prefix-caching + +"${VENV_ROOT}/bin/python" "${RUNNER_DIR}/analyze_qwen235_v020_campaign.py" \ + --campaign-root "${BASE_CAMPAIGN_ROOT}" --sim-root "${OUTPUT_ROOT}" \ + --json-output "${OUTPUT_ROOT}/analysis/comparison.json" \ + --markdown-output "${OUTPUT_ROOT}/analysis/comparison.md" +find "${OUTPUT_ROOT}" -type f ! -path '*/provenance/artifacts.sha256' -print0 \ + | sort -z | xargs -0 sha256sum > "${OUTPUT_ROOT}/provenance/artifacts.sha256" +date -u +END_UTC=%Y-%m-%dT%H:%M:%SZ +echo Q235_COLLECTIVE_ABLATION_COMPLETE diff --git a/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py b/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py index c48ba2b..0c4c298 100644 --- a/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py +++ b/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py @@ -26,6 +26,28 @@ def load(name: str): class FidelityEnvelopeTest(unittest.TestCase): + @staticmethod + def write_allreduce_csv(path: Path, tps: tuple[int, ...]) -> None: + fields = [ + "time_stats.all_reduce.median", + "num_workers", + "size", + "collective", + ] + with path.open("w", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fields, lineterminator="\n") + writer.writeheader() + for tp in tps: + for size in (8192, 65536): + writer.writerow( + { + "time_stats.all_reduce.median": 0.1 * tp, + "num_workers": tp, + "size": size, + "collective": "all_reduce", + } + ) + def test_block_identities_are_parent_sensitive_and_prefix_stable(self) -> None: module = load("prepare_exact_trace.py") prefix = list(range(32)) @@ -86,6 +108,103 @@ class FidelityEnvelopeTest(unittest.TestCase): str(4 + 8192 / 1000), ) + def test_allreduce_profile_requires_every_selected_tp(self) -> None: + module = load("run_frontier_qwen30_exact_trace_surface.py") + with tempfile.TemporaryDirectory() as temporary: + profile = Path(temporary) / "allreduce.csv" + self.write_allreduce_csv(profile, (4,)) + with self.assertRaisesRegex( + ValueError, "coverage missing TPs \\[8\\].*fallback is forbidden" + ): + module.validate_allreduce_profile(profile, {4, 8}) + + self.write_allreduce_csv(profile, (4, 8)) + validated = module.validate_allreduce_profile(profile, {4, 8}) + self.assertEqual(validated["required_tp_coverage"], [4, 8]) + self.assertEqual(validated["rows_by_tp"], {"4": 2, "8": 2}) + + def test_collective_analytical_fallback_is_runtime_failure(self) -> None: + module = load("run_frontier_qwen30_exact_trace_surface.py") + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / "stdout.log").write_text( + "All-reduce profiling data is empty after filtering\n" + ) + (root / "stderr.log").write_text("unrelated warning\n") + evidence = module.collective_fallback_evidence(root) + self.assertEqual(len(evidence), 1) + self.assertEqual(evidence[0]["log"], "stdout.log") + + def test_materialize_qwen235_allreduce_requires_serving_contract(self) -> None: + module = load("materialize_qwen235_v020_allreduce.py") + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + inputs = [] + for tp in (4, 8): + path = root / f"tp{tp}.json" + rows = [] + for tokens in module.TOKEN_POINTS: + fusion_limit = {4: 2 * 1024 * 1024, 8: 512 * 1024}[tp] + payload_bytes = tokens * module.HIDDEN_DIM * 2 + median = tp / 10 + tokens / 10000 + rows.append( + { + "tensor_parallel_size": tp, + "num_tokens": tokens, + "hidden_dim": module.HIDDEN_DIM, + "payload_bytes": payload_bytes, + "selected_backend": ( + "flashinfer_trtllm_fused_projection" + if payload_bytes <= fusion_limit + else "pynccl" + ), + "real_fusion_limit_bytes": fusion_limit, + "trials": 3, + "repeats_per_trial": 20, + "per_trial_rank_samples_ms": [ + [[median] * 20 for _ in range(tp)] for _ in range(3) + ], + "critical_path_median_ms": median, + } + ) + path.write_text( + json.dumps( + { + "schema_version": "vllm020_allreduce_raw.v2", + "environment": { + "vllm_version": "0.20.0", + "vllm_source_commit": module.VLLM_COMMIT, + "gpu": "NVIDIA H20", + "model": "/models/Qwen3-235B-A22B-FP8", + "collective_contract": "qwen235-serving-projected", + "disable_custom_all_reduce": True, + "backend_env": { + "VLLM_ALLREDUCE_USE_FLASHINFER": "1", + "VLLM_ALLREDUCE_USE_SYMM_MEM": "1", + }, + }, + "rows": rows, + } + ) + ) + inputs.append(path) + output = root / "allreduce.csv" + manifest_path = root / "manifest.json" + manifest = module.materialize(inputs, output, manifest_path) + self.assertEqual(manifest["tp_coverage"], [4, 8]) + self.assertEqual(manifest["rows"], 2 * len(module.TOKEN_POINTS)) + self.assertEqual( + manifest["observed_backends_by_tp"], + { + "4": ["flashinfer_trtllm_fused_projection", "pynccl"], + "8": ["flashinfer_trtllm_fused_projection", "pynccl"], + }, + ) + with output.open(newline="") as handle: + rows = list(csv.DictReader(handle)) + self.assertEqual(rows[0]["num_workers"], "4") + self.assertEqual(rows[-1]["num_workers"], "8") + def test_batch_profile_retains_one_single_request_anchor_per_tp(self) -> None: runner = REPO_ROOT / "runs/frontier-phase-factorial-v0/run_frontier_qwen30_prefill_surface.py" spec = importlib.util.spec_from_file_location("qwen30_surface_runner", runner) diff --git a/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_allreduce.py b/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_allreduce.py index 09b3752..63309a1 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_allreduce.py +++ b/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_allreduce.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Profile vLLM 0.20 TP all-reduce and assert FlashInfer TRTLLM dispatch.""" +"""Profile the vLLM 0.20 TP all-reduce path under a frozen runtime contract.""" from __future__ import annotations @@ -27,8 +27,14 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--output", type=Path, required=True) parser.add_argument("--num-tokens", type=int, nargs="+", default=[8]) parser.add_argument("--hidden-dim", type=int, default=2048) + parser.add_argument( + "--collective-contract", + choices=("flashinfer-trtllm", "qwen235-serving-projected"), + default="flashinfer-trtllm", + ) parser.add_argument("--warmup-iters", type=int, default=3) parser.add_argument("--repeats", type=int, default=10) + parser.add_argument("--trials", type=int, default=1) return parser.parse_args() @@ -55,10 +61,19 @@ def main() -> None: source_head = git_head(args.vllm_source) if source_head != VLLM_COMMIT: raise SystemExit(f"expected vLLM source {VLLM_COMMIT}, got {source_head}") + if args.repeats <= 0 or args.trials <= 0 or args.warmup_iters < 0: + raise SystemExit("warmup/repeats/trials must be non-negative/positive") if os.getenv("VLLM_ALLREDUCE_USE_FLASHINFER") != "1": raise SystemExit("VLLM_ALLREDUCE_USE_FLASHINFER must equal 1") if os.getenv("VLLM_FLASHINFER_ALLREDUCE_BACKEND") != "trtllm": raise SystemExit("VLLM_FLASHINFER_ALLREDUCE_BACKEND must equal trtllm") + expected_symm_mem = ( + "1" if args.collective_contract == "qwen235-serving-projected" else "0" + ) + if os.getenv("VLLM_ALLREDUCE_USE_SYMM_MEM") != expected_symm_mem: + raise SystemExit( + f"VLLM_ALLREDUCE_USE_SYMM_MEM must equal {expected_symm_mem}" + ) if "RANK" not in os.environ or "WORLD_SIZE" not in os.environ: raise SystemExit("launch with torchrun") @@ -67,6 +82,7 @@ def main() -> None: destroy_model_parallel, init_distributed_environment, initialize_model_parallel, + set_custom_all_reduce, tensor_model_parallel_all_reduce, ) from vllm.distributed.parallel_state import get_tp_group @@ -80,11 +96,15 @@ def main() -> None: rank = int(os.environ["RANK"]) local_rank = int(os.environ["LOCAL_RANK"]) world_size = int(os.environ["WORLD_SIZE"]) - if world_size not in (2, 4): - raise SystemExit(f"expected TP world size 2 or 4, got {world_size}") + if world_size not in (2, 4, 8): + raise SystemExit(f"expected TP world size 2, 4, or 8, got {world_size}") device = torch.device(f"cuda:{local_rank}") torch.accelerator.set_device_index(device) torch.set_default_device(device) + disable_custom_all_reduce = ( + args.collective_contract == "qwen235-serving-projected" + ) + set_custom_all_reduce(not disable_custom_all_reduce) init_distributed_environment() model_config = ModelConfig( model=str(args.model), @@ -95,11 +115,80 @@ def main() -> None: ) vllm_config = VllmConfig( model_config=model_config, - parallel_config=ParallelConfig(tensor_parallel_size=world_size) + parallel_config=ParallelConfig( + tensor_parallel_size=world_size, + disable_custom_all_reduce=disable_custom_all_reduce, + ) ) with set_current_vllm_config(vllm_config): initialize_model_parallel(tensor_model_parallel_size=world_size) + from vllm.distributed.device_communicators.all_reduce_utils import ( + should_nccl_symm_mem_allreduce, + ) + + def resolve_profile_operation(input_tensor: torch.Tensor): + communicator = get_tp_group().device_communicator + if args.collective_contract == "qwen235-serving-projected": + from vllm.compilation.passes.fusion.allreduce_rms_fusion import ( + FI_ALLREDUCE_FUSION_MAX_SIZE_MB, + ) + + major, minor = torch.cuda.get_device_capability(device) + capability = major * 10 + minor + try: + fusion_limit_bytes = int( + FI_ALLREDUCE_FUSION_MAX_SIZE_MB[capability][world_size] + * 1024 + * 1024 + ) + except KeyError as error: + raise SystemExit( + f"no FlashInfer fusion limit for SM{capability}, TP{world_size}" + ) from error + if input_tensor.nbytes <= fusion_limit_bytes: + fi_comm = communicator.fi_ar_comm + if fi_comm is None or fi_comm.disabled: + raise SystemExit("FlashInfer all-reduce communicator is unavailable") + if not fi_comm.should_use_fi_ar(input_tensor): + raise SystemExit("FlashInfer rejected a fusion-eligible payload") + return ( + "flashinfer_trtllm_fused_projection", + fi_comm.all_reduce, + fusion_limit_bytes, + ) + if ( + communicator.pynccl_comm is not None + and should_nccl_symm_mem_allreduce(world_size, input_tensor) + ): + return ( + "pynccl_symmetric_with_copy", + torch.ops.vllm.all_reduce_symmetric_with_copy, + fusion_limit_bytes, + ) + if ( + communicator.symm_mem_comm is not None + and communicator.symm_mem_comm.should_use_symm_mem(input_tensor) + ): + return ( + "torch_symmetric_memory", + communicator.symm_mem_comm.all_reduce, + fusion_limit_bytes, + ) + if communicator.pynccl_comm is not None: + return "pynccl", communicator.pynccl_comm.all_reduce, fusion_limit_bytes + return "torch_distributed", tensor_model_parallel_all_reduce, fusion_limit_bytes + + fi_comm = communicator.fi_ar_comm + selected_backend = ( + "flashinfer_trtllm" + if fi_comm is not None + and not fi_comm.disabled + and fi_comm.should_use_fi_ar(input_tensor) + else "non_flashinfer_fallback" + ) + return selected_backend, tensor_model_parallel_all_reduce, None + rows: list[dict[str, Any]] = [] expected_sum = world_size * (world_size + 1) / 2 try: @@ -110,8 +199,11 @@ def main() -> None: dtype=torch.bfloat16, device=device, ) + selected_backend, profile_operation, fusion_limit_bytes = ( + resolve_profile_operation(input_tensor) + ) for _ in range(args.warmup_iters): - output = tensor_model_parallel_all_reduce(input_tensor) + output = profile_operation(input_tensor) torch.accelerator.synchronize() torch.testing.assert_close( output, @@ -120,30 +212,55 @@ def main() -> None: rtol=0.0, ) - communicator = get_tp_group().device_communicator - fi_comm = communicator.fi_ar_comm - if fi_comm is None or fi_comm.disabled: + if args.collective_contract == "flashinfer-trtllm": + if selected_backend != "flashinfer_trtllm": + raise SystemExit( + f"expected FlashInfer TRTLLM, got {selected_backend} at " + f"TP={world_size}, tokens={num_tokens}" + ) + elif selected_backend not in { + "flashinfer_trtllm_fused_projection", + "pynccl_symmetric_with_copy", + "torch_symmetric_memory", + "pynccl", + }: raise SystemExit( - f"FlashInfer all-reduce was not selected at TP={world_size}, " - f"tokens={num_tokens}" + f"unexpected Qwen235 serving backend: {selected_backend}" ) - uses_flashinfer = fi_comm.should_use_fi_ar(input_tensor) - samples: list[float] = [] - for _ in range(args.repeats): - dist.barrier() - start = torch.cuda.Event(enable_timing=True) - end = torch.cuda.Event(enable_timing=True) - start.record() - output = tensor_model_parallel_all_reduce(input_tensor) - end.record() - torch.accelerator.synchronize() - samples.append(float(start.elapsed_time(end))) + all_trial_rank_samples: list[list[list[float]]] = [] + critical_path_samples: list[float] = [] + for _ in range(args.trials): + samples: list[float] = [] + for _ in range(args.repeats): + dist.barrier() + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + start.record() + output = profile_operation(input_tensor) + end.record() + torch.accelerator.synchronize() + samples.append(float(start.elapsed_time(end))) - gathered: list[list[float] | None] = [None] * world_size - dist.all_gather_object(gathered, samples) + gathered: list[list[float] | None] = [None] * world_size + dist.all_gather_object(gathered, samples) + if rank == 0: + rank_samples = [item for item in gathered if item is not None] + all_trial_rank_samples.append(rank_samples) + critical_path_samples.extend( + max(per_rank[index] for per_rank in rank_samples) + for index in range(args.repeats) + ) if rank == 0: - per_rank = [stats_ms(item) for item in gathered if item is not None] + flattened_by_rank = [ + [ + sample + for trial in all_trial_rank_samples + for sample in trial[rank_index] + ] + for rank_index in range(world_size) + ] + per_rank = [stats_ms(samples) for samples in flattened_by_rank] row = { "tensor_parallel_size": world_size, "num_tokens": num_tokens, @@ -153,12 +270,16 @@ def main() -> None: * torch.tensor([], dtype=torch.bfloat16).element_size(), "dtype": "bfloat16", "communicator": "vllm.tensor_model_parallel_all_reduce", - "selected_backend": ( - "flashinfer_trtllm" if uses_flashinfer else "nccl_fallback" - ), + "collective_contract": args.collective_contract, + "selected_backend": selected_backend, + "real_fusion_limit_bytes": fusion_limit_bytes, + "trials": args.trials, + "repeats_per_trial": args.repeats, + "per_trial_rank_samples_ms": all_trial_rank_samples, "per_rank_time_ms": per_rank, - "critical_path_median_ms": max( - rank_stats["median"] for rank_stats in per_rank + "critical_path_time_ms": stats_ms(critical_path_samples), + "critical_path_median_ms": statistics.median( + critical_path_samples ), } rows.append(row) @@ -169,7 +290,11 @@ def main() -> None: if rank == 0: payload = { - "schema_version": "qwen30_vllm020_allreduce_raw.v1", + "schema_version": ( + "qwen30_vllm020_allreduce_raw.v1" + if args.collective_contract == "flashinfer-trtllm" + else "vllm020_allreduce_raw.v2" + ), "environment": { "vllm_version": vllm.__version__, "vllm_source_commit": source_head, @@ -178,9 +303,18 @@ def main() -> None: "gpu": torch.cuda.get_device_name(device), "model": str(args.model), "backend_env": { - "VLLM_ALLREDUCE_USE_FLASHINFER": "1", - "VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm", + "VLLM_ALLREDUCE_USE_FLASHINFER": os.getenv( + "VLLM_ALLREDUCE_USE_FLASHINFER", "0" + ), + "VLLM_FLASHINFER_ALLREDUCE_BACKEND": os.getenv( + "VLLM_FLASHINFER_ALLREDUCE_BACKEND" + ), + "VLLM_ALLREDUCE_USE_SYMM_MEM": os.getenv( + "VLLM_ALLREDUCE_USE_SYMM_MEM", "1" + ), }, + "collective_contract": args.collective_contract, + "disable_custom_all_reduce": disable_custom_all_reduce, }, "rows": rows, } diff --git a/runs/frontier-qwen30-vllm020-profile-v1/run_allreduce_profile.sh b/runs/frontier-qwen30-vllm020-profile-v1/run_allreduce_profile.sh index 0cf4565..fef602d 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/run_allreduce_profile.sh +++ b/runs/frontier-qwen30-vllm020-profile-v1/run_allreduce_profile.sh @@ -2,10 +2,11 @@ set -euo pipefail -TP="${TP:?TP must be set to 2 or 4}" +TP="${TP:?TP must be set to 2, 4, or 8}" case "${TP}" in 2) HARD_GPU_CAP="0.40_H20h" ;; 4) HARD_GPU_CAP="0.80_H20h" ;; + 8) HARD_GPU_CAP="1.60_H20h" ;; *) echo "ERROR: invalid TP=${TP}" >&2; exit 1 ;; esac OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT must be set}" @@ -13,6 +14,10 @@ VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" VLLM_SOURCE="${VLLM_SOURCE:-/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build}" MODEL="${MODEL:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" NUM_TOKENS="${NUM_TOKENS:-8}" +HIDDEN_DIM="${HIDDEN_DIM:-2048}" +COLLECTIVE_CONTRACT="${COLLECTIVE_CONTRACT:-flashinfer-trtllm}" +TRIALS="${TRIALS:-1}" +REPEATS="${REPEATS:-10}" mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" "${OUTPUT_ROOT}/raw" exec > >(tee -a "${OUTPUT_ROOT}/logs/profile.log") 2>&1 @@ -22,9 +27,20 @@ if [[ "${#GPU_IDS[@]}" -ne "${TP}" ]]; then exit 1 fi -export VLLM_ALLREDUCE_USE_FLASHINFER=1 -export VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm -echo "PROFILE_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL} runtime=vLLM-0.20.0+cu129 operator=tensor_model_parallel_all_reduce backend=FlashInfer-TRTLLM tp=${TP} tokens=${NUM_TOKENS} hidden=2048 dtype=BF16 output=${OUTPUT_ROOT} expected_wall=2-6m hard_wall=720s hard_gpu_cap=${HARD_GPU_CAP}" +case "${COLLECTIVE_CONTRACT}" in + flashinfer-trtllm) + export VLLM_ALLREDUCE_USE_FLASHINFER=1 + export VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm + export VLLM_ALLREDUCE_USE_SYMM_MEM=0 + ;; + qwen235-serving-projected) + export VLLM_ALLREDUCE_USE_FLASHINFER=1 + export VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm + export VLLM_ALLREDUCE_USE_SYMM_MEM=1 + ;; + *) echo "ERROR: invalid COLLECTIVE_CONTRACT=${COLLECTIVE_CONTRACT}" >&2; exit 1 ;; +esac +echo "PROFILE_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL} runtime=vLLM-0.20.0+cu129 operator=tensor_model_parallel_all_reduce contract=${COLLECTIVE_CONTRACT} tp=${TP} tokens=${NUM_TOKENS} hidden=${HIDDEN_DIM} dtype=BF16 trials=${TRIALS} repeats=${REPEATS} output=${OUTPUT_ROOT} expected_wall=2-8m hard_wall=720s hard_gpu_cap=${HARD_GPU_CAP}" date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ" nvidia-smi --query-gpu=index,name,driver_version,memory.used,utilization.gpu --format=csv,noheader @@ -46,8 +62,11 @@ timeout --signal=TERM --kill-after=30s 600 \ --model "${MODEL}" \ --output "${OUTPUT_ROOT}/raw/allreduce-tp${TP}.json" \ --num-tokens "${TOKEN_ARGS[@]}" \ + --hidden-dim "${HIDDEN_DIM}" \ + --collective-contract "${COLLECTIVE_CONTRACT}" \ --warmup-iters 3 \ - --repeats 10 + --repeats "${REPEATS}" \ + --trials "${TRIALS}" test -s "${OUTPUT_ROOT}/raw/allreduce-tp${TP}.json" sha256sum "${OUTPUT_ROOT}/raw/allreduce-tp${TP}.json" \