use serde::Deserialize; use std::path::Path; #[derive(Debug, Clone, Deserialize)] pub struct RopeScaling { pub rope_type: Option, pub factor: Option, pub original_max_position_embeddings: Option, pub beta_fast: Option, pub beta_slow: Option, } #[derive(Debug, Clone, Deserialize)] pub struct ModelConfig { pub architectures: Option>, pub model_type: Option, // Modern HF naming #[serde(default)] pub hidden_size: Option, #[serde(default)] pub intermediate_size: Option, #[serde(default)] pub num_attention_heads: Option, #[serde(default)] pub num_key_value_heads: Option, #[serde(default)] pub num_hidden_layers: Option, pub vocab_size: usize, #[serde(default)] pub max_position_embeddings: Option, // GPT-2 naming #[serde(default)] pub n_embd: Option, #[serde(default)] pub n_head: Option, #[serde(default)] pub n_layer: Option, #[serde(default)] pub n_positions: Option, #[serde(default)] pub n_inner: Option, // Normalization #[serde(default)] pub layer_norm_eps: Option, #[serde(default)] pub layer_norm_epsilon: Option, #[serde(default)] pub rms_norm_eps: Option, // Other #[serde(default)] pub rope_theta: Option, #[serde(default)] pub tie_word_embeddings: Option, // MoE (gpt-oss) #[serde(default)] pub num_local_experts: Option, #[serde(default)] pub num_experts_per_tok: Option, #[serde(default)] pub layer_types: Option>, #[serde(default)] pub sliding_window: Option, #[serde(default)] pub attention_bias: Option, #[serde(default, rename = "head_dim")] pub explicit_head_dim: Option, #[serde(default)] pub rope_scaling: Option, #[serde(default)] pub swiglu_limit: Option, #[serde(default)] pub geglu_alpha: Option, #[serde(default)] pub hidden_act: Option, } impl ModelConfig { pub fn from_file(path: &Path) -> Self { let data = std::fs::read_to_string(path) .unwrap_or_else(|e| panic!("failed to read {}: {e}", path.display())); serde_json::from_str(&data) .unwrap_or_else(|e| panic!("failed to parse {}: {e}", path.display())) } pub fn hidden(&self) -> usize { self.hidden_size.or(self.n_embd).expect("hidden_size or n_embd required") } pub fn num_heads(&self) -> usize { self.num_attention_heads.or(self.n_head).expect("num_attention_heads or n_head required") } pub fn num_layers(&self) -> usize { self.num_hidden_layers.or(self.n_layer).expect("num_hidden_layers or n_layer required") } pub fn max_seq_len(&self) -> usize { self.max_position_embeddings.or(self.n_positions).unwrap_or(2048) } pub fn ffn_hidden(&self) -> usize { self.intermediate_size.or(self.n_inner).unwrap_or(self.hidden() * 4) } pub fn num_kv_heads(&self) -> usize { self.num_key_value_heads.unwrap_or(self.num_heads()) } pub fn head_dim(&self) -> usize { self.explicit_head_dim.unwrap_or_else(|| self.hidden() / self.num_heads()) } pub fn ln_eps(&self) -> f32 { self.layer_norm_eps .or(self.layer_norm_epsilon) .unwrap_or(1e-5) as f32 } pub fn tied_embeddings(&self) -> bool { self.tie_word_embeddings.unwrap_or(true) } pub fn num_experts(&self) -> usize { self.num_local_experts.unwrap_or(0) } pub fn experts_per_token(&self) -> usize { self.num_experts_per_tok.unwrap_or(1) } pub fn is_moe(&self) -> bool { self.num_local_experts.unwrap_or(0) > 1 } pub fn is_sliding_layer(&self, layer_idx: usize) -> bool { self.layer_types .as_ref() .and_then(|lt| lt.get(layer_idx)) .map(|t| t == "sliding_attention") .unwrap_or(false) } pub fn window_size(&self) -> usize { self.sliding_window.unwrap_or(0) } pub fn geglu_alpha(&self) -> f32 { self.geglu_alpha.unwrap_or(1.702) as f32 } }