3 lines
1.4 KiB
Plaintext
3 lines
1.4 KiB
Plaintext
{'name': 'Qwen3-235B-A22B-FP8', 'num_layers': 94, 'num_q_heads': 64, 'num_kv_heads': 4, 'embedding_dim': 4096, 'mlp_hidden_dim': 1536, 'max_position_embeddings': 40960, 'use_gated_mlp': True, 'use_bias': False, 'use_qkv_bias': False, 'activation': 'silu', 'norm': 'rms_norm', 'post_attn_norm': True, 'vocab_size': 151936, 'is_neox_style': True, 'rope_theta': 1000000.0, 'rope_scaling': None, 'partial_rotary_factor': 1.0, 'no_tensor_parallel': False, 'is_moe': True, 'num_experts': 128, 'num_experts_per_tok': 8, 'moe_layers_enum': None, 'use_qk_norm': True, 'attn_output_gate': False, 'rms_norm_eps': 1e-06, 'dtype': 'BF16', 'model_type': 'qwen3_moe', 'fused_add_norm_capability': True, 'model_arch': 'generic', 'share_expert_dim': None, 'share_q_dim': None, 'head_dim': 128, 'quantization_config': {'quant_method': 'fp8', 'activation_scheme': 'dynamic', 'is_checkpoint_fp8_serialized': True, 'weight_block_size': (128, 128), 'ignored_layers': []}, 'tie_word_embeddings': False}
|
|
{'tp_size': 4, 'attn_enabled': True, 'ffn_enabled': True, 'attn_sharded_enabled': True, 'ffn_sharded_enabled': True, 'replicated_enabled': True, 'disable_replicated': False, 'enabled_ops': ['input_layernorm', 'post_attention_layernorm', 'add', 'emb', 'attn_pre_proj', 'attn_rope', 'attn_post_proj'], 'disabled_ops': [], 'replicated_ops': ['input_layernorm', 'post_attention_layernorm', 'add', 'emb'], 'padded_n_embd': 4096, 'padded_n_expanded_embd': 1536, 'skip_reasons': []}
|