File tree Expand file tree Collapse file tree 1 file changed +7
-0
lines changed Expand file tree Collapse file tree 1 file changed +7
-0
lines changed Original file line number Diff line number Diff line change @@ -605,5 +605,12 @@ def get_layer_types(self) -> Optional[List[LayerTypeCpp]]:
605605 def get_num_attention_layers (self ):
606606 if is_nemotron_hybrid (self .pretrained_config ):
607607 return self .pretrained_config .hybrid_override_pattern .count ("*" )
608+ elif hasattr (
609+ self .pretrained_config , "architectures"
610+ ) and self .pretrained_config .architectures is not None and self .pretrained_config .architectures [
611+ 0 ] in ["Qwen3NextForCausalLM" ]:
612+ # Qwen3NextForCausalLM has hybrid attention pattern(1:3 full attention:linear attention),
613+ # we need to calculate the number of fullattention layers
614+ return self .pretrained_config .num_hidden_layers // self .pretrained_config .full_attention_interval
608615 else :
609616 return self .pretrained_config .num_hidden_layers
You can’t perform that action at this time.
0 commit comments