File tree Expand file tree Collapse file tree 1 file changed +7
-0
lines changed Expand file tree Collapse file tree 1 file changed +7
-0
lines changed Original file line number Diff line number Diff line change @@ -642,5 +642,12 @@ def get_layer_types(self) -> Optional[List[LayerTypeCpp]]:
642642 def get_num_attention_layers (self ):
643643 if is_nemotron_hybrid (self .pretrained_config ):
644644 return self .pretrained_config .hybrid_override_pattern .count ("*" )
645+ elif hasattr (self .pretrained_config , "architectures"
646+ ) and self .pretrained_config .architectures [0 ] in [
647+ "Qwen3NextForCausalLM"
648+ ]:
649+ # Qwen3NextForCausalLM has hybrid attention pattern(1:3 full attention:linear attention),
650+ # we need to calculate the number of fullattention layers
651+ return self .pretrained_config .num_hidden_layers // self .pretrained_config .full_attention_interval
645652 else :
646653 return self .pretrained_config .num_hidden_layers
You can’t perform that action at this time.
0 commit comments