2727 AutoTokenizer ,
2828 GenerationConfig ,
2929 GotOcr2ImageProcessor ,
30- InternVLConfig ,
3130 InternVLForConditionalGeneration ,
3231 InternVLProcessor ,
3332 InternVLVideoProcessor ,
34- InternVLVisionConfig ,
35- LlamaConfig ,
36- Qwen2Config ,
3733)
3834from transformers .utils import SAFE_WEIGHTS_INDEX_NAME , SAFE_WEIGHTS_NAME , WEIGHTS_INDEX_NAME , WEIGHTS_NAME
3935
5753 "OpenGVLab/InternVL3-78B" : "qwen2" ,
5854}
5955
60- UNNECESSARY_CONFIG_KEYS = [ "_name_or_path" , "_attn_implementation_autoset" , "auto_map" , "use_bfloat16" , "use_flash_attn" , "bias" , "laux_allreduce" , "moe_coeff_ratio" , "moe_intermediate_size" , "moe_output_scale" , "noisy_gate_policy" , "shared_expert_intermediate_size" , "use_residual" , "use_moe" , "use_rts" , "use_weighted_residual" , "moe_config" , "num_experts" , "num_routed_experts" , "num_shared_experts" , "capacity_factor" , "eval_capacity_factor" , "drop_path_rate" ] # fmt: skip
61-
6256# fmt: off
6357ORIGINAL_TO_CONVERTED_KEY_MAPPING_VISION = {
6458 # Vision encoder mapping
127121# fmt: on
128122
129123CONTEXT_LENGTH = 8192
124+ # Old InternVL2-1B/2B checkpoints use a different tokenizer base than newer models.
130125OLD_INTERNVL2_CHECKPOINTS = {"OpenGVLab/InternVL2-1B" , "OpenGVLab/InternVL2-2B" }
131126OLD_INTERNVL2_QWEN2_CHECKPOINTS = {"OpenGVLab/InternVL2-1B" }
132127BASE_SPECIAL_TOKENS = ["<img>" , "</img>" , "<IMG_CONTEXT>" , "<quad>" , "</quad>" , "<ref>" , "</ref>" , "<box>" , "</box>" ]
133128
134129
135- def is_old_internvl2_checkpoint (path : str | None ) -> bool :
136- return path in OLD_INTERNVL2_CHECKPOINTS
137-
138-
139- def get_original_config (path : str ):
140- return AutoConfig .from_pretrained (path , trust_remote_code = True )
141-
142-
143130def get_tokenizer_base_model (path : str ) -> tuple [str , bool ]:
144131 if path in OLD_INTERNVL2_QWEN2_CHECKPOINTS :
145132 return "Qwen/Qwen2-0.5B-Instruct" , False
@@ -155,7 +142,7 @@ def get_tokenizer_special_tokens(path: str) -> tuple[list[str], dict[str, str]]:
155142 "end_image_token" : "</img>" ,
156143 "context_image_token" : "<IMG_CONTEXT>" ,
157144 }
158- if not is_old_internvl2_checkpoint ( path ) :
145+ if path not in OLD_INTERNVL2_CHECKPOINTS :
159146 special_tokens .append ("<video>" )
160147 model_specific_tokens ["video_token" ] = "<video>"
161148
@@ -225,22 +212,17 @@ def get_lm_type(path: str) -> Literal["qwen2", "llama"]:
225212 Determine the type of language model (either 'qwen2' or 'llama') based on a given model path.
226213 """
227214 if path not in LM_TYPE_CORRESPONDENCE :
228- base_config = get_original_config (path )
229-
230- lm_arch = base_config .llm_config .architectures [0 ]
231-
232- if lm_arch == "InternLM2ForCausalLM" :
233- lm_type = "llama"
234- elif lm_arch == "Qwen2ForCausalLM" :
235- lm_type = "qwen2"
215+ config = AutoConfig .from_pretrained (path )
216+ text_model_type = config .text_config .model_type
217+ if text_model_type == "qwen2" :
218+ return "qwen2"
219+ elif text_model_type in ("llama" ,):
220+ return "llama"
236221 else :
237222 raise ValueError (
238- f"Architecture ' { lm_arch } Qwen2ForCausalLM ' and 'InternLM2ForCausalLM ' are recognized."
223+ f"Text model type ' { text_model_type } qwen2 ' and 'llama ' are recognized."
239224 )
240- else :
241- lm_type : Literal ["qwen2" , "llama" ] = LM_TYPE_CORRESPONDENCE [path ]
242-
243- return lm_type
225+ return LM_TYPE_CORRESPONDENCE [path ]
244226
245227
246228def convert_old_keys_to_new_keys (state_dict_keys : dict | None = None , path : str | None = None ):
@@ -291,52 +273,30 @@ def load_original_state_dict(input_base_path):
291273
292274
293275def get_internvl_config (input_base_path ):
294- base_config = get_original_config (input_base_path )
276+ # Load the config natively — InternVL2 (internvl_chat) configs are remapped automatically.
277+ config = AutoConfig .from_pretrained (input_base_path )
295278 tokenizer = build_tokenizer (input_base_path )
296- llm_config = base_config .llm_config .to_dict ()
297- vision_config = base_config .vision_config .to_dict ()
298- vision_config ["use_absolute_position_embeddings" ] = True
299- if get_lm_type (input_base_path ) == "qwen2" :
300- language_config_class = Qwen2Config
301- else :
302- language_config_class = LlamaConfig
303279
304- image_size = getattr (base_config , "force_image_size" , None ) or vision_config ["image_size" ]
305- patch_size = vision_config ["patch_size" ]
306- if isinstance (image_size , list | tuple ):
280+ # Fix image_token_id from the actual tokenizer built for this checkpoint.
281+ config .image_token_id = tokenizer .context_image_token_id
282+
283+ # Recompute image_seq_length from the resolved image_size and downsample_ratio.
284+ image_size = config .vision_config .image_size
285+ patch_size = config .vision_config .patch_size
286+ if isinstance (image_size , (list , tuple )):
307287 image_size = image_size [0 ]
308- if isinstance (patch_size , list | tuple ):
288+ if isinstance (patch_size , ( list , tuple ) ):
309289 patch_size = patch_size [0 ]
310- image_seq_length = int ((image_size // patch_size ) ** 2 * ( base_config .downsample_ratio ** 2 ) )
290+ config . image_seq_length = int ((image_size // patch_size ) ** 2 * config .downsample_ratio ** 2 )
311291
312- llm_config = {k : v for k , v in llm_config .items () if k not in UNNECESSARY_CONFIG_KEYS }
313- # Force use_cache to True
314- llm_config ["use_cache" ] = True
315- # Force correct eos_token_id for InternVL3
292+ # Force use_cache to True for generation.
293+ config .text_config .use_cache = True
294+ # Force correct eos_token_id for InternVL3 Qwen2 models.
316295 if "InternVL3" in input_base_path and get_lm_type (input_base_path ) == "qwen2" :
317- llm_config ["eos_token_id" ] = 151645
318-
319- vision_config = {k : v for k , v in vision_config .items () if k not in UNNECESSARY_CONFIG_KEYS }
320- if "attention_probs_dropout_prob" in vision_config :
321- attention_dropout = vision_config .pop ("attention_probs_dropout_prob" )
322- vision_config ["attention_dropout" ] = attention_dropout
323- vision_config ["projection_dropout" ] = attention_dropout
324- if "qk_normalization" in vision_config :
325- use_qk_norm = vision_config .pop ("qk_normalization" )
326- vision_config ["use_qk_norm" ] = use_qk_norm
327- if "qkv_bias" in vision_config :
328- attention_bias = vision_config .pop ("qkv_bias" )
329- vision_config ["attention_bias" ] = attention_bias
330-
331- return InternVLConfig (
332- text_config = language_config_class (** llm_config ),
333- vision_config = InternVLVisionConfig (** vision_config ),
334- image_token_id = tokenizer .context_image_token_id ,
335- image_seq_length = image_seq_length ,
336- downsample_ratio = base_config .downsample_ratio ,
337- vision_feature_layer = getattr (base_config , "select_layer" , - 1 ),
338- tie_word_embeddings = getattr (base_config , "tie_word_embeddings" , llm_config .get ("tie_word_embeddings" , True )),
339- )
296+ config .text_config .eos_token_id = 151645
297+
298+ config .architectures = ["InternVLForConditionalGeneration" ]
299+ return config
340300
341301
342302def write_model (
@@ -348,7 +308,6 @@ def write_model(
348308 os .makedirs (model_path , exist_ok = True )
349309
350310 config = get_internvl_config (input_base_path )
351- config .architectures = ["InternVLForConditionalGeneration" ]
352311 config .save_pretrained (model_path )
353312 if push_to_hub :
354313 model_name = (hub_dir or model_path ).split (os .path .sep )[- 1 ]
0 commit comments