Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit df99b36

Browse files
yonigozlanyonigozlan
authored
Merge 06d6b8c into 57f9936 
2 parents 57f9936 + 06d6b8c commit df99b36

100 files changed

Lines changed: 9158 additions & 5507 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎src/transformers/conversion_mapping.py‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,13 @@
3636

3737

3838
_MODEL_TO_CONVERSION_PATTERN = {
39+
# ViT-style vision models (old HuggingFace checkpoint format → new modular format)
40+
"audio-spectrogram-transformer": "vit",
41+
"deit": "vit",
42+
"ijepa": "vit",
43+
"vit_mae": "vit",
44+
"vit_msn": "vit",
45+
"vivit": "vit",
3946
# Mixtral-style MoE
4047
"minimax": "mixtral",
4148
"minimax_m2": "mixtral",
@@ -95,6 +102,58 @@
95102

96103
def _build_checkpoint_conversion_mapping():
97104
mapping = {
105+
"vit": [
106+
WeightRenaming(r"encoder\.layer\.", "layers."),
107+
WeightRenaming("attention.query", "q_proj"),
108+
WeightRenaming("attention.key", "k_proj"),
109+
WeightRenaming("attention.value", "v_proj"),
110+
WeightRenaming("attention.output.dense", "attention.o_proj"),
111+
WeightRenaming("intermediate.dense", "mlp.fc1"),
112+
WeightRenaming("output.dense", "mlp.fc2"),
113+
],
114+
"lw_detr": [
115+
WeightRenaming("attention.attention.query", "attention.q_proj"),
116+
WeightRenaming("attention.attention.key", "attention.k_proj"),
117+
WeightRenaming("attention.attention.value", "attention.v_proj"),
118+
WeightRenaming("attention.output", "attention.o_proj"),
119+
],
120+
"segformer": [
121+
# Structural: legacy `encoder.*` lists → SegformerModel.stages (no `encoder` submodule on SegformerModel)
122+
WeightRenaming(r"encoder.patch_embeddings.(\d+).", r"stages.\1.patch_embeddings."),
123+
WeightRenaming(r"encoder.block.(\d+).", r"stages.\1.blocks."),
124+
WeightRenaming(r"encoder.layer_norm.(\d+)", r"stages.\1.layer_norm"),
125+
# Attention projection renames
126+
WeightRenaming("attention.self.query", "attention.q_proj"),
127+
WeightRenaming("attention.self.key", "attention.k_proj"),
128+
WeightRenaming("attention.self.value", "attention.v_proj"),
129+
WeightRenaming("attention.self.sr", "attention.sequence_reduction.sequence_reduction"),
130+
WeightRenaming("attention.self.layer_norm", "attention.sequence_reduction.layer_norm"),
131+
WeightRenaming("attention.output.dense", "attention.o_proj"),
132+
# MLP renames
133+
WeightRenaming("mlp.dense1", "mlp.fc1"),
134+
WeightRenaming("mlp.dense2", "mlp.fc2"),
135+
# LayerNorm renames
136+
WeightRenaming("layer_norm_1", "layernorm_before"),
137+
WeightRenaming("layer_norm_2", "layernorm_after"),
138+
# Decode head: legacy checkpoints name the MLP list `linear_c`
139+
WeightRenaming("decode_head.linear_c", "decode_head.linear_projections"),
140+
],
141+
"swin": [
142+
# Attention projection renames (SwinSelfAttention.{query,key,value} → SwinAttention.{q,k,v}_proj)
143+
WeightRenaming("attention.self.query", "attention.q_proj"),
144+
WeightRenaming("attention.self.key", "attention.k_proj"),
145+
WeightRenaming("attention.self.value", "attention.v_proj"),
146+
# Relative position bias: SwinSelfAttention → dedicated SwinRelativePositionBias submodule
147+
WeightRenaming(
148+
"attention.self.relative_position_bias_table",
149+
"attention.relative_position_bias.relative_position_bias_table",
150+
),
151+
# Output projection rename (SwinSelfOutput.dense → SwinAttention.o_proj)
152+
WeightRenaming("attention.output.dense", "attention.o_proj"),
153+
# MLP renames (SwinIntermediate.dense → SwinMLP.fc1, SwinOutput.dense → SwinMLP.fc2)
154+
WeightRenaming("intermediate.dense", "mlp.fc1"),
155+
WeightRenaming("output.dense", "mlp.fc2"),
156+
],
98157
"altclip": [
99158
WeightRenaming(source_patterns=r"layer\.", target_patterns="layers."),
100159
],
@@ -344,6 +403,17 @@ def _build_checkpoint_conversion_mapping():
344403
operations=[ErnieFuseAndSplitTextVisionExperts(stack_dim=0, concat_dim=1)],
345404
),
346405
],
406+
# MaskFormer embeds both a Swin backbone (model_type="swin") and a DETR decoder
407+
# (model_type="detr") as submodules. Both get their mappings collected automatically, but
408+
# the Swin reverse mapping "mlp.fc1 → intermediate.dense" would corrupt DETR decoder keys
409+
# if it runs before the DETR reverse "layers.N.mlp.fc1 → layers.N.fc1".
410+
# By adding a "maskformer"-level mapping with the DETR fc1 rename, it is collected first
411+
# (model-level before submodule-level), so its reverse runs first and removes "mlp.fc1"
412+
# from DETR decoder paths before the Swin reverse can match them.
413+
"maskformer": [
414+
WeightRenaming(r"layers.(\d+).fc1", r"layers.\1.mlp.fc1"),
415+
WeightRenaming(r"layers.(\d+).fc2", r"layers.\1.mlp.fc2"),
416+
],
347417
"detr": [
348418
WeightRenaming("backbone.conv_encoder", "backbone"),
349419
WeightRenaming("out_proj", "o_proj"),
@@ -585,6 +655,40 @@ def _build_checkpoint_conversion_mapping():
585655
mapping["ernie4_5_moe"] += [
586656
WeightRenaming("mlp.moe_statics.e_score_correction_bias", "mlp.gate.moe_statics.e_score_correction_bias")
587657
]
658+
mapping["beit"] = [
659+
WeightRenaming("attention.query", "q_proj"),
660+
WeightRenaming("attention.key", "k_proj"),
661+
WeightRenaming("attention.value", "v_proj"),
662+
WeightRenaming("attention.output.dense", "attention.o_proj"),
663+
WeightRenaming("intermediate.dense", "mlp.fc1"),
664+
WeightRenaming("output.dense", "mlp.fc2"),
665+
WeightRenaming("encoder.relative_position_bias", "shared_position_bias"),
666+
WeightRenaming(r"attention.attention.relative_position_bias\.", r"relative_position_bias."),
667+
WeightRenaming("decode_head.bottleneck.", "decode_head.psp_bottleneck."),
668+
WeightRenaming(r"(?<!psp_modules\.[0-9]\.1\.)bn\.", "normalization."),
669+
WeightRenaming(r"(?<!psp_modules\.[0-9]\.1\.)conv\.weight", "convolution.weight"),
670+
WeightRenaming(
671+
r"decode_head\.psp_modules\.(\d+)\.1\.conv\.weight",
672+
r"decode_head.psp_modules.blocks.\1.conv.convolution.weight",
673+
),
674+
WeightRenaming(
675+
r"decode_head\.psp_modules\.(\d+)\.1\.bn\.",
676+
r"decode_head.psp_modules.blocks.\1.conv.normalization.",
677+
),
678+
WeightRenaming(r"^fpn1\.0\.", "fpn.fpn1.conv_transpose1."),
679+
WeightRenaming(r"^fpn1\.1\.", "fpn.fpn1.normalization."),
680+
WeightRenaming(r"^fpn1\.3\.", "fpn.fpn1.conv_transpose2."),
681+
WeightRenaming(r"^fpn2\.0\.", "fpn.fpn2."),
682+
*mapping["vit"].copy(),
683+
WeightRenaming(r"^encoder\.", "beit."),
684+
]
685+
686+
mapping["pixio"] = mapping["vit"].copy()
687+
mapping["pixio"] += [
688+
WeightRenaming("norm1", "layernorm_before"),
689+
WeightRenaming("norm2", "layernorm_after"),
690+
WeightRenaming(r"^encoder\.", "pixio."),
691+
]
588692

589693
mapping["minimax_m2"] = mapping["mixtral"].copy()
590694
mapping["minimax_m2"] += [

‎src/transformers/loss/loss_utils.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,31 @@ def ForTokenClassification(logits: torch.Tensor, labels, config, **kwargs):
142142
return fixed_cross_entropy(logits, labels, **kwargs)
143143

144144

145+
def ForSemanticSegmentationLoss(
146+
logits: torch.Tensor,
147+
labels: torch.Tensor,
148+
ignore_index: int = 255,
149+
num_items_in_batch: torch.Tensor | None = None,
150+
auxiliary_logits: torch.Tensor | None = None,
151+
auxiliary_loss_weight: float = 0.4,
152+
**kwargs,
153+
) -> torch.Tensor:
154+
upsampled_logits = nn.functional.interpolate(logits, size=labels.shape[-2:], mode="bilinear", align_corners=False)
155+
loss = fixed_cross_entropy(
156+
upsampled_logits, labels, num_items_in_batch=num_items_in_batch, ignore_index=ignore_index
157+
)
158+
if auxiliary_logits is not None:
159+
upsampled_auxiliary_logits = nn.functional.interpolate(
160+
auxiliary_logits, size=labels.shape[-2:], mode="bilinear", align_corners=False
161+
)
162+
loss = loss + auxiliary_loss_weight * fixed_cross_entropy(
163+
upsampled_auxiliary_logits, labels, num_items_in_batch=num_items_in_batch, ignore_index=ignore_index
164+
)
165+
return loss
166+
167+
145168
LOSS_MAPPING = {
169+
"ForSemanticSegmentation": ForSemanticSegmentationLoss,
146170
"ForCausalLM": ForCausalLMLoss,
147171
"ForMaskedLM": ForMaskedLMLoss,
148172
"ForQuestionAnswering": ForQuestionAnsweringLoss,

‎src/transformers/models/aimv2/modeling_aimv2.py‎

Lines changed: 56 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,54 @@ def forward(self, x):
113113
return down_proj
114114

115115

116+
def build_2d_sinusoidal_position_embedding(
117+
height: int,
118+
width: int,
119+
embed_dim: int = 256,
120+
temperature: float = 10000.0,
121+
cls_token: bool = False,
122+
device: torch.device | None = None,
123+
dtype: torch.dtype = torch.float32,
124+
) -> torch.Tensor:
125+
"""2D sinusoidal position embeddings for an image patch grid.
126+
127+
Each (h, w) position gets an ``embed_dim``-dimensional vector laid out as
128+
``[sin_h | cos_h | sin_w | cos_w]``, with row-major (H-outer) patch ordering.
129+
130+
Args:
131+
height: Grid height in patches.
132+
width: Grid width in patches.
133+
embed_dim: Total embedding dimension; must be divisible by 4.
134+
temperature: Base for the frequency decay.
135+
cls_token: If `True`, prepend a zero row for a CLS token.
136+
device: Target device; defaults to CPU.
137+
dtype: Output dtype; frequency arithmetic uses float64 internally.
138+
139+
Returns:
140+
Tensor of shape ``(height * width [+1], embed_dim)``.
141+
"""
142+
if embed_dim % 4 != 0:
143+
raise ValueError(f"`embed_dim` must be divisible by 4, got {embed_dim}")
144+
145+
pos_dim = embed_dim // 4
146+
omega = torch.arange(pos_dim, dtype=torch.float64, device=device) / pos_dim
147+
omega = 1.0 / temperature**omega # (D/4,)
148+
149+
grid_h = torch.arange(height, dtype=torch.float64, device=device)
150+
grid_w = torch.arange(width, dtype=torch.float64, device=device)
151+
grid_h, grid_w = torch.meshgrid(grid_h, grid_w, indexing="ij") # (H, W) each
152+
153+
emb_h = grid_h.flatten().outer(omega) # (H*W, D/4)
154+
emb_w = grid_w.flatten().outer(omega) # (H*W, D/4)
155+
156+
pos_embed = torch.cat([emb_h.sin(), emb_h.cos(), emb_w.sin(), emb_w.cos()], dim=1)
157+
158+
if cls_token:
159+
pos_embed = torch.cat([torch.zeros(1, embed_dim, dtype=torch.float64, device=device), pos_embed], dim=0)
160+
161+
return pos_embed.to(dtype)
162+
163+
116164
class Aimv2VisionEmbeddings(nn.Module):
117165
def __init__(self, config: Aimv2VisionConfig):
118166
super().__init__()
@@ -128,36 +176,24 @@ def __init__(self, config: Aimv2VisionConfig):
128176
self.position_embedding = nn.Embedding(num_patches, config.hidden_size)
129177
self.register_buffer("position_ids", torch.arange(num_patches).expand((1, -1)), persistent=False)
130178

131-
@staticmethod
132-
def build_2d_sincos_position_embedding(
133-
height, width, embed_dim=256, temperature=10000.0, device="cpu", dtype=torch.float32
134-
) -> torch.Tensor:
135-
grid_w = torch.arange(int(width), dtype=dtype, device=device)
136-
grid_h = torch.arange(int(height), dtype=dtype, device=device)
137-
grid_h, grid_w = torch.meshgrid(grid_w, grid_h, indexing="xy")
138-
139-
pos_dim = embed_dim // 4
140-
omega = torch.arange(pos_dim, dtype=dtype, device=device) / pos_dim
141-
omega = 1.0 / (temperature**omega)
142-
143-
out_h = grid_h.flatten()[..., None] @ omega[None, :]
144-
out_w = grid_w.flatten()[..., None] @ omega[None, :]
145-
146-
return torch.concat([out_h.sin(), out_h.cos(), out_w.sin(), out_w.cos()], dim=1)[None, :, :]
147-
148179
def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
149180
_, _, height, width = pixel_values.size()
150181
hidden_states = self.patch_embed(pixel_values).flatten(2).transpose(1, 2)
151182
hidden_states = self.rms_norm(hidden_states)
152183

153184
if self.config.is_native:
154-
pos_embed = self.build_2d_sincos_position_embedding(
155-
height // self.patch_size,
156-
width // self.patch_size,
185+
pos_embed = build_2d_sinusoidal_position_embedding(
186+
height=height // self.patch_size,
187+
width=width // self.patch_size,
157188
embed_dim=self.config.hidden_size,
158189
device=hidden_states.device,
159190
dtype=hidden_states.dtype,
160191
)
192+
# AIMv2 was trained with [sin_w|cos_w|sin_h|cos_h] layout (matching ViT-MAE's
193+
# original naming-bug convention); rotate the canonical h-first embedding to match.
194+
half = pos_embed.shape[-1] // 2
195+
pos_embed = torch.cat([pos_embed[..., half:], pos_embed[..., :half]], dim=-1)
196+
pos_embed = pos_embed.unsqueeze(0)
161197
else:
162198
pos_embed = self.position_embedding(self.position_ids)
163199

‎src/transformers/models/aimv2/modular_aimv2.py‎

Lines changed: 9 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
from ..llama.modeling_llama import LlamaMLP, LlamaRMSNorm
3636
from ..siglip.configuration_siglip import SiglipConfig, SiglipTextConfig, SiglipVisionConfig
3737
from ..siglip.modeling_siglip import SiglipAttention, SiglipEncoder, SiglipOutput
38+
from ..vit_mae.modeling_vit_mae import build_2d_sinusoidal_position_embedding
3839

3940

4041
@auto_docstring(checkpoint="apple/aimv2-large-patch14-224-lit")
@@ -164,36 +165,24 @@ def __init__(self, config: Aimv2VisionConfig):
164165
self.position_embedding = nn.Embedding(num_patches, config.hidden_size)
165166
self.register_buffer("position_ids", torch.arange(num_patches).expand((1, -1)), persistent=False)
166167

167-
@staticmethod
168-
def build_2d_sincos_position_embedding(
169-
height, width, embed_dim=256, temperature=10000.0, device="cpu", dtype=torch.float32
170-
) -> torch.Tensor:
171-
grid_w = torch.arange(int(width), dtype=dtype, device=device)
172-
grid_h = torch.arange(int(height), dtype=dtype, device=device)
173-
grid_h, grid_w = torch.meshgrid(grid_w, grid_h, indexing="xy")
174-
175-
pos_dim = embed_dim // 4
176-
omega = torch.arange(pos_dim, dtype=dtype, device=device) / pos_dim
177-
omega = 1.0 / (temperature**omega)
178-
179-
out_h = grid_h.flatten()[..., None] @ omega[None, :]
180-
out_w = grid_w.flatten()[..., None] @ omega[None, :]
181-
182-
return torch.concat([out_h.sin(), out_h.cos(), out_w.sin(), out_w.cos()], dim=1)[None, :, :]
183-
184168
def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
185169
_, _, height, width = pixel_values.size()
186170
hidden_states = self.patch_embed(pixel_values).flatten(2).transpose(1, 2)
187171
hidden_states = self.rms_norm(hidden_states)
188172

189173
if self.config.is_native:
190-
pos_embed = self.build_2d_sincos_position_embedding(
191-
height // self.patch_size,
192-
width // self.patch_size,
174+
pos_embed = build_2d_sinusoidal_position_embedding(
175+
height=height // self.patch_size,
176+
width=width // self.patch_size,
193177
embed_dim=self.config.hidden_size,
194178
device=hidden_states.device,
195179
dtype=hidden_states.dtype,
196180
)
181+
# AIMv2 was trained with [sin_w|cos_w|sin_h|cos_h] layout (matching ViT-MAE's
182+
# original naming-bug convention); rotate the canonical h-first embedding to match.
183+
half = pos_embed.shape[-1] // 2
184+
pos_embed = torch.cat([pos_embed[..., half:], pos_embed[..., :half]], dim=-1)
185+
pos_embed = pos_embed.unsqueeze(0)
197186
else:
198187
pos_embed = self.position_embedding(self.position_ids)
199188

0 commit comments

Comments
 (0)