Skip to content

Commit

Permalink
[Pixtral-Large] Pixtral actually has no bias in vision-lang adapter (v…
Browse files Browse the repository at this point in the history
…llm-project#10449)

Signed-off-by: Tyler Michael Smith <[email protected]>
  • Loading branch information
patrickvonplaten authored and tlrmchlsmth committed Nov 23, 2024
1 parent ee45dd4 commit e26f7ad
Showing 1 changed file with 3 additions and 2 deletions.
5 changes: 3 additions & 2 deletions vllm/model_executor/models/pixtral.py
Original file line number Diff line number Diff line change
Expand Up @@ -331,6 +331,7 @@ class VisionEncoderArgs:
num_attention_heads: int
rope_theta: float # for rope-2D
image_token_id: int
adapter_bias: bool = True


def _reshape_for_broadcast(freqs_cis: torch.Tensor,
Expand Down Expand Up @@ -595,10 +596,10 @@ def __init__(self, args: VisionEncoderArgs, dim: int):
self.w_in = nn.Linear(
args.hidden_size,
dim,
bias=True,
bias=args.adapter_bias,
)
self.gelu = nn.GELU()
self.w_out = nn.Linear(dim, dim, bias=True)
self.w_out = nn.Linear(dim, dim, bias=args.adapter_bias)

def forward(self, x: torch.Tensor) -> torch.Tensor:
return self.w_out(self.gelu(self.w_in(x)))
Expand Down

0 comments on commit e26f7ad

Please sign in to comment.