[V1][VLM] V1 support for selected single-image models.

docs/source/models/supported_models.md

@@ -570,28 +570,28 @@ See [this page](#generative-models) for more information on how to use generativ
   - `rhymes-ai/Aria`
   -
   - ✅︎
-  -
+  - ✅︎
 * - `Blip2ForConditionalGeneration`
   - BLIP-2
   - T + I<sup>E</sup>
   - `Salesforce/blip2-opt-2.7b`, `Salesforce/blip2-opt-6.7b`, etc.
   -
   - ✅︎
-  -
+  - ✅︎
 * - `ChameleonForConditionalGeneration`
   - Chameleon
   - T + I
   - `facebook/chameleon-7b` etc.
   -
   - ✅︎
-  -
+  - ✅︎
 * - `FuyuForCausalLM`
   - Fuyu
   - T + I
   - `adept/fuyu-8b` etc.
   -
   - ✅︎
-  -
+  - ✅︎
 * - `ChatGLMModel`
   - GLM-4V
   - T + I
@@ -633,7 +633,7 @@ See [this page](#generative-models) for more information on how to use generativ
   - `llava-hf/llava-v1.6-mistral-7b-hf`, `llava-hf/llava-v1.6-vicuna-7b-hf`, etc.
   -
   - ✅︎
-  -
+  - ✅︎
 * - `LlavaNextVideoForConditionalGeneration`
   - LLaVA-NeXT-Video
   - T + V

examples/offline_inference_vision_language.py

@@ -28,6 +28,7 @@ def run_aria(question: str, modality: str):
               tokenizer_mode="slow",
               trust_remote_code=True,
               dtype="bfloat16",
+              max_num_seqs=2,
               disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
 
     prompt = (f"<|im_start|>user\n<fim_prefix><|img|><fim_suffix>\n{question}"
@@ -191,8 +192,10 @@ def run_llava_next(question: str, modality: str):
 
     prompt = f"[INST] <image>\n{question} [/INST]"
     llm = LLM(model="llava-hf/llava-v1.6-mistral-7b-hf",
-              max_model_len=8192,
-              disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
+              max_num_batched_tokens=32768,
+              disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
+              limit_mm_per_prompt={"image": 4},
+              enable_prefix_caching=False)
     stop_token_ids = None
     return llm, prompt, stop_token_ids
 
@@ -591,7 +594,7 @@ def main(args):
 
     # We set temperature to 0.2 so that outputs can be different
     # even when all prompts are identical when running batch inference.
-    sampling_params = SamplingParams(temperature=0.2,
+    sampling_params = SamplingParams(temperature=0,
                                      max_tokens=64,
                                      stop_token_ids=stop_token_ids)
 

vllm/model_executor/models/aria.py

@@ -1,24 +1,27 @@
 import math
-from typing import Iterable, List, Optional, Set, Tuple, TypedDict, Union
+from typing import (Iterable, List, Mapping, Optional, Set, Tuple, TypedDict,
+                    Union)
 
 import torch
 import torch.nn as nn
+from PIL import Image
 from torch.nn.init import trunc_normal_
 from transformers import LlamaConfig
 
 from vllm.attention import AttentionMetadata
 from vllm.config import CacheConfig, QuantizationConfig, VllmConfig
 from vllm.distributed import get_tensor_model_parallel_rank
-from vllm.inputs import INPUT_REGISTRY, token_inputs
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
+                         InputContext, token_inputs)
 from vllm.model_executor.layers.activation import get_act_fn
 from vllm.model_executor.layers.fused_moe import FusedMoE
 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
                                                RowParallelLinear)
 from vllm.model_executor.layers.logits_processor import LogitsProcessor
 from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
     get_compressed_tensors_cache_scale)
-from vllm.model_executor.layers.sampler import (Sampler, SamplerOutput,
-                                                SamplingMetadata)
+from vllm.model_executor.layers.sampler import (SamplerOutput,
+                                                SamplingMetadata, get_sampler)
 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
 from vllm.model_executor.model_loader.weight_utils import (
     default_weight_loader, maybe_remap_kv_scale_name)
@@ -35,10 +38,12 @@
 from vllm.multimodal.image import cached_get_image_processor
 from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
 from vllm.multimodal.utils import (cached_get_tokenizer,
+                                   consecutive_placeholder_ranges,
                                    repeat_and_pad_placeholder_tokens)
-from vllm.sequence import IntermediateTensors
+from vllm.sequence import IntermediateTensors, SequenceData
 from vllm.transformers_utils.configs.aria import (AriaMoELMConfig,
                                                   AriaVisionConfig)
+from vllm.utils import is_list_of
 
 from .utils import flatten_bn
 
@@ -445,15 +450,74 @@ def build_mm_projector(config):
     )
 
 
-def get_max_multimodal_tokens(ctx):
-    return max(ctx.model_config.hf_config.image_size2tokens.values())
-
-
-def input_mapper_for_aria(ctx, data):
-    return MultiModalKwargs(data)
-
-
-def input_processor(ctx, llm_inputs):
+def get_aria_max_multimodal_tokens(ctx: InputContext):
+    hf_config = ctx.get_hf_config()
+    image_size2tokens = {
+        int(math.sqrt(k) * hf_config.vision_config.patch_size): v
+        for k, v in hf_config.projector_patch_to_query_dict.items()
+    }
+    return max(image_size2tokens.values())
+
+
+def dummy_seq_data_for_aria(ctx: InputContext, seq_len: int, num_images: int):
+    image_feature_size = get_aria_max_multimodal_tokens(ctx)
+    hf_config = ctx.get_hf_config()
+    return SequenceData.from_prompt_token_counts(
+        (hf_config.image_token_index, image_feature_size * num_images),
+        (0, seq_len - image_feature_size * num_images),
+    ), {
+        "image":
+        consecutive_placeholder_ranges(num_items=num_images,
+                                       item_size=image_feature_size)
+    }
+
+
+def dummy_image_for_aria(
+    ctx: InputContext,
+    num_images: int,
+):
+    hf_config = ctx.get_hf_config()
+    max_image_size = hf_config.vision_config.image_size
+    image = Image.new("RGB", (max_image_size, max_image_size), color=0)
+    images = [image] * num_images
+
+    return {"image": images}
+
+
+def dummy_data_for_aria(ctx: InputContext, seq_len: int,
+                        mm_counts: Mapping[str, int]):
+    num_images = mm_counts["image"]
+    seq_data, ranges = dummy_seq_data_for_aria(ctx, seq_len, num_images)
+    mm_data = dummy_image_for_aria(ctx, num_images)
+    return DummyData(seq_data, mm_data, ranges)
+
+
+def input_mapper_for_aria(ctx: InputContext, data: object):
+    data_list = data if isinstance(data, list) else [data]
+
+    # For profiling with dummy image data
+    if is_list_of(data_list, Image.Image):
+        hf_config = ctx.get_hf_config()
+        max_image_size = hf_config.vision_config.image_size
+        model_config = ctx.model_config
+        image_processor = cached_get_image_processor(
+            model_config.model,
+            trust_remote_code=model_config.trust_remote_code)
+        image_inputs = image_processor.preprocess(
+            data_list,
+            max_image_size=max_image_size,
+            split_image=False,
+            return_tensors="pt").data
+        image_inputs['pixel_values'] = image_inputs['pixel_values'].to(
+            ctx.model_config.dtype)
+        return MultiModalKwargs(image_inputs)
+
+    # For actual inference when image has been processed with
+    # prompt in input processor
+    return MultiModalKwargs(data_list[0])
+
+
+def input_processor_for_aria(ctx: InputContext, llm_inputs: DecoderOnlyInputs):
     multi_modal_data = llm_inputs.get("multi_modal_data")
     # if it is pure text input, use it as is
     if multi_modal_data is None or "image" not in multi_modal_data:
@@ -494,9 +558,12 @@ def input_processor(ctx, llm_inputs):
             repeat_count=num_crops,
         )
 
-    repeat_count = [hf_config.image_size2tokens[max_image_size]
-                    ] * sum(num_crops).item()
-    new_prompt, new_token_ids, _ = repeat_and_pad_placeholder_tokens(
+    image_size2tokens = {
+        int(math.sqrt(k) * hf_config.vision_config.patch_size): v
+        for k, v in hf_config.projector_patch_to_query_dict.items()
+    }
+    repeat_count = [image_size2tokens[max_image_size]] * sum(num_crops).item()
+    new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
         tokenizer,
         None,
         prompt_token_ids,
@@ -508,12 +575,14 @@ def input_processor(ctx, llm_inputs):
         prompt_token_ids=new_token_ids,
         prompt=new_prompt,
         multi_modal_data={"image": image_inputs},
+        multi_modal_placeholders={"image": ranges},
     )
 
 
-@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_multimodal_tokens)
+@MULTIMODAL_REGISTRY.register_max_image_tokens(get_aria_max_multimodal_tokens)
 @MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_aria)
-@INPUT_REGISTRY.register_input_processor(input_processor)
+@INPUT_REGISTRY.register_dummy_data(dummy_data_for_aria)
+@INPUT_REGISTRY.register_input_processor(input_processor_for_aria)
 class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
     """
     Aria model for conditional generation tasks.
@@ -540,12 +609,6 @@ def __init__(
         config = vllm_config.model_config.hf_config
         quant_config = vllm_config.quant_config
 
-        # prepare the image_size to tokens mapping for the image preprocess, see
-        # input_processor
-        config.image_size2tokens = {
-            int(math.sqrt(k) * config.vision_config.patch_size): v
-            for k, v in config.projector_patch_to_query_dict.items()
-        }
         self.config = config
         self.vision_tower = AriaVisionModel(config.vision_config)
         self.multi_modal_projector = build_mm_projector(config)
@@ -566,7 +629,7 @@ def __init__(
         logit_scale = getattr(config, "logit_scale", 1.0)
         self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
                                                 self.vocab_size, logit_scale)
-        self.sampler = Sampler()
+        self.sampler = get_sampler()
 
     def _validate_image_sizes(
             self, images: List[torch.Tensor]) -> List[torch.Tensor]:

vllm/model_executor/models/blip2.py

@@ -16,7 +16,7 @@
 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
 from vllm.model_executor.sampling_metadata import SamplingMetadata
 from vllm.multimodal import MULTIMODAL_REGISTRY
-from vllm.multimodal.inputs import NestedTensors
+from vllm.multimodal.inputs import NestedTensors, PlaceholderRange
 from vllm.multimodal.utils import consecutive_placeholder_ranges
 from vllm.sequence import IntermediateTensors, SequenceData
 
@@ -468,14 +468,18 @@ def input_processor_for_blip2(ctx: InputContext, inputs: DecoderOnlyInputs):
     # https://github.com/huggingface/transformers/blob/v4.41.2/src/transformers/models/blip_2/modeling_blip_2.py#L1514
     new_token_ids = [BLIP2_IMAGE_TOKEN_ID] * image_feature_size
     new_token_ids += inputs["prompt_token_ids"]
+    placeholder_ranges = [
+        PlaceholderRange(offset=0, length=image_feature_size)
+    ]
 
     new_prompt = inputs.get("prompt")
     if new_prompt is not None:
         new_prompt = BLIP2_IMAGE_TOKEN * image_feature_size + new_prompt
 
     return token_inputs(prompt_token_ids=new_token_ids,
                         prompt=new_prompt,
-                        multi_modal_data=multi_modal_data)
+                        multi_modal_data=multi_modal_data,
+                        multi_modal_placeholders={"image": placeholder_ranges})
 
 
 @MULTIMODAL_REGISTRY.register_image_input_mapper()

vllm/model_executor/models/chameleon.py

@@ -153,7 +153,8 @@ def input_processor_for_chameleon(ctx: InputContext,
     # NOTE: Create a defensive copy of the original inputs
     return token_inputs(prompt_token_ids=new_token_ids,
                         prompt=new_prompt,
-                        multi_modal_data=multi_modal_data)
+                        multi_modal_data=multi_modal_data,
+                        multi_modal_placeholders={"image": ranges})
 
 
 class ChameleonLayerNorm(nn.LayerNorm):