Merge pull request #16 from rhymes-ai/chengen/vllm_readme

add stop for vllm readme
rhymes-ai · Oct 9, 2024 · cbf51d3 · cbf51d3
2 parents acd9304 + e1cdeea
commit cbf51d3
Showing 1 changed file with 3 additions and 2 deletions.
diff --git a/docs/inference.md b/docs/inference.md
@@ -103,11 +103,12 @@ _MULTIMODAL_MODELS["AriaForConditionalGeneration"] = (
 def main():
     llm = LLM(
         model="rhymes-ai/Aria",
+        tokenizer="rhymes-ai/Aria",
+        tokenizer_mode="slow",
         dtype="bfloat16",
         limit_mm_per_prompt={"image": 256},
         enforce_eager=True,
         trust_remote_code=True,
-        skip_tokenizer_init=True,
     )
 
     tokenizer = AutoTokenizer.from_pretrained(
@@ -143,7 +144,7 @@ def main():
                 "split_image": True,  # [Optional] whether to split the images, default `False`
             },
         },
-        sampling_params=SamplingParams(max_tokens=200, top_k=1),
+        sampling_params=SamplingParams(max_tokens=200, top_k=1, stop=["<|im_end|>"]),
     )
 
     for o in outputs: