From c411def234b0e85a349c8d95b5f32eade4aa1ed6 Mon Sep 17 00:00:00 2001 From: youkaichao Date: Wed, 27 Nov 2024 10:16:10 -0800 Subject: [PATCH] [torch.compile] fix shape specialization (#10722) Signed-off-by: youkaichao --- vllm/config.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/vllm/config.py b/vllm/config.py index 68f73bf4b4dc9..cd24e9ffdf598 100644 --- a/vllm/config.py +++ b/vllm/config.py @@ -2151,7 +2151,7 @@ class CompilationConfig(BaseModel): use_inductor: bool = True inductor_specialize_for_cudagraph_no_more_than: Optional[int] = None - inductor_compile_sizes: Optional[List[int]] = Field(default_factory=dict) + inductor_compile_sizes: Optional[List[int]] = Field(default=None) inductor_compile_config: Dict = Field(default_factory=dict) inductor_passes: Dict[str, str] = Field(default_factory=dict) @@ -2290,9 +2290,8 @@ def init_during_runtime(self): if x <= self.inductor_specialize_for_cudagraph_no_more_than ] else: - assert self.inductor_compile_sizes is not None, ( - "inductor_compile_sizes should not be None when " - "inductor_specialize_for_cudagraph_no_more_than is None") + if self.inductor_compile_sizes is None: + self.inductor_compile_sizes = [] self.compile_sizes = self.inductor_compile_sizes