From 7c2134beda9a4f72c71c4faffcca22cebd4e1c3c Mon Sep 17 00:00:00 2001 From: Jee Jee Li Date: Mon, 25 Nov 2024 15:04:21 +0800 Subject: [PATCH] [torch.compile] force inductor threads (#10620) Signed-off-by: Jee Jee Li --- vllm/plugins/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vllm/plugins/__init__.py b/vllm/plugins/__init__.py index d5056b18fe968..bd4764c5cc79c 100644 --- a/vllm/plugins/__init__.py +++ b/vllm/plugins/__init__.py @@ -3,6 +3,8 @@ from contextlib import contextmanager from typing import TYPE_CHECKING, Optional +import torch + import vllm.envs as envs if TYPE_CHECKING: @@ -26,7 +28,8 @@ def load_general_plugins(): # see https://github.com/vllm-project/vllm/issues/10480 os.environ['TORCHINDUCTOR_COMPILE_THREADS'] = '1' - + # see https://github.com/vllm-project/vllm/issues/10619 + torch._inductor.config.compile_threads = 1 global plugins_loaded if plugins_loaded: return