From a2d7e3ac0de4b2c10d5628df44e61118271b0a83 Mon Sep 17 00:00:00 2001 From: Konrad Zawora Date: Wed, 4 Dec 2024 14:21:57 +0200 Subject: [PATCH 1/2] Avoid torch.compile for HPU lazy backend Signed-off-by: Konrad Zawora --- vllm/plugins/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/vllm/plugins/__init__.py b/vllm/plugins/__init__.py index 81ee9975cdc4a..68eba4f93e138 100644 --- a/vllm/plugins/__init__.py +++ b/vllm/plugins/__init__.py @@ -29,6 +29,20 @@ def load_general_plugins(): if current_platform.is_xpu(): # see https://github.com/pytorch/pytorch/blob/8cada5cbe5450e17c26fb8b358116785324537b2/torch/_dynamo/config.py#L158 # noqa os.environ['TORCH_COMPILE_DISABLE'] = 'True' + if current_platform.is_hpu(): + # NOTE(kzawora): PT HPU lazy backend (PT_HPU_LAZY_MODE = 1) + # does not support torch.compile + # Eager backend (PT_HPU_LAZY_MODE = 0) must be selected for + # torch.compile support + is_lazy = os.environ.get('PT_HPU_LAZY_MODE', '1') == '1' + if is_lazy: + # see https://github.com/pytorch/pytorch/blob/43c5f59/torch/_dynamo/config.py#L158 + torch._dynamo.config.disable = True + # NOTE(kzawora) mutli-HPU inference with HPUGraphs (lazy-only) + # requires enabling lazy collectives + # see https://docs.habana.ai/en/latest/PyTorch/Inference_on_PyTorch/Inference_Using_HPU_Graphs.html # noqa: E501 + os.environ['PT_HPU_ENABLE_LAZY_COLLECTIVES'] = 'true' + global plugins_loaded if plugins_loaded: return From a6d9c380a229a69758149a330348f2e724580fa3 Mon Sep 17 00:00:00 2001 From: Konrad Zawora Date: Wed, 4 Dec 2024 14:46:36 +0200 Subject: [PATCH 2/2] formatting fix Signed-off-by: Konrad Zawora --- vllm/plugins/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vllm/plugins/__init__.py b/vllm/plugins/__init__.py index 68eba4f93e138..ae6e5c0a3481f 100644 --- a/vllm/plugins/__init__.py +++ b/vllm/plugins/__init__.py @@ -30,7 +30,7 @@ def load_general_plugins(): # see https://github.com/pytorch/pytorch/blob/8cada5cbe5450e17c26fb8b358116785324537b2/torch/_dynamo/config.py#L158 # noqa os.environ['TORCH_COMPILE_DISABLE'] = 'True' if current_platform.is_hpu(): - # NOTE(kzawora): PT HPU lazy backend (PT_HPU_LAZY_MODE = 1) + # NOTE(kzawora): PT HPU lazy backend (PT_HPU_LAZY_MODE = 1) # does not support torch.compile # Eager backend (PT_HPU_LAZY_MODE = 0) must be selected for # torch.compile support @@ -38,7 +38,7 @@ def load_general_plugins(): if is_lazy: # see https://github.com/pytorch/pytorch/blob/43c5f59/torch/_dynamo/config.py#L158 torch._dynamo.config.disable = True - # NOTE(kzawora) mutli-HPU inference with HPUGraphs (lazy-only) + # NOTE(kzawora) multi-HPU inference with HPUGraphs (lazy-only) # requires enabling lazy collectives # see https://docs.habana.ai/en/latest/PyTorch/Inference_on_PyTorch/Inference_Using_HPU_Graphs.html # noqa: E501 os.environ['PT_HPU_ENABLE_LAZY_COLLECTIVES'] = 'true'