diff --git a/vllm/worker/worker.py b/vllm/worker/worker.py
index f51b51d433d3d..9b7155ba3799d 100644
--- a/vllm/worker/worker.py
+++ b/vllm/worker/worker.py
@@ -207,7 +207,8 @@ def determine_num_available_blocks(self) -> Tuple[int, int]:
         memory_for_current_instance = total_gpu_memory * \
             self.cache_config.gpu_memory_utilization
         available_kv_cache_memory = (memory_for_current_instance -
-                                     result.non_kv_cache_memory_in_bytes)
+                                     result.non_kv_cache_memory_in_bytes -
+                                     result.baseline_memory_in_bytes)
 
         # Calculate the number of blocks that can be allocated with the
         # profiled peak memory.