fix: fix MAX_NUM_BATCHED_TOKENS default

hommayushi3 · Aug 21, 2024 · 2c58904 · 2c58904
1 parent 34a5d4b
commit 2c58904
Showing 1 changed file with 4 additions and 4 deletions.
diff --git a/endpoints-entrypoint.sh b/endpoints-entrypoint.sh
@@ -2,7 +2,7 @@
 NUM_SHARD=${NUM_SHARD:-$(nvidia-smi --list-gpus | wc -l)}
 MODEL_PATH=${MODEL_PATH:-"/repository"}
 MAX_MODEL_LEN=${MAX_MODEL_LEN:-1}
-MAX_NUM_BATCHED_TOKENS=${MAX_NUM_BATCHED_TOKENS:-0}
+MAX_NUM_BATCHED_TOKENS=${MAX_NUM_BATCHED_TOKENS:-1}
 ENABLE_CHUNKED_PREFILL=${ENABLE_CHUNKED_PREFILL:-false}
 ENABLE_PREFIX_CACHING=${ENABLE_PREFIX_CACHING:-false}
 DISABLE_SLIDING_WINDOW=${DISABLE_SLIDING_WINDOW:-false}
@@ -25,6 +25,9 @@ CMD="$CMD --gpu-memory-utilization $GPU_MEMORY_UTILIZATION"
 if [ "$MAX_MODEL_LEN" -ne -1 ]; then
     CMD="$CMD --max-model-len $MAX_MODEL_LEN"
 fi
+if [ "MAX_NUM_BATCHED_TOKENS" -ne 1 ]; then
+    CMD="$CMD --max-num-batched-tokens $MAX_NUM_BATCHED_TOKENS"
+fi
 if [ "$ENABLE_PREFIX_CACHING" = true ]; then
     CMD="$CMD --enable-prefix-caching"
 fi
@@ -43,9 +46,6 @@ fi
 if [ "$ENFORCE_EAGER" = true ]; then
     CMD="$CMD --enforce-eager"
 fi
-if [ "MAX_NUM_BATCHED_TOKENS" -ne 0 ]; then
-    CMD="$CMD --max-num-batched-tokens $MAX_NUM_BATCHED_TOKENS"
-fi
 
 # Execute the command
 eval $CMD