Generator: Make sampler settings optional instead of default arg

turboderp-org · Nov 12, 2024 · 16cd5ef · 16cd5ef
1 parent 9089596
commit 16cd5ef
Showing 1 changed file with 5 additions and 2 deletions.
diff --git a/exllamav2/generator/dynamic.py b/exllamav2/generator/dynamic.py
@@ -655,7 +655,7 @@ def generate(
                 assert False, "Unexpected type in prompt"
 
             if gen_settings is None:
-                p_settings = ExLlamaV2Sampler.Settings()
+                p_settings = None
             elif isinstance(gen_settings, ExLlamaV2Sampler.Settings):
                 p_settings = gen_settings
             elif isinstance(gen_settings, list):
@@ -1536,7 +1536,7 @@ def __init__(
         max_new_tokens: int,
         min_new_tokens: int = 0,
         max_skips: int | None = 4,
-        gen_settings: ExLlamaV2Sampler.Settings = ExLlamaV2Sampler.Settings(),
+        gen_settings: ExLlamaV2Sampler.Settings | None = None,
         seed: int = None,
         stop_conditions: list | tuple | set = None,
         decode_special_tokens: bool = False,
@@ -1629,6 +1629,9 @@ def __init__(
         self.max_skips = max_skips
         self.allocated_pages = None
 
+        if gen_settings is None:
+            gen_settings = ExLlamaV2Sampler.Settings()
+
         # Prepare sequences
 
         if not isinstance(input_ids, list):