better testing + api checks before starting the search

src/metahyper/api.py

@@ -10,7 +10,7 @@
 from copy import deepcopy
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, List
+from typing import Any, Iterable
 
 from ._locker import Locker
 from .utils import YamlSerializer, find_files, non_empty_file
@@ -156,6 +156,9 @@ def _process_sampler_info(
                 else:
                     # If the file is empty or doesn't exist, write the sampler_info
                     serializer.dump(sampler_info, sampler_info_file, sort_keys=False)
+            except ValueError as ve:
+                # Handle specific value error
+                raise ve
             except Exception as e:
                 raise RuntimeError(f"Error during data saving: {e}") from e
             finally:
@@ -423,7 +426,7 @@ def run(
     logger=None,
     post_evaluation_hook=None,
     overwrite_optimization_dir=False,
-    pre_load_hooks: List = [],
+    pre_load_hooks: Iterable | None = None,
 ):
     serializer = YamlSerializer(sampler.load_config)
     if logger is None:

src/neps/api.py

@@ -6,7 +6,7 @@
 import logging
 import warnings
 from pathlib import Path
-from typing import Callable, List, Literal
+from typing import Callable, Iterable, Literal
 
 import ConfigSpace as CS
 
@@ -110,7 +110,7 @@ def run(
     ignore_errors: bool = False,
     loss_value_on_error: None | float = None,
     cost_value_on_error: None | float = None,
-    pre_load_hooks: List = [],
+    pre_load_hooks: Iterable | None = None,
     searcher: Literal[
         "default",
         "bayesian_optimization",
@@ -171,7 +171,6 @@ def run(
     Raises:
         ValueError: If deprecated argument working_directory is used.
         ValueError: If root_directory is None.
-        TypeError: If pipeline_space has invalid type.
 
 
     Example:
@@ -207,6 +206,9 @@ def run(
         max_cost_total = searcher_kwargs["budget"]
         del searcher_kwargs["budget"]
 
+    if pre_load_hooks is None:
+        pre_load_hooks = []
+
     logger = logging.getLogger("neps")
     logger.info(f"Starting neps.run using root directory {root_directory}")
 
@@ -215,15 +217,17 @@ def run(
     searcher_info = {
         "searcher_name": "",
         "searcher_alg": "",
-        "searcher_selection_source": "",
-        "searcher_modified_arguments": {},
+        "searcher_selection": "",
+        "neps_decision_tree": True,
+        "searcher_args": {},
     }
 
     if isinstance(searcher, BaseOptimizer):
         searcher_instance = searcher
-        searcher_info["searcher_name"] = "custom"
+        searcher_info["searcher_name"] = "baseoptimizer"
         searcher_info["searcher_alg"] = searcher.whoami()
-        searcher_info["searcher_selection_source"] = "Custom-BaseOptimizer"
+        searcher_info["searcher_selection"] = "user-instantiation"
+        searcher_info["neps_decision_tree"] = False
     else:
         (
             searcher_instance,
@@ -248,8 +252,10 @@ def run(
             f"Unrecognized `searcher` of type {type(searcher)}. Not str or BaseOptimizer."
         )
     elif isinstance(searcher, BaseOptimizer):
-        logger.warning(
-            "An instantiated optimizer is provided. All kwargs are not supported"
+        # This check is not strict when a user-defined neps.optimizer is provided
+        logger.warn(
+            "An instantiated optimizer is provided. The safety checks of NePS will be "
+            "skipped. Accurate continuation of runs can no longer be guaranteed!"
         )
 
     metahyper.run(
@@ -329,7 +335,8 @@ def _run_args(
         logging.info("Preparing to run user created searcher")
 
         config = get_searcher_data(searcher, searcher_path)
-        searcher_info["searcher_selection_source"] = "Custom-User_Yaml"
+        searcher_info["searcher_selection"] = "user-yaml"
+        searcher_info["neps_decision_tree"] = False
     else:
         if searcher in ["default", None]:
             # NePS decides the searcher according to the pipeline space.
@@ -341,12 +348,11 @@ def _run_args(
                     if pipeline_space.has_fidelity
                     else "bayesian_optimization"
                 )
-            searcher_info[
-                "searcher_selection_source"
-            ] = "Default_Searcher-NePS_Decision_Tree"
+            searcher_info["searcher_selection"] = "neps-default"
         else:
             # Users choose one of NePS searchers.
-            searcher_info["searcher_selection_source"] = "Default_Searcher-User_Choice"
+            searcher_info["neps_decision_tree"] = False
+            searcher_info["searcher_selection"] = "neps-default"
         # Fetching the searcher data, throws an error when the searcher is not found
         config = get_searcher_data(searcher)
 
@@ -365,17 +371,13 @@ def _run_args(
 
     # Updating searcher arguments from searcher_kwargs
     for key, value in searcher_kwargs.items():
-        if (
-            searcher_info["searcher_selection_source"] == "Default_Searcher-User_Choice"
-            or searcher_info["searcher_selection_source"] == "Custom-User_Yaml"
-        ):
+        if not searcher_info["neps_decision_tree"]:
             if key not in searcher_config or searcher_config[key] != value:
                 searcher_config[key] = value
                 logger.info(
                     f"Updating the current searcher argument '{key}'"
                     f" with the value '{get_value(value)}'"
                 )
-                searcher_info["searcher_modified_arguments"][key] = get_value(value)
             else:
                 logger.info(
                     f"The searcher argument '{key}' has the same"
@@ -390,6 +392,8 @@ def _run_args(
                 f" because NePS chose the searcher"
             )
 
+    searcher_info["searcher_args"] = get_value(searcher_config)
+
     searcher_config.update(
         {
             "loss_value_on_error": loss_value_on_error,

tests/test_neps_api/solution_yamls/bo_custom_created.yaml

@@ -0,0 +1,5 @@
+searcher_name: baseoptimizer
+searcher_alg: BayesianOptimization
+searcher_selection: user-instantiation
+neps_decision_tree: false
+searcher_args: {}

tests/test_neps_api/solution_yamls/bo_neps_decided.yaml

@@ -0,0 +1,13 @@
+searcher_name: bayesian_optimization
+searcher_alg: bayesian_optimization
+searcher_selection: neps-default
+neps_decision_tree: true
+searcher_args:
+  initial_design_size: 10
+  surrogate_model: gp
+  acquisition: EI
+  log_prior_weighted: false
+  acquisition_sampler: mutation
+  random_interleave_prob: 0.0
+  disable_priors: true
+  sample_default_first: false

tests/test_neps_api/solution_yamls/bo_user_decided.yaml

@@ -0,0 +1,29 @@
+searcher_name: bayesian_optimization
+searcher_alg: bayesian_optimization
+searcher_selection: neps-default
+neps_decision_tree: false
+searcher_args:
+  initial_design_size: 10
+  surrogate_model: ComprehensiveGPHierarchy
+  acquisition: EI
+  log_prior_weighted: false
+  acquisition_sampler: mutation
+  random_interleave_prob: 0.0
+  disable_priors: true
+  sample_default_first: false
+  surrogate_model_args:
+    graph_kernels:
+    - WeisfeilerLehman
+    - WeisfeilerLehman
+    - WeisfeilerLehman
+    - WeisfeilerLehman
+    - WeisfeilerLehman
+    hp_kernels: []
+    verbose: false
+    hierarchy_consider:
+    - 0
+    - 1
+    - 2
+    - 3
+    d_graph_features: 0
+    vectorial_features: null

tests/test_neps_api/solution_yamls/hyperband_custom_created.yaml

@@ -0,0 +1,5 @@
+searcher_name: baseoptimizer
+searcher_alg: Hyperband
+searcher_selection: user-instantiation
+neps_decision_tree: false
+searcher_args: {}

tests/test_neps_api/solution_yamls/hyperband_neps_decided.yaml

@@ -0,0 +1,11 @@
+searcher_name: hyperband
+searcher_alg: hyperband
+searcher_selection: neps-default
+neps_decision_tree: true
+searcher_args:
+  eta: 3
+  initial_design_type: max_budget
+  use_priors: false
+  random_interleave_prob: 0.0
+  sample_default_first: false
+  sample_default_at_target: false

tests/test_neps_api/solution_yamls/pibo_neps_decided.yaml

@@ -0,0 +1,14 @@
+searcher_name: pibo
+searcher_alg: bayesian_optimization
+searcher_selection: neps-default
+neps_decision_tree: true
+searcher_args:
+  initial_design_size: 10
+  surrogate_model: gp
+  acquisition: EI
+  log_prior_weighted: false
+  acquisition_sampler: mutation
+  random_interleave_prob: 0.0
+  disable_priors: false
+  prior_confidence: medium
+  sample_default_first: false

tests/test_neps_api/solution_yamls/priorband_bo_user_decided.yaml

@@ -0,0 +1,23 @@
+searcher_name: priorband_bo
+searcher_alg: priorband
+searcher_selection: neps-default
+neps_decision_tree: false
+searcher_args:
+  eta: 3
+  initial_design_type: max_budget
+  prior_confidence: medium
+  random_interleave_prob: 0.0
+  sample_default_first: true
+  sample_default_at_target: false
+  prior_weight_type: geometric
+  inc_sample_type: mutation
+  inc_mutation_rate: 0.5
+  inc_mutation_std: 0.25
+  inc_style: dynamic
+  model_based: true
+  modelling_type: joint
+  initial_design_size: 5
+  surrogate_model: gp
+  acquisition: EI
+  log_prior_weighted: false
+  acquisition_sampler: mutation

tests/test_neps_api/solution_yamls/priorband_neps_decided.yaml

@@ -0,0 +1,17 @@
+searcher_name: priorband
+searcher_alg: priorband
+searcher_selection: neps-default
+neps_decision_tree: true
+searcher_args:
+  eta: 3
+  initial_design_type: max_budget
+  prior_confidence: medium
+  random_interleave_prob: 0.0
+  sample_default_first: true
+  sample_default_at_target: false
+  prior_weight_type: geometric
+  inc_sample_type: mutation
+  inc_mutation_rate: 0.5
+  inc_mutation_std: 0.25
+  inc_style: dynamic
+  model_based: false

tests/test_neps_api/solution_yamls/user_yaml_bo.yaml

@@ -0,0 +1,14 @@
+searcher_name: optimizer_test
+searcher_alg: bayesian_optimization
+searcher_selection: user-yaml
+neps_decision_tree: false
+searcher_args:
+  initial_design_size: 5
+  surrogate_model: gp
+  acquisition: EI
+  log_prior_weighted: false
+  acquisition_sampler: random
+  random_interleave_prob: 0.1
+  disable_priors: false
+  prior_confidence: high
+  sample_default_first: false