diff --git a/src/metahyper/api.py b/src/metahyper/api.py index 763e3fd7..7c662ba2 100644 --- a/src/metahyper/api.py +++ b/src/metahyper/api.py @@ -10,7 +10,7 @@ from copy import deepcopy from dataclasses import dataclass from pathlib import Path -from typing import Any, List +from typing import Any, Iterable from ._locker import Locker from .utils import YamlSerializer, find_files, non_empty_file @@ -156,6 +156,9 @@ def _process_sampler_info( else: # If the file is empty or doesn't exist, write the sampler_info serializer.dump(sampler_info, sampler_info_file, sort_keys=False) + except ValueError as ve: + # Handle specific value error + raise ve except Exception as e: raise RuntimeError(f"Error during data saving: {e}") from e finally: @@ -423,7 +426,7 @@ def run( logger=None, post_evaluation_hook=None, overwrite_optimization_dir=False, - pre_load_hooks: List = [], + pre_load_hooks: Iterable | None = None, ): serializer = YamlSerializer(sampler.load_config) if logger is None: diff --git a/src/neps/api.py b/src/neps/api.py index 24f80bf3..5c135e42 100644 --- a/src/neps/api.py +++ b/src/neps/api.py @@ -6,7 +6,7 @@ import logging import warnings from pathlib import Path -from typing import Callable, List, Literal +from typing import Callable, Iterable, Literal import ConfigSpace as CS @@ -110,7 +110,7 @@ def run( ignore_errors: bool = False, loss_value_on_error: None | float = None, cost_value_on_error: None | float = None, - pre_load_hooks: List = [], + pre_load_hooks: Iterable | None = None, searcher: Literal[ "default", "bayesian_optimization", @@ -171,7 +171,6 @@ def run( Raises: ValueError: If deprecated argument working_directory is used. ValueError: If root_directory is None. - TypeError: If pipeline_space has invalid type. Example: @@ -207,6 +206,9 @@ def run( max_cost_total = searcher_kwargs["budget"] del searcher_kwargs["budget"] + if pre_load_hooks is None: + pre_load_hooks = [] + logger = logging.getLogger("neps") logger.info(f"Starting neps.run using root directory {root_directory}") @@ -215,15 +217,17 @@ def run( searcher_info = { "searcher_name": "", "searcher_alg": "", - "searcher_selection_source": "", - "searcher_modified_arguments": {}, + "searcher_selection": "", + "neps_decision_tree": True, + "searcher_args": {}, } if isinstance(searcher, BaseOptimizer): searcher_instance = searcher - searcher_info["searcher_name"] = "custom" + searcher_info["searcher_name"] = "baseoptimizer" searcher_info["searcher_alg"] = searcher.whoami() - searcher_info["searcher_selection_source"] = "Custom-BaseOptimizer" + searcher_info["searcher_selection"] = "user-instantiation" + searcher_info["neps_decision_tree"] = False else: ( searcher_instance, @@ -248,8 +252,10 @@ def run( f"Unrecognized `searcher` of type {type(searcher)}. Not str or BaseOptimizer." ) elif isinstance(searcher, BaseOptimizer): - logger.warning( - "An instantiated optimizer is provided. All kwargs are not supported" + # This check is not strict when a user-defined neps.optimizer is provided + logger.warn( + "An instantiated optimizer is provided. The safety checks of NePS will be " + "skipped. Accurate continuation of runs can no longer be guaranteed!" ) metahyper.run( @@ -329,7 +335,8 @@ def _run_args( logging.info("Preparing to run user created searcher") config = get_searcher_data(searcher, searcher_path) - searcher_info["searcher_selection_source"] = "Custom-User_Yaml" + searcher_info["searcher_selection"] = "user-yaml" + searcher_info["neps_decision_tree"] = False else: if searcher in ["default", None]: # NePS decides the searcher according to the pipeline space. @@ -341,12 +348,11 @@ def _run_args( if pipeline_space.has_fidelity else "bayesian_optimization" ) - searcher_info[ - "searcher_selection_source" - ] = "Default_Searcher-NePS_Decision_Tree" + searcher_info["searcher_selection"] = "neps-default" else: # Users choose one of NePS searchers. - searcher_info["searcher_selection_source"] = "Default_Searcher-User_Choice" + searcher_info["neps_decision_tree"] = False + searcher_info["searcher_selection"] = "neps-default" # Fetching the searcher data, throws an error when the searcher is not found config = get_searcher_data(searcher) @@ -365,17 +371,13 @@ def _run_args( # Updating searcher arguments from searcher_kwargs for key, value in searcher_kwargs.items(): - if ( - searcher_info["searcher_selection_source"] == "Default_Searcher-User_Choice" - or searcher_info["searcher_selection_source"] == "Custom-User_Yaml" - ): + if not searcher_info["neps_decision_tree"]: if key not in searcher_config or searcher_config[key] != value: searcher_config[key] = value logger.info( f"Updating the current searcher argument '{key}'" f" with the value '{get_value(value)}'" ) - searcher_info["searcher_modified_arguments"][key] = get_value(value) else: logger.info( f"The searcher argument '{key}' has the same" @@ -390,6 +392,8 @@ def _run_args( f" because NePS chose the searcher" ) + searcher_info["searcher_args"] = get_value(searcher_config) + searcher_config.update( { "loss_value_on_error": loss_value_on_error, diff --git a/tests/test_neps_api/solution_yamls/bo_custom_created.yaml b/tests/test_neps_api/solution_yamls/bo_custom_created.yaml new file mode 100644 index 00000000..c808c699 --- /dev/null +++ b/tests/test_neps_api/solution_yamls/bo_custom_created.yaml @@ -0,0 +1,5 @@ +searcher_name: baseoptimizer +searcher_alg: BayesianOptimization +searcher_selection: user-instantiation +neps_decision_tree: false +searcher_args: {} diff --git a/tests/test_neps_api/solution_yamls/bo_neps_decided.yaml b/tests/test_neps_api/solution_yamls/bo_neps_decided.yaml new file mode 100644 index 00000000..76935d6c --- /dev/null +++ b/tests/test_neps_api/solution_yamls/bo_neps_decided.yaml @@ -0,0 +1,13 @@ +searcher_name: bayesian_optimization +searcher_alg: bayesian_optimization +searcher_selection: neps-default +neps_decision_tree: true +searcher_args: + initial_design_size: 10 + surrogate_model: gp + acquisition: EI + log_prior_weighted: false + acquisition_sampler: mutation + random_interleave_prob: 0.0 + disable_priors: true + sample_default_first: false diff --git a/tests/test_neps_api/solution_yamls/bo_user_decided.yaml b/tests/test_neps_api/solution_yamls/bo_user_decided.yaml new file mode 100644 index 00000000..c87b923a --- /dev/null +++ b/tests/test_neps_api/solution_yamls/bo_user_decided.yaml @@ -0,0 +1,29 @@ +searcher_name: bayesian_optimization +searcher_alg: bayesian_optimization +searcher_selection: neps-default +neps_decision_tree: false +searcher_args: + initial_design_size: 10 + surrogate_model: ComprehensiveGPHierarchy + acquisition: EI + log_prior_weighted: false + acquisition_sampler: mutation + random_interleave_prob: 0.0 + disable_priors: true + sample_default_first: false + surrogate_model_args: + graph_kernels: + - WeisfeilerLehman + - WeisfeilerLehman + - WeisfeilerLehman + - WeisfeilerLehman + - WeisfeilerLehman + hp_kernels: [] + verbose: false + hierarchy_consider: + - 0 + - 1 + - 2 + - 3 + d_graph_features: 0 + vectorial_features: null diff --git a/tests/test_neps_api/solution_yamls/hyperband_custom_created.yaml b/tests/test_neps_api/solution_yamls/hyperband_custom_created.yaml new file mode 100644 index 00000000..602b965c --- /dev/null +++ b/tests/test_neps_api/solution_yamls/hyperband_custom_created.yaml @@ -0,0 +1,5 @@ +searcher_name: baseoptimizer +searcher_alg: Hyperband +searcher_selection: user-instantiation +neps_decision_tree: false +searcher_args: {} diff --git a/tests/test_neps_api/solution_yamls/hyperband_neps_decided.yaml b/tests/test_neps_api/solution_yamls/hyperband_neps_decided.yaml new file mode 100644 index 00000000..29bf8dec --- /dev/null +++ b/tests/test_neps_api/solution_yamls/hyperband_neps_decided.yaml @@ -0,0 +1,11 @@ +searcher_name: hyperband +searcher_alg: hyperband +searcher_selection: neps-default +neps_decision_tree: true +searcher_args: + eta: 3 + initial_design_type: max_budget + use_priors: false + random_interleave_prob: 0.0 + sample_default_first: false + sample_default_at_target: false diff --git a/tests/test_neps_api/solution_yamls/pibo_neps_decided.yaml b/tests/test_neps_api/solution_yamls/pibo_neps_decided.yaml new file mode 100644 index 00000000..dce4d40e --- /dev/null +++ b/tests/test_neps_api/solution_yamls/pibo_neps_decided.yaml @@ -0,0 +1,14 @@ +searcher_name: pibo +searcher_alg: bayesian_optimization +searcher_selection: neps-default +neps_decision_tree: true +searcher_args: + initial_design_size: 10 + surrogate_model: gp + acquisition: EI + log_prior_weighted: false + acquisition_sampler: mutation + random_interleave_prob: 0.0 + disable_priors: false + prior_confidence: medium + sample_default_first: false diff --git a/tests/test_neps_api/solution_yamls/priorband_bo_user_decided.yaml b/tests/test_neps_api/solution_yamls/priorband_bo_user_decided.yaml new file mode 100644 index 00000000..cd7c82ec --- /dev/null +++ b/tests/test_neps_api/solution_yamls/priorband_bo_user_decided.yaml @@ -0,0 +1,23 @@ +searcher_name: priorband_bo +searcher_alg: priorband +searcher_selection: neps-default +neps_decision_tree: false +searcher_args: + eta: 3 + initial_design_type: max_budget + prior_confidence: medium + random_interleave_prob: 0.0 + sample_default_first: true + sample_default_at_target: false + prior_weight_type: geometric + inc_sample_type: mutation + inc_mutation_rate: 0.5 + inc_mutation_std: 0.25 + inc_style: dynamic + model_based: true + modelling_type: joint + initial_design_size: 5 + surrogate_model: gp + acquisition: EI + log_prior_weighted: false + acquisition_sampler: mutation diff --git a/tests/test_neps_api/solution_yamls/priorband_neps_decided.yaml b/tests/test_neps_api/solution_yamls/priorband_neps_decided.yaml new file mode 100644 index 00000000..eb3b0179 --- /dev/null +++ b/tests/test_neps_api/solution_yamls/priorband_neps_decided.yaml @@ -0,0 +1,17 @@ +searcher_name: priorband +searcher_alg: priorband +searcher_selection: neps-default +neps_decision_tree: true +searcher_args: + eta: 3 + initial_design_type: max_budget + prior_confidence: medium + random_interleave_prob: 0.0 + sample_default_first: true + sample_default_at_target: false + prior_weight_type: geometric + inc_sample_type: mutation + inc_mutation_rate: 0.5 + inc_mutation_std: 0.25 + inc_style: dynamic + model_based: false diff --git a/tests/test_neps_api/solution_yamls/user_yaml_bo.yaml b/tests/test_neps_api/solution_yamls/user_yaml_bo.yaml new file mode 100644 index 00000000..156d67e4 --- /dev/null +++ b/tests/test_neps_api/solution_yamls/user_yaml_bo.yaml @@ -0,0 +1,14 @@ +searcher_name: optimizer_test +searcher_alg: bayesian_optimization +searcher_selection: user-yaml +neps_decision_tree: false +searcher_args: + initial_design_size: 5 + surrogate_model: gp + acquisition: EI + log_prior_weighted: false + acquisition_sampler: random + random_interleave_prob: 0.1 + disable_priors: false + prior_confidence: high + sample_default_first: false diff --git a/tests/test_neps_api/test_api.py b/tests/test_neps_api/test_api.py index acd94322..a50b91d1 100644 --- a/tests/test_neps_api/test_api.py +++ b/tests/test_neps_api/test_api.py @@ -21,95 +21,11 @@ def use_tmpdir(tmp_path, request): def no_logs_gte_error(caplog): yield errors = [ - record - for record in caplog.get_records("call") - if record.levelno >= logging.ERROR + record for record in caplog.get_records("call") if record.levelno >= logging.ERROR ] assert not errors -# Expected outcomes of the optimizer YAML according to different cases of neps.run -# all based on the examples in tests/test_neps_api/examples_test_api.py -expected_dicts = { - "priorband_bo_user_decided": { - "searcher_name": "priorband_bo", - "searcher_alg": "priorband", - "searcher_selection_source": "Default_Searcher-User_Choice", - "searcher_modified_arguments": { - "initial_design_size": 5, - }, - }, - "bo_user_decided": { - "searcher_name": "bayesian_optimization", - "searcher_alg": "bayesian_optimization", - "searcher_selection_source": "Default_Searcher-User_Choice", - "searcher_modified_arguments": { - "surrogate_model": "ComprehensiveGPHierarchy", - "surrogate_model_args": { - "graph_kernels": [ - "WeisfeilerLehman", - "WeisfeilerLehman", - "WeisfeilerLehman", - "WeisfeilerLehman", - "WeisfeilerLehman", - ], - "hp_kernels": [], - "verbose": False, - "hierarchy_consider": [0, 1, 2, 3], - "d_graph_features": 0, - "vectorial_features": None, - }, - }, - }, - "priorband_neps_decided": { - "searcher_name": "priorband", - "searcher_alg": "priorband", - "searcher_selection_source": "Default_Searcher-NePS_Decision_Tree", - "searcher_modified_arguments": {}, - }, - "bo_neps_decided": { - "searcher_name": "bayesian_optimization", - "searcher_alg": "bayesian_optimization", - "searcher_selection_source": "Default_Searcher-NePS_Decision_Tree", - "searcher_modified_arguments": {}, - }, - "pibo_neps_decided": { - "searcher_name": "pibo", - "searcher_alg": "bayesian_optimization", - "searcher_selection_source": "Default_Searcher-NePS_Decision_Tree", - "searcher_modified_arguments": {}, - }, - "hyperband_neps_decided": { - "searcher_name": "hyperband", - "searcher_alg": "hyperband", - "searcher_selection_source": "Default_Searcher-NePS_Decision_Tree", - "searcher_modified_arguments": {}, - }, - "bo_custom_created": { - "searcher_name": "custom", - "searcher_alg": "BayesianOptimization", - "searcher_selection_source": "Custom-BaseOptimizer", - "searcher_modified_arguments": {}, - }, - "hyperband_custom_created": { - "searcher_name": "custom", - "searcher_alg": "Hyperband", - "searcher_selection_source": "Custom-BaseOptimizer", - "searcher_modified_arguments": {}, - }, - "user_yaml_bo": { - "searcher_name": "optimizer_test", - "searcher_alg": "bayesian_optimization", - "searcher_selection_source": "Custom-User_Yaml", - "searcher_modified_arguments": { - "initial_design_size": 5, - }, - }, -} - - -# Run locally and on github actions - testing_scripts = [ "default_neps", "baseoptimizer_neps", @@ -117,6 +33,7 @@ def no_logs_gte_error(caplog): ] examples_folder = Path(__file__, "..", "testing_scripts").resolve() +solution_folder = Path(__file__, "..", "solution_yamls").resolve() neps_api_example_script = [ examples_folder / f"{example}.py" for example in testing_scripts ] @@ -147,7 +64,10 @@ def test_default_examples(tmp_path): with open(str(info_yaml_path)) as yaml_config: loaded_data = yaml.safe_load(yaml_config) - assert loaded_data == expected_dicts[folder_name] + with open(str(solution_folder / (folder_name + ".yaml"))) as solution_yaml: + expected_data = yaml.safe_load(solution_yaml) + + assert loaded_data == expected_data @pytest.mark.neps_api @@ -175,7 +95,10 @@ def test_baseoptimizer_examples(tmp_path): with open(str(info_yaml_path)) as yaml_config: loaded_data = yaml.safe_load(yaml_config) - assert loaded_data == expected_dicts[folder_name] + with open(str(solution_folder / (folder_name + ".yaml"))) as solution_yaml: + expected_data = yaml.safe_load(solution_yaml) + + assert loaded_data == expected_data @pytest.mark.neps_api @@ -201,4 +124,7 @@ def test_user_created_yaml_examples(tmp_path): with open(str(info_yaml_path)) as yaml_config: loaded_data = yaml.safe_load(yaml_config) - assert loaded_data == expected_dicts[folder_name] + with open(str(solution_folder / (folder_name + ".yaml"))) as solution_yaml: + expected_data = yaml.safe_load(solution_yaml) + + assert loaded_data == expected_data