diff --git a/CHANGELOG.md b/CHANGELOG.md index db009bf97..499353674 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ ## Improvements - Add logger information on handling of stopIteration error (#960) - Replace deprecated ConfigSpace methods (#1139) +- Separated Wallclock time measurements from CPU time measurements and storing them under new 'cpu_time' variable (#????) ## Dependencies - Allow numpy >= 2.x (#1146) diff --git a/smac/main/smbo.py b/smac/main/smbo.py index dd6bfd354..3ed86c10b 100644 --- a/smac/main/smbo.py +++ b/smac/main/smbo.py @@ -81,6 +81,7 @@ def __init__( # Stats variables self._start_time: float | None = None self._used_target_function_walltime = 0.0 + self._used_target_function_cputime = 0.0 # Set walltime used method for intensifier self._intensifier.used_walltime = lambda: self.used_walltime # type: ignore @@ -108,7 +109,7 @@ def remaining_walltime(self) -> float: @property def remaining_cputime(self) -> float: """Subtracts the target function running budget with the used time.""" - return self._scenario.cputime_limit - self._used_target_function_walltime + return self._scenario.cputime_limit - self._used_target_function_cputime @property def remaining_trials(self) -> int: @@ -137,6 +138,11 @@ def used_target_function_walltime(self) -> float: """Returns how much walltime the target function spend so far.""" return self._used_target_function_walltime + @property + def used_target_function_cputime(self) -> float: + """Returns how much cputime the target function spend so far.""" + return self._used_target_function_cputime + def ask(self) -> TrialInfo: """Asks the intensifier for the next trial. @@ -204,6 +210,7 @@ def tell( config=info.config, cost=value.cost, time=value.time, + cpu_time=value.cpu_time, status=value.status, instance=info.instance, seed=info.seed, @@ -355,6 +362,7 @@ def optimize(self, *, data_to_scatter: dict[str, Any] | None = None) -> Configur def reset(self) -> None: """Resets the internal variables of the optimizer, intensifier, and runhistory.""" self._used_target_function_walltime = 0 + self._used_target_function_cputime = 0 self._finished = False # We also reset runhistory and intensifier here @@ -398,6 +406,7 @@ def load(self) -> None: self._intensifier.load(intensifier_fn) self._used_target_function_walltime = data["used_target_function_walltime"] + self._used_target_function_cputime = data["used_target_function_cputime"] self._finished = data["finished"] self._start_time = time.time() - data["used_walltime"] @@ -409,6 +418,7 @@ def save(self) -> None: data = { "used_walltime": self.used_walltime, "used_target_function_walltime": self.used_target_function_walltime, + "used_target_function_cputime": self.used_target_function_cputime, "last_update": time.time(), "finished": self._finished, } @@ -442,6 +452,7 @@ def _add_results(self) -> None: # Update SMAC stats self._used_target_function_walltime += float(trial_value.time) + self._used_target_function_cputime += float(trial_value.cpu_time) # Gracefully end optimization if termination cost is reached if self._scenario.termination_cost_threshold != np.inf: @@ -582,7 +593,7 @@ def validate( # TODO: Use submit run for faster evaluation # self._runner.submit_trial(trial_info=trial) - _, cost, _, _ = self._runner.run(config, **kwargs) + _, cost, _, _, _ = self._runner.run(config, **kwargs) costs += [cost] np_costs = np.array(costs) @@ -600,5 +611,7 @@ def print_stats(self) -> None: f"--- Used wallclock time: {round(self.used_walltime)} / {self._scenario.walltime_limit} sec\n" "--- Used target function runtime: " f"{round(self.used_target_function_walltime, 2)} / {self._scenario.cputime_limit} sec\n" + "--- Used target function CPU time: " + f"{round(self.used_target_function_cputime, 2)} / {self._scenario.cputime_limit} sec\n" f"----------------------------------------------------" ) diff --git a/smac/runhistory/dataclasses.py b/smac/runhistory/dataclasses.py index 585459cb3..1ac9df4bb 100644 --- a/smac/runhistory/dataclasses.py +++ b/smac/runhistory/dataclasses.py @@ -98,6 +98,7 @@ class TrialValue: ---------- cost : float | list[float] time : float, defaults to 0.0 + cpu_time : float, defaults to 0.0 status : StatusType, defaults to StatusType.SUCCESS starttime : float, defaults to 0.0 endtime : float, defaults to 0.0 @@ -106,6 +107,7 @@ class TrialValue: cost: float | list[float] time: float = 0.0 + cpu_time: float = 0.0 status: StatusType = StatusType.SUCCESS starttime: float = 0.0 endtime: float = 0.0 diff --git a/smac/runhistory/runhistory.py b/smac/runhistory/runhistory.py index ba6979224..7c27ff731 100644 --- a/smac/runhistory/runhistory.py +++ b/smac/runhistory/runhistory.py @@ -173,6 +173,7 @@ def add( config: Configuration, cost: int | float | list[int | float], time: float = 0.0, + cpu_time: float = 0.0, status: StatusType = StatusType.SUCCESS, instance: str | None = None, seed: int | None = None, @@ -191,6 +192,8 @@ def add( Cost of the evaluated trial. Might be a list in case of multi-objective. time : float How much time was needed to evaluate the trial. + cpu_time : float + How much CPU time was needed to evaluate the trial. status : StatusType, defaults to StatusType.SUCCESS The status of the trial. instance : str | None, defaults to none @@ -254,6 +257,7 @@ def add( v = TrialValue( cost=c, time=time, + cpu_time=cpu_time, status=status, starttime=starttime, endtime=endtime, @@ -269,6 +273,7 @@ def add( ("budget", budget), ("cost", c), ("time", time), + ("cpu_time", cpu_time), ("status", status), ("starttime", starttime), ("endtime", endtime), @@ -310,6 +315,7 @@ def add_trial(self, info: TrialInfo, value: TrialValue) -> None: config=info.config, cost=value.cost, time=value.time, + cpu_time=value.cpu_time, status=value.status, instance=info.instance, seed=info.seed, @@ -331,6 +337,7 @@ def add_running_trial(self, trial: TrialInfo) -> None: config=trial.config, cost=float(MAXINT), time=0.0, + cpu_time=0.0, status=StatusType.RUNNING, instance=trial.instance, seed=trial.seed, @@ -771,6 +778,7 @@ def save(self, filename: str | Path = "runhistory.json") -> None: float(k.budget) if k.budget is not None else None, v.cost, v.time, + v.cpu_time, v.status, v.starttime, v.endtime, @@ -848,6 +856,7 @@ def load(self, filename: str | Path, configspace: ConfigurationSpace) -> None: self._n_id = len(self._config_ids) # Important to use add method to use all data structure correctly + # NOTE: These hardcoded indices can easily lead to trouble for entry in data["data"]: # Set n_objectives first if self._n_objectives == -1: @@ -866,13 +875,14 @@ def load(self, filename: str | Path, configspace: ConfigurationSpace) -> None: config=self._ids_config[int(entry[0])], cost=cost, time=float(entry[5]), - status=StatusType(entry[6]), + cpu_time=float(entry[6]), + status=StatusType(entry[7]), instance=entry[1], seed=entry[2], budget=entry[3], - starttime=entry[7], - endtime=entry[8], - additional_info=entry[9], + starttime=entry[8], + endtime=entry[9], + additional_info=entry[10], ) # Although adding trials should give us the same stats, the trajectory might be different @@ -916,6 +926,7 @@ def update(self, runhistory: RunHistory) -> None: config=config, cost=value.cost, time=value.time, + cpu_time=value.cpu_time, status=value.status, instance=key.instance, starttime=value.starttime, diff --git a/smac/runner/abstract_runner.py b/smac/runner/abstract_runner.py index 4e15cf4b5..dd038d055 100644 --- a/smac/runner/abstract_runner.py +++ b/smac/runner/abstract_runner.py @@ -105,9 +105,9 @@ def run_wrapper( Contains information about the status/performance of config. """ start = time.time() - + cpu_time = time.process_time() try: - status, cost, runtime, additional_info = self.run( + status, cost, runtime, cpu_time, additional_info = self.run( config=trial_info.config, instance=trial_info.instance, budget=trial_info.budget, @@ -117,6 +117,7 @@ def run_wrapper( except Exception as e: status = StatusType.CRASHED cost = self._crash_cost + cpu_time = time.process_time() - cpu_time runtime = time.time() - start # Add context information to the error message @@ -148,6 +149,7 @@ def run_wrapper( status=status, cost=cost, time=runtime, + cpu_time=cpu_time, additional_info=additional_info, starttime=start, endtime=end, @@ -185,7 +187,7 @@ def run( instance: str | None = None, budget: float | None = None, seed: int | None = None, - ) -> tuple[StatusType, float | list[float], float, dict]: + ) -> tuple[StatusType, float | list[float], float, float, dict]: """Runs the target function with a configuration on a single instance-budget-seed combination (aka trial). @@ -208,6 +210,8 @@ def run( Resulting cost(s) of the trial. runtime : float The time the target function took to run. + cpu_time : float + The CPU time the target function took to run. additional_info : dict All further additional trial information. """ diff --git a/smac/runner/dask_runner.py b/smac/runner/dask_runner.py index d4bb528bd..760665170 100644 --- a/smac/runner/dask_runner.py +++ b/smac/runner/dask_runner.py @@ -163,7 +163,7 @@ def run( budget: float | None = None, seed: int | None = None, **dask_data_to_scatter: dict[str, Any], - ) -> tuple[StatusType, float | list[float], float, dict]: # noqa: D102 + ) -> tuple[StatusType, float | list[float], float, float, dict]: # noqa: D102 return self._single_worker.run( config=config, instance=instance, seed=seed, budget=budget, **dask_data_to_scatter ) diff --git a/smac/runner/target_function_runner.py b/smac/runner/target_function_runner.py index 8a14c2fc9..c9c5d46c8 100644 --- a/smac/runner/target_function_runner.py +++ b/smac/runner/target_function_runner.py @@ -112,7 +112,7 @@ def run( budget: float | None = None, seed: int | None = None, **dask_data_to_scatter: dict[str, Any], - ) -> tuple[StatusType, float | list[float], float, dict]: + ) -> tuple[StatusType, float | list[float], float, float, dict]: """Calls the target function with pynisher if algorithm wall time limit or memory limit is set. Otherwise, the function is called directly. @@ -143,6 +143,8 @@ def run( Resulting cost(s) of the trial. runtime : float The time the target function took to run. + cpu_time : float + The CPU time the target function took to run. additional_info : dict All further additional trial information. """ @@ -162,6 +164,7 @@ def run( # Presetting cost: float | list[float] = self._crash_cost runtime = 0.0 + cpu_time = runtime additional_info = {} status = StatusType.CRASHED @@ -183,7 +186,9 @@ def run( # Call target function try: start_time = time.time() + cpu_time = time.process_time() rval = self(config_copy, target_function, kwargs) + cpu_time = time.process_time() - cpu_time runtime = time.time() - start_time status = StatusType.SUCCESS except WallTimeoutException: @@ -199,7 +204,7 @@ def run( status = StatusType.CRASHED if status != StatusType.SUCCESS: - return status, cost, runtime, additional_info + return status, cost, runtime, cpu_time, additional_info if isinstance(rval, tuple): result, additional_info = rval @@ -240,7 +245,7 @@ def run( # We want to get either a float or a list of floats. cost = np.asarray(cost).squeeze().tolist() - return status, cost, runtime, additional_info + return status, cost, runtime, cpu_time, additional_info def __call__( self, diff --git a/smac/runner/target_function_script_runner.py b/smac/runner/target_function_script_runner.py index 258c545e7..f17b13d39 100644 --- a/smac/runner/target_function_script_runner.py +++ b/smac/runner/target_function_script_runner.py @@ -83,7 +83,7 @@ def run( instance: str | None = None, budget: float | None = None, seed: int | None = None, - ) -> tuple[StatusType, float | list[float], float, dict]: + ) -> tuple[StatusType, float | list[float], float, float, dict]: """Calls the target function. Parameters @@ -105,6 +105,8 @@ def run( Resulting cost(s) of the trial. runtime : float The time the target function took to run. + cpu_time : float + The CPU time the target function took to run. additional_info : dict All further additional trial information. """ @@ -128,6 +130,7 @@ def run( # Presetting cost: float | list[float] = self._crash_cost runtime = 0.0 + cpu_time = runtime additional_info = {} status = StatusType.SUCCESS @@ -139,7 +142,9 @@ def run( # Call target function start_time = time.time() + cpu_time = time.process_time() output, error = self(kwargs) + cpu_time = time.process_time() - cpu_time runtime = time.time() - start_time # Now we have to parse the std output @@ -181,6 +186,10 @@ def run( if "runtime" in outputs: runtime = float(outputs["runtime"]) + # Overwrite CPU time + if "cpu_time" in outputs: + cpu_time = float(outputs["cpu_time"]) + # Add additional info if "additional_info" in outputs: additional_info["additional_info"] = outputs["additional_info"] @@ -194,7 +203,7 @@ def run( "The target function crashed but returned a cost. The cost is ignored and replaced by crash cost." ) - return status, cost, runtime, additional_info + return status, cost, runtime, cpu_time, additional_info def __call__( self, diff --git a/tests/test_runner/test_script_target_algorithm_runner.py b/tests/test_runner/test_script_target_algorithm_runner.py index 61d87f331..8464608a6 100644 --- a/tests/test_runner/test_script_target_algorithm_runner.py +++ b/tests/test_runner/test_script_target_algorithm_runner.py @@ -17,7 +17,7 @@ def test_success(configspace, make_scenario): runner = TargetFunctionScriptRunner(script, scenario, required_arguments=["seed", "instance"]) config = configspace.get_default_configuration() - status, cost, runtime, additional_info = runner.run(config, instance=scenario.instances[0], seed=0) + status, cost, runtime, cpu_time, additional_info = runner.run(config, instance=scenario.instances[0], seed=0) assert status == StatusType.SUCCESS assert cost == config["x0"] @@ -30,7 +30,7 @@ def test_success_multi_objective(configspace, make_scenario): runner = TargetFunctionScriptRunner(script, scenario, required_arguments=["seed", "instance"]) config = configspace.get_default_configuration() - status, cost, runtime, additional_info = runner.run(config, instance=scenario.instances[0], seed=0) + status, cost, runtime, cpu_time, additional_info = runner.run(config, instance=scenario.instances[0], seed=0) assert status == StatusType.SUCCESS assert cost == [config["x0"], config["x0"]] @@ -43,7 +43,7 @@ def test_exit(configspace, make_scenario): runner = TargetFunctionScriptRunner(script, scenario, required_arguments=["seed", "instance"]) config = configspace.get_default_configuration() - status, cost, runtime, additional_info = runner.run(config, instance=scenario.instances[0], seed=0) + status, cost, runtime, cpu_time, additional_info = runner.run(config, instance=scenario.instances[0], seed=0) assert status == StatusType.CRASHED assert "error" in additional_info @@ -55,7 +55,7 @@ def test_crashed(configspace, make_scenario): runner = TargetFunctionScriptRunner(script, scenario, required_arguments=["seed", "instance"]) config = configspace.get_default_configuration() - status, cost, runtime, additional_info = runner.run(config, instance=scenario.instances[0], seed=0) + status, cost, runtime, cpu_time, additional_info = runner.run(config, instance=scenario.instances[0], seed=0) assert status == StatusType.CRASHED assert cost == np.inf @@ -67,7 +67,7 @@ def test_python(configspace, make_scenario): runner = TargetFunctionScriptRunner(script, scenario, required_arguments=["seed", "instance"]) config = configspace.get_default_configuration() - status, cost, runtime, additional_info = runner.run(config, instance=scenario.instances[0], seed=0) + status, cost, runtime, cpu_time, additional_info = runner.run(config, instance=scenario.instances[0], seed=0) assert status == StatusType.SUCCESS assert cost == config["x0"] diff --git a/tests/test_runner/test_target_algorithm_runner.py b/tests/test_runner/test_target_algorithm_runner.py index 44fa7cc01..724b2aa01 100644 --- a/tests/test_runner/test_target_algorithm_runner.py +++ b/tests/test_runner/test_target_algorithm_runner.py @@ -148,7 +148,7 @@ def test_call(make_runner: Callable[..., TargetFunctionRunner]) -> None: config = runner._scenario.configspace.get_default_configuration() SEED = 2345 - status, cost, _, _ = runner.run(config=config, instance=None, seed=SEED, budget=None) + status, cost, _, _, _ = runner.run(config=config, instance=None, seed=SEED, budget=None) assert cost == SEED assert status == StatusType.SUCCESS @@ -163,7 +163,7 @@ def test_multi_objective(make_runner: Callable[..., TargetFunctionRunner]) -> No config = runner._scenario.configspace.get_default_configuration() SEED = 2345 - status, cost, _, _ = runner.run(config=config, instance=None, seed=SEED, budget=None) + status, cost, _, _, _ = runner.run(config=config, instance=None, seed=SEED, budget=None) assert isinstance(cost, list) assert cost == [SEED, SEED]