code sorting

examples/branin.py

@@ -0,0 +1,31 @@
+
+"""
+Branin
+^^^^^^
+"""
+
+import hydra
+import numpy as np
+from omegaconf import DictConfig
+
+__copyright__ = "Copyright 2022, AutoML.org Freiburg-Hannover"
+__license__ = "3-clause BSD"
+
+
+@hydra.main(config_path="configs", config_name="branin", version_base="1.1")
+def branin(cfg: DictConfig):
+    x0 = cfg.x0
+    x1 = cfg.x1
+    a = 1.0
+    b = 5.1 / (4.0 * np.pi**2)
+    c = 5.0 / np.pi
+    r = 6.0
+    s = 10.0
+    t = 1.0 / (8.0 * np.pi)
+    ret = a * (x1 - b * x0**2 + c * x0 - r) ** 2 + s * (1 - t) * np.cos(x0) + s
+
+    return ret
+
+
+if __name__ == "__main__":
+    branin()

examples/configs/branin.yaml

@@ -0,0 +1,44 @@
+
+defaults:
+  - _self_
+  - cluster/local
+  - override hydra/sweeper: SMAC
+
+hydra:
+  sweeper:
+    smac_class: smac.facade.blackbox_facade.BlackBoxFacade
+    scenario:
+      seed: 42
+      n_trials: 100
+      deterministic: true
+      n_workers: 4
+    smac_kwargs:
+      dask_client:
+        _target_: dask.distributed.Client
+        address: ${create_cluster:${cluster},${hydra.sweeper.scenario.n_workers}}
+      logging_level: 20  # 10 DEBUG, 20 INFO
+    search_space:  # TODO adjust search space
+      hyperparameters:
+        x0:
+          type: uniform_float
+          lower: -5
+          upper: 10
+          log: false
+        x1:
+          type: uniform_float
+          lower: 0
+          upper: 15
+          log: false
+          default_value: 2
+  run:
+    dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S}
+  sweep:
+    dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S}
+
+x0: 3
+x1: 4
+
+seed: None
+budget: None  # TODO document if used: add this to config
+
+spurious_var: 3.14

examples/configs/mlp.yaml

@@ -0,0 +1,90 @@
+
+defaults:
+  - _self_
+  - hpc
+  - override hydra/sweeper: SMAC
+
+
+learning_rate: constant
+learning_rate_init: 0.001
+batch_size: 200
+n_neurons: 10
+n_layer: 1
+solver: adam
+activation: tanh
+
+seed: 42
+epochs: 10  # Default number of epochs
+budget_variable: epochs  # Tells SMAC what variable to adjust for multi-fidelity
+
+hydra:
+  sweeper:
+    smac_class: smac.facade.multi_fidelity_facade.MultiFidelityFacade
+    scenario:
+      n_trials: 45
+      seed: ${seed}
+      min_budget: 5
+      max_budget: 50
+      deterministic: true
+      n_workers: 1
+    smac_kwargs:
+      dask_client:
+        _target_: dask.distributed.Client
+        address: ${create_cluster:${cluster},${hydra.sweeper.scenario.n_workers}}
+      intensifier: ${get_method:smac.facade.multi_fidelity_facade.MultiFidelityFacade.get_intensifier}
+      intensifier_kwargs:
+        eta: 3
+    search_space:
+      hyperparameters:
+        n_layer:
+          type: uniform_int
+          lower: 1
+          upper: 5
+          default: ${n_layer}
+        n_neurons:
+          type: uniform_int
+          lower: 8
+          upper: 1024
+          log: true
+          default_value: ${n_neurons}
+        activation:
+          type: categorical
+          choices: [ logistic, tanh, relu ]
+          default_value: ${activation}
+        solver:
+          type: categorical
+          choices: [ lbfgs, sgd, adam ]
+          default_value: ${solver}
+        batch_size:
+          type: uniform_int
+          lower: 30
+          upper: 300
+          default_value: ${batch_size}
+        learning_rate:
+          type: categorical
+          choices: [ constant, invscaling, adaptive ]
+          default_value: ${learning_rate}
+        learning_rate_init:
+          type: uniform_float
+          lower: 0.0001
+          upper: 1
+          default_value: ${learning_rate_init}
+          log: true
+      conditions:
+        - child: batch_size
+          parent: solver
+          type: IN
+          values: [ sgd, adam ]
+        - child: learning_rate
+          parent: solver
+          type: EQ
+          value: sgd
+        - child: learning_rate_init
+          parent: solver
+          type: IN
+          values: [ sgd, adam ]
+
+  run:
+    dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S}
+  sweep:
+    dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S}

examples/mlp.py

@@ -0,0 +1,76 @@
+
+"""
+MLP with Multi-Fidelity
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Example for optimizing a Multi-Layer Perceptron (MLP) using multiple budgets.
+Since we want to take advantage of Multi-Fidelity, the SMAC4MF facade is a good choice. By default,
+SMAC4MF internally runs with `hyperband <https://arxiv.org/abs/1603.06560>`_, which is a combination of an
+aggressive racing mechanism and successive halving.
+
+MLP is a deep neural network, and therefore, we choose epochs as fidelity type. The digits dataset
+is chosen to optimize the average accuracy on 5-fold cross validation.
+
+This example is adapted from `<https://github.com/automl/SMAC3/blob/main/examples/2_multi_fidelity/1_mlp_epochs.py>`_.
+"""
+__copyright__ = "Copyright 2022, AutoML.org Freiburg-Hannover"
+__license__ = "3-clause BSD"
+
+
+import warnings
+
+import hydra
+import numpy as np
+from omegaconf import DictConfig
+from sklearn.datasets import load_digits
+from sklearn.exceptions import ConvergenceWarning
+from sklearn.model_selection import StratifiedKFold, cross_val_score
+from sklearn.neural_network import MLPClassifier
+
+digits = load_digits()
+
+
+# Target Algorithm
+@hydra.main(config_path="configs", config_name="mlp", version_base="1.1")
+def mlp_from_cfg(cfg: DictConfig):
+    """
+    Creates a MLP classifier from sklearn and fits the given data on it.
+
+    Parameters
+    ----------
+    cfg: Configuration
+        configuration chosen by smac
+
+    Returns
+    -------
+    float
+    """
+    # For deactivated parameters, the configuration stores None-values.
+    # This is not accepted by the MLP, so we replace them with placeholder values.
+    lr = cfg.learning_rate or "constant"
+    lr_init = cfg.learning_rate_init or 0.001
+    batch_size = cfg.batch_size or 200
+
+    with warnings.catch_warnings():
+        warnings.filterwarnings("ignore", category=ConvergenceWarning)
+
+        mlp = MLPClassifier(
+            hidden_layer_sizes=[cfg.n_neurons] * cfg.n_layer,
+            solver=cfg.solver,
+            batch_size=batch_size,
+            activation=cfg.activation,
+            learning_rate=lr,
+            learning_rate_init=lr_init,
+            max_iter=int(np.ceil(cfg.epochs)),
+            random_state=cfg.seed,
+        )
+
+        # returns the cross validation accuracy
+        cv = StratifiedKFold(n_splits=5, random_state=cfg.seed, shuffle=True)  # to make CV splits consistent
+        score = cross_val_score(mlp, digits.data, digits.target, cv=cv, error_score="raise")
+
+    return 1 - np.mean(score)
+
+
+if __name__ == "__main__":
+    mlp_from_cfg()

hydra_plugins/hydra_smac_sweeper/hydra_smac.py

@@ -117,8 +117,9 @@ def __init__(
         self.max_parallel = min(job_array_size_limit, max(1, int(max_parallelization * n_trials)))
 
         self.min_budget = min_budget
-        self.iteration = 0
+        self.trials_run = 0
         self.n_trials = n_trials
+        self.iteration = 0
         self.opt_time = 0
         self.incumbent = []
         self.history = {}
@@ -198,7 +199,6 @@ def run_configs(self, configs, budgets, seeds):
             The incurred costs.
         """
         # Generate overrides
-        #TODO: handle budget correctly
         overrides = []
         for i in range(len(configs)):
             names = (list(configs[0].keys()) + [self.budget_arg_name] + [self.save_arg_name])
@@ -251,6 +251,7 @@ def run_configs(self, configs, budgets, seeds):
                 try:
                     res[j].return_value
                     done = True
+                    self.trials_run += 1
                 except:
                     done = False
 
@@ -351,7 +352,7 @@ def run(self, verbose=False):
         if verbose:
             log.info("Starting SMAC Sweep")
         self.start = time.time()
-        while self.iteration <= self.n_trials:
+        while self.trials_run <= self.n_trials:
             opt_time_start = time.time()
             configs = []
             budgets = []