device id passthrough

README.md

@@ -45,29 +45,10 @@ Running `dvc repro` in that folder will reproduce the experiment outlined in the
 To build the package (optional and is a very rough draft):
 
 ```
-######################################
-# Ubuntu 22.04, 20.04
-sudo apt update
-sudo apt install python3-venv python3-pip python3-dev python3-setuptools
-sudo apt install software-properties-common -y
-sudo add-apt-repository ppa:deadsnakes/ppa
-sudo apt install python3.9 -y
-sudo apt install msttcorefonts -qqpython3-distutils #fonts (optional)
-export SETUPTOOLS_USE_DISTUTILS=stdlib
-######################################
-python3 -m venv env
-source env/bin/activate
-git clone --recurse-submodules -j8 https://github.com/simplymathematics/deckard.git
-# git submodule update --init --recursive # To just update the submodules
-python3 -m pip install deckard/adversarial-robustness-toolbox/
-python3 -m pip install -e deckard/
-python3 -m pip install pyinstaller
-python3 -m pip install -u numba pip setuptools
-cd deckard && pyinstaller --onefile deckard.py -n deckard
-```
 
-After adding it to your path, you can then run deckard like a package:
+
+After adding it to your path, you can then run it as a module:
 
 ```
-deckard examples/sklearn
+python -m deckard --config_name mnist.yaml --config_folder examples/power
 ```

build_instructions.md

@@ -0,0 +1,19 @@
+######################################
+# Ubuntu 22.04, 20.04
+sudo apt update
+sudo apt install python3-venv python3-pip python3-dev python3-setuptools
+sudo apt install software-properties-common -y
+sudo add-apt-repository ppa:deadsnakes/ppa
+sudo apt install python3.9 -y
+sudo apt install msttcorefonts -qqpython3-distutils #fonts (optional)
+export SETUPTOOLS_USE_DISTUTILS=stdlib
+######################################
+python3 -m venv env
+source env/bin/activate
+git clone --recurse-submodules -j8 https://github.com/simplymathematics/deckard.git
+# git submodule update --init --recursive # To just update the submodules
+python3 -m pip install deckard/adversarial-robustness-toolbox/
+python3 -m pip install -e deckard/
+python3 -m pip install pyinstaller
+python3 -m pip install -u numba pip setuptools
+cd deckard && pyinstaller --onefile deckard.py -n deckard

deckard/__main__.py

@@ -40,11 +40,11 @@ def run_submodule(submodule, args):
             return 0
 
 
-def parse_and_repro(args, default_config="default.yaml"):
+def parse_and_repro(args, default_config="default.yaml", config_dir="conf"):
     if len(args) == 0:
         assert (
             save_params_file(
-                config_dir=Path(Path(), "conf"),
+                config_dir=Path(Path(), config_dir) if not Path(config_dir).is_absolute() else Path(config_dir),
                 config_file=default_config,
             )
             is None
@@ -84,6 +84,7 @@ def parse_and_repro(args, default_config="default.yaml"):
         type=str,
         help="default hydra configuration file that you would like to reproduce with dvc repro.",
     )
+    parser.add_argument("--config_dir", type=str, default="conf")
     parser.add_argument("other_args", type=str, nargs="*")
     args = parser.parse_args()
     submodule = args.submodule
@@ -98,6 +99,6 @@ def parse_and_repro(args, default_config="default.yaml"):
     else:
         other_args = []
     if submodule is None:
-        assert parse_and_repro(other_args, args.config_file) == 0
+        assert parse_and_repro(other_args, args.config_file, config_dir=args.config_dir) == 0
     else:
         assert run_submodule(submodule, other_args) == 0

deckard/base/experiment/experiment.py

@@ -29,6 +29,7 @@ class Experiment:
     name: Union[str, None] = field(default_factory=str)
     stage: Union[str, None] = field(default_factory=str)
     optimizers: Union[list, None] = field(default_factory=list)
+    device_id : str = 'cpu'
     kwargs: Union[dict, None] = field(default_factory=dict)
 
     def __init__(
@@ -37,6 +38,7 @@ def __init__(
         model: Model,
         scorers: ScorerDict,
         files: list,
+        device_id : str = 'cpu',
         attack: Attack = None,
         name=None,
         stage=None,
@@ -100,6 +102,7 @@ def __init__(
         else:  # pragma: no cover
             raise ValueError("attack must be a dict, DictConfig, or Attack object.")
         assert isinstance(self.attack, (Attack, type(None)))
+        self.device_id = device_id
         self.stage = stage
         self.optimizers = optimizers
         self.kwargs = kwargs
@@ -251,6 +254,7 @@ def __call__(self):
                     old_score_dict = self.data.load(files["score_dict_file"])
                     old_score_dict.update(**score_dict)
                     score_dict = old_score_dict
+                score_dict.update({"device_id" : self.device_id})
                 self.data.save(score_dict, files["score_dict_file"])
         else:  # pragma: no cover
             raise ValueError("Scorer is None. Please specify a scorer.")

deckard/base/model/model.py

@@ -190,7 +190,7 @@ def __call__(self, data: list, model: object, library=None):
             else:
                 raise e
         time_dict = {
-            "train_time": end,
+            "train_time": (start - end) / 1e9,
             "train_time_per_sample": end / (len(data[0]) * 1e9),
             "train_time_start": start,
             "train_time_end": end,

examples/power/conf/cifar10.yaml

@@ -7,6 +7,7 @@ defaults:
     - scorers: default
     - override hydra/sweeper : optuna
     - override hydra/sweeper/sampler : tpe
+device_id : ${oc.env:DECKARD_DEVICE,cpu}
 stage : '???'
 direction : 
   - maximize
@@ -18,7 +19,7 @@ direction :
 optimizers: 
    - accuracy
    - train_time
-   - predict_time
+   - predict_proba_time
    - adv_accuracy
    - adv_fit_time
    - adv_predict_time
@@ -45,7 +46,7 @@ hydra:
     params:
       ++data.sample.random_state: int(range(0, 1))
       ++model.art.initialize.optimizer.lr: range(0.0001, 1)
-      ++model.trainer.nb_epoch: int(range(1, 100))
+      ++model.trainer.nb_epoch: int(range(1, 50))
       ++model.trainer.batch_size: int(range(10, 10000))
       ++attack.init.eps : range(0.01, 1.0)
       ++attack.init.eps_step : ${eval:'(.1)*${attack.init.eps}'}

examples/power/conf/cifar100.yaml

@@ -8,6 +8,7 @@ defaults:
     - override hydra/sweeper : optuna
     - override hydra/sweeper/sampler : tpe
     # - override hydra/launcher : rq
+device_id : ${oc.env:DECKARD_DEVICE,cpu}
 stage : '???'
 direction : 
   - maximize
@@ -19,7 +20,7 @@ direction :
 optimizers: 
    - accuracy
    - train_time
-   - predict_time
+   - predict_proba_time
    - adv_accuracy
    - adv_fit_time
    - adv_predict_time
@@ -46,7 +47,7 @@ hydra:
     params:
       ++data.sample.random_state: int(range(0, 1))
       ++model.art.initialize.optimizer.lr: range(0.0001, 1)
-      ++model.trainer.nb_epoch: int(range(1, 100))
+      ++model.trainer.nb_epoch: int(range(1, 50))
       ++model.trainer.batch_size: int(range(10, 10000))
       ++attack.init.eps : range(0.01, 1.0)
       ++attack.init.eps_step : ${eval:'(.1)*${attack.init.eps}'}

examples/power/conf/data/torch_cifar.yaml

@@ -0,0 +1,16 @@
+_target_: deckard.base.data.Data
+generate:
+    _target_: deckard.base.data.generator.DataGenerator
+    name: torch_cifar10
+sample:
+    _target_: deckard.base.data.sampler.SklearnDataSampler
+    random_state : 0
+    stratify: True
+    train_size : 100
+    test_size : 100
+sklearn_pipeline:
+    _target_: deckard.base.data.sklearn_pipeline.SklearnDataPipeline
+    preprocessor:
+        name: sklearn.preprocessing.StandardScaler
+        with_mean: True
+        with_std: True

examples/power/conf/data/torch_cifar100.yaml

@@ -0,0 +1,16 @@
+_target_: deckard.base.data.Data
+generate:
+    _target_: deckard.base.data.generator.DataGenerator
+    name: torch_cifar100
+sample:
+    _target_: deckard.base.data.sampler.SklearnDataSampler
+    random_state : 0
+    stratify: True
+    train_size : 100
+    test_size : 100
+sklearn_pipeline:
+    _target_: deckard.base.data.sklearn_pipeline.SklearnDataPipeline
+    preprocessor:
+        name: sklearn.preprocessing.StandardScaler
+        with_mean: True
+        with_std: True

examples/power/conf/mnist.yaml

@@ -8,6 +8,7 @@ defaults:
     - override hydra/sweeper : optuna
     - override hydra/sweeper/sampler : tpe
     # - override hydra/launcher : rq
+device_id : ${oc.env:DECKARD_DEVICE,cpu}
 stage : '???'
 direction : 
   - maximize
@@ -19,7 +20,7 @@ direction :
 optimizers: 
    - accuracy
    - train_time
-   - predict_time
+   - predict_proba_time
    - adv_accuracy
    - adv_fit_time
    - adv_predict_time
@@ -46,7 +47,7 @@ hydra:
     params:
       ++data.sample.random_state: int(range(0, 1))
       ++model.art.initialize.optimizer.lr: range(0.0001, 1)
-      ++model.trainer.nb_epoch: int(range(1, 100))
+      ++model.trainer.nb_epoch: int(range(1, 50))
       ++model.trainer.batch_size: int(range(10, 10000))
       ++attack.init.eps : range(0.01, 1.0)
       ++attack.init.eps_step : ${eval:'(.1)*${attack.init.eps}'}

examples/power/conf/model/art/initialize/default.yaml

@@ -5,3 +5,4 @@ optimizer:
     lr : 0.01
     momentum : 0.9
 clip_values : [0, 255]
+device_type : ${oc.env:DECKARD_DEVICE_TYPE,cpu}