Reproduce

.gitignore

@@ -142,4 +142,3 @@ deckard/deckard.egg-info/*
 examples/**/params.yaml
 examples/**/multirun/*
 *log.txt
-examples/power/mnist/plots/adv_accuracy_vs_attack_type.pdf

deckard/__main__.py

@@ -55,11 +55,7 @@ def parse_and_repro(args, default_config="default.yaml", config_dir="conf"):
     else:
         cmd = f"python -m deckard.layers.parse {args} --config_file {default_config}"
         # error = f"error parsing command: {cmd} {args}"
-        with subprocess.Popen(
-            cmd,
-            stdout=subprocess.PIPE,
-            shell=True,
-        ) as proc:
+        with subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) as proc:
             for line in proc.stdout:
                 print(line.rstrip().decode("utf-8"))
     if Path(Path(), "dvc.yaml").exists():

deckard/base/experiment/experiment.py

@@ -6,7 +6,7 @@
 
 import numpy as np
 from hydra.utils import instantiate
-from omegaconf import DictConfig, OmegaConf, ListConfig
+from omegaconf import DictConfig, OmegaConf
 
 from ..attack import Attack
 from ..data import Data
@@ -258,28 +258,7 @@ def __call__(self):
                 self.data.save(score_dict, files["score_dict_file"])
         else:  # pragma: no cover
             raise ValueError("Scorer is None. Please specify a scorer.")
-        #########################################################################
-        # Returns score if scorer is not None, otherwise returns status
-        #########################################################################
-        if self.optimizers is not None and self.optimizers != []:
-            self.optimizers = (
-                [self.optimizers]
-                if not isinstance(self.optimizers, (list, ListConfig))
-                else self.optimizers
-            )
-            scores = {}
-            for scorer in self.optimizers:
-                try:
-                    score = score_dict[scorer]
-                except KeyError:  # pragma: no cover
-                    raise KeyError(
-                        f"Scorer {scorer} not found in score_dict. Available self.optimizers: {score_dict.keys()}",
-                    )
-                scores[scorer] = score
-                logger.info(f"Score for id : {self.get_name()} : {scorer}: {score}")
-        else:
-            scores = score_dict
-            logger.info(f"Score for id : {self.get_name()}: {score_dict}")
+        logger.info(f"Score for id : {self.get_name()}: {score_dict}")
         logger.info("Finished running experiment with id: {}".format(self.get_name()))
         new_name = self.get_name()
         assert (
@@ -288,7 +267,7 @@ def __call__(self):
         logger.debug(
             f"Experiment deckard hash changed from {old_hash} to {my_hash(self)}.",
         )
-        return scores
+        return score_dict
 
     def _set_name(self):
         if self.files.name is not None:

deckard/base/files/files.py

@@ -54,21 +54,25 @@ def __init__(
         for need in needs:
             assert need is not None, f"Need to specify {need}"
         files.update(kwargs)
-        self.reports = str(Path(reports).as_posix())
-        self.data_dir = str(Path(data_dir).as_posix())
-        self.model_dir = str(Path(model_dir).as_posix())
-        self.attack_dir = str(Path(attack_dir).as_posix())
-        self.data_type = data_type
-        self.model_type = model_type
-        self.attack_type = attack_type
+        self.reports = str(Path(reports).as_posix()) if reports else None
+        self.data_dir = str(Path(data_dir).as_posix()) if data_dir else None
+        self.model_dir = str(Path(model_dir).as_posix()) if model_dir else None
+        self.attack_dir = str(Path(attack_dir).as_posix()) if attack_dir else None
+        self.data_type = data_type if data_type else None
+        self.model_type = model_type if model_type else None
+        self.attack_type = attack_type if attack_type else None
         self.directory = (
-            Path(directory).as_posix()
-            if Path(directory).is_absolute()
-            else Path(Path(), directory).as_posix()
+            (
+                Path(directory).as_posix()
+                if Path(directory).is_absolute()
+                else Path(Path(), directory).as_posix()
+            )
+            if directory
+            else None
         )
-        self.name = name
-        self.stage = stage
-        self.files = files
+        self.name = name if name else None
+        self.stage = stage if stage else None
+        self.files = files if files else {}
         logger.debug(f"FileConfig init: {self.files}")
 
     def __call__(self):
@@ -95,10 +99,10 @@ def _set_filenames(self, **kwargs):
         data_type = self.data_type
         model_type = self.model_type
         attack_type = self.attack_type
-        reports = str(Path(directory, reports).as_posix())
-        data_dir = str(Path(directory, data_dir).as_posix())
-        model_dir = str(Path(directory, model_dir).as_posix())
-        attack_dir = str(Path(directory, attack_dir).as_posix())
+        reports = str(Path(directory, reports).as_posix()) if reports else None
+        data_dir = str(Path(directory, data_dir).as_posix()) if data_dir else None
+        model_dir = str(Path(directory, model_dir).as_posix()) if model_dir else None
+        attack_dir = str(Path(directory, attack_dir).as_posix()) if attack_dir else None
         if name is None and stage is None:
             path = Path(reports)
         elif name is not None and stage is None:
@@ -109,17 +113,17 @@ def _set_filenames(self, **kwargs):
             path = Path(reports, stage, name)
         for kwarg in files:
             name = files.get(kwarg)
-            if "data_file" == kwarg:
+            if "data_file" == kwarg and data_dir is not None:
                 new_path = Path(data_dir, name)
                 if new_path.suffix != data_type:
                     new_path = Path(data_dir, Path(name).stem + data_type)
                 new_files[kwarg] = str(new_path.as_posix())
-            elif "model_file" == kwarg:
+            elif "model_file" == kwarg and model_dir is not None:
                 new_path = Path(model_dir, name)
                 if new_path.suffix != model_type:
                     new_path = Path(model_dir, Path(name).stem + model_type)
                 new_files[kwarg] = str(new_path.as_posix())
-            elif "attack_file" == kwarg:
+            elif "attack_file" == kwarg and attack_dir is not None:
                 new_path = Path(attack_dir, name)
                 if new_path.suffix != attack_type:
                     new_path = Path(attack_dir, Path(name).stem + attack_type)

deckard/base/model/art_pipeline.py

@@ -63,8 +63,15 @@ def __call__(self):
             model,
             tuple(torch_dict.values()),
         ):
+            import torch
+
+            device_type = "gpu" if torch.cuda.is_available() else "cpu"
             model = TorchInitializer(
-                data=data, model=model, library=library, **kwargs
+                data=data,
+                model=model,
+                library=library,
+                device_type=device_type,
+                **kwargs,
             )()
         elif "keras" in str(library) and not isinstance(
             model,