Fix cifar100 accuracy

.gitignore

@@ -140,5 +140,10 @@ env/
 # screenlog
 screenlog.*
 
-# tmp.py
-tmp.py
+# env310
+env310/*
+# dvc read-write protect files in  the examples folder
+examples/*/.dvc/tmp/lock
+examples/*/.dvc/tmp/lock
+examples/*/.dvc/tmp/rwlock
+examples/*/.dvc/tmp/rwlock.lock

deckard/base/attack/attack.py

@@ -202,15 +202,15 @@ def __call__(
             )
         except TypeError as e:
             logger.error(f"Failed to compute success rate. Error: {e}")
-        if attack_file is not None:
+        if attack_file is not None and not Path(attack_file).exists():
             self.data.save(samples, attack_file)
         if adv_predictions_file is not None and Path(adv_predictions_file).exists():
             adv_predictions = self.data.load(adv_predictions_file)
             results["adv_predictions"] = np.array(adv_predictions)
         else:
             adv_predictions = model.predict(samples)
             results["adv_predictions"] = np.array(adv_predictions)
-        if adv_predictions_file is not None:
+        if adv_predictions_file is not None and not Path(adv_predictions_file).exists():
             self.data.save(adv_predictions, adv_predictions_file)
         if adv_probabilities_file is not None and Path(adv_probabilities_file).exists():
             adv_probabilities = self.data.load(adv_probabilities_file)

deckard/base/data/generator.py

@@ -11,7 +11,11 @@
     make_moons,
     make_circles,
 )
-from torchvision.io import read_image, read_file
+
+try:
+    from torchvision.io import read_image, read_file
+except ImportError:
+    pass
 from art.utils import load_mnist, load_cifar10, load_diabetes, to_categorical
 from ..utils import my_hash
 
@@ -225,8 +229,6 @@ def __call__(self):
             return TorchDataGenerator(self.name, **self.kwargs)()
         elif self.name in KERAS_DATASETS:
             return KerasDataGenerator(self.name, **self.kwargs)()
-        elif isinstance(self.name, str) and Path(self.name).exists():
-            return SklearnDataGenerator(self.name, **self.kwargs)()
         else:  # pragma: no cover
             raise ValueError(
                 f"Invalid name {self.name}. Please choose from {ALL_DATASETS}",

deckard/base/data/sklearn_pipeline.py

@@ -26,6 +26,8 @@ def __call__(self, X_train, X_test, y_train, y_test):
         name = self.kwargs.pop("_target_", self.name)
         dict_ = {"_target_": name}
         dict_.update(**self.kwargs)
+        while "kwargs" in dict_:
+            dict_.update(**dict_.pop("kwargs"))
         obj = instantiate(dict_)
         X_train = obj.fit(X_train).transform(X_train)
         X_test = obj.transform(X_test)

deckard/base/experiment/experiment.py

@@ -112,7 +112,7 @@ def __hash__(self):
         name = str(self.name).encode("utf-8")
         return int.from_bytes(name, "little")
 
-    def __call__(self):
+    def __call__(self, **kwargs):
         """Runs the experiment. If the experiment has already been run, it will load the results from disk. If scorer is not None, it will return the score for the specified scorer. If scorer is None, it will return the score for the first scorer in the ScorerDict.
         :param scorer: The scorer to return the score for. If None, the score for the first scorer in the ScorerDict will be returned.
         :type scorer: str
@@ -134,7 +134,8 @@ def __call__(self):
         else:
             score_dict = {}
         results = {}
-        results["score_dict_file"] = score_dict
+        results["score_dict"] = score_dict
+        files.update(**results)
         #########################################################################
         # Load or generate data
         #########################################################################
@@ -143,10 +144,10 @@ def __call__(self):
         # Load or train model
         #########################################################################
         if self.model is not None:
-            model_results = self.model(data, **files)
+            model_results = self.model(**files)
             score_dict.update(**model_results.pop("time_dict", {}))
             score_dict.update(**model_results.pop("score_dict", {}))
-            model = model_results["model"]
+            files.update(**model_results)
             # Prefer probabilities, then loss_files, then predictions
             if (
                 "probabilities" in model_results
@@ -174,8 +175,6 @@ def __call__(self):
         ##########################################################################
         if self.attack is not None:
             adv_results = self.attack(
-                data,
-                model,
                 **files,
             )
             if "adv_predictions" in adv_results:
@@ -195,6 +194,7 @@ def __call__(self):
             if "adv_success" in adv_results:
                 adv_success = adv_results["adv_success"]
                 score_dict.update({"adv_success": adv_success})
+            files.update(**adv_results)
         ##########################################################################
         # Score results
         #########################################################################
@@ -216,7 +216,7 @@ def __call__(self):
                 logger.debug(f" len(preds) : {len(preds)}")
                 new_score_dict = self.scorers(ground_truth, preds)
                 score_dict.update(**new_score_dict)
-                results["score_dict_file"] = score_dict
+                results["score_dict"] = score_dict
             if "adv_preds" in locals():
                 ground_truth = data[3][: len(adv_preds)]
                 adv_preds = adv_preds[: len(ground_truth)]
@@ -225,7 +225,7 @@ def __call__(self):
                     f"adv_{key}": value for key, value in adv_score_dict.items()
                 }
                 score_dict.update(**adv_score_dict)
-                results["score_dict_file"] = score_dict
+                results["score_dict"] = score_dict
             # # Save results
             if "score_dict_file" in files and files["score_dict_file"] is not None:
                 if Path(files["score_dict_file"]).exists():

deckard/base/files/files.py

@@ -83,17 +83,17 @@ def __call__(self):
         files = dict(self.get_filenames())
         return files
 
-    def get_filenames(self):
+    def get_filenames(self, **kwargs):
         files = deepcopy(self.files)
+        files.update(**kwargs)
         files = self._set_filenames(**files)
         return files
 
     def _set_filenames(self, **kwargs):
         name = self.name
         stage = self.stage
-        if hasattr(self, "files"):
-            kwargs.update(self.files)
-        files = dict(kwargs)
+        files = self.files
+        files.update(**kwargs)
         new_files = {}
         directory = self.directory
         reports = self.reports

deckard/base/model/art_pipeline.py

@@ -34,7 +34,6 @@ class ArtPipelineStage:
     kwargs: dict = field(default_factory=dict)
 
     def __init__(self, name=None, **kwargs):
-        logger.info(f"Creating pipeline stage: {name} kwargs: {kwargs}")
         self.name = name
         kwargs.update(**kwargs.pop("kwargs", {}))
         self.kwargs = kwargs
@@ -69,7 +68,6 @@ def __call__(self):
 
             device_type = "gpu" if torch.cuda.is_available() else "cpu"
             if device_type == "gpu":
-                logger.info("Using GPU")
                 number_of_devices = torch.cuda.device_count()
                 num = randint(0, number_of_devices - 1)
                 device = torch.device(f"cuda:{num}")
@@ -205,7 +203,7 @@ def __call__(self, model: object, data: list) -> BaseEstimator:
             name = params.pop("name", None)
             kwargs = params.pop("kwargs", {})
         else:
-            name = None
+            name = self.library
             kwargs = {}
         pre_def = []
         post_def = []

deckard/base/model/model.py

@@ -76,7 +76,10 @@ def __call__(self, data: list, model: object, library=None):
         if library in sklearn_dict.keys():
             pass
         elif library in torch_dict.keys():
-            pass
+            trainer["nb_epochs"] = trainer.pop(
+                "nb_epochs",
+                trainer.pop("epochs", trainer.pop("nb_epoch", 10)),
+            )
         elif library in keras_dict.keys():
             pass
         elif library in tensorflow_dict.keys():
@@ -142,7 +145,7 @@ def __call__(self, data: list, model: object, library=None):
                 model.fit(data[0], data[2], **trainer)
                 end = process_time_ns()
                 end_timestamp = time()
-            except Exception as e:
+            except AttributeError as e:
                 raise e
         except RuntimeError as e:  # pragma: no cover
             if "eager mode" in str(e) and library in tensorflow_dict.keys():
@@ -176,6 +179,45 @@ def __call__(self, data: list, model: object, library=None):
                 model.fit(data[0], data[2], **trainer)
                 end = process_time_ns()
                 end_timestamp = time()
+            elif "disable eager execution" in str(e):
+                logger.warning("Disabling eager execution for Tensorflow.")
+                import tensorflow as tf
+
+                tf.compat.v1.disable_eager_execution()
+                start = process_time_ns()
+                start_timestamp = time()
+                model.fit(data[0], data[2], **trainer)
+                end = process_time_ns()
+                end_timestamp = time()
+            elif "out of memory" in set(e).lower() and library in torch_dict.keys():
+                import torch
+
+                torch.cuda.empty_cache()
+                device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+                if device == "cuda":
+                    # Pick the GPU with the most free memory
+                    free_memory = [
+                        torch.cuda.memory_reserved(i) - torch.cuda.memory_allocated(i)
+                        for i in range(torch.cuda.device_count())
+                    ]
+                    device = f"cuda:{free_memory.index(max(free_memory))}"
+                data[0] = torch.from_numpy(data[0])
+                data[1] = torch.from_numpy(data[1])
+                data[0] = torch.Tensor.float(data[0])
+                data[1] = torch.Tensor.float(data[1])
+                data[0].to(device)
+                data[2] = torch.from_numpy(data[2])
+                data[3] = torch.from_numpy(data[3])
+                data[2] = torch.Tensor.float(data[2])
+                data[3] = torch.Tensor.float(data[3])
+                data[2].to(device)
+                model.model.to(device) if hasattr(model, "model") else model.to(device)
+                start = process_time_ns()
+                start_timestamp = time()
+                model.fit(data[0], data[2], **trainer)
+                end = process_time_ns()
+                end_timestamp = time()
+
             else:
                 raise e
         time_dict = {