Reproduce

.github/workflows/docker-test.yml

@@ -1,5 +1,5 @@
 #
-name: Create and publish a Docker image
+name: Test Docker Image
 
 # Configures this workflow to run every time a change is pushed to the branch called `release`.
 on:

.gitignore

@@ -94,7 +94,6 @@ ipython_config.py
 *.json
 *.npz
 *.npy
-examples/pytorch/output/models/model.optimizer.pt
 
 
 # PyTorch Model Checkpoints
@@ -129,3 +128,6 @@ examples/*/output/*
 **/*.egg-info/*
 **/*.egg-info
 deckard/deckard.egg-info/*
+
+*log.txt
+examples/*/*/*/*/*/params.yaml

Dockerfile

@@ -5,7 +5,8 @@ RUN apt-get install -y python3 python3-distutils python3-pip ffmpeg libavcodec-e
 RUN python3 -m pip install nvidia-pyindex nvidia-cuda-runtime-cu11 
 RUN git clone https://github.com/simplymathematics/deckard.git
 WORKDIR /deckard
-RUN python3 -m pip install --editable .[pytorch_image]
+RUN python3 -m pip install --editable .[torch,torchvision,tensorflow]
+RUN python3 -m pip install pytest
 RUN git clone https://github.com/Trusted-AI/adversarial-robustness-toolbox.git
 RUN cd adversarial-robustness-toolbox && python3 -m pip install .
 RUN apt install python-is-python3

deckard.code-workspace

@@ -3,15 +3,13 @@
 		{
 			"path": "."
 		},
-		{
-			"path": "../../../Home/staff/cmeyers/.art"
-		}
+
 	],
 	"settings": {
 		"python.testing.pytestEnabled": true,
     	"python.testing.pytestPath": "${workspaceFolder}/deckard/test/",
 		"python.envFile": "${workspaceFolder}/.env",
 		"yaml.schemas": {
-			"https://raw.githubusercontent.com/iterative/dvcyaml-schema/master/schema.json":		}
+			"https://raw.githubusercontent.com/iterative/dvcyaml-schema/master/schema.json": 	["dvc.yaml"]	}
 	}
 }

deckard/base/attack/attack.py

@@ -7,7 +7,7 @@
 from time import process_time_ns
 from omegaconf import DictConfig, OmegaConf
 from hydra.utils import instantiate
-from art.utils import to_categorical
+from art.utils import to_categorical, compute_success
 from ..data import Data
 from ..model import Model
 from ..utils import my_hash
@@ -58,8 +58,6 @@ def __call__(self, model=None, data=None, attack_size=-1):
         for thing in pop_list:
             kwargs.pop(thing, None)
         logger.info(f"Initializing attack {name} with parameters {kwargs}")
-        self.data = data
-        self.model = model
         if "x_train" in kwargs:
             assert (
                 data is not None
@@ -92,15 +90,16 @@ def __call__(self, model=None, data=None, attack_size=-1):
             logger.info("Attempting black-box attack.")
             config = {"_target_": name}
             config.update(**kwargs)
-            attack = instantiate(config)
+            attack = instantiate(config, model)
         except TypeError as e:
             if "verbose" in str(e):
                 config.pop("verbose", None)
-                attack = instantiate(config)
+                attack = instantiate(config, model)
+            else:
+                raise e
         except Exception as e:
-            if "estimator" or "classifier" in str(e):
-                logger.warning(f"Black-box attack failed with error: {e}")
-                logger.info("Attempting white-box attack.")
+            if "has not been fitted correctly" in str(e):
+                model, _ = self.model.fit(data=data, model=model)
                 attack = instantiate(config, model)
             else:
                 raise e
@@ -141,28 +140,44 @@ def __call__(
     ):
         time_dict = {}
         results = {}
+        kwargs = deepcopy(self.init.kwargs)
+        scale_max = kwargs.pop("scale_max", 1)
+        targeted = kwargs.pop("targeted", False)
+        ben_samples = data[1][: self.attack_size]
         if attack_file is not None and Path(attack_file).exists():
             samples = self.data.load(attack_file)
         else:
-            ben_samples = data[0][: self.attack_size]
-            start = process_time_ns()
             atk = self.init(model=model, attack_size=self.attack_size)
-            kwargs = deepcopy(self.init.kwargs)
-            scale_max = kwargs.pop("scale_max", 1)
-            targeted = kwargs.pop("targeted", False)
+
             if targeted is True:
                 kwargs.update({"y": data[2][: self.attack_size]})
             if "AdversarialPatch" in self.name:
-                patches, masks = atk.generate(ben_samples, **kwargs)
-                samples = atk.apply_patch(ben_samples, scale=scale_max)
-            else:
-                samples = atk.generate(ben_samples)
-                end = process_time_ns()
-                time_dict.update({"adv_fit_time": (end - start) / 1e9})
-                time_dict.update(
-                    {"adv_fit_time_per_sample": (end - start) / (len(samples) * 1e9)},
+                start = process_time_ns()
+                patches, _ = atk.generate(ben_samples, **kwargs)
+                samples = atk.apply_patch(
+                    ben_samples,
+                    scale=scale_max,
+                    patch_external=patches,
                 )
+            else:
+                start = process_time_ns()
+                samples = atk.generate(ben_samples, **kwargs)
+            end = process_time_ns()
+            time_dict.update({"adv_fit_time": (end - start) / 1e9})
+            time_dict.update(
+                {"adv_fit_time_per_sample": (end - start) / (len(samples) * 1e9)},
+            )
         results["adv_samples"] = samples
+        try:
+            results["adv_success"] = compute_success(
+                classifier=model,
+                x_clean=ben_samples,
+                labels=data[3][: self.attack_size],
+                x_adv=samples,
+                targeted=self.kwargs.pop("targeted", False),
+            )
+        except TypeError as e:
+            logger.error(f"Failed to compute success rate. Error: {e}")
         if attack_file is not None:
             self.data.save(samples, attack_file)
         if adv_predictions_file is not None and Path(adv_predictions_file).exists():
@@ -234,8 +249,8 @@ def __call__(
                 adv_loss = log_loss(data[3][: self.attack_size], preds)
             self.data.save(adv_loss, adv_losses_file)
             results["adv_loss"] = adv_loss
-        results["time_dict"] = time_dict
-
+        if len(time_dict) > 0:
+            results["time_dict"] = time_dict
         return results
 
 
@@ -331,6 +346,7 @@ def __call__(
                         x_trigger = torch.tensor(x_trigger, device=device)
                         y_trigger = y_trigger.to(torch.long)
                         y_trigger = y_trigger.to(torch.long)
+                        start = process_time_ns()
                         samples, _ = atk.poison(
                             x_trigger=x_trigger,
                             y_trigger=y_trigger,
@@ -722,7 +738,7 @@ def __init__(
         self.method = method
         self.attack_size = attack_size
         if isinstance(kwargs, DictConfig):
-            kwargs = OmegaConf.to_container(kwargs)
+            kwargs = OmegaConf.to_container(kwargs, resolve=True)
         elif isinstance(kwargs, dict):
             pass
         else:
@@ -817,6 +833,7 @@ def __call__(
             )
         else:
             raise NotImplementedError(f"Attack method {self.method} not implemented.")
+
         return result
 
     def __hash__(self):

deckard/base/data/data.py

@@ -6,7 +6,7 @@
 from typing import Union
 
 import numpy as np
-from pandas import DataFrame, read_csv, read_excel
+from pandas import DataFrame, read_csv, read_excel, Series
 
 from ..utils import my_hash
 from .generator import DataGenerator
@@ -97,14 +97,24 @@ def initialize(self):
                 assert len(result) == 4, f"Data is not generated: {self.name}"
         else:
             result = self.load(self.name)
-            if len(result) == 1:
-                assert self.target is not None, "Target is not specified"
+            if isinstance(result, list) and len(result) == 2:
+                result = self.sample(*result)
+            elif isinstance(result, DataFrame) and self.target is not None:
+                if not isinstance(result, DataFrame):
+                    result = DataFrame(result)
+                assert (
+                    self.target in result
+                ), f"Target {self.target} not in data with columns {result.columns}"
                 y = result[self.target]
-                X = result.drop(self.target, axis=1)
+                if isinstance(result, DataFrame):
+                    X = result.drop(self.target, axis=1)
+                else:
+                    X = result[~self.target]
                 result = self.sample(X, y)
-            if len(result) == 2:
-                result = self.sample(*result)
-            assert len(result) == 4
+            else:
+                assert len(result) == 4
+        if self.sklearn_pipeline is not None:
+            result = self.sklearn_pipeline(*result)
         return result
 
     def load(self, filename) -> DataFrame:
@@ -118,8 +128,7 @@ def load(self, filename) -> DataFrame:
             with open(filename, "r") as f:
                 data = json.load(f)
         elif suffix in [".csv"]:
-            data = read_csv(filename)
-            data = data.to_numpy()
+            data = read_csv(filename, delimiter=",", header=0)
         elif suffix in [".pkl", ".pickle"]:
             with open(filename, "rb") as f:
                 data = pickle.load(f)
@@ -141,6 +150,8 @@ def save(self, data, filename):
             if suffix in [".json"]:
                 if isinstance(data, DataFrame):
                     data = data.to_dict(orient="records")
+                elif isinstance(data, Series):
+                    data = data.to_dict()
                 elif isinstance(data, np.ndarray):
                     data = data.tolist()
                 with open(filename, "w") as f:

deckard/base/data/generator.py

@@ -11,7 +11,7 @@
 from dataclasses import dataclass, field
 from ..utils import my_hash
 import numpy as np
-from art.utils import load_mnist, load_cifar10
+from art.utils import load_mnist, load_cifar10, load_diabetes
 
 __all__ = [
     "SklearnDataGenerator",
@@ -21,10 +21,26 @@
 ]
 logger = logging.getLogger(__name__)
 
+SKLEARN_DATASETS = [
+    "classification",
+    "regression",
+    "blobs",
+    "moons",
+    "circles",
+    "biclusters",
+]
+
 
 @dataclass
 class SklearnDataGenerator:
-    name: Literal["classification", "regression"] = "classification"
+    name: Literal[
+        "classification",
+        "regression",
+        "blobs",
+        "moons",
+        "circles",
+        "biclusters",
+    ] = "classification"
     kwargs: dict = field(default_factory=dict)
 
     def __init__(self, name, **kwargs):
@@ -55,9 +71,12 @@ def __hash__(self):
         return int(my_hash(self), 16)
 
 
+TORCH_DATASETS = ["torch_mnist", "torch_cifar10", "torch_diabetes"]
+
+
 @dataclass
 class TorchDataGenerator:
-    name: Literal["torch_mnist", "torch_cifar10"] = "torch_mnist"
+    name: Literal["torch_mnist", "torch_cifar10", "torch_diabetes"] = "torch_mnist"
     path = None
     kwargs: dict = field(default_factory=dict)
 
@@ -84,6 +103,10 @@ def __call__(self):
             X_test = np.transpose(X_test, (0, 3, 1, 2)).astype(np.float32)
             X = np.concatenate((X_train, X_test))
             y = np.concatenate((y_train, y_test))
+        elif self.name == "torch_diabetes":
+            (X_train, y_train), (X_test, y_test), _, _ = load_diabetes()
+            X = np.concatenate((X_train, X_test))
+            y = np.concatenate((y_train, y_test))
         else:
             raise ValueError(f"Unknown dataset name {self.name}")
         return [X, y]
@@ -92,9 +115,12 @@ def __hash__(self):
         return int(my_hash(self), 16)
 
 
+KERAS_DATASETS = ["keras_mnist", "keras_cifar10", "mnist", "cifar10", "diabetes"]
+
+
 @dataclass
 class KerasDataGenerator:
-    name: Literal["mnist", "cifar10"] = "mnist"
+    name: Literal["mnist", "cifar10", "diabetes"] = "mnist"
     kwargs: dict = field(default_factory=dict)
 
     def __init__(self, name, **kwargs):
@@ -113,6 +139,10 @@ def __call__(self):
             (X_train, y_train), (X_test, y_test), _, _ = load_mnist()
             X = np.concatenate((X_train, X_test))
             y = np.concatenate((y_train, y_test))
+        elif "diabetes" in self.name:
+            (X_train, y_train), (X_test, y_test), _, _ = load_diabetes()
+            X = np.concatenate((X_train, X_test))
+            y = np.concatenate((y_train, y_test))
         else:
             raise ValueError(f"Unknown dataset name {self.name}")
         return [X, y]
@@ -131,11 +161,11 @@ def __init__(self, name, **kwargs):
         self.kwargs = {k: v for k, v in kwargs.items() if v is not None}
 
     def __call__(self):
-        if self.name in ["classification", "regression"]:
+        if self.name in SKLEARN_DATASETS:
             return SklearnDataGenerator(self.name, **self.kwargs)()
-        elif self.name in ["torch_mnist", "torch_cifar10"]:
+        elif self.name in TORCH_DATASETS:
             return TorchDataGenerator(self.name, **self.kwargs)()
-        elif self.name in ["keras_mnist", "keras_cifar10", "mnist", "cifar10"]:
+        elif self.name in KERAS_DATASETS:
             return KerasDataGenerator(self.name, **self.kwargs)()
         else:
             raise ValueError(f"Invalid name {self.name}. Please choose from ")