Update pytorch example (#182)

* update pytorch example

deckard/layers/afr.py

@@ -28,14 +28,6 @@
 
 logger = logging.getLogger(__name__)
 
-__all__ = [
-    "afr_main",
-    "survival_probability_calibration",
-    "fit_aft",
-    "plot_aft",
-    "afr_parser",
-]
-
 
 # Modified from https://github.com/CamDavidsonPilon/lifelines/blob/master/lifelines/calibration.py
 def survival_probability_calibration(
@@ -118,7 +110,7 @@ def ccl(p):
                 crc.fit_interval_censoring(prediction_df, T, E, regressors=regressors)
             else:
                 crc.fit(prediction_df, T, E, regressors=regressors)
-        except ConvergenceError as e:
+        except (ConvergenceError, AttributeError) as e:
             if "delta contains nan value(s)" in str(e):
                 fit_options = {
                     "step_size": 0.1,
@@ -155,7 +147,7 @@ def ccl(p):
                     )
                 else:
                     crc.fit(prediction_df, T, E, regressors=regressors)
-            except ConvergenceError as e:
+            except (ConvergenceError, AttributeError) as e:
                 logger.error(f"Could not fit CRC model. due to {e}")
                 return ax, np.nan, np.nan
 
@@ -282,9 +274,7 @@ def fit_aft(
         kwarg_dict["timeline"] = timeline
     try:
         aft.fit(df, **kwarg_dict)
-    except AttributeError as e:
-        raise ConvergenceError(f"Could not fit {mtype} model due to {e}")
-    except ConvergenceError as e:
+    except (ConvergenceError, AttributeError) as e:
         if "delta contains nan value(s)" in str(e):
             fit_options = {
                 "step_size": 0.1,
@@ -295,18 +285,14 @@ def fit_aft(
             logger.info(
                 "Reducing the step size to 0.1 and increasing the max steps to 1000",
             )
-            input("Inside the fit function")
         else:
             logger.info("Trying to fit with SLSQP")
             aft._scipy_fit_method = "SLSQP"
         try:
             aft.fit(df, **kwarg_dict)
-        except ConvergenceError as e:
+        except (ConvergenceError, AttributeError) as e:
             logger.error(f"Could not fit {mtype} model due to {e}")
             raise ConvergenceError(f"Could not fit {mtype} model due to {e}")
-
-    else:
-        logger.info(f"Fitted {mtype} model")
     if summary_file is not None:
         summary = pd.DataFrame(aft.summary).copy()
         if folder is None:
@@ -880,7 +866,7 @@ def calculate_raw_failures(args, data, config):
     return data
 
 
-def afr_main(args):
+def main(args):
     target = args.target
     duration_col = args.duration_col
     dataset = args.dataset
@@ -937,4 +923,4 @@ def afr_main(args):
     afr_parser.add_argument("--config_file", type=str, default="afr.yaml")
     afr_parser.add_argument("--plots_folder", type=str, default="plots")
     args = afr_parser.parse_args()
-    afr_main(args)
+    main(args)

deckard/layers/clean_data.py

@@ -124,6 +124,7 @@ def calculate_failure_rate(data):
         failure_rate = (
             (1 - data.loc[:, "accuracy"]) * data.loc[:, "attack.attack_size"]
         ) / data.loc[:, "predict_time"]
+        survival_time = data.loc[:, "predict_time"] * data.loc[:, "accuracy"]
     elif "predict_proba_time" in data.columns:
         data.loc[:, "predict_proba_time"] = pd.to_numeric(
             data.loc[:, "predict_proba_time"],
@@ -133,17 +134,35 @@ def calculate_failure_rate(data):
         ) / data.loc[:, "predict_proba_time"]
     else:
         raise ValueError("predict_time or predict_proba_time not in data.columns")
-    adv_failure_rate = (
-        (1 - data.loc[:, "adv_accuracy"])
-        * data.loc[:, "attack.attack_size"]
-        / data.loc[:, "predict_time"]
-    )
-
+    if "adv_fit_time" in data.columns:
+        assert "adv_accuracy" in data.columns, "adv_accuracy not in data.columns"
+        if "predict_time" in data.columns:
+            adv_failure_rate = (
+                (1 - data.loc[:, "adv_accuracy"])
+                * data.loc[:, "attack.attack_size"]
+                / data.loc[:, "adv_fit_time"]
+            )
+            adv_survival_time = (
+                data.loc[:, "predict_time"] * data.loc[:, "adv_accuracy"]
+            )
+        elif "predict_proba_time" in data.columns:
+            adv_failure_rate = (
+                (1 - data.loc[:, "adv_accuracy"])
+                * data.loc[:, "attack.attack_size"]
+                / data.loc[:, "adv_fit_time"]
+            )
+            adv_survival_time = (
+                data.loc[:, "predict_proba_time"] * data.loc[:, "adv_accuracy"]
+            )
+        else:
+            raise ValueError("predict_time or predict_proba_time not in data.columns")
+    data = data.assign(adv_survival_time=adv_survival_time)
+    data = data.assign(survival_time=survival_time)
     data = data.assign(adv_failure_rate=adv_failure_rate)
     data = data.assign(failure_rate=failure_rate)
-    training_time_per_failure = data.loc[:, "train_time"] / data.loc[:, "failure_rate"]
+    training_time_per_failure = data.loc[:, "train_time"] / data.loc[:, "survival_time"]
     training_time_per_adv_failure = (
-        data.loc[:, "train_time"] * data.loc[:, "adv_failure_rate"]
+        data.loc[:, "train_time"] * data.loc[:, "adv_survival_time"]
     )
     data = data.assign(training_time_per_failure=training_time_per_failure)
     data = data.assign(training_time_per_adv_failure=training_time_per_adv_failure)

deckard/layers/compile.py

@@ -179,7 +179,7 @@ def load_results(results_file, results_folder) -> pd.DataFrame:
     elif suffix == ".html":
         results = pd.read_html(results_file, index_col=0)
     elif suffix == ".json":
-        results = pd.read_json(results_file, index_col=0)
+        results = pd.read_json(results_file)
     elif suffix == ".tex":
         pd.read_csv(
             results_file,
@@ -188,7 +188,6 @@ def load_results(results_file, results_folder) -> pd.DataFrame:
             skiprows=4,
             skipfooter=3,
             engine="python",
-            index_col=0,
         )
     else:
         raise ValueError(f"File type {suffix} not supported.")

examples/gzip/.gitignore

@@ -1,3 +1,4 @@
+adversarial-robustness-toolbox
 output
 *.db
 kdd_nsl

examples/gzip/params.yaml

@@ -0,0 +1,88 @@
+data:
+  _target_: deckard.base.data.Data
+  name: https://gist.githubusercontent.com/simplymathematics/8c6c04bd151950d5ea9e62825db97fdd/raw/d6a22cdb42a1db624c89f0298cb4f654d3812703/kdd_nsl.csv
+  sample:
+    _target_: deckard.base.data.SklearnDataSampler
+    random_state: 0
+    stratify: true
+    test_size: 100
+    train_size: 100
+  sklearn_pipeline:
+    encoder:
+      handle_unknown: use_encoded_value
+      name: sklearn.preprocessing.OrdinalEncoder
+      unknown_value: -1
+    preprocessor:
+      name: sklearn.preprocessing.StandardScaler
+      with_mean: true
+      with_std: true
+  target: label
+dataset: kdd_nsl
+direction:
+- maximize
+files:
+  _target_: deckard.base.files.FileConfig
+  attack_dir: attacks
+  attack_file: attack
+  attack_type: .pkl
+  data_dir: data
+  data_file: data
+  data_type: .pkl
+  directory: output
+  model_dir: model
+  model_file: model
+  model_type: .pkl
+  name: default
+  params_file: params.yaml
+  predictions_file: predictions.json
+  reports: reports
+  score_dict_file: score_dict.json
+model:
+  _target_: deckard.base.model.Model
+  art:
+    _target_: deckard.base.model.art_pipeline.ArtPipeline
+    initialize:
+      nb_classes: 3
+    library: sklearn
+  data:
+    _target_: deckard.base.data.Data
+    name: https://gist.githubusercontent.com/simplymathematics/8c6c04bd151950d5ea9e62825db97fdd/raw/d6a22cdb42a1db624c89f0298cb4f654d3812703/kdd_nsl.csv
+    sample:
+      _target_: deckard.base.data.SklearnDataSampler
+      random_state: 0
+      stratify: true
+      test_size: 100
+      train_size: 100
+    sklearn_pipeline:
+      encoder:
+        handle_unknown: use_encoded_value
+        name: sklearn.preprocessing.OrdinalEncoder
+        unknown_value: -1
+      preprocessor:
+        name: sklearn.preprocessing.StandardScaler
+        with_mean: true
+        with_std: true
+    target: label
+  init:
+    _target_: deckard.base.model.ModelInitializer
+    compressor: gzip
+    distance_matrix: output/model/kdd_nsl/gzip_classifier/gzip/0-100.npz
+    k: 1
+    m: -1
+    method: random
+    name: gzip_classifier.GzipClassifier
+  library: sklearn
+model_name: gzip_classifier
+optimizers:
+- accuracy
+scorers:
+  _target_: deckard.base.scorer.ScorerDict
+  accuracy:
+    _target_: deckard.base.scorer.ScorerConfig
+    direction: maximize
+    name: sklearn.metrics.accuracy_score
+  log_loss:
+    _target_: deckard.base.scorer.ScorerConfig
+    direction: minimize
+    name: sklearn.metrics.log_loss
+stage: train

examples/power/conf/afr.yaml

@@ -2,14 +2,14 @@ covariates:
   - accuracy
   - train_time
   - predict_proba_time
-  - model.trainer.nb_epoch
+  - model.trainer.np_epochs
   - model.trainer.batch_size
   - data.sample.random_state
   - adv_fit_time
   - attack.init.eps
   - adv_failures
 fillna:
-  model.trainer.nb_epoch: 20
+  model.trainer.np_epochs: 20
   model.trainer.batch_size: 1024
   model.art.preprocessor.bit_depth: 32
 weibull:
@@ -28,13 +28,13 @@ weibull:
       "adv_fit_time: lambda_": "$t_{attack}$"
       "adv_failure_rate: lambda_": "$h_{adv.}(t;\\theta)$"
       "failure_rate: lambda_": "$h_{ben.}(t;\\theta)$"
-      "model.trainer.nb_epoch: lambda_": "No. of Epochs"
+      "model.trainer.np_epochs: lambda_": "No. of Epochs"
       "model.trainer.batch_size: lambda_": "Batch Size"
       "def_gen": "Defence"
       "attack.init.eps: lambda_": "$\\varepsilon$"
   partial_effect:
     - "file": "weibull_epochs_partial_effect.pdf"
-      "covariate_array": "model.trainer.nb_epoch"
+      "covariate_array": "model.trainer.np_epochs"
       "values_array": [1,10,25,50]
       "title": "$S(t)$ for Weibull AFR"
       "ylabel": "$\\mathbb{P}~(T>t)$"
@@ -57,13 +57,13 @@ cox:
       "adv_fit_time": "$t_{attack}$"
       "adv_failure_rate": "$h_{adv.}(t;\\theta)$"
       "failure_rate": "$h_{ben.}(t;\\theta)$"
-      "model.trainer.nb_epoch": "No. of Epochs"
+      "model.trainer.np_epochs": "No. of Epochs"
       "model.trainer.batch_size": "Batch Size"
       "def_gen": "Defence"
       "attack.init.eps": "$\\varepsilon$"
   partial_effect:
     - "file": "cox_epochs_partial_effect.pdf"
-      "covariate_array": "model.trainer.nb_epoch"
+      "covariate_array": "model.trainer.np_epochs"
       "values_array": [1,10,25,50]
       "title": "$S(t)$ for Cox AFR"
       "ylabel": "$\\mathbb{P}~(T>t)$"
@@ -88,13 +88,13 @@ log_logistic:
       "adv_fit_time: alpha_": "$t_{attack}$"
       "adv_failure_rate: alpha_": "$h_{adv.}(t;\\theta)$"
       "failure_rate: alpha_": "$h_{ben.}(t;\\theta)$"
-      "model.trainer.nb_epoch: alpha_": "No. of Epochs"
+      "model.trainer.np_epochs: alpha_": "No. of Epochs"
       "model.trainer.batch_size: alpha_": "Batch Size"
       "def_gen": "Defence"
       "attack.init.eps: alpha_": "$\\varepsilon$"
   partial_effect:
     - "file": "log_logistic_epochs_partial_effect.pdf"
-      "covariate_array": "model.trainer.nb_epoch"
+      "covariate_array": "model.trainer.np_epochs"
       "values_array": [1,10,25,50]
       "title": "$S(t)$ for Log-Logistic AFR"
       "ylabel": "$\\mathbb{P}~(T>t)$"
@@ -118,14 +118,14 @@ log_normal:
       "adv_fit_time: mu_": "$t_{attack}$"
       "adv_failure_rate: mu_": "$h_{adv.}(t;\\theta)$"
       "failure_rate: mu_": "$h_{ben.}(t;\\theta)$"
-      "model.trainer.nb_epoch: mu_": "No. of Epochs"
+      "model.trainer.np_epochs: mu_": "No. of Epochs"
       "model.trainer.batch_size: mu_": "Batch Size"
       "def_gen": "Defence"
       "attack.init.eps: mu_": "$\\varepsilon$"
       "data.sample.random_state: mu_": "Random State"
   partial_effect:
     - "file": "log_normal_epochs_partial_effect.pdf"
-      "covariate_array": "model.trainer.nb_epoch"
+      "covariate_array": "model.trainer.np_epochs"
       "values_array": [1,10,25,50]
       "title": "$S(t)$ for Log-Normal AFR"
       "ylabel": "$\\mathbb{P}~(T>t)$"

examples/power/conf/bit_depth/torch_cifar10.yaml

@@ -44,7 +44,7 @@ hydra:
     params:
       ++data.sample.random_state: int(range(0, 1))
       ++model.art.initialize.optimizer.lr: tag(log, interval(0.000001, 1))
-      ++model.trainer.nb_epoch: int(interval(1, 50))
+      ++model.trainer.np_epochs: int(interval(1, 50))
       ++model.trainer.batch_size:  int(interval(1, 10000))
       ++attack.init.eps : interval(0.01, 1.0)
       ++model.art.preprocessor.params.bit_depth: choice(4,8,16,32,64)

examples/power/conf/bit_depth/torch_cifar100.yaml

@@ -44,7 +44,7 @@ hydra:
     params:
       ++data.sample.random_state: int(range(0, 1))
       ++model.art.initialize.optimizer.lr: tag(log, interval(0.000001, 100))
-      ++model.trainer.nb_epoch: tag(log, int(interval(1, 100)))
+      ++model.trainer.np_epochs: tag(log, int(interval(1, 100)))
       ++model.trainer.batch_size: tag(log, int(interval(10, 10000)))
       ++attack.init.eps : interval(0.01, 1.0)
       ++model.art.preprocessor.params.bit_depth: choice(4,8,16,32,64)

examples/power/conf/bit_depth/torch_mnist.yaml

@@ -45,7 +45,7 @@ hydra:
     params:
       ++data.sample.random_state: int(range(0, 10))
       ++model.art.initialize.optimizer.lr: tag(log, interval(0.000001, 1))
-      ++model.trainer.nb_epoch: int(interval(1, 50))
+      ++model.trainer.np_epochs: int(interval(1, 50))
       ++model.trainer.batch_size: int(interval(1, 10000))
       ++attack.init.eps : interval(0.01, 1.0)
       ++model.art.preprocessor.params.bit_depth: choice(4,8,16,32,64)

examples/power/conf/clean.yaml

@@ -15,7 +15,7 @@ params:
   DeepFool: attack.init.eps
   FSQ: model.art.preprocessor.bit_depth
   Control: model_layers
-  Epochs: model.trainer.nb_epoch
+  Epochs: model.trainer.np_epochs
   Batch_Size: model.trainer.batch_size
 fillna:
   Epochs: 20

examples/power/conf/combined_afr.yaml

@@ -2,7 +2,7 @@ covariates:
   - accuracy
   - train_time
   - predict_proba_time
-  - model.trainer.nb_epoch
+  - model.trainer.np_epochs
   - model.trainer.batch_size
   - attack.init.eps
   - data.sample.random_state
@@ -11,7 +11,7 @@ covariates:
   - adv_fit_time
   - adv_failures
 fillna:
-  model.trainer.nb_epoch: 20
+  model.trainer.np_epochs: 20
   model.trainer.batch_size: 1024
   model.art.preprocessor.bit_depth: 32
 weibull:
@@ -50,13 +50,13 @@ weibull:
 #     "adv_fit_time": "$T_{a}$"
 #     "adv_failure_rate": "$h_{adv.}(t;\\theta)$"
 #     "failure_rate": "$h_{ben.}(t;\\theta)$"
-#     "model.trainer.nb_epoch": "No. of Epochs"
+#     "model.trainer.np_epochs": "No. of Epochs"
 #     "model.trainer.batch_size": "Batch Size"
 #     "def_gen": "Defence"
 #     "attack.init.eps": "$\\varepsilon$"
 #   partial_effect:
 #     - "file": "cox_epochs_partial_effect.pdf"
-#       "covariate_array": "model.trainer.nb_epoch"
+#       "covariate_array": "model.trainer.np_epochs"
 #       "values_array": [1,10,25,50]
 #       "title": "$S(t)$ for Cox AFR"
 #       "ylabel": "$\\mathbb{P}~(T>t)$"