diff --git a/deckard/layers/afr.py b/deckard/layers/afr.py index c69e7887..449ae744 100644 --- a/deckard/layers/afr.py +++ b/deckard/layers/afr.py @@ -28,14 +28,6 @@ logger = logging.getLogger(__name__) -__all__ = [ - "afr_main", - "survival_probability_calibration", - "fit_aft", - "plot_aft", - "afr_parser", -] - # Modified from https://github.com/CamDavidsonPilon/lifelines/blob/master/lifelines/calibration.py def survival_probability_calibration( @@ -118,7 +110,7 @@ def ccl(p): crc.fit_interval_censoring(prediction_df, T, E, regressors=regressors) else: crc.fit(prediction_df, T, E, regressors=regressors) - except ConvergenceError as e: + except (ConvergenceError, AttributeError) as e: if "delta contains nan value(s)" in str(e): fit_options = { "step_size": 0.1, @@ -155,7 +147,7 @@ def ccl(p): ) else: crc.fit(prediction_df, T, E, regressors=regressors) - except ConvergenceError as e: + except (ConvergenceError, AttributeError) as e: logger.error(f"Could not fit CRC model. due to {e}") return ax, np.nan, np.nan @@ -282,9 +274,7 @@ def fit_aft( kwarg_dict["timeline"] = timeline try: aft.fit(df, **kwarg_dict) - except AttributeError as e: - raise ConvergenceError(f"Could not fit {mtype} model due to {e}") - except ConvergenceError as e: + except (ConvergenceError, AttributeError) as e: if "delta contains nan value(s)" in str(e): fit_options = { "step_size": 0.1, @@ -295,18 +285,14 @@ def fit_aft( logger.info( "Reducing the step size to 0.1 and increasing the max steps to 1000", ) - input("Inside the fit function") else: logger.info("Trying to fit with SLSQP") aft._scipy_fit_method = "SLSQP" try: aft.fit(df, **kwarg_dict) - except ConvergenceError as e: + except (ConvergenceError, AttributeError) as e: logger.error(f"Could not fit {mtype} model due to {e}") raise ConvergenceError(f"Could not fit {mtype} model due to {e}") - - else: - logger.info(f"Fitted {mtype} model") if summary_file is not None: summary = pd.DataFrame(aft.summary).copy() if folder is None: @@ -880,7 +866,7 @@ def calculate_raw_failures(args, data, config): return data -def afr_main(args): +def main(args): target = args.target duration_col = args.duration_col dataset = args.dataset @@ -937,4 +923,4 @@ def afr_main(args): afr_parser.add_argument("--config_file", type=str, default="afr.yaml") afr_parser.add_argument("--plots_folder", type=str, default="plots") args = afr_parser.parse_args() - afr_main(args) + main(args) diff --git a/deckard/layers/clean_data.py b/deckard/layers/clean_data.py index 615a563b..69a272ef 100644 --- a/deckard/layers/clean_data.py +++ b/deckard/layers/clean_data.py @@ -124,6 +124,7 @@ def calculate_failure_rate(data): failure_rate = ( (1 - data.loc[:, "accuracy"]) * data.loc[:, "attack.attack_size"] ) / data.loc[:, "predict_time"] + survival_time = data.loc[:, "predict_time"] * data.loc[:, "accuracy"] elif "predict_proba_time" in data.columns: data.loc[:, "predict_proba_time"] = pd.to_numeric( data.loc[:, "predict_proba_time"], @@ -133,17 +134,35 @@ def calculate_failure_rate(data): ) / data.loc[:, "predict_proba_time"] else: raise ValueError("predict_time or predict_proba_time not in data.columns") - adv_failure_rate = ( - (1 - data.loc[:, "adv_accuracy"]) - * data.loc[:, "attack.attack_size"] - / data.loc[:, "predict_time"] - ) - + if "adv_fit_time" in data.columns: + assert "adv_accuracy" in data.columns, "adv_accuracy not in data.columns" + if "predict_time" in data.columns: + adv_failure_rate = ( + (1 - data.loc[:, "adv_accuracy"]) + * data.loc[:, "attack.attack_size"] + / data.loc[:, "adv_fit_time"] + ) + adv_survival_time = ( + data.loc[:, "predict_time"] * data.loc[:, "adv_accuracy"] + ) + elif "predict_proba_time" in data.columns: + adv_failure_rate = ( + (1 - data.loc[:, "adv_accuracy"]) + * data.loc[:, "attack.attack_size"] + / data.loc[:, "adv_fit_time"] + ) + adv_survival_time = ( + data.loc[:, "predict_proba_time"] * data.loc[:, "adv_accuracy"] + ) + else: + raise ValueError("predict_time or predict_proba_time not in data.columns") + data = data.assign(adv_survival_time=adv_survival_time) + data = data.assign(survival_time=survival_time) data = data.assign(adv_failure_rate=adv_failure_rate) data = data.assign(failure_rate=failure_rate) - training_time_per_failure = data.loc[:, "train_time"] / data.loc[:, "failure_rate"] + training_time_per_failure = data.loc[:, "train_time"] / data.loc[:, "survival_time"] training_time_per_adv_failure = ( - data.loc[:, "train_time"] * data.loc[:, "adv_failure_rate"] + data.loc[:, "train_time"] * data.loc[:, "adv_survival_time"] ) data = data.assign(training_time_per_failure=training_time_per_failure) data = data.assign(training_time_per_adv_failure=training_time_per_adv_failure) diff --git a/deckard/layers/compile.py b/deckard/layers/compile.py index 28a33a56..b365c879 100644 --- a/deckard/layers/compile.py +++ b/deckard/layers/compile.py @@ -179,7 +179,7 @@ def load_results(results_file, results_folder) -> pd.DataFrame: elif suffix == ".html": results = pd.read_html(results_file, index_col=0) elif suffix == ".json": - results = pd.read_json(results_file, index_col=0) + results = pd.read_json(results_file) elif suffix == ".tex": pd.read_csv( results_file, @@ -188,7 +188,6 @@ def load_results(results_file, results_folder) -> pd.DataFrame: skiprows=4, skipfooter=3, engine="python", - index_col=0, ) else: raise ValueError(f"File type {suffix} not supported.") diff --git a/examples/gzip/.gitignore b/examples/gzip/.gitignore index 14be55ba..d1676d84 100644 --- a/examples/gzip/.gitignore +++ b/examples/gzip/.gitignore @@ -1,3 +1,4 @@ +adversarial-robustness-toolbox output *.db kdd_nsl diff --git a/examples/gzip/params.yaml b/examples/gzip/params.yaml new file mode 100644 index 00000000..43dbcb17 --- /dev/null +++ b/examples/gzip/params.yaml @@ -0,0 +1,88 @@ +data: + _target_: deckard.base.data.Data + name: https://gist.githubusercontent.com/simplymathematics/8c6c04bd151950d5ea9e62825db97fdd/raw/d6a22cdb42a1db624c89f0298cb4f654d3812703/kdd_nsl.csv + sample: + _target_: deckard.base.data.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 100 + train_size: 100 + sklearn_pipeline: + encoder: + handle_unknown: use_encoded_value + name: sklearn.preprocessing.OrdinalEncoder + unknown_value: -1 + preprocessor: + name: sklearn.preprocessing.StandardScaler + with_mean: true + with_std: true + target: label +dataset: kdd_nsl +direction: +- maximize +files: + _target_: deckard.base.files.FileConfig + attack_dir: attacks + attack_file: attack + attack_type: .pkl + data_dir: data + data_file: data + data_type: .pkl + directory: output + model_dir: model + model_file: model + model_type: .pkl + name: default + params_file: params.yaml + predictions_file: predictions.json + reports: reports + score_dict_file: score_dict.json +model: + _target_: deckard.base.model.Model + art: + _target_: deckard.base.model.art_pipeline.ArtPipeline + initialize: + nb_classes: 3 + library: sklearn + data: + _target_: deckard.base.data.Data + name: https://gist.githubusercontent.com/simplymathematics/8c6c04bd151950d5ea9e62825db97fdd/raw/d6a22cdb42a1db624c89f0298cb4f654d3812703/kdd_nsl.csv + sample: + _target_: deckard.base.data.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 100 + train_size: 100 + sklearn_pipeline: + encoder: + handle_unknown: use_encoded_value + name: sklearn.preprocessing.OrdinalEncoder + unknown_value: -1 + preprocessor: + name: sklearn.preprocessing.StandardScaler + with_mean: true + with_std: true + target: label + init: + _target_: deckard.base.model.ModelInitializer + compressor: gzip + distance_matrix: output/model/kdd_nsl/gzip_classifier/gzip/0-100.npz + k: 1 + m: -1 + method: random + name: gzip_classifier.GzipClassifier + library: sklearn +model_name: gzip_classifier +optimizers: +- accuracy +scorers: + _target_: deckard.base.scorer.ScorerDict + accuracy: + _target_: deckard.base.scorer.ScorerConfig + direction: maximize + name: sklearn.metrics.accuracy_score + log_loss: + _target_: deckard.base.scorer.ScorerConfig + direction: minimize + name: sklearn.metrics.log_loss +stage: train diff --git a/examples/power/conf/afr.yaml b/examples/power/conf/afr.yaml index 7fcba74d..386cf4b1 100644 --- a/examples/power/conf/afr.yaml +++ b/examples/power/conf/afr.yaml @@ -2,14 +2,14 @@ covariates: - accuracy - train_time - predict_proba_time - - model.trainer.nb_epoch + - model.trainer.np_epochs - model.trainer.batch_size - data.sample.random_state - adv_fit_time - attack.init.eps - adv_failures fillna: - model.trainer.nb_epoch: 20 + model.trainer.np_epochs: 20 model.trainer.batch_size: 1024 model.art.preprocessor.bit_depth: 32 weibull: @@ -28,13 +28,13 @@ weibull: "adv_fit_time: lambda_": "$t_{attack}$" "adv_failure_rate: lambda_": "$h_{adv.}(t;\\theta)$" "failure_rate: lambda_": "$h_{ben.}(t;\\theta)$" - "model.trainer.nb_epoch: lambda_": "No. of Epochs" + "model.trainer.np_epochs: lambda_": "No. of Epochs" "model.trainer.batch_size: lambda_": "Batch Size" "def_gen": "Defence" "attack.init.eps: lambda_": "$\\varepsilon$" partial_effect: - "file": "weibull_epochs_partial_effect.pdf" - "covariate_array": "model.trainer.nb_epoch" + "covariate_array": "model.trainer.np_epochs" "values_array": [1,10,25,50] "title": "$S(t)$ for Weibull AFR" "ylabel": "$\\mathbb{P}~(T>t)$" @@ -57,13 +57,13 @@ cox: "adv_fit_time": "$t_{attack}$" "adv_failure_rate": "$h_{adv.}(t;\\theta)$" "failure_rate": "$h_{ben.}(t;\\theta)$" - "model.trainer.nb_epoch": "No. of Epochs" + "model.trainer.np_epochs": "No. of Epochs" "model.trainer.batch_size": "Batch Size" "def_gen": "Defence" "attack.init.eps": "$\\varepsilon$" partial_effect: - "file": "cox_epochs_partial_effect.pdf" - "covariate_array": "model.trainer.nb_epoch" + "covariate_array": "model.trainer.np_epochs" "values_array": [1,10,25,50] "title": "$S(t)$ for Cox AFR" "ylabel": "$\\mathbb{P}~(T>t)$" @@ -88,13 +88,13 @@ log_logistic: "adv_fit_time: alpha_": "$t_{attack}$" "adv_failure_rate: alpha_": "$h_{adv.}(t;\\theta)$" "failure_rate: alpha_": "$h_{ben.}(t;\\theta)$" - "model.trainer.nb_epoch: alpha_": "No. of Epochs" + "model.trainer.np_epochs: alpha_": "No. of Epochs" "model.trainer.batch_size: alpha_": "Batch Size" "def_gen": "Defence" "attack.init.eps: alpha_": "$\\varepsilon$" partial_effect: - "file": "log_logistic_epochs_partial_effect.pdf" - "covariate_array": "model.trainer.nb_epoch" + "covariate_array": "model.trainer.np_epochs" "values_array": [1,10,25,50] "title": "$S(t)$ for Log-Logistic AFR" "ylabel": "$\\mathbb{P}~(T>t)$" @@ -118,14 +118,14 @@ log_normal: "adv_fit_time: mu_": "$t_{attack}$" "adv_failure_rate: mu_": "$h_{adv.}(t;\\theta)$" "failure_rate: mu_": "$h_{ben.}(t;\\theta)$" - "model.trainer.nb_epoch: mu_": "No. of Epochs" + "model.trainer.np_epochs: mu_": "No. of Epochs" "model.trainer.batch_size: mu_": "Batch Size" "def_gen": "Defence" "attack.init.eps: mu_": "$\\varepsilon$" "data.sample.random_state: mu_": "Random State" partial_effect: - "file": "log_normal_epochs_partial_effect.pdf" - "covariate_array": "model.trainer.nb_epoch" + "covariate_array": "model.trainer.np_epochs" "values_array": [1,10,25,50] "title": "$S(t)$ for Log-Normal AFR" "ylabel": "$\\mathbb{P}~(T>t)$" diff --git a/examples/power/conf/bit_depth/torch_cifar10.yaml b/examples/power/conf/bit_depth/torch_cifar10.yaml index 531705bf..3706844f 100644 --- a/examples/power/conf/bit_depth/torch_cifar10.yaml +++ b/examples/power/conf/bit_depth/torch_cifar10.yaml @@ -44,7 +44,7 @@ hydra: params: ++data.sample.random_state: int(range(0, 1)) ++model.art.initialize.optimizer.lr: tag(log, interval(0.000001, 1)) - ++model.trainer.nb_epoch: int(interval(1, 50)) + ++model.trainer.np_epochs: int(interval(1, 50)) ++model.trainer.batch_size: int(interval(1, 10000)) ++attack.init.eps : interval(0.01, 1.0) ++model.art.preprocessor.params.bit_depth: choice(4,8,16,32,64) diff --git a/examples/power/conf/bit_depth/torch_cifar100.yaml b/examples/power/conf/bit_depth/torch_cifar100.yaml index 8691df7b..667ce7e1 100644 --- a/examples/power/conf/bit_depth/torch_cifar100.yaml +++ b/examples/power/conf/bit_depth/torch_cifar100.yaml @@ -44,7 +44,7 @@ hydra: params: ++data.sample.random_state: int(range(0, 1)) ++model.art.initialize.optimizer.lr: tag(log, interval(0.000001, 100)) - ++model.trainer.nb_epoch: tag(log, int(interval(1, 100))) + ++model.trainer.np_epochs: tag(log, int(interval(1, 100))) ++model.trainer.batch_size: tag(log, int(interval(10, 10000))) ++attack.init.eps : interval(0.01, 1.0) ++model.art.preprocessor.params.bit_depth: choice(4,8,16,32,64) diff --git a/examples/power/conf/bit_depth/torch_mnist.yaml b/examples/power/conf/bit_depth/torch_mnist.yaml index adbb17d5..143d0857 100644 --- a/examples/power/conf/bit_depth/torch_mnist.yaml +++ b/examples/power/conf/bit_depth/torch_mnist.yaml @@ -45,7 +45,7 @@ hydra: params: ++data.sample.random_state: int(range(0, 10)) ++model.art.initialize.optimizer.lr: tag(log, interval(0.000001, 1)) - ++model.trainer.nb_epoch: int(interval(1, 50)) + ++model.trainer.np_epochs: int(interval(1, 50)) ++model.trainer.batch_size: int(interval(1, 10000)) ++attack.init.eps : interval(0.01, 1.0) ++model.art.preprocessor.params.bit_depth: choice(4,8,16,32,64) diff --git a/examples/power/conf/clean.yaml b/examples/power/conf/clean.yaml index c8f2b5c7..a960e865 100644 --- a/examples/power/conf/clean.yaml +++ b/examples/power/conf/clean.yaml @@ -15,7 +15,7 @@ params: DeepFool: attack.init.eps FSQ: model.art.preprocessor.bit_depth Control: model_layers - Epochs: model.trainer.nb_epoch + Epochs: model.trainer.np_epochs Batch_Size: model.trainer.batch_size fillna: Epochs: 20 diff --git a/examples/power/conf/combined_afr.yaml b/examples/power/conf/combined_afr.yaml index f65984b9..2e8709d8 100644 --- a/examples/power/conf/combined_afr.yaml +++ b/examples/power/conf/combined_afr.yaml @@ -2,7 +2,7 @@ covariates: - accuracy - train_time - predict_proba_time - - model.trainer.nb_epoch + - model.trainer.np_epochs - model.trainer.batch_size - attack.init.eps - data.sample.random_state @@ -11,7 +11,7 @@ covariates: - adv_fit_time - adv_failures fillna: - model.trainer.nb_epoch: 20 + model.trainer.np_epochs: 20 model.trainer.batch_size: 1024 model.art.preprocessor.bit_depth: 32 weibull: @@ -50,13 +50,13 @@ weibull: # "adv_fit_time": "$T_{a}$" # "adv_failure_rate": "$h_{adv.}(t;\\theta)$" # "failure_rate": "$h_{ben.}(t;\\theta)$" -# "model.trainer.nb_epoch": "No. of Epochs" +# "model.trainer.np_epochs": "No. of Epochs" # "model.trainer.batch_size": "Batch Size" # "def_gen": "Defence" # "attack.init.eps": "$\\varepsilon$" # partial_effect: # - "file": "cox_epochs_partial_effect.pdf" -# "covariate_array": "model.trainer.nb_epoch" +# "covariate_array": "model.trainer.np_epochs" # "values_array": [1,10,25,50] # "title": "$S(t)$ for Cox AFR" # "ylabel": "$\\mathbb{P}~(T>t)$" diff --git a/examples/power/conf/model/torch_cifar.yaml b/examples/power/conf/model/torch_cifar.yaml index fb0414cc..8484c553 100644 --- a/examples/power/conf/model/torch_cifar.yaml +++ b/examples/power/conf/model/torch_cifar.yaml @@ -8,6 +8,6 @@ init: num_classes: 10 _target_: deckard.base.model.Model trainer: - nb_epoch: 1 + np_epochs: 1 batch_size: 1024 library : pytorch diff --git a/examples/power/conf/model/torch_cifar100.yaml b/examples/power/conf/model/torch_cifar100.yaml index d7d0f91d..78ef6775 100644 --- a/examples/power/conf/model/torch_cifar100.yaml +++ b/examples/power/conf/model/torch_cifar100.yaml @@ -8,6 +8,6 @@ init: num_classes: 100 _target_: deckard.base.model.Model trainer: - nb_epoch: 1 + np_epochs: 1 batch_size: 1024 library : pytorch diff --git a/examples/power/conf/model/torch_mnist.yaml b/examples/power/conf/model/torch_mnist.yaml index 618c6de6..a1fec778 100644 --- a/examples/power/conf/model/torch_mnist.yaml +++ b/examples/power/conf/model/torch_mnist.yaml @@ -7,6 +7,6 @@ init: name : torch_example.ResNet18 _target_: deckard.base.model.Model trainer: - nb_epoch: 1 + np_epochs: 1 batch_size: 1024 library : pytorch diff --git a/examples/power/conf/torch_cifar10.yaml b/examples/power/conf/torch_cifar10.yaml index 374e53e6..a122c32b 100644 --- a/examples/power/conf/torch_cifar10.yaml +++ b/examples/power/conf/torch_cifar10.yaml @@ -44,7 +44,7 @@ hydra: params: ++data.sample.random_state: int(range(0, 9)) ++model.art.initialize.optimizer.lr: tag(log, interval(0.000001, 1)) - ++model.trainer.nb_epoch: int(interval(1, 50)) + ++model.trainer.np_epochs: int(interval(1, 50)) ++model.trainer.batch_size: int(interval(1, 10000)) ++attack.init.eps : interval(0.01, 1.0) _target_: hydra_plugins.hydra_optuna_sweeper.optuna_sweeper.OptunaSweeper diff --git a/examples/power/conf/torch_cifar100.yaml b/examples/power/conf/torch_cifar100.yaml index 16660da4..e14bc09c 100644 --- a/examples/power/conf/torch_cifar100.yaml +++ b/examples/power/conf/torch_cifar100.yaml @@ -44,7 +44,7 @@ hydra: params: ++data.sample.random_state: int(range(0, 9)) ++model.art.initialize.optimizer.lr: tag(log, interval(0.000001, 100)) - ++model.trainer.nb_epoch: tag(log, int(interval(1, 100))) + ++model.trainer.np_epochs: tag(log, int(interval(1, 100))) ++model.trainer.batch_size: tag(log, int(interval(10, 10000))) ++attack.init.eps : interval(0.01, 1.0) _target_: hydra_plugins.hydra_optuna_sweeper.optuna_sweeper.OptunaSweeper diff --git a/examples/power/conf/torch_mnist.yaml b/examples/power/conf/torch_mnist.yaml index 986d350b..9aaf8720 100644 --- a/examples/power/conf/torch_mnist.yaml +++ b/examples/power/conf/torch_mnist.yaml @@ -45,7 +45,7 @@ hydra: params: ++data.sample.random_state: int(range(0, 9)) ++model.art.initialize.optimizer.lr: tag(log, interval(0.000001, 1)) - ++model.trainer.nb_epoch: int(interval(1, 50)) + ++model.trainer.np_epochs: int(interval(1, 50)) ++model.trainer.batch_size: int(interval(1, 10000)) ++attack.init.eps : interval(0.01, 1.0) _target_: hydra_plugins.hydra_optuna_sweeper.optuna_sweeper.OptunaSweeper diff --git a/examples/power/params.yaml b/examples/power/params.yaml index 81856502..f35272b5 100644 --- a/examples/power/params.yaml +++ b/examples/power/params.yaml @@ -74,7 +74,7 @@ attack: library: pytorch trainer: batch_size: 1024 - nb_epoch: 1 + np_epochs: 1 name: art.attacks.evasion.FastGradientMethod targeted: false method: evasion @@ -130,7 +130,7 @@ attack: library: pytorch trainer: batch_size: 1024 - nb_epoch: 1 + np_epochs: 1 data: _target_: deckard.base.data.Data generate: @@ -222,7 +222,7 @@ model: library: pytorch trainer: batch_size: 1024 - nb_epoch: 1 + np_epochs: 1 optimizers: - accuracy - train_time diff --git a/examples/power/plots/combined_plots.py b/examples/power/plots/combined_plots.py index 97f7f6ff..6a8e3bb1 100644 --- a/examples/power/plots/combined_plots.py +++ b/examples/power/plots/combined_plots.py @@ -59,7 +59,7 @@ "nvidia-tesla-v100": 2.55, "nvidia-l4": 0.81, } -epochs = "model.trainer.nb_epoch" +epochs = "model.trainer.np_epochs" batch_size = "model.trainer.batch_size" bit_depth = "model.art.preprocessor.bit_depth" resolution = "n_pixels" diff --git a/examples/power/plots/data/bit_depth/cifar/power.csv b/examples/power/plots/data/bit_depth/cifar/power.csv index 67285ddb..8db04d9c 100644 --- a/examples/power/plots/data/bit_depth/cifar/power.csv +++ b/examples/power/plots/data/bit_depth/cifar/power.csv @@ -1,4 +1,4 @@ -,stage,_target_,attack,data,device_id,files,model,name,scorers,train_time,train_time_per_sample,train_start_time,train_end_time,train_device,predict_time,predict_time_per_sample,predict_start_time,predict_end_time,predict_device,predict_proba_time,predict_proba_time_per_sample,predict_proba_start_time,predict_proba_end_time,predict_proba_device,predict_log_proba_time,predict_log_proba_time_per_sample,predict_log_proba_start_time,predict_log_proba_end_time,predict_log_device,adv_fit_time_per_sample,adv_fit_time,adv_fit_start_time,adv_fit_end_time,adv_fit_device,adv_predict_time_per_sample,adv_predict_time,adv_predict_start_time,adv_predict_end_time,adv_predict_device,adv_success,accuracy,log_loss,adv_accuracy,adv_log_loss,predictions,adv_predictions,exception,stage.1,_target_.1,attack._target_,attack.attack_size,attack.method,attack.data._target_,attack.data.generate._target_,attack.data.generate.name,attack.data.sample._target_,attack.data.sample.random_state,attack.data.sample.stratify,attack.data.sklearn_pipeline._target_,attack.data.sklearn_pipeline.preprocessor.name,attack.data.sklearn_pipeline.preprocessor.with_mean,attack.data.sklearn_pipeline.preprocessor.with_std,attack.init._target_,attack.init.batch_size,attack.init.eps,attack.init.minimal,attack.init.model._target_,attack.init.model.art._target_,attack.init.model.art.data._target_,attack.init.model.art.data.generate._target_,attack.init.model.art.data.generate.name,attack.init.model.art.data.sample._target_,attack.init.model.art.data.sample.random_state,attack.init.model.art.data.sample.stratify,attack.init.model.art.data.sklearn_pipeline._target_,attack.init.model.art.data.sklearn_pipeline.preprocessor.name,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.art.initialize.clip_values,attack.init.model.art.initialize.criterion.name,attack.init.model.art.initialize.optimizer.lr,attack.init.model.art.initialize.optimizer.momentum,attack.init.model.art.initialize.optimizer.name,attack.init.model.art.library,attack.init.model.art.preprocessor.bit_depth,attack.init.model.art.preprocessor.clip_values,attack.init.model.art.preprocessor.name,attack.init.model.art.preprocessor.params.bit_depth,attack.init.model.data._target_,attack.init.model.data.generate._target_,attack.init.model.data.generate.name,attack.init.model.data.sample._target_,attack.init.model.data.sample.random_state,attack.init.model.data.sample.stratify,attack.init.model.data.sklearn_pipeline._target_,attack.init.model.data.sklearn_pipeline.preprocessor.name,attack.init.model.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.init._target_,attack.init.model.init.name,attack.init.model.init.num_channels,attack.init.model.init.num_classes,attack.init.model.library,attack.init.model.trainer.batch_size,attack.init.model.trainer.nb_epoch,attack.init.name,attack.init.targeted,attack.model._target_,attack.model.art._target_,attack.model.art.data._target_,attack.model.art.data.generate._target_,attack.model.art.data.generate.name,attack.model.art.data.sample._target_,attack.model.art.data.sample.random_state,attack.model.art.data.sample.stratify,attack.model.art.data.sklearn_pipeline._target_,attack.model.art.data.sklearn_pipeline.preprocessor.name,attack.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.model.art.initialize.clip_values,attack.model.art.initialize.criterion.name,attack.model.art.initialize.optimizer.lr,attack.model.art.initialize.optimizer.momentum,attack.model.art.initialize.optimizer.name,attack.model.art.library,attack.model.art.preprocessor.bit_depth,attack.model.art.preprocessor.clip_values,attack.model.art.preprocessor.name,attack.model.art.preprocessor.params.bit_depth,attack.model.data._target_,attack.model.data.generate._target_,attack.model.data.generate.name,attack.model.data.sample._target_,attack.model.data.sample.random_state,attack.model.data.sample.stratify,attack.model.data.sklearn_pipeline._target_,attack.model.data.sklearn_pipeline.preprocessor.name,attack.model.data.sklearn_pipeline.preprocessor.with_mean,attack.model.data.sklearn_pipeline.preprocessor.with_std,attack.model.init._target_,attack.model.init.name,attack.model.init.num_channels,attack.model.init.num_classes,attack.model.library,attack.model.trainer.batch_size,attack.model.trainer.nb_epoch,data._target_,data.generate._target_,data.generate.name,data.sample._target_,data.sample.random_state,data.sample.stratify,data.sklearn_pipeline._target_,data.sklearn_pipeline.preprocessor.name,data.sklearn_pipeline.preprocessor.with_mean,data.sklearn_pipeline.preprocessor.with_std,device_id.1,files._target_,files.adv_predictions_file,files.attack_dir,files.attack_file,files.attack_type,files.data_dir,files.data_file,files.data_type,files.directory,files.name,files.predictions_file,files.reports,files.score_dict_file,files.stage,files.model_dir,files.model_file,files.model_type,files.params_file,model._target_,model.library,model.art._target_,model.art.data._target_,model.art.data.generate._target_,model.art.data.generate.name,model.art.data.sample._target_,model.art.data.sample.random_state,model.art.data.sample.stratify,model.art.data.sklearn_pipeline._target_,model.art.data.sklearn_pipeline.preprocessor.name,model.art.data.sklearn_pipeline.preprocessor.with_mean,model.art.data.sklearn_pipeline.preprocessor.with_std,model.art.initialize.clip_values,model.art.initialize.criterion.name,model.art.initialize.optimizer.lr,model.art.initialize.optimizer.momentum,model.art.initialize.optimizer.name,model.art.library,model.art.preprocessor.bit_depth,model.art.preprocessor.clip_values,model.art.preprocessor.name,model.art.preprocessor.params.bit_depth,model.data._target_,model.data.generate._target_,model.data.generate.name,model.data.sample._target_,model.data.sample.random_state,model.data.sample.stratify,model.data.sklearn_pipeline._target_,model.data.sklearn_pipeline.preprocessor.name,model.data.sklearn_pipeline.preprocessor.with_mean,model.data.sklearn_pipeline.preprocessor.with_std,model.init._target_,model.init.name,model.init.num_channels,model.init.num_classes,model.trainer.batch_size,model.trainer.nb_epoch,name.1,scorers._target_,scorers.accuracy._target_,scorers.accuracy.direction,scorers.accuracy.name,scorers.log_loss._target_,scorers.log_loss.direction,scorers.log_loss.name,train_time.1,train_time_per_sample.1,train_start_time.1,train_end_time.1,train_device.1,predict_time.1,predict_time_per_sample.1,predict_start_time.1,predict_end_time.1,predict_device.1,predict_proba_time.1,predict_proba_time_per_sample.1,predict_proba_start_time.1,predict_proba_end_time.1,predict_proba_device.1,predict_log_proba_time.1,predict_log_proba_time_per_sample.1,predict_log_proba_start_time.1,predict_log_proba_end_time.1,predict_log_device.1,adv_fit_time_per_sample.1,adv_fit_time.1,adv_fit_start_time.1,adv_fit_end_time.1,adv_fit_device.1,adv_predict_time_per_sample.1,adv_predict_time.1,adv_predict_start_time.1,adv_predict_end_time.1,adv_predict_device.1,adv_success.1,accuracy.1,log_loss.1,adv_accuracy.1,adv_log_loss.1,predictions.1,adv_predictions.1,exception.1,train_power,peak_power,predict_power,predict_proba_power,predict_log_proba_power,adv_fit_power,adv_predict_power +,stage,_target_,attack,data,device_id,files,model,name,scorers,train_time,train_time_per_sample,train_start_time,train_end_time,train_device,predict_time,predict_time_per_sample,predict_start_time,predict_end_time,predict_device,predict_proba_time,predict_proba_time_per_sample,predict_proba_start_time,predict_proba_end_time,predict_proba_device,predict_log_proba_time,predict_log_proba_time_per_sample,predict_log_proba_start_time,predict_log_proba_end_time,predict_log_device,adv_fit_time_per_sample,adv_fit_time,adv_fit_start_time,adv_fit_end_time,adv_fit_device,adv_predict_time_per_sample,adv_predict_time,adv_predict_start_time,adv_predict_end_time,adv_predict_device,adv_success,accuracy,log_loss,adv_accuracy,adv_log_loss,predictions,adv_predictions,exception,stage.1,_target_.1,attack._target_,attack.attack_size,attack.method,attack.data._target_,attack.data.generate._target_,attack.data.generate.name,attack.data.sample._target_,attack.data.sample.random_state,attack.data.sample.stratify,attack.data.sklearn_pipeline._target_,attack.data.sklearn_pipeline.preprocessor.name,attack.data.sklearn_pipeline.preprocessor.with_mean,attack.data.sklearn_pipeline.preprocessor.with_std,attack.init._target_,attack.init.batch_size,attack.init.eps,attack.init.minimal,attack.init.model._target_,attack.init.model.art._target_,attack.init.model.art.data._target_,attack.init.model.art.data.generate._target_,attack.init.model.art.data.generate.name,attack.init.model.art.data.sample._target_,attack.init.model.art.data.sample.random_state,attack.init.model.art.data.sample.stratify,attack.init.model.art.data.sklearn_pipeline._target_,attack.init.model.art.data.sklearn_pipeline.preprocessor.name,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.art.initialize.clip_values,attack.init.model.art.initialize.criterion.name,attack.init.model.art.initialize.optimizer.lr,attack.init.model.art.initialize.optimizer.momentum,attack.init.model.art.initialize.optimizer.name,attack.init.model.art.library,attack.init.model.art.preprocessor.bit_depth,attack.init.model.art.preprocessor.clip_values,attack.init.model.art.preprocessor.name,attack.init.model.art.preprocessor.params.bit_depth,attack.init.model.data._target_,attack.init.model.data.generate._target_,attack.init.model.data.generate.name,attack.init.model.data.sample._target_,attack.init.model.data.sample.random_state,attack.init.model.data.sample.stratify,attack.init.model.data.sklearn_pipeline._target_,attack.init.model.data.sklearn_pipeline.preprocessor.name,attack.init.model.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.init._target_,attack.init.model.init.name,attack.init.model.init.num_channels,attack.init.model.init.num_classes,attack.init.model.library,attack.init.model.trainer.batch_size,attack.init.model.trainer.np_epochs,attack.init.name,attack.init.targeted,attack.model._target_,attack.model.art._target_,attack.model.art.data._target_,attack.model.art.data.generate._target_,attack.model.art.data.generate.name,attack.model.art.data.sample._target_,attack.model.art.data.sample.random_state,attack.model.art.data.sample.stratify,attack.model.art.data.sklearn_pipeline._target_,attack.model.art.data.sklearn_pipeline.preprocessor.name,attack.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.model.art.initialize.clip_values,attack.model.art.initialize.criterion.name,attack.model.art.initialize.optimizer.lr,attack.model.art.initialize.optimizer.momentum,attack.model.art.initialize.optimizer.name,attack.model.art.library,attack.model.art.preprocessor.bit_depth,attack.model.art.preprocessor.clip_values,attack.model.art.preprocessor.name,attack.model.art.preprocessor.params.bit_depth,attack.model.data._target_,attack.model.data.generate._target_,attack.model.data.generate.name,attack.model.data.sample._target_,attack.model.data.sample.random_state,attack.model.data.sample.stratify,attack.model.data.sklearn_pipeline._target_,attack.model.data.sklearn_pipeline.preprocessor.name,attack.model.data.sklearn_pipeline.preprocessor.with_mean,attack.model.data.sklearn_pipeline.preprocessor.with_std,attack.model.init._target_,attack.model.init.name,attack.model.init.num_channels,attack.model.init.num_classes,attack.model.library,attack.model.trainer.batch_size,attack.model.trainer.np_epochs,data._target_,data.generate._target_,data.generate.name,data.sample._target_,data.sample.random_state,data.sample.stratify,data.sklearn_pipeline._target_,data.sklearn_pipeline.preprocessor.name,data.sklearn_pipeline.preprocessor.with_mean,data.sklearn_pipeline.preprocessor.with_std,device_id.1,files._target_,files.adv_predictions_file,files.attack_dir,files.attack_file,files.attack_type,files.data_dir,files.data_file,files.data_type,files.directory,files.name,files.predictions_file,files.reports,files.score_dict_file,files.stage,files.model_dir,files.model_file,files.model_type,files.params_file,model._target_,model.library,model.art._target_,model.art.data._target_,model.art.data.generate._target_,model.art.data.generate.name,model.art.data.sample._target_,model.art.data.sample.random_state,model.art.data.sample.stratify,model.art.data.sklearn_pipeline._target_,model.art.data.sklearn_pipeline.preprocessor.name,model.art.data.sklearn_pipeline.preprocessor.with_mean,model.art.data.sklearn_pipeline.preprocessor.with_std,model.art.initialize.clip_values,model.art.initialize.criterion.name,model.art.initialize.optimizer.lr,model.art.initialize.optimizer.momentum,model.art.initialize.optimizer.name,model.art.library,model.art.preprocessor.bit_depth,model.art.preprocessor.clip_values,model.art.preprocessor.name,model.art.preprocessor.params.bit_depth,model.data._target_,model.data.generate._target_,model.data.generate.name,model.data.sample._target_,model.data.sample.random_state,model.data.sample.stratify,model.data.sklearn_pipeline._target_,model.data.sklearn_pipeline.preprocessor.name,model.data.sklearn_pipeline.preprocessor.with_mean,model.data.sklearn_pipeline.preprocessor.with_std,model.init._target_,model.init.name,model.init.num_channels,model.init.num_classes,model.trainer.batch_size,model.trainer.np_epochs,name.1,scorers._target_,scorers.accuracy._target_,scorers.accuracy.direction,scorers.accuracy.name,scorers.log_loss._target_,scorers.log_loss.direction,scorers.log_loss.name,train_time.1,train_time_per_sample.1,train_start_time.1,train_end_time.1,train_device.1,predict_time.1,predict_time_per_sample.1,predict_start_time.1,predict_end_time.1,predict_device.1,predict_proba_time.1,predict_proba_time_per_sample.1,predict_proba_start_time.1,predict_proba_end_time.1,predict_proba_device.1,predict_log_proba_time.1,predict_log_proba_time_per_sample.1,predict_log_proba_start_time.1,predict_log_proba_end_time.1,predict_log_device.1,adv_fit_time_per_sample.1,adv_fit_time.1,adv_fit_start_time.1,adv_fit_end_time.1,adv_fit_device.1,adv_predict_time_per_sample.1,adv_predict_time.1,adv_predict_start_time.1,adv_predict_end_time.1,adv_predict_device.1,adv_success.1,accuracy.1,log_loss.1,adv_accuracy.1,adv_log_loss.1,predictions.1,adv_predictions.1,exception.1,train_power,peak_power,predict_power,predict_proba_power,predict_log_proba_power,adv_fit_power,adv_predict_power 000ebefbc197fb3d43a3dc8aadf502c6,attack,deckard.base.experiment.Experiment,000ebefbc197fb3d43a3dc8aadf502c6,000ebefbc197fb3d43a3dc8aadf502c6,nvidia-l4,000ebefbc197fb3d43a3dc8aadf502c6,000ebefbc197fb3d43a3dc8aadf502c6,000ebefbc197fb3d43a3dc8aadf502c6,000ebefbc197fb3d43a3dc8aadf502c6,20.798746585,0.0004333072205208,1701756798.393697,1701756819.0929022,cuda:0,1.42415573,2.9669911041666667e-05,1701756819.0943675,1701756820.5177956,cuda:0,1.460288112,3.0422669e-05,1701756820.7342632,1701756822.1942945,cuda:0,1.432434553,2.984238652083333e-05,1701756822.194462,1701756823.626309,cuda:0,0.00105851306,0.105851306,1701756826.2200809,1701756826.3259609,cuda:0,0.00014055318,0.014055318,1701756828.1861413,1701756828.200218,cuda:0,1.0,0.4733333333333333,4.364713044121862,0.05,19.462214255332945,/result/bit_depth/cifar/reports/attack/000ebefbc197fb3d43a3dc8aadf502c6/predictions.json,/result/bit_depth/cifar/reports/attack/000ebefbc197fb3d43a3dc8aadf502c6/adv_predictions.json,,attack,deckard.base.experiment.Experiment,deckard.base.attack.Attack,100,evasion,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.attack.AttackInitializer,4319,0.147903936667764,True,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0024263516470858,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,16.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,10,pytorch,4319,22,art.attacks.evasion.FastGradientMethod,False,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0024263516470858,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,16.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,10,pytorch,4319,22,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,nvidia-l4,deckard.base.files.FileConfig,adv_predictions.json,attacks,0a5cfe082fdb5973b9653a431489d120,.pkl,data,ea1a0afb95689dc7b59820b2cd11c56c,.pkl,/result/bit_depth/cifar/,000ebefbc197fb3d43a3dc8aadf502c6,predictions.json,reports,score_dict.json,attack,,,,,deckard.base.model.Model,pytorch,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0024263516470858,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,16.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,10,4319,22,000ebefbc197fb3d43a3dc8aadf502c6,deckard.base.scorer.ScorerDict,deckard.base.scorer.ScorerConfig,maximize,sklearn.metrics.accuracy_score,deckard.base.scorer.ScorerConfig,minimize,sklearn.metrics.log_loss,20.798746585,0.0004333072205208,1701756798.393697,1701756819.0929022,cuda:0,1.42415573,2.9669911041666667e-05,1701756819.0943675,1701756820.5177956,cuda:0,1.460288112,3.0422669e-05,1701756820.7342632,1701756822.1942945,cuda:0,1.432434553,2.984238652083333e-05,1701756822.194462,1701756823.626309,cuda:0,0.00105851306,0.105851306,1701756826.2200809,1701756826.3259609,cuda:0,0.00014055318,0.014055318,1701756828.1861413,1701756828.200218,cuda:0,1.0,0.4733333333333333,4.364713044121862,0.05,19.462214255332945,/result/bit_depth/cifar/reports/attack/000ebefbc197fb3d43a3dc8aadf502c6/predictions.json,/result/bit_depth/cifar/reports/attack/000ebefbc197fb3d43a3dc8aadf502c6/adv_predictions.json,,0.4429629904270172,72,0.02425236726284027,0.028514410722255707,0.03211346131038666,0.0021150593204498293,0.00026554305267333984 00415dd483128ccc7df4cec50fc2d173,attack,deckard.base.experiment.Experiment,00415dd483128ccc7df4cec50fc2d173,00415dd483128ccc7df4cec50fc2d173,nvidia-l4,00415dd483128ccc7df4cec50fc2d173,00415dd483128ccc7df4cec50fc2d173,00415dd483128ccc7df4cec50fc2d173,00415dd483128ccc7df4cec50fc2d173,20.342539462,0.0004238029054583,1701870879.9091127,1701870900.1457748,cuda:0,1.25711817,2.6189961875e-05,1701870900.1462655,1701870901.4031055,cuda:0,1.280774315,2.6682798229166667e-05,1701870901.8503766,1701870903.1337695,cuda:0,1.291406695,2.6904306145833333e-05,1701870903.1338584,1701870904.4248807,cuda:0,0.00103959274,0.103959274,1701870906.9498286,1701870907.0537786,cuda:0,0.00013942684,0.013942684,1701870908.6542792,1701870908.6682274,cuda:0,0.0,0.1004166666666666,20.09881012827158,0.1,19.528728548288345,/result/bit_depth/cifar/reports/attack/00415dd483128ccc7df4cec50fc2d173/predictions.json,/result/bit_depth/cifar/reports/attack/00415dd483128ccc7df4cec50fc2d173/adv_predictions.json,,attack,deckard.base.experiment.Experiment,deckard.base.attack.Attack,100,evasion,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.attack.AttackInitializer,5491,0.0631488095614281,True,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,3.349114123256118e-06,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,8.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,10,pytorch,5491,23,art.attacks.evasion.FastGradientMethod,False,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,3.349114123256118e-06,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,8.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,10,pytorch,5491,23,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,nvidia-l4,deckard.base.files.FileConfig,adv_predictions.json,attacks,6becd7d70329a74b287e07182a5cc1a5,.pkl,data,ea1a0afb95689dc7b59820b2cd11c56c,.pkl,/result/bit_depth/cifar/,00415dd483128ccc7df4cec50fc2d173,predictions.json,reports,score_dict.json,attack,,,,,deckard.base.model.Model,pytorch,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,3.349114123256118e-06,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,8.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,10,5491,23,00415dd483128ccc7df4cec50fc2d173,deckard.base.scorer.ScorerDict,deckard.base.scorer.ScorerConfig,maximize,sklearn.metrics.accuracy_score,deckard.base.scorer.ScorerConfig,minimize,sklearn.metrics.log_loss,20.342539462,0.0004238029054583,1701870879.9091127,1701870900.1457748,cuda:0,1.25711817,2.6189961875e-05,1701870900.1462655,1701870901.4031055,cuda:0,1.280774315,2.6682798229166667e-05,1701870901.8503766,1701870903.1337695,cuda:0,1.291406695,2.6904306145833333e-05,1701870903.1338584,1701870904.4248807,cuda:0,0.00103959274,0.103959274,1701870906.9498286,1701870907.0537786,cuda:0,0.00013942684,0.013942684,1701870908.6542792,1701870908.6682274,cuda:0,0.0,0.1004166666666666,20.09881012827158,0.1,19.528728548288345,/result/bit_depth/cifar/reports/attack/00415dd483128ccc7df4cec50fc2d173/predictions.json,/result/bit_depth/cifar/reports/attack/00415dd483128ccc7df4cec50fc2d173/adv_predictions.json,,0.4768567068891525,72,0.02164781199836731,0.030593520097732543,0.027886081695556644,0.0017189175910949709,0.00023310235404968262 004a896f341b6e25d958d148ef4a156a,attack,deckard.base.experiment.Experiment,004a896f341b6e25d958d148ef4a156a,004a896f341b6e25d958d148ef4a156a,nvidia-l4,004a896f341b6e25d958d148ef4a156a,004a896f341b6e25d958d148ef4a156a,004a896f341b6e25d958d148ef4a156a,004a896f341b6e25d958d148ef4a156a,23.952431565,0.0004990089909375,1701822810.0836835,1701822833.9221852,cuda:0,3.804299213,7.925623360416667e-05,1701822833.9227624,1701822837.7266448,cuda:0,3.714964562,7.739509504166666e-05,1701822837.9962673,1701822841.7022254,cuda:0,3.674934266,7.656113054166667e-05,1701822841.7023735,1701822845.3761077,cuda:0,0.00296175067,0.296175067,1701822847.919942,1701822848.2160606,cuda:0,0.00041924827,0.041924827,1701822855.1166472,1701822855.1586,cuda:0,1.0,0.1150833333333333,20.70036181043622,0.11,16.78006957769394,/result/bit_depth/cifar/reports/attack/004a896f341b6e25d958d148ef4a156a/predictions.json,/result/bit_depth/cifar/reports/attack/004a896f341b6e25d958d148ef4a156a/adv_predictions.json,,attack,deckard.base.experiment.Experiment,deckard.base.attack.Attack,100,evasion,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.attack.AttackInitializer,7934,0.2839038156185892,True,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,1.678776039415729e-05,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,32.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,10,pytorch,7934,21,art.attacks.evasion.FastGradientMethod,False,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,1.678776039415729e-05,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,32.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,10,pytorch,7934,21,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,nvidia-l4,deckard.base.files.FileConfig,adv_predictions.json,attacks,102ac4821fd4d9b7e9a4c6f6184de99f,.pkl,data,6e5857478e8d10445dfe508f8632e60d,.pkl,/result/bit_depth/cifar/,004a896f341b6e25d958d148ef4a156a,predictions.json,reports,score_dict.json,attack,,,,,deckard.base.model.Model,pytorch,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,1.678776039415729e-05,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,32.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,10,7934,21,004a896f341b6e25d958d148ef4a156a,deckard.base.scorer.ScorerDict,deckard.base.scorer.ScorerConfig,maximize,sklearn.metrics.accuracy_score,deckard.base.scorer.ScorerConfig,minimize,sklearn.metrics.log_loss,23.952431565,0.0004990089909375,1701822810.0836835,1701822833.9221852,cuda:0,3.804299213,7.925623360416667e-05,1701822833.9227624,1701822837.7266448,cuda:0,3.714964562,7.739509504166666e-05,1701822837.9962673,1701822841.7022254,cuda:0,3.674934266,7.656113054166667e-05,1701822841.7023735,1701822845.3761077,cuda:0,0.00296175067,0.296175067,1701822847.919942,1701822848.2160606,cuda:0,0.00041924827,0.041924827,1701822855.1166472,1701822855.1586,cuda:0,1.0,0.1150833333333333,20.70036181043622,0.11,16.78006957769394,/result/bit_depth/cifar/reports/attack/004a896f341b6e25d958d148ef4a156a/predictions.json,/result/bit_depth/cifar/reports/attack/004a896f341b6e25d958d148ef4a156a/adv_predictions.json,,0.530645047659874,72,0.07839040768432617,0.07019084694385527,0.06637702931022645,0.006163415376663208,0.0008189196014404297 diff --git a/examples/power/plots/data/bit_depth/cifar100/power.csv b/examples/power/plots/data/bit_depth/cifar100/power.csv index 2bc3c42c..44cf0ee7 100644 --- a/examples/power/plots/data/bit_depth/cifar100/power.csv +++ b/examples/power/plots/data/bit_depth/cifar100/power.csv @@ -1,4 +1,4 @@ -,stage,_target_,attack,data,device_id,files,model,name,scorers,train_time,train_time_per_sample,train_start_time,train_end_time,train_device,predict_time,predict_time_per_sample,predict_start_time,predict_end_time,predict_device,predict_proba_time,predict_proba_time_per_sample,predict_proba_start_time,predict_proba_end_time,predict_proba_device,predict_log_proba_time,predict_log_proba_time_per_sample,predict_log_proba_start_time,predict_log_proba_end_time,predict_log_device,adv_fit_time_per_sample,adv_fit_time,adv_fit_start_time,adv_fit_end_time,adv_fit_device,adv_predict_time_per_sample,adv_predict_time,adv_predict_start_time,adv_predict_end_time,adv_predict_device,adv_success,accuracy,log_loss,adv_accuracy,adv_log_loss,predictions,adv_predictions,stage.1,_target_.1,attack._target_,attack.attack_size,attack.method,attack.data._target_,attack.data.generate._target_,attack.data.generate.name,attack.data.generate.path,attack.data.sample._target_,attack.data.sample.random_state,attack.data.sample.stratify,attack.data.sklearn_pipeline._target_,attack.data.sklearn_pipeline.preprocessor.name,attack.data.sklearn_pipeline.preprocessor.with_mean,attack.data.sklearn_pipeline.preprocessor.with_std,attack.init._target_,attack.init.batch_size,attack.init.eps,attack.init.minimal,attack.init.model._target_,attack.init.model.art._target_,attack.init.model.art.data._target_,attack.init.model.art.data.generate._target_,attack.init.model.art.data.generate.name,attack.init.model.art.data.generate.path,attack.init.model.art.data.sample._target_,attack.init.model.art.data.sample.random_state,attack.init.model.art.data.sample.stratify,attack.init.model.art.data.sklearn_pipeline._target_,attack.init.model.art.data.sklearn_pipeline.preprocessor.name,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.art.initialize.clip_values,attack.init.model.art.initialize.criterion.name,attack.init.model.art.initialize.optimizer.lr,attack.init.model.art.initialize.optimizer.momentum,attack.init.model.art.initialize.optimizer.name,attack.init.model.art.library,attack.init.model.art.preprocessor.bit_depth,attack.init.model.art.preprocessor.clip_values,attack.init.model.art.preprocessor.name,attack.init.model.art.preprocessor.params.bit_depth,attack.init.model.data._target_,attack.init.model.data.generate._target_,attack.init.model.data.generate.name,attack.init.model.data.generate.path,attack.init.model.data.sample._target_,attack.init.model.data.sample.random_state,attack.init.model.data.sample.stratify,attack.init.model.data.sklearn_pipeline._target_,attack.init.model.data.sklearn_pipeline.preprocessor.name,attack.init.model.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.init._target_,attack.init.model.init.name,attack.init.model.init.num_channels,attack.init.model.init.num_classes,attack.init.model.library,attack.init.model.trainer.batch_size,attack.init.model.trainer.nb_epoch,attack.init.name,attack.init.targeted,attack.model._target_,attack.model.art._target_,attack.model.art.data._target_,attack.model.art.data.generate._target_,attack.model.art.data.generate.name,attack.model.art.data.generate.path,attack.model.art.data.sample._target_,attack.model.art.data.sample.random_state,attack.model.art.data.sample.stratify,attack.model.art.data.sklearn_pipeline._target_,attack.model.art.data.sklearn_pipeline.preprocessor.name,attack.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.model.art.initialize.clip_values,attack.model.art.initialize.criterion.name,attack.model.art.initialize.optimizer.lr,attack.model.art.initialize.optimizer.momentum,attack.model.art.initialize.optimizer.name,attack.model.art.library,attack.model.art.preprocessor.bit_depth,attack.model.art.preprocessor.clip_values,attack.model.art.preprocessor.name,attack.model.art.preprocessor.params.bit_depth,attack.model.data._target_,attack.model.data.generate._target_,attack.model.data.generate.name,attack.model.data.generate.path,attack.model.data.sample._target_,attack.model.data.sample.random_state,attack.model.data.sample.stratify,attack.model.data.sklearn_pipeline._target_,attack.model.data.sklearn_pipeline.preprocessor.name,attack.model.data.sklearn_pipeline.preprocessor.with_mean,attack.model.data.sklearn_pipeline.preprocessor.with_std,attack.model.init._target_,attack.model.init.name,attack.model.init.num_channels,attack.model.init.num_classes,attack.model.library,attack.model.trainer.batch_size,attack.model.trainer.nb_epoch,data._target_,data.generate._target_,data.generate.name,data.generate.path,data.sample._target_,data.sample.random_state,data.sample.stratify,data.sklearn_pipeline._target_,data.sklearn_pipeline.preprocessor.name,data.sklearn_pipeline.preprocessor.with_mean,data.sklearn_pipeline.preprocessor.with_std,device_id.1,files._target_,files.adv_predictions_file,files.attack_dir,files.attack_file,files.attack_type,files.data_dir,files.data_file,files.data_type,files.directory,files.name,files.predictions_file,files.reports,files.score_dict_file,files.stage,files.model_dir,files.model_file,files.model_type,files.params_file,model._target_,model.library,model.art._target_,model.art.data._target_,model.art.data.generate._target_,model.art.data.generate.name,model.art.data.generate.path,model.art.data.sample._target_,model.art.data.sample.random_state,model.art.data.sample.stratify,model.art.data.sklearn_pipeline._target_,model.art.data.sklearn_pipeline.preprocessor.name,model.art.data.sklearn_pipeline.preprocessor.with_mean,model.art.data.sklearn_pipeline.preprocessor.with_std,model.art.initialize.clip_values,model.art.initialize.criterion.name,model.art.initialize.optimizer.lr,model.art.initialize.optimizer.momentum,model.art.initialize.optimizer.name,model.art.library,model.art.preprocessor.bit_depth,model.art.preprocessor.clip_values,model.art.preprocessor.name,model.art.preprocessor.params.bit_depth,model.data._target_,model.data.generate._target_,model.data.generate.name,model.data.generate.path,model.data.sample._target_,model.data.sample.random_state,model.data.sample.stratify,model.data.sklearn_pipeline._target_,model.data.sklearn_pipeline.preprocessor.name,model.data.sklearn_pipeline.preprocessor.with_mean,model.data.sklearn_pipeline.preprocessor.with_std,model.init._target_,model.init.name,model.init.num_channels,model.init.num_classes,model.trainer.batch_size,model.trainer.nb_epoch,name.1,scorers._target_,scorers.accuracy._target_,scorers.accuracy.direction,scorers.accuracy.name,scorers.log_loss._target_,scorers.log_loss.direction,scorers.log_loss.name,train_time.1,train_time_per_sample.1,train_start_time.1,train_end_time.1,train_device.1,predict_time.1,predict_time_per_sample.1,predict_start_time.1,predict_end_time.1,predict_device.1,predict_proba_time.1,predict_proba_time_per_sample.1,predict_proba_start_time.1,predict_proba_end_time.1,predict_proba_device.1,predict_log_proba_time.1,predict_log_proba_time_per_sample.1,predict_log_proba_start_time.1,predict_log_proba_end_time.1,predict_log_device.1,adv_fit_time_per_sample.1,adv_fit_time.1,adv_fit_start_time.1,adv_fit_end_time.1,adv_fit_device.1,adv_predict_time_per_sample.1,adv_predict_time.1,adv_predict_start_time.1,adv_predict_end_time.1,adv_predict_device.1,adv_success.1,accuracy.1,log_loss.1,adv_accuracy.1,adv_log_loss.1,predictions.1,adv_predictions.1,train_power,peak_power,predict_power,predict_proba_power,predict_log_proba_power,adv_fit_power,adv_predict_power +,stage,_target_,attack,data,device_id,files,model,name,scorers,train_time,train_time_per_sample,train_start_time,train_end_time,train_device,predict_time,predict_time_per_sample,predict_start_time,predict_end_time,predict_device,predict_proba_time,predict_proba_time_per_sample,predict_proba_start_time,predict_proba_end_time,predict_proba_device,predict_log_proba_time,predict_log_proba_time_per_sample,predict_log_proba_start_time,predict_log_proba_end_time,predict_log_device,adv_fit_time_per_sample,adv_fit_time,adv_fit_start_time,adv_fit_end_time,adv_fit_device,adv_predict_time_per_sample,adv_predict_time,adv_predict_start_time,adv_predict_end_time,adv_predict_device,adv_success,accuracy,log_loss,adv_accuracy,adv_log_loss,predictions,adv_predictions,stage.1,_target_.1,attack._target_,attack.attack_size,attack.method,attack.data._target_,attack.data.generate._target_,attack.data.generate.name,attack.data.generate.path,attack.data.sample._target_,attack.data.sample.random_state,attack.data.sample.stratify,attack.data.sklearn_pipeline._target_,attack.data.sklearn_pipeline.preprocessor.name,attack.data.sklearn_pipeline.preprocessor.with_mean,attack.data.sklearn_pipeline.preprocessor.with_std,attack.init._target_,attack.init.batch_size,attack.init.eps,attack.init.minimal,attack.init.model._target_,attack.init.model.art._target_,attack.init.model.art.data._target_,attack.init.model.art.data.generate._target_,attack.init.model.art.data.generate.name,attack.init.model.art.data.generate.path,attack.init.model.art.data.sample._target_,attack.init.model.art.data.sample.random_state,attack.init.model.art.data.sample.stratify,attack.init.model.art.data.sklearn_pipeline._target_,attack.init.model.art.data.sklearn_pipeline.preprocessor.name,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.art.initialize.clip_values,attack.init.model.art.initialize.criterion.name,attack.init.model.art.initialize.optimizer.lr,attack.init.model.art.initialize.optimizer.momentum,attack.init.model.art.initialize.optimizer.name,attack.init.model.art.library,attack.init.model.art.preprocessor.bit_depth,attack.init.model.art.preprocessor.clip_values,attack.init.model.art.preprocessor.name,attack.init.model.art.preprocessor.params.bit_depth,attack.init.model.data._target_,attack.init.model.data.generate._target_,attack.init.model.data.generate.name,attack.init.model.data.generate.path,attack.init.model.data.sample._target_,attack.init.model.data.sample.random_state,attack.init.model.data.sample.stratify,attack.init.model.data.sklearn_pipeline._target_,attack.init.model.data.sklearn_pipeline.preprocessor.name,attack.init.model.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.init._target_,attack.init.model.init.name,attack.init.model.init.num_channels,attack.init.model.init.num_classes,attack.init.model.library,attack.init.model.trainer.batch_size,attack.init.model.trainer.np_epochs,attack.init.name,attack.init.targeted,attack.model._target_,attack.model.art._target_,attack.model.art.data._target_,attack.model.art.data.generate._target_,attack.model.art.data.generate.name,attack.model.art.data.generate.path,attack.model.art.data.sample._target_,attack.model.art.data.sample.random_state,attack.model.art.data.sample.stratify,attack.model.art.data.sklearn_pipeline._target_,attack.model.art.data.sklearn_pipeline.preprocessor.name,attack.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.model.art.initialize.clip_values,attack.model.art.initialize.criterion.name,attack.model.art.initialize.optimizer.lr,attack.model.art.initialize.optimizer.momentum,attack.model.art.initialize.optimizer.name,attack.model.art.library,attack.model.art.preprocessor.bit_depth,attack.model.art.preprocessor.clip_values,attack.model.art.preprocessor.name,attack.model.art.preprocessor.params.bit_depth,attack.model.data._target_,attack.model.data.generate._target_,attack.model.data.generate.name,attack.model.data.generate.path,attack.model.data.sample._target_,attack.model.data.sample.random_state,attack.model.data.sample.stratify,attack.model.data.sklearn_pipeline._target_,attack.model.data.sklearn_pipeline.preprocessor.name,attack.model.data.sklearn_pipeline.preprocessor.with_mean,attack.model.data.sklearn_pipeline.preprocessor.with_std,attack.model.init._target_,attack.model.init.name,attack.model.init.num_channels,attack.model.init.num_classes,attack.model.library,attack.model.trainer.batch_size,attack.model.trainer.np_epochs,data._target_,data.generate._target_,data.generate.name,data.generate.path,data.sample._target_,data.sample.random_state,data.sample.stratify,data.sklearn_pipeline._target_,data.sklearn_pipeline.preprocessor.name,data.sklearn_pipeline.preprocessor.with_mean,data.sklearn_pipeline.preprocessor.with_std,device_id.1,files._target_,files.adv_predictions_file,files.attack_dir,files.attack_file,files.attack_type,files.data_dir,files.data_file,files.data_type,files.directory,files.name,files.predictions_file,files.reports,files.score_dict_file,files.stage,files.model_dir,files.model_file,files.model_type,files.params_file,model._target_,model.library,model.art._target_,model.art.data._target_,model.art.data.generate._target_,model.art.data.generate.name,model.art.data.generate.path,model.art.data.sample._target_,model.art.data.sample.random_state,model.art.data.sample.stratify,model.art.data.sklearn_pipeline._target_,model.art.data.sklearn_pipeline.preprocessor.name,model.art.data.sklearn_pipeline.preprocessor.with_mean,model.art.data.sklearn_pipeline.preprocessor.with_std,model.art.initialize.clip_values,model.art.initialize.criterion.name,model.art.initialize.optimizer.lr,model.art.initialize.optimizer.momentum,model.art.initialize.optimizer.name,model.art.library,model.art.preprocessor.bit_depth,model.art.preprocessor.clip_values,model.art.preprocessor.name,model.art.preprocessor.params.bit_depth,model.data._target_,model.data.generate._target_,model.data.generate.name,model.data.generate.path,model.data.sample._target_,model.data.sample.random_state,model.data.sample.stratify,model.data.sklearn_pipeline._target_,model.data.sklearn_pipeline.preprocessor.name,model.data.sklearn_pipeline.preprocessor.with_mean,model.data.sklearn_pipeline.preprocessor.with_std,model.init._target_,model.init.name,model.init.num_channels,model.init.num_classes,model.trainer.batch_size,model.trainer.np_epochs,name.1,scorers._target_,scorers.accuracy._target_,scorers.accuracy.direction,scorers.accuracy.name,scorers.log_loss._target_,scorers.log_loss.direction,scorers.log_loss.name,train_time.1,train_time_per_sample.1,train_start_time.1,train_end_time.1,train_device.1,predict_time.1,predict_time_per_sample.1,predict_start_time.1,predict_end_time.1,predict_device.1,predict_proba_time.1,predict_proba_time_per_sample.1,predict_proba_start_time.1,predict_proba_end_time.1,predict_proba_device.1,predict_log_proba_time.1,predict_log_proba_time_per_sample.1,predict_log_proba_start_time.1,predict_log_proba_end_time.1,predict_log_device.1,adv_fit_time_per_sample.1,adv_fit_time.1,adv_fit_start_time.1,adv_fit_end_time.1,adv_fit_device.1,adv_predict_time_per_sample.1,adv_predict_time.1,adv_predict_start_time.1,adv_predict_end_time.1,adv_predict_device.1,adv_success.1,accuracy.1,log_loss.1,adv_accuracy.1,adv_log_loss.1,predictions.1,adv_predictions.1,train_power,peak_power,predict_power,predict_proba_power,predict_log_proba_power,adv_fit_power,adv_predict_power 0134dd382db493953f3dbe49acad5992,attack,deckard.base.experiment.Experiment,0134dd382db493953f3dbe49acad5992,0134dd382db493953f3dbe49acad5992,nvidia-l4,0134dd382db493953f3dbe49acad5992,0134dd382db493953f3dbe49acad5992,0134dd382db493953f3dbe49acad5992,0134dd382db493953f3dbe49acad5992,734.173509267,0.0152952814430625,1702072069.2122278,1702072802.6832693,cuda:0,1.567335093,3.26528144375e-05,1702072802.6867418,1702072804.2533498,cuda:0,1.53759657,3.2033261875e-05,1702072806.1664176,1702072807.7034106,cuda:0,1.544730235,3.2181879895833334e-05,1702072807.703498,1702072809.2476144,cuda:0,0.00395196246,0.395196246,1702072814.2535863,1702072814.648875,cuda:0,0.00016669583,0.016669583,1702072816.8321471,1702072816.8488245,cuda:0,0.99,0.4274166666666666,4.31787318434318,0.02,20.92195061922073,/result/bit_depth/cifar100/reports/attack/0134dd382db493953f3dbe49acad5992/predictions.json,/result/bit_depth/cifar100/reports/attack/0134dd382db493953f3dbe49acad5992/adv_predictions.json,attack,deckard.base.experiment.Experiment,deckard.base.attack.Attack,100,evasion,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.attack.AttackInitializer,54,0.8730770044270277,True,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0366166076617675,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,64.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,100,pytorch,54,4,art.attacks.evasion.FastGradientMethod,False,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0366166076617675,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,64.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,100,pytorch,54,4,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,nvidia-l4,deckard.base.files.FileConfig,adv_predictions.json,attacks,2dd7e0a7fa9c3ae3b0a925062d3fceee,.pkl,data,a34af6e396d29867f56b8d059aeed97d,.pkl,/result/bit_depth/cifar100/,0134dd382db493953f3dbe49acad5992,predictions.json,reports,score_dict.json,attack,,,,,deckard.base.model.Model,pytorch,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0366166076617675,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,64.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,100,54,4,0134dd382db493953f3dbe49acad5992,deckard.base.scorer.ScorerDict,deckard.base.scorer.ScorerConfig,maximize,sklearn.metrics.accuracy_score,deckard.base.scorer.ScorerConfig,minimize,sklearn.metrics.log_loss,734.173509267,0.0152952814430625,1702072069.2122278,1702072802.6832693,cuda:0,1.567335093,3.26528144375e-05,1702072802.6867418,1702072804.2533498,cuda:0,1.53759657,3.2033261875e-05,1702072806.1664176,1702072807.7034106,cuda:0,1.544730235,3.2181879895833334e-05,1702072807.703498,1702072809.2476144,cuda:0,0.00395196246,0.395196246,1702072814.2535863,1702072814.648875,cuda:0,0.00016669583,0.016669583,1702072816.8321471,1702072816.8488245,cuda:0,0.99,0.4274166666666666,4.31787318434318,0.02,20.92195061922073,/result/bit_depth/cifar100/reports/attack/0134dd382db493953f3dbe49acad5992/predictions.json,/result/bit_depth/cifar100/reports/attack/0134dd382db493953f3dbe49acad5992/adv_predictions.json,12.781466368150234,72,0.03611031329631806,0.0321446721611023,0.02651248025417328,0.009235525322914124,0.00027317547798156737 0159054e647e48d345be73830fca5f18,attack,deckard.base.experiment.Experiment,0159054e647e48d345be73830fca5f18,0159054e647e48d345be73830fca5f18,nvidia-l4,0159054e647e48d345be73830fca5f18,0159054e647e48d345be73830fca5f18,0159054e647e48d345be73830fca5f18,0159054e647e48d345be73830fca5f18,29.738640305,0.0006195550063541,1701911782.0541124,1701911811.0839908,cuda:0,1.049919694,2.187332695833333e-05,1701911811.0842588,1701911812.1338742,cuda:0,1.041458552,2.1697053166666666e-05,1701911813.9797916,1701911815.0209675,cuda:0,1.032438512,2.1509135666666662e-05,1701911815.021042,1701911816.053167,cuda:0,0.00155912063,0.155912063,1701911820.9773595,1701911821.1333466,cuda:0,0.00010296244,0.010296244,1701911822.251412,1701911822.2617147,cuda:0,0.99,0.3295,5.108585190276305,0.02,21.628998374938966,/result/bit_depth/cifar100/reports/attack/0159054e647e48d345be73830fca5f18/predictions.json,/result/bit_depth/cifar100/reports/attack/0159054e647e48d345be73830fca5f18/adv_predictions.json,attack,deckard.base.experiment.Experiment,deckard.base.attack.Attack,100,evasion,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.attack.AttackInitializer,663,0.8790577631413385,True,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0675517597532005,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,64.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,100,pytorch,663,5,art.attacks.evasion.FastGradientMethod,False,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0675517597532005,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,64.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,100,pytorch,663,5,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,nvidia-l4,deckard.base.files.FileConfig,adv_predictions.json,attacks,4268da1707255fd58db9b5f0d3b9fcef,.pkl,data,a34af6e396d29867f56b8d059aeed97d,.pkl,/result/bit_depth/cifar100/,0159054e647e48d345be73830fca5f18,predictions.json,reports,score_dict.json,attack,,,,,deckard.base.model.Model,pytorch,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0675517597532005,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,64.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,100,663,5,0159054e647e48d345be73830fca5f18,deckard.base.scorer.ScorerDict,deckard.base.scorer.ScorerConfig,maximize,sklearn.metrics.accuracy_score,deckard.base.scorer.ScorerConfig,minimize,sklearn.metrics.log_loss,29.738640305,0.0006195550063541,1701911782.0541124,1701911811.0839908,cuda:0,1.049919694,2.187332695833333e-05,1701911811.0842588,1701911812.1338742,cuda:0,1.041458552,2.1697053166666666e-05,1701911813.9797916,1701911815.0209675,cuda:0,1.032438512,2.1509135666666662e-05,1701911815.021042,1701911816.053167,cuda:0,0.00155912063,0.155912063,1701911820.9773595,1701911821.1333466,cuda:0,0.00010296244,0.010296244,1701911822.251412,1701911822.2617147,cuda:0,0.99,0.3295,5.108585190276305,0.02,21.628998374938966,/result/bit_depth/cifar100/reports/attack/0159054e647e48d345be73830fca5f18/predictions.json,/result/bit_depth/cifar100/reports/attack/0159054e647e48d345be73830fca5f18/adv_predictions.json,0.546864848883152,72,0.023981612274169922,0.018707847534179686,0.024023741411209106,0.002650219542980194,0.00019956488847732543 01b68ffaf4ae57f2a1bd9e0df87fe19a,attack,deckard.base.experiment.Experiment,01b68ffaf4ae57f2a1bd9e0df87fe19a,01b68ffaf4ae57f2a1bd9e0df87fe19a,nvidia-l4,01b68ffaf4ae57f2a1bd9e0df87fe19a,01b68ffaf4ae57f2a1bd9e0df87fe19a,01b68ffaf4ae57f2a1bd9e0df87fe19a,01b68ffaf4ae57f2a1bd9e0df87fe19a,19.679277864,0.0004099849555,1701917817.9849825,1701917836.938026,cuda:0,1.083913805,2.2581537604166668e-05,1701917836.9407887,1701917838.0243511,cuda:0,1.083481602,2.2572533375e-05,1701917839.8616965,1701917840.944965,cuda:0,1.086234976,2.262989533333333e-05,1701917840.9450345,1701917842.0310452,cuda:0,0.0009751751,0.09751751,1701917846.959299,1701917847.0568178,cuda:0,9.812236e-05,0.009812236,1701917848.2390256,1701917848.2488434,cuda:0,1.0,0.081,11.689022366046906,0.01,18.99685847043991,/result/bit_depth/cifar100/reports/attack/01b68ffaf4ae57f2a1bd9e0df87fe19a/predictions.json,/result/bit_depth/cifar100/reports/attack/01b68ffaf4ae57f2a1bd9e0df87fe19a/adv_predictions.json,attack,deckard.base.experiment.Experiment,deckard.base.attack.Attack,100,evasion,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.attack.AttackInitializer,9972,0.3772749664989439,True,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.2240192946635449,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,8.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,100,pytorch,9972,93,art.attacks.evasion.FastGradientMethod,False,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.2240192946635449,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,8.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,100,pytorch,9972,93,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,nvidia-l4,deckard.base.files.FileConfig,adv_predictions.json,attacks,ebe92bb5bc6097f6b64d551fbe3288cc,.pkl,data,a34af6e396d29867f56b8d059aeed97d,.pkl,/result/bit_depth/cifar100/,01b68ffaf4ae57f2a1bd9e0df87fe19a,predictions.json,reports,score_dict.json,attack,,,,,deckard.base.model.Model,pytorch,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.2240192946635449,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,8.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,100,9972,93,01b68ffaf4ae57f2a1bd9e0df87fe19a,deckard.base.scorer.ScorerDict,deckard.base.scorer.ScorerConfig,maximize,sklearn.metrics.accuracy_score,deckard.base.scorer.ScorerConfig,minimize,sklearn.metrics.log_loss,19.679277864,0.0004099849555,1701917817.9849825,1701917836.938026,cuda:0,1.083913805,2.2581537604166668e-05,1701917836.9407887,1701917838.0243511,cuda:0,1.083481602,2.2572533375e-05,1701917839.8616965,1701917840.944965,cuda:0,1.086234976,2.262989533333333e-05,1701917840.9450345,1701917842.0310452,cuda:0,0.0009751751,0.09751751,1701917846.959299,1701917847.0568178,cuda:0,9.812236e-05,0.009812236,1701917848.2390256,1701917848.2488434,cuda:0,1.0,0.081,11.689022366046906,0.01,18.99685847043991,/result/bit_depth/cifar100/reports/attack/01b68ffaf4ae57f2a1bd9e0df87fe19a/predictions.json,/result/bit_depth/cifar100/reports/attack/01b68ffaf4ae57f2a1bd9e0df87fe19a/adv_predictions.json,0.3195483127498627,72,0.024156939568519595,0.02446020056247711,0.02113594013643265,0.0016560622658729552,0.00018734399747848512 diff --git a/examples/power/plots/data/bit_depth/mnist/power.csv b/examples/power/plots/data/bit_depth/mnist/power.csv index d5b8ed43..6f33cf9d 100644 --- a/examples/power/plots/data/bit_depth/mnist/power.csv +++ b/examples/power/plots/data/bit_depth/mnist/power.csv @@ -1,4 +1,4 @@ -,stage,_target_,attack,data,device_id,files,model,name,scorers,train_time,train_time_per_sample,train_start_time,train_end_time,train_device,predict_time,predict_time_per_sample,predict_start_time,predict_end_time,predict_device,predict_proba_time,predict_proba_time_per_sample,predict_proba_start_time,predict_proba_end_time,predict_proba_device,predict_log_proba_time,predict_log_proba_time_per_sample,predict_log_proba_start_time,predict_log_proba_end_time,predict_log_device,adv_fit_time_per_sample,adv_fit_time,adv_fit_start_time,adv_fit_end_time,adv_fit_device,adv_predict_time_per_sample,adv_predict_time,adv_predict_start_time,adv_predict_end_time,adv_predict_device,adv_success,accuracy,log_loss,adv_accuracy,adv_log_loss,predictions,adv_predictions,stage.1,_target_.1,attack._target_,attack.attack_size,attack.method,attack.data._target_,attack.data.generate._target_,attack.data.generate.name,attack.data.sample._target_,attack.data.sample.random_state,attack.data.sample.stratify,attack.data.sklearn_pipeline._target_,attack.data.sklearn_pipeline.preprocessor.name,attack.data.sklearn_pipeline.preprocessor.with_mean,attack.data.sklearn_pipeline.preprocessor.with_std,attack.init._target_,attack.init.batch_size,attack.init.eps,attack.init.minimal,attack.init.model._target_,attack.init.model.art._target_,attack.init.model.art.data._target_,attack.init.model.art.data.generate._target_,attack.init.model.art.data.generate.name,attack.init.model.art.data.sample._target_,attack.init.model.art.data.sample.random_state,attack.init.model.art.data.sample.stratify,attack.init.model.art.data.sklearn_pipeline._target_,attack.init.model.art.data.sklearn_pipeline.preprocessor.name,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.art.initialize.clip_values,attack.init.model.art.initialize.criterion.name,attack.init.model.art.initialize.optimizer.lr,attack.init.model.art.initialize.optimizer.momentum,attack.init.model.art.initialize.optimizer.name,attack.init.model.art.library,attack.init.model.art.preprocessor.bit_depth,attack.init.model.art.preprocessor.clip_values,attack.init.model.art.preprocessor.name,attack.init.model.art.preprocessor.params.bit_depth,attack.init.model.data._target_,attack.init.model.data.generate._target_,attack.init.model.data.generate.name,attack.init.model.data.sample._target_,attack.init.model.data.sample.random_state,attack.init.model.data.sample.stratify,attack.init.model.data.sklearn_pipeline._target_,attack.init.model.data.sklearn_pipeline.preprocessor.name,attack.init.model.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.init._target_,attack.init.model.init.name,attack.init.model.init.num_channels,attack.init.model.library,attack.init.model.trainer.batch_size,attack.init.model.trainer.nb_epoch,attack.init.name,attack.init.targeted,attack.model._target_,attack.model.art._target_,attack.model.art.data._target_,attack.model.art.data.generate._target_,attack.model.art.data.generate.name,attack.model.art.data.sample._target_,attack.model.art.data.sample.random_state,attack.model.art.data.sample.stratify,attack.model.art.data.sklearn_pipeline._target_,attack.model.art.data.sklearn_pipeline.preprocessor.name,attack.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.model.art.initialize.clip_values,attack.model.art.initialize.criterion.name,attack.model.art.initialize.optimizer.lr,attack.model.art.initialize.optimizer.momentum,attack.model.art.initialize.optimizer.name,attack.model.art.library,attack.model.art.preprocessor.bit_depth,attack.model.art.preprocessor.clip_values,attack.model.art.preprocessor.name,attack.model.art.preprocessor.params.bit_depth,attack.model.data._target_,attack.model.data.generate._target_,attack.model.data.generate.name,attack.model.data.sample._target_,attack.model.data.sample.random_state,attack.model.data.sample.stratify,attack.model.data.sklearn_pipeline._target_,attack.model.data.sklearn_pipeline.preprocessor.name,attack.model.data.sklearn_pipeline.preprocessor.with_mean,attack.model.data.sklearn_pipeline.preprocessor.with_std,attack.model.init._target_,attack.model.init.name,attack.model.init.num_channels,attack.model.library,attack.model.trainer.batch_size,attack.model.trainer.nb_epoch,data._target_,data.generate._target_,data.generate.name,data.sample._target_,data.sample.random_state,data.sample.stratify,data.sklearn_pipeline._target_,data.sklearn_pipeline.preprocessor.name,data.sklearn_pipeline.preprocessor.with_mean,data.sklearn_pipeline.preprocessor.with_std,device_id.1,files._target_,files.adv_predictions_file,files.attack_dir,files.attack_file,files.attack_type,files.data_dir,files.data_file,files.data_type,files.directory,files.name,files.predictions_file,files.reports,files.score_dict_file,files.stage,files.model_dir,files.model_file,files.model_type,files.params_file,model._target_,model.library,model.art._target_,model.art.data._target_,model.art.data.generate._target_,model.art.data.generate.name,model.art.data.sample._target_,model.art.data.sample.random_state,model.art.data.sample.stratify,model.art.data.sklearn_pipeline._target_,model.art.data.sklearn_pipeline.preprocessor.name,model.art.data.sklearn_pipeline.preprocessor.with_mean,model.art.data.sklearn_pipeline.preprocessor.with_std,model.art.initialize.clip_values,model.art.initialize.criterion.name,model.art.initialize.optimizer.lr,model.art.initialize.optimizer.momentum,model.art.initialize.optimizer.name,model.art.library,model.art.preprocessor.bit_depth,model.art.preprocessor.clip_values,model.art.preprocessor.name,model.art.preprocessor.params.bit_depth,model.data._target_,model.data.generate._target_,model.data.generate.name,model.data.sample._target_,model.data.sample.random_state,model.data.sample.stratify,model.data.sklearn_pipeline._target_,model.data.sklearn_pipeline.preprocessor.name,model.data.sklearn_pipeline.preprocessor.with_mean,model.data.sklearn_pipeline.preprocessor.with_std,model.init._target_,model.init.name,model.init.num_channels,model.trainer.batch_size,model.trainer.nb_epoch,name.1,scorers._target_,scorers.accuracy._target_,scorers.accuracy.direction,scorers.accuracy.name,scorers.log_loss._target_,scorers.log_loss.direction,scorers.log_loss.name,train_time.1,train_time_per_sample.1,train_start_time.1,train_end_time.1,train_device.1,predict_time.1,predict_time_per_sample.1,predict_start_time.1,predict_end_time.1,predict_device.1,predict_proba_time.1,predict_proba_time_per_sample.1,predict_proba_start_time.1,predict_proba_end_time.1,predict_proba_device.1,predict_log_proba_time.1,predict_log_proba_time_per_sample.1,predict_log_proba_start_time.1,predict_log_proba_end_time.1,predict_log_device.1,adv_fit_time_per_sample.1,adv_fit_time.1,adv_fit_start_time.1,adv_fit_end_time.1,adv_fit_device.1,adv_predict_time_per_sample.1,adv_predict_time.1,adv_predict_start_time.1,adv_predict_end_time.1,adv_predict_device.1,adv_success.1,accuracy.1,log_loss.1,adv_accuracy.1,adv_log_loss.1,predictions.1,adv_predictions.1,train_power,peak_power,predict_power,predict_proba_power,predict_log_proba_power,adv_fit_power,adv_predict_power +,stage,_target_,attack,data,device_id,files,model,name,scorers,train_time,train_time_per_sample,train_start_time,train_end_time,train_device,predict_time,predict_time_per_sample,predict_start_time,predict_end_time,predict_device,predict_proba_time,predict_proba_time_per_sample,predict_proba_start_time,predict_proba_end_time,predict_proba_device,predict_log_proba_time,predict_log_proba_time_per_sample,predict_log_proba_start_time,predict_log_proba_end_time,predict_log_device,adv_fit_time_per_sample,adv_fit_time,adv_fit_start_time,adv_fit_end_time,adv_fit_device,adv_predict_time_per_sample,adv_predict_time,adv_predict_start_time,adv_predict_end_time,adv_predict_device,adv_success,accuracy,log_loss,adv_accuracy,adv_log_loss,predictions,adv_predictions,stage.1,_target_.1,attack._target_,attack.attack_size,attack.method,attack.data._target_,attack.data.generate._target_,attack.data.generate.name,attack.data.sample._target_,attack.data.sample.random_state,attack.data.sample.stratify,attack.data.sklearn_pipeline._target_,attack.data.sklearn_pipeline.preprocessor.name,attack.data.sklearn_pipeline.preprocessor.with_mean,attack.data.sklearn_pipeline.preprocessor.with_std,attack.init._target_,attack.init.batch_size,attack.init.eps,attack.init.minimal,attack.init.model._target_,attack.init.model.art._target_,attack.init.model.art.data._target_,attack.init.model.art.data.generate._target_,attack.init.model.art.data.generate.name,attack.init.model.art.data.sample._target_,attack.init.model.art.data.sample.random_state,attack.init.model.art.data.sample.stratify,attack.init.model.art.data.sklearn_pipeline._target_,attack.init.model.art.data.sklearn_pipeline.preprocessor.name,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.art.initialize.clip_values,attack.init.model.art.initialize.criterion.name,attack.init.model.art.initialize.optimizer.lr,attack.init.model.art.initialize.optimizer.momentum,attack.init.model.art.initialize.optimizer.name,attack.init.model.art.library,attack.init.model.art.preprocessor.bit_depth,attack.init.model.art.preprocessor.clip_values,attack.init.model.art.preprocessor.name,attack.init.model.art.preprocessor.params.bit_depth,attack.init.model.data._target_,attack.init.model.data.generate._target_,attack.init.model.data.generate.name,attack.init.model.data.sample._target_,attack.init.model.data.sample.random_state,attack.init.model.data.sample.stratify,attack.init.model.data.sklearn_pipeline._target_,attack.init.model.data.sklearn_pipeline.preprocessor.name,attack.init.model.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.init._target_,attack.init.model.init.name,attack.init.model.init.num_channels,attack.init.model.library,attack.init.model.trainer.batch_size,attack.init.model.trainer.np_epochs,attack.init.name,attack.init.targeted,attack.model._target_,attack.model.art._target_,attack.model.art.data._target_,attack.model.art.data.generate._target_,attack.model.art.data.generate.name,attack.model.art.data.sample._target_,attack.model.art.data.sample.random_state,attack.model.art.data.sample.stratify,attack.model.art.data.sklearn_pipeline._target_,attack.model.art.data.sklearn_pipeline.preprocessor.name,attack.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.model.art.initialize.clip_values,attack.model.art.initialize.criterion.name,attack.model.art.initialize.optimizer.lr,attack.model.art.initialize.optimizer.momentum,attack.model.art.initialize.optimizer.name,attack.model.art.library,attack.model.art.preprocessor.bit_depth,attack.model.art.preprocessor.clip_values,attack.model.art.preprocessor.name,attack.model.art.preprocessor.params.bit_depth,attack.model.data._target_,attack.model.data.generate._target_,attack.model.data.generate.name,attack.model.data.sample._target_,attack.model.data.sample.random_state,attack.model.data.sample.stratify,attack.model.data.sklearn_pipeline._target_,attack.model.data.sklearn_pipeline.preprocessor.name,attack.model.data.sklearn_pipeline.preprocessor.with_mean,attack.model.data.sklearn_pipeline.preprocessor.with_std,attack.model.init._target_,attack.model.init.name,attack.model.init.num_channels,attack.model.library,attack.model.trainer.batch_size,attack.model.trainer.np_epochs,data._target_,data.generate._target_,data.generate.name,data.sample._target_,data.sample.random_state,data.sample.stratify,data.sklearn_pipeline._target_,data.sklearn_pipeline.preprocessor.name,data.sklearn_pipeline.preprocessor.with_mean,data.sklearn_pipeline.preprocessor.with_std,device_id.1,files._target_,files.adv_predictions_file,files.attack_dir,files.attack_file,files.attack_type,files.data_dir,files.data_file,files.data_type,files.directory,files.name,files.predictions_file,files.reports,files.score_dict_file,files.stage,files.model_dir,files.model_file,files.model_type,files.params_file,model._target_,model.library,model.art._target_,model.art.data._target_,model.art.data.generate._target_,model.art.data.generate.name,model.art.data.sample._target_,model.art.data.sample.random_state,model.art.data.sample.stratify,model.art.data.sklearn_pipeline._target_,model.art.data.sklearn_pipeline.preprocessor.name,model.art.data.sklearn_pipeline.preprocessor.with_mean,model.art.data.sklearn_pipeline.preprocessor.with_std,model.art.initialize.clip_values,model.art.initialize.criterion.name,model.art.initialize.optimizer.lr,model.art.initialize.optimizer.momentum,model.art.initialize.optimizer.name,model.art.library,model.art.preprocessor.bit_depth,model.art.preprocessor.clip_values,model.art.preprocessor.name,model.art.preprocessor.params.bit_depth,model.data._target_,model.data.generate._target_,model.data.generate.name,model.data.sample._target_,model.data.sample.random_state,model.data.sample.stratify,model.data.sklearn_pipeline._target_,model.data.sklearn_pipeline.preprocessor.name,model.data.sklearn_pipeline.preprocessor.with_mean,model.data.sklearn_pipeline.preprocessor.with_std,model.init._target_,model.init.name,model.init.num_channels,model.trainer.batch_size,model.trainer.np_epochs,name.1,scorers._target_,scorers.accuracy._target_,scorers.accuracy.direction,scorers.accuracy.name,scorers.log_loss._target_,scorers.log_loss.direction,scorers.log_loss.name,train_time.1,train_time_per_sample.1,train_start_time.1,train_end_time.1,train_device.1,predict_time.1,predict_time_per_sample.1,predict_start_time.1,predict_end_time.1,predict_device.1,predict_proba_time.1,predict_proba_time_per_sample.1,predict_proba_start_time.1,predict_proba_end_time.1,predict_proba_device.1,predict_log_proba_time.1,predict_log_proba_time_per_sample.1,predict_log_proba_start_time.1,predict_log_proba_end_time.1,predict_log_device.1,adv_fit_time_per_sample.1,adv_fit_time.1,adv_fit_start_time.1,adv_fit_end_time.1,adv_fit_device.1,adv_predict_time_per_sample.1,adv_predict_time.1,adv_predict_start_time.1,adv_predict_end_time.1,adv_predict_device.1,adv_success.1,accuracy.1,log_loss.1,adv_accuracy.1,adv_log_loss.1,predictions.1,adv_predictions.1,train_power,peak_power,predict_power,predict_proba_power,predict_log_proba_power,adv_fit_power,adv_predict_power 0064251bfdfc4a9cddb27abeb12ce450,attack,deckard.base.experiment.Experiment,0064251bfdfc4a9cddb27abeb12ce450,0064251bfdfc4a9cddb27abeb12ce450,nvidia-l4,0064251bfdfc4a9cddb27abeb12ce450,0064251bfdfc4a9cddb27abeb12ce450,0064251bfdfc4a9cddb27abeb12ce450,0064251bfdfc4a9cddb27abeb12ce450,31.2411115,0.0005578769910714,1701687540.831812,1701687571.9951878,cuda:0,1.971648677,3.5208012089285714e-05,1701687571.9976866,1701687573.9690738,cuda:0,2.036850342,3.6372327535714286e-05,1701687574.20858,1701687576.2447393,cuda:0,2.024160306,3.614571975e-05,1701687576.2448645,1701687578.2685492,cuda:0,0.00240592733,0.240592733,1701687579.930968,1701687580.1714814,cuda:0,0.00019938792,0.019938792,1701687583.4263215,1701687583.4462671,cuda:0,0.78,0.9722142857142856,1.2880083298009952,0.09,13.013279651999474,/result/bit_depth/mnist/reports/attack/0064251bfdfc4a9cddb27abeb12ce450/predictions.json,/result/bit_depth/mnist/reports/attack/0064251bfdfc4a9cddb27abeb12ce450/adv_predictions.json,attack,deckard.base.experiment.Experiment,deckard.base.attack.Attack,100,evasion,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.attack.AttackInitializer,2435,0.4864120234576462,True,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0346865819033803,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,8.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,pytorch,2435,10,art.attacks.evasion.FastGradientMethod,False,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0346865819033803,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,8.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,pytorch,2435,10,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,nvidia-l4,deckard.base.files.FileConfig,adv_predictions.json,attacks,6ed9cac2983a494cc4e19fb3f77a6b45,.pkl,data,e10a287ea198f3b54c4d3c142f24a456,.pkl,/result/bit_depth/mnist/,0064251bfdfc4a9cddb27abeb12ce450,predictions.json,reports,score_dict.json,attack,,,,,deckard.base.model.Model,pytorch,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0346865819033803,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,8.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,2435,10,0064251bfdfc4a9cddb27abeb12ce450,deckard.base.scorer.ScorerDict,deckard.base.scorer.ScorerConfig,maximize,sklearn.metrics.accuracy_score,deckard.base.scorer.ScorerConfig,minimize,sklearn.metrics.log_loss,31.2411115,0.0005578769910714,1701687540.831812,1701687571.9951878,cuda:0,1.971648677,3.5208012089285714e-05,1701687571.9976866,1701687573.9690738,cuda:0,2.036850342,3.6372327535714286e-05,1701687574.20858,1701687576.2447393,cuda:0,2.024160306,3.614571975e-05,1701687576.2448645,1701687578.2685492,cuda:0,0.00240592733,0.240592733,1701687579.930968,1701687580.1714814,cuda:0,0.00019938792,0.019938792,1701687583.4263215,1701687583.4462671,cuda:0,0.78,0.9722142857142856,1.2880083298009952,0.09,13.013279651999474,/result/bit_depth/mnist/reports/attack/0064251bfdfc4a9cddb27abeb12ce450/predictions.json,/result/bit_depth/mnist/reports/attack/0064251bfdfc4a9cddb27abeb12ce450/adv_predictions.json,0.8170289228027342,72,0.05243258969535827,0.0400325203125,0.05038975002765656,0.006431518714141846,0.0003974603885650635 00a48db95f8faaa83dd0525f44c1855a,attack,deckard.base.experiment.Experiment,00a48db95f8faaa83dd0525f44c1855a,00a48db95f8faaa83dd0525f44c1855a,nvidia-l4,00a48db95f8faaa83dd0525f44c1855a,00a48db95f8faaa83dd0525f44c1855a,00a48db95f8faaa83dd0525f44c1855a,00a48db95f8faaa83dd0525f44c1855a,19.702645352,0.0003518329527142,1701660379.3342788,1701660398.95372,cuda:0,1.188783624,2.1228279e-05,1701660398.9541066,1701660400.1424274,cuda:0,1.133770342,2.024589896428572e-05,1701660400.4525745,1701660401.5914068,cuda:0,1.184063142,2.114398467857143e-05,1701660401.591488,1701660402.7754624,cuda:0,0.00137451447,0.137451447,1701660404.4032514,1701660404.5407138,cuda:0,0.00011982578,0.011982578,1701660406.2865784,1701660406.2985675,cuda:0,1.0,0.6505,3.3935765696046136,0.15,6.529011898040771,/result/bit_depth/mnist/reports/attack/00a48db95f8faaa83dd0525f44c1855a/predictions.json,/result/bit_depth/mnist/reports/attack/00a48db95f8faaa83dd0525f44c1855a/adv_predictions.json,attack,deckard.base.experiment.Experiment,deckard.base.attack.Attack,100,evasion,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,9,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.attack.AttackInitializer,4877,0.4997503597621268,True,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,9,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,6.621698552904389e-05,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,32.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,9,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,pytorch,4877,13,art.attacks.evasion.FastGradientMethod,False,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,9,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,6.621698552904389e-05,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,32.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,9,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,pytorch,4877,13,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,9,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,nvidia-l4,deckard.base.files.FileConfig,adv_predictions.json,attacks,50ddff1f7c875102844c02ba3bd5ad09,.pkl,data,148b964031a53868653088dc184aff04,.pkl,/result/bit_depth/mnist/,00a48db95f8faaa83dd0525f44c1855a,predictions.json,reports,score_dict.json,attack,,,,,deckard.base.model.Model,pytorch,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,9,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,6.621698552904389e-05,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,32.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,9,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,4877,13,00a48db95f8faaa83dd0525f44c1855a,deckard.base.scorer.ScorerDict,deckard.base.scorer.ScorerConfig,maximize,sklearn.metrics.accuracy_score,deckard.base.scorer.ScorerConfig,minimize,sklearn.metrics.log_loss,19.702645352,0.0003518329527142,1701660379.3342788,1701660398.95372,cuda:0,1.188783624,2.1228279e-05,1701660398.9541066,1701660400.1424274,cuda:0,1.133770342,2.024589896428572e-05,1701660400.4525745,1701660401.5914068,cuda:0,1.184063142,2.114398467857143e-05,1701660401.591488,1701660402.7754624,cuda:0,0.00137451447,0.137451447,1701660404.4032514,1701660404.5407138,cuda:0,0.00011982578,0.011982578,1701660406.2865784,1701660406.2985675,cuda:0,1.0,0.6505,3.3935765696046136,0.15,6.529011898040771,/result/bit_depth/mnist/reports/attack/00a48db95f8faaa83dd0525f44c1855a/predictions.json,/result/bit_depth/mnist/reports/attack/00a48db95f8faaa83dd0525f44c1855a/adv_predictions.json,0.46356815834712983,72,0.032834731765937805,0.025612794650459292,0.032277039351081845,0.00320705225315094,0.0002466784732818604 00bb2bbf8bcbd0dec970f551d536630b,attack,deckard.base.experiment.Experiment,00bb2bbf8bcbd0dec970f551d536630b,00bb2bbf8bcbd0dec970f551d536630b,nvidia-l4,00bb2bbf8bcbd0dec970f551d536630b,00bb2bbf8bcbd0dec970f551d536630b,00bb2bbf8bcbd0dec970f551d536630b,00bb2bbf8bcbd0dec970f551d536630b,19.292816381,0.0003445145782321,1701652841.3548884,1701652860.5475855,cuda:0,0.618399955,1.1042856339285717e-05,1701652860.549899,1701652861.168158,cuda:0,0.635787219,1.1353343196428571e-05,1701652861.4213982,1701652862.0572605,cuda:0,0.600712895,1.0727015982142858e-05,1701652862.0573397,1701652862.6579626,cuda:0,0.00075029189,0.075029189,1701652864.252877,1701652864.3278887,cuda:0,6.265753e-05,0.006265753,1701652865.0728152,1701652865.0790927,cuda:0,0.5,0.9743571428571428,1.1329994824799776,0.11,20.688893682956696,/result/bit_depth/mnist/reports/attack/00bb2bbf8bcbd0dec970f551d536630b/predictions.json,/result/bit_depth/mnist/reports/attack/00bb2bbf8bcbd0dec970f551d536630b/adv_predictions.json,attack,deckard.base.experiment.Experiment,deckard.base.attack.Attack,100,evasion,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.attack.AttackInitializer,9269,0.6339545055346733,True,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0094909588900287,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,64.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,pytorch,9269,49,art.attacks.evasion.FastGradientMethod,False,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0094909588900287,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,64.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,pytorch,9269,49,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,nvidia-l4,deckard.base.files.FileConfig,adv_predictions.json,attacks,82bc7b63558d84d675d446570e821128,.pkl,data,20761683b94eca729b58b63071f98327,.pkl,/result/bit_depth/mnist/,00bb2bbf8bcbd0dec970f551d536630b,predictions.json,reports,score_dict.json,attack,,,,,deckard.base.model.Model,pytorch,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0094909588900287,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,64.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,9269,49,00bb2bbf8bcbd0dec970f551d536630b,deckard.base.scorer.ScorerDict,deckard.base.scorer.ScorerConfig,maximize,sklearn.metrics.accuracy_score,deckard.base.scorer.ScorerConfig,minimize,sklearn.metrics.log_loss,19.292816381,0.0003445145782321,1701652841.3548884,1701652860.5475855,cuda:0,0.618399955,1.1042856339285717e-05,1701652860.549899,1701652861.168158,cuda:0,0.635787219,1.1353343196428571e-05,1701652861.4213982,1701652862.0572605,cuda:0,0.600712895,1.0727015982142858e-05,1701652862.0573397,1701652862.6579626,cuda:0,0.00075029189,0.075029189,1701652864.252877,1701652864.3278887,cuda:0,6.265753e-05,0.006265753,1701652865.0728152,1701652865.0790927,cuda:0,0.5,0.9743571428571428,1.1329994824799776,0.11,20.688893682956696,/result/bit_depth/mnist/reports/attack/00bb2bbf8bcbd0dec970f551d536630b/predictions.json,/result/bit_depth/mnist/reports/attack/00bb2bbf8bcbd0dec970f551d536630b/adv_predictions.json,0.551966612948227,72,0.014817441373443601,0.01792826614379883,0.015696438420295715,0.0015547231246948242,0.00014446928215026855 diff --git a/examples/power/plots/data/bit_depth/mnist/raw.csv b/examples/power/plots/data/bit_depth/mnist/raw.csv index 63f37f51..fa423ff9 100644 --- a/examples/power/plots/data/bit_depth/mnist/raw.csv +++ b/examples/power/plots/data/bit_depth/mnist/raw.csv @@ -1,4 +1,4 @@ -,stage,_target_,attack,data,device_id,files,model,name,scorers,train_time,train_time_per_sample,train_start_time,train_end_time,train_device,predict_time,predict_time_per_sample,predict_start_time,predict_end_time,predict_device,predict_proba_time,predict_proba_time_per_sample,predict_proba_start_time,predict_proba_end_time,predict_proba_device,predict_log_proba_time,predict_log_proba_time_per_sample,predict_log_proba_start_time,predict_log_proba_end_time,predict_log_device,adv_fit_time_per_sample,adv_fit_time,adv_fit_start_time,adv_fit_end_time,adv_fit_device,adv_predict_time_per_sample,adv_predict_time,adv_predict_start_time,adv_predict_end_time,adv_predict_device,adv_success,accuracy,log_loss,adv_accuracy,adv_log_loss,predictions,adv_predictions,stage,_target_,attack._target_,attack.attack_size,attack.method,attack.data._target_,attack.data.generate._target_,attack.data.generate.name,attack.data.sample._target_,attack.data.sample.random_state,attack.data.sample.stratify,attack.data.sklearn_pipeline._target_,attack.data.sklearn_pipeline.preprocessor.name,attack.data.sklearn_pipeline.preprocessor.with_mean,attack.data.sklearn_pipeline.preprocessor.with_std,attack.init._target_,attack.init.batch_size,attack.init.eps,attack.init.minimal,attack.init.model._target_,attack.init.model.art._target_,attack.init.model.art.data._target_,attack.init.model.art.data.generate._target_,attack.init.model.art.data.generate.name,attack.init.model.art.data.sample._target_,attack.init.model.art.data.sample.random_state,attack.init.model.art.data.sample.stratify,attack.init.model.art.data.sklearn_pipeline._target_,attack.init.model.art.data.sklearn_pipeline.preprocessor.name,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.art.initialize.clip_values,attack.init.model.art.initialize.criterion.name,attack.init.model.art.initialize.optimizer.lr,attack.init.model.art.initialize.optimizer.momentum,attack.init.model.art.initialize.optimizer.name,attack.init.model.art.library,attack.init.model.art.preprocessor.bit_depth,attack.init.model.art.preprocessor.clip_values,attack.init.model.art.preprocessor.name,attack.init.model.art.preprocessor.params.bit_depth,attack.init.model.data._target_,attack.init.model.data.generate._target_,attack.init.model.data.generate.name,attack.init.model.data.sample._target_,attack.init.model.data.sample.random_state,attack.init.model.data.sample.stratify,attack.init.model.data.sklearn_pipeline._target_,attack.init.model.data.sklearn_pipeline.preprocessor.name,attack.init.model.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.init._target_,attack.init.model.init.name,attack.init.model.init.num_channels,attack.init.model.library,attack.init.model.trainer.batch_size,attack.init.model.trainer.nb_epoch,attack.init.name,attack.init.targeted,attack.model._target_,attack.model.art._target_,attack.model.art.data._target_,attack.model.art.data.generate._target_,attack.model.art.data.generate.name,attack.model.art.data.sample._target_,attack.model.art.data.sample.random_state,attack.model.art.data.sample.stratify,attack.model.art.data.sklearn_pipeline._target_,attack.model.art.data.sklearn_pipeline.preprocessor.name,attack.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.model.art.initialize.clip_values,attack.model.art.initialize.criterion.name,attack.model.art.initialize.optimizer.lr,attack.model.art.initialize.optimizer.momentum,attack.model.art.initialize.optimizer.name,attack.model.art.library,attack.model.art.preprocessor.bit_depth,attack.model.art.preprocessor.clip_values,attack.model.art.preprocessor.name,attack.model.art.preprocessor.params.bit_depth,attack.model.data._target_,attack.model.data.generate._target_,attack.model.data.generate.name,attack.model.data.sample._target_,attack.model.data.sample.random_state,attack.model.data.sample.stratify,attack.model.data.sklearn_pipeline._target_,attack.model.data.sklearn_pipeline.preprocessor.name,attack.model.data.sklearn_pipeline.preprocessor.with_mean,attack.model.data.sklearn_pipeline.preprocessor.with_std,attack.model.init._target_,attack.model.init.name,attack.model.init.num_channels,attack.model.library,attack.model.trainer.batch_size,attack.model.trainer.nb_epoch,data._target_,data.generate._target_,data.generate.name,data.sample._target_,data.sample.random_state,data.sample.stratify,data.sklearn_pipeline._target_,data.sklearn_pipeline.preprocessor.name,data.sklearn_pipeline.preprocessor.with_mean,data.sklearn_pipeline.preprocessor.with_std,device_id,files._target_,files.adv_predictions_file,files.attack_dir,files.attack_file,files.attack_type,files.data_dir,files.data_file,files.data_type,files.directory,files.name,files.predictions_file,files.reports,files.score_dict_file,files.stage,files.model_dir,files.model_file,files.model_type,files.params_file,model._target_,model.library,model.art._target_,model.art.data._target_,model.art.data.generate._target_,model.art.data.generate.name,model.art.data.sample._target_,model.art.data.sample.random_state,model.art.data.sample.stratify,model.art.data.sklearn_pipeline._target_,model.art.data.sklearn_pipeline.preprocessor.name,model.art.data.sklearn_pipeline.preprocessor.with_mean,model.art.data.sklearn_pipeline.preprocessor.with_std,model.art.initialize.clip_values,model.art.initialize.criterion.name,model.art.initialize.optimizer.lr,model.art.initialize.optimizer.momentum,model.art.initialize.optimizer.name,model.art.library,model.art.preprocessor.bit_depth,model.art.preprocessor.clip_values,model.art.preprocessor.name,model.art.preprocessor.params.bit_depth,model.data._target_,model.data.generate._target_,model.data.generate.name,model.data.sample._target_,model.data.sample.random_state,model.data.sample.stratify,model.data.sklearn_pipeline._target_,model.data.sklearn_pipeline.preprocessor.name,model.data.sklearn_pipeline.preprocessor.with_mean,model.data.sklearn_pipeline.preprocessor.with_std,model.init._target_,model.init.name,model.init.num_channels,model.trainer.batch_size,model.trainer.nb_epoch,name,scorers._target_,scorers.accuracy._target_,scorers.accuracy.direction,scorers.accuracy.name,scorers.log_loss._target_,scorers.log_loss.direction,scorers.log_loss.name,train_time,train_time_per_sample,train_start_time,train_end_time,train_device,predict_time,predict_time_per_sample,predict_start_time,predict_end_time,predict_device,predict_proba_time,predict_proba_time_per_sample,predict_proba_start_time,predict_proba_end_time,predict_proba_device,predict_log_proba_time,predict_log_proba_time_per_sample,predict_log_proba_start_time,predict_log_proba_end_time,predict_log_device,adv_fit_time_per_sample,adv_fit_time,adv_fit_start_time,adv_fit_end_time,adv_fit_device,adv_predict_time_per_sample,adv_predict_time,adv_predict_start_time,adv_predict_end_time,adv_predict_device,adv_success,accuracy,log_loss,adv_accuracy,adv_log_loss,predictions,adv_predictions +,stage,_target_,attack,data,device_id,files,model,name,scorers,train_time,train_time_per_sample,train_start_time,train_end_time,train_device,predict_time,predict_time_per_sample,predict_start_time,predict_end_time,predict_device,predict_proba_time,predict_proba_time_per_sample,predict_proba_start_time,predict_proba_end_time,predict_proba_device,predict_log_proba_time,predict_log_proba_time_per_sample,predict_log_proba_start_time,predict_log_proba_end_time,predict_log_device,adv_fit_time_per_sample,adv_fit_time,adv_fit_start_time,adv_fit_end_time,adv_fit_device,adv_predict_time_per_sample,adv_predict_time,adv_predict_start_time,adv_predict_end_time,adv_predict_device,adv_success,accuracy,log_loss,adv_accuracy,adv_log_loss,predictions,adv_predictions,stage,_target_,attack._target_,attack.attack_size,attack.method,attack.data._target_,attack.data.generate._target_,attack.data.generate.name,attack.data.sample._target_,attack.data.sample.random_state,attack.data.sample.stratify,attack.data.sklearn_pipeline._target_,attack.data.sklearn_pipeline.preprocessor.name,attack.data.sklearn_pipeline.preprocessor.with_mean,attack.data.sklearn_pipeline.preprocessor.with_std,attack.init._target_,attack.init.batch_size,attack.init.eps,attack.init.minimal,attack.init.model._target_,attack.init.model.art._target_,attack.init.model.art.data._target_,attack.init.model.art.data.generate._target_,attack.init.model.art.data.generate.name,attack.init.model.art.data.sample._target_,attack.init.model.art.data.sample.random_state,attack.init.model.art.data.sample.stratify,attack.init.model.art.data.sklearn_pipeline._target_,attack.init.model.art.data.sklearn_pipeline.preprocessor.name,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.art.initialize.clip_values,attack.init.model.art.initialize.criterion.name,attack.init.model.art.initialize.optimizer.lr,attack.init.model.art.initialize.optimizer.momentum,attack.init.model.art.initialize.optimizer.name,attack.init.model.art.library,attack.init.model.art.preprocessor.bit_depth,attack.init.model.art.preprocessor.clip_values,attack.init.model.art.preprocessor.name,attack.init.model.art.preprocessor.params.bit_depth,attack.init.model.data._target_,attack.init.model.data.generate._target_,attack.init.model.data.generate.name,attack.init.model.data.sample._target_,attack.init.model.data.sample.random_state,attack.init.model.data.sample.stratify,attack.init.model.data.sklearn_pipeline._target_,attack.init.model.data.sklearn_pipeline.preprocessor.name,attack.init.model.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.init._target_,attack.init.model.init.name,attack.init.model.init.num_channels,attack.init.model.library,attack.init.model.trainer.batch_size,attack.init.model.trainer.np_epochs,attack.init.name,attack.init.targeted,attack.model._target_,attack.model.art._target_,attack.model.art.data._target_,attack.model.art.data.generate._target_,attack.model.art.data.generate.name,attack.model.art.data.sample._target_,attack.model.art.data.sample.random_state,attack.model.art.data.sample.stratify,attack.model.art.data.sklearn_pipeline._target_,attack.model.art.data.sklearn_pipeline.preprocessor.name,attack.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.model.art.initialize.clip_values,attack.model.art.initialize.criterion.name,attack.model.art.initialize.optimizer.lr,attack.model.art.initialize.optimizer.momentum,attack.model.art.initialize.optimizer.name,attack.model.art.library,attack.model.art.preprocessor.bit_depth,attack.model.art.preprocessor.clip_values,attack.model.art.preprocessor.name,attack.model.art.preprocessor.params.bit_depth,attack.model.data._target_,attack.model.data.generate._target_,attack.model.data.generate.name,attack.model.data.sample._target_,attack.model.data.sample.random_state,attack.model.data.sample.stratify,attack.model.data.sklearn_pipeline._target_,attack.model.data.sklearn_pipeline.preprocessor.name,attack.model.data.sklearn_pipeline.preprocessor.with_mean,attack.model.data.sklearn_pipeline.preprocessor.with_std,attack.model.init._target_,attack.model.init.name,attack.model.init.num_channels,attack.model.library,attack.model.trainer.batch_size,attack.model.trainer.np_epochs,data._target_,data.generate._target_,data.generate.name,data.sample._target_,data.sample.random_state,data.sample.stratify,data.sklearn_pipeline._target_,data.sklearn_pipeline.preprocessor.name,data.sklearn_pipeline.preprocessor.with_mean,data.sklearn_pipeline.preprocessor.with_std,device_id,files._target_,files.adv_predictions_file,files.attack_dir,files.attack_file,files.attack_type,files.data_dir,files.data_file,files.data_type,files.directory,files.name,files.predictions_file,files.reports,files.score_dict_file,files.stage,files.model_dir,files.model_file,files.model_type,files.params_file,model._target_,model.library,model.art._target_,model.art.data._target_,model.art.data.generate._target_,model.art.data.generate.name,model.art.data.sample._target_,model.art.data.sample.random_state,model.art.data.sample.stratify,model.art.data.sklearn_pipeline._target_,model.art.data.sklearn_pipeline.preprocessor.name,model.art.data.sklearn_pipeline.preprocessor.with_mean,model.art.data.sklearn_pipeline.preprocessor.with_std,model.art.initialize.clip_values,model.art.initialize.criterion.name,model.art.initialize.optimizer.lr,model.art.initialize.optimizer.momentum,model.art.initialize.optimizer.name,model.art.library,model.art.preprocessor.bit_depth,model.art.preprocessor.clip_values,model.art.preprocessor.name,model.art.preprocessor.params.bit_depth,model.data._target_,model.data.generate._target_,model.data.generate.name,model.data.sample._target_,model.data.sample.random_state,model.data.sample.stratify,model.data.sklearn_pipeline._target_,model.data.sklearn_pipeline.preprocessor.name,model.data.sklearn_pipeline.preprocessor.with_mean,model.data.sklearn_pipeline.preprocessor.with_std,model.init._target_,model.init.name,model.init.num_channels,model.trainer.batch_size,model.trainer.np_epochs,name,scorers._target_,scorers.accuracy._target_,scorers.accuracy.direction,scorers.accuracy.name,scorers.log_loss._target_,scorers.log_loss.direction,scorers.log_loss.name,train_time,train_time_per_sample,train_start_time,train_end_time,train_device,predict_time,predict_time_per_sample,predict_start_time,predict_end_time,predict_device,predict_proba_time,predict_proba_time_per_sample,predict_proba_start_time,predict_proba_end_time,predict_proba_device,predict_log_proba_time,predict_log_proba_time_per_sample,predict_log_proba_start_time,predict_log_proba_end_time,predict_log_device,adv_fit_time_per_sample,adv_fit_time,adv_fit_start_time,adv_fit_end_time,adv_fit_device,adv_predict_time_per_sample,adv_predict_time,adv_predict_start_time,adv_predict_end_time,adv_predict_device,adv_success,accuracy,log_loss,adv_accuracy,adv_log_loss,predictions,adv_predictions 0064251bfdfc4a9cddb27abeb12ce450,attack,deckard.base.experiment.Experiment,0064251bfdfc4a9cddb27abeb12ce450,0064251bfdfc4a9cddb27abeb12ce450,nvidia-l4,0064251bfdfc4a9cddb27abeb12ce450,0064251bfdfc4a9cddb27abeb12ce450,0064251bfdfc4a9cddb27abeb12ce450,0064251bfdfc4a9cddb27abeb12ce450,31.2411115,0.0005578769910714286,1701687540.831812,1701687571.9951878,cuda:0,1.971648677,3.5208012089285714e-05,1701687571.9976866,1701687573.9690738,cuda:0,2.036850342,3.6372327535714286e-05,1701687574.20858,1701687576.2447393,cuda:0,2.024160306,3.614571975e-05,1701687576.2448645,1701687578.2685492,cuda:0,0.00240592733,0.240592733,1701687579.930968,1701687580.1714814,cuda:0,0.00019938792,0.019938792,1701687583.4263217,1701687583.4462671,cuda:0,0.78,0.9722142857142857,1.2880083298009952,0.09,13.013279651999474,/result/bit_depth/mnist/reports/attack/0064251bfdfc4a9cddb27abeb12ce450/predictions.json,/result/bit_depth/mnist/reports/attack/0064251bfdfc4a9cddb27abeb12ce450/adv_predictions.json,attack,deckard.base.experiment.Experiment,deckard.base.attack.Attack,100,evasion,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.attack.AttackInitializer,2435,0.4864120234576462,True,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.03468658190338035,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,8.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,pytorch,2435,10,art.attacks.evasion.FastGradientMethod,False,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.03468658190338035,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,8.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,pytorch,2435,10,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,nvidia-l4,deckard.base.files.FileConfig,adv_predictions.json,attacks,6ed9cac2983a494cc4e19fb3f77a6b45,.pkl,data,e10a287ea198f3b54c4d3c142f24a456,.pkl,/result/bit_depth/mnist/,0064251bfdfc4a9cddb27abeb12ce450,predictions.json,reports,score_dict.json,attack,,,,,deckard.base.model.Model,pytorch,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.03468658190338035,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,8.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,2435,10,0064251bfdfc4a9cddb27abeb12ce450,deckard.base.scorer.ScorerDict,deckard.base.scorer.ScorerConfig,maximize,sklearn.metrics.accuracy_score,deckard.base.scorer.ScorerConfig,minimize,sklearn.metrics.log_loss,31.2411115,0.0005578769910714286,1701687540.831812,1701687571.9951878,cuda:0,1.971648677,3.5208012089285714e-05,1701687571.9976866,1701687573.9690738,cuda:0,2.036850342,3.6372327535714286e-05,1701687574.20858,1701687576.2447393,cuda:0,2.024160306,3.614571975e-05,1701687576.2448645,1701687578.2685492,cuda:0,0.00240592733,0.240592733,1701687579.930968,1701687580.1714814,cuda:0,0.00019938792,0.019938792,1701687583.4263217,1701687583.4462671,cuda:0,0.78,0.9722142857142857,1.2880083298009952,0.09,13.013279651999474,/result/bit_depth/mnist/reports/attack/0064251bfdfc4a9cddb27abeb12ce450/predictions.json,/result/bit_depth/mnist/reports/attack/0064251bfdfc4a9cddb27abeb12ce450/adv_predictions.json 00a48db95f8faaa83dd0525f44c1855a,attack,deckard.base.experiment.Experiment,00a48db95f8faaa83dd0525f44c1855a,00a48db95f8faaa83dd0525f44c1855a,nvidia-l4,00a48db95f8faaa83dd0525f44c1855a,00a48db95f8faaa83dd0525f44c1855a,00a48db95f8faaa83dd0525f44c1855a,00a48db95f8faaa83dd0525f44c1855a,19.702645352,0.0003518329527142857,1701660379.3342788,1701660398.9537199,cuda:0,1.188783624,2.1228279e-05,1701660398.9541066,1701660400.1424274,cuda:0,1.133770342,2.0245898964285716e-05,1701660400.4525743,1701660401.5914068,cuda:0,1.184063142,2.114398467857143e-05,1701660401.591488,1701660402.7754624,cuda:0,0.00137451447,0.137451447,1701660404.4032514,1701660404.5407138,cuda:0,0.00011982578,0.011982578,1701660406.2865784,1701660406.2985675,cuda:0,1.0,0.6505,3.3935765696046136,0.15,6.529011898040771,/result/bit_depth/mnist/reports/attack/00a48db95f8faaa83dd0525f44c1855a/predictions.json,/result/bit_depth/mnist/reports/attack/00a48db95f8faaa83dd0525f44c1855a/adv_predictions.json,attack,deckard.base.experiment.Experiment,deckard.base.attack.Attack,100,evasion,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,9,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.attack.AttackInitializer,4877,0.4997503597621268,True,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,9,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,6.621698552904389e-05,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,32.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,9,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,pytorch,4877,13,art.attacks.evasion.FastGradientMethod,False,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,9,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,6.621698552904389e-05,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,32.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,9,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,pytorch,4877,13,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,9,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,nvidia-l4,deckard.base.files.FileConfig,adv_predictions.json,attacks,50ddff1f7c875102844c02ba3bd5ad09,.pkl,data,148b964031a53868653088dc184aff04,.pkl,/result/bit_depth/mnist/,00a48db95f8faaa83dd0525f44c1855a,predictions.json,reports,score_dict.json,attack,,,,,deckard.base.model.Model,pytorch,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,9,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,6.621698552904389e-05,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,32.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,9,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,4877,13,00a48db95f8faaa83dd0525f44c1855a,deckard.base.scorer.ScorerDict,deckard.base.scorer.ScorerConfig,maximize,sklearn.metrics.accuracy_score,deckard.base.scorer.ScorerConfig,minimize,sklearn.metrics.log_loss,19.702645352,0.0003518329527142857,1701660379.3342788,1701660398.9537199,cuda:0,1.188783624,2.1228279e-05,1701660398.9541066,1701660400.1424274,cuda:0,1.133770342,2.0245898964285716e-05,1701660400.4525743,1701660401.5914068,cuda:0,1.184063142,2.114398467857143e-05,1701660401.591488,1701660402.7754624,cuda:0,0.00137451447,0.137451447,1701660404.4032514,1701660404.5407138,cuda:0,0.00011982578,0.011982578,1701660406.2865784,1701660406.2985675,cuda:0,1.0,0.6505,3.3935765696046136,0.15,6.529011898040771,/result/bit_depth/mnist/reports/attack/00a48db95f8faaa83dd0525f44c1855a/predictions.json,/result/bit_depth/mnist/reports/attack/00a48db95f8faaa83dd0525f44c1855a/adv_predictions.json 00bb2bbf8bcbd0dec970f551d536630b,attack,deckard.base.experiment.Experiment,00bb2bbf8bcbd0dec970f551d536630b,00bb2bbf8bcbd0dec970f551d536630b,nvidia-l4,00bb2bbf8bcbd0dec970f551d536630b,00bb2bbf8bcbd0dec970f551d536630b,00bb2bbf8bcbd0dec970f551d536630b,00bb2bbf8bcbd0dec970f551d536630b,19.292816381,0.00034451457823214286,1701652841.3548884,1701652860.5475855,cuda:0,0.618399955,1.1042856339285715e-05,1701652860.5498993,1701652861.168158,cuda:0,0.635787219,1.1353343196428571e-05,1701652861.4213982,1701652862.0572603,cuda:0,0.600712895,1.0727015982142858e-05,1701652862.0573397,1701652862.6579626,cuda:0,0.00075029189,0.075029189,1701652864.252877,1701652864.3278887,cuda:0,6.265753e-05,0.006265753,1701652865.0728152,1701652865.0790927,cuda:0,0.5,0.9743571428571428,1.1329994824799776,0.11,20.688893682956696,/result/bit_depth/mnist/reports/attack/00bb2bbf8bcbd0dec970f551d536630b/predictions.json,/result/bit_depth/mnist/reports/attack/00bb2bbf8bcbd0dec970f551d536630b/adv_predictions.json,attack,deckard.base.experiment.Experiment,deckard.base.attack.Attack,100,evasion,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.attack.AttackInitializer,9269,0.6339545055346733,True,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.009490958890028784,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,64.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,pytorch,9269,49,art.attacks.evasion.FastGradientMethod,False,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.009490958890028784,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,64.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,pytorch,9269,49,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,nvidia-l4,deckard.base.files.FileConfig,adv_predictions.json,attacks,82bc7b63558d84d675d446570e821128,.pkl,data,20761683b94eca729b58b63071f98327,.pkl,/result/bit_depth/mnist/,00bb2bbf8bcbd0dec970f551d536630b,predictions.json,reports,score_dict.json,attack,,,,,deckard.base.model.Model,pytorch,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.009490958890028784,0.9,torch.optim.SGD,pytorch,32,"[0, 255]",art.defences.preprocessor.FeatureSqueezing,64.0,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,9269,49,00bb2bbf8bcbd0dec970f551d536630b,deckard.base.scorer.ScorerDict,deckard.base.scorer.ScorerConfig,maximize,sklearn.metrics.accuracy_score,deckard.base.scorer.ScorerConfig,minimize,sklearn.metrics.log_loss,19.292816381,0.00034451457823214286,1701652841.3548884,1701652860.5475855,cuda:0,0.618399955,1.1042856339285715e-05,1701652860.5498993,1701652861.168158,cuda:0,0.635787219,1.1353343196428571e-05,1701652861.4213982,1701652862.0572603,cuda:0,0.600712895,1.0727015982142858e-05,1701652862.0573397,1701652862.6579626,cuda:0,0.00075029189,0.075029189,1701652864.252877,1701652864.3278887,cuda:0,6.265753e-05,0.006265753,1701652865.0728152,1701652865.0790927,cuda:0,0.5,0.9743571428571428,1.1329994824799776,0.11,20.688893682956696,/result/bit_depth/mnist/reports/attack/00bb2bbf8bcbd0dec970f551d536630b/predictions.json,/result/bit_depth/mnist/reports/attack/00bb2bbf8bcbd0dec970f551d536630b/adv_predictions.json diff --git a/examples/power/plots/data/cifar/power.csv b/examples/power/plots/data/cifar/power.csv index 92accbc6..128a83f1 100644 --- a/examples/power/plots/data/cifar/power.csv +++ b/examples/power/plots/data/cifar/power.csv @@ -1,4 +1,4 @@ -,stage,_target_,attack,data,device_id,files,model,name,scorers,train_time,train_time_per_sample,train_start_time,train_end_time,train_device,predict_time,predict_time_per_sample,predict_start_time,predict_end_time,predict_device,predict_proba_time,predict_proba_time_per_sample,predict_proba_start_time,predict_proba_end_time,predict_proba_device,predict_log_proba_time,predict_log_proba_time_per_sample,predict_log_proba_start_time,predict_log_proba_end_time,predict_log_device,adv_fit_time_per_sample,adv_fit_time,adv_fit_start_time,adv_fit_end_time,adv_fit_device,adv_predict_time_per_sample,adv_predict_time,adv_predict_start_time,adv_predict_end_time,adv_predict_device,adv_success,accuracy,log_loss,adv_accuracy,adv_log_loss,predictions,adv_predictions,exception,stage.1,_target_.1,attack._target_,attack.attack_size,attack.method,attack.data._target_,attack.data.generate._target_,attack.data.generate.name,attack.data.sample._target_,attack.data.sample.random_state,attack.data.sample.stratify,attack.data.sklearn_pipeline._target_,attack.data.sklearn_pipeline.preprocessor.name,attack.data.sklearn_pipeline.preprocessor.with_mean,attack.data.sklearn_pipeline.preprocessor.with_std,attack.init._target_,attack.init.batch_size,attack.init.eps,attack.init.minimal,attack.init.model._target_,attack.init.model.art._target_,attack.init.model.art.data._target_,attack.init.model.art.data.generate._target_,attack.init.model.art.data.generate.name,attack.init.model.art.data.sample._target_,attack.init.model.art.data.sample.random_state,attack.init.model.art.data.sample.stratify,attack.init.model.art.data.sklearn_pipeline._target_,attack.init.model.art.data.sklearn_pipeline.preprocessor.name,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.art.initialize.clip_values,attack.init.model.art.initialize.criterion.name,attack.init.model.art.initialize.optimizer.lr,attack.init.model.art.initialize.optimizer.momentum,attack.init.model.art.initialize.optimizer.name,attack.init.model.art.library,attack.init.model.data._target_,attack.init.model.data.generate._target_,attack.init.model.data.generate.name,attack.init.model.data.sample._target_,attack.init.model.data.sample.random_state,attack.init.model.data.sample.stratify,attack.init.model.data.sklearn_pipeline._target_,attack.init.model.data.sklearn_pipeline.preprocessor.name,attack.init.model.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.init._target_,attack.init.model.init.name,attack.init.model.init.num_channels,attack.init.model.init.num_classes,attack.init.model.library,attack.init.model.trainer.batch_size,attack.init.model.trainer.nb_epoch,attack.init.name,attack.init.targeted,attack.model._target_,attack.model.art._target_,attack.model.art.data._target_,attack.model.art.data.generate._target_,attack.model.art.data.generate.name,attack.model.art.data.sample._target_,attack.model.art.data.sample.random_state,attack.model.art.data.sample.stratify,attack.model.art.data.sklearn_pipeline._target_,attack.model.art.data.sklearn_pipeline.preprocessor.name,attack.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.model.art.initialize.clip_values,attack.model.art.initialize.criterion.name,attack.model.art.initialize.optimizer.lr,attack.model.art.initialize.optimizer.momentum,attack.model.art.initialize.optimizer.name,attack.model.art.library,attack.model.data._target_,attack.model.data.generate._target_,attack.model.data.generate.name,attack.model.data.sample._target_,attack.model.data.sample.random_state,attack.model.data.sample.stratify,attack.model.data.sklearn_pipeline._target_,attack.model.data.sklearn_pipeline.preprocessor.name,attack.model.data.sklearn_pipeline.preprocessor.with_mean,attack.model.data.sklearn_pipeline.preprocessor.with_std,attack.model.init._target_,attack.model.init.name,attack.model.init.num_channels,attack.model.init.num_classes,attack.model.library,attack.model.trainer.batch_size,attack.model.trainer.nb_epoch,data._target_,data.generate._target_,data.generate.name,data.sample._target_,data.sample.random_state,data.sample.stratify,data.sklearn_pipeline._target_,data.sklearn_pipeline.preprocessor.name,data.sklearn_pipeline.preprocessor.with_mean,data.sklearn_pipeline.preprocessor.with_std,device_id.1,files._target_,files.adv_predictions_file,files.attack_dir,files.attack_file,files.attack_type,files.data_dir,files.data_file,files.data_type,files.directory,files.name,files.predictions_file,files.reports,files.score_dict_file,files.stage,files.model_dir,files.model_file,files.model_type,files.params_file,model._target_,model.library,model.art._target_,model.art.data._target_,model.art.data.generate._target_,model.art.data.generate.name,model.art.data.sample._target_,model.art.data.sample.random_state,model.art.data.sample.stratify,model.art.data.sklearn_pipeline._target_,model.art.data.sklearn_pipeline.preprocessor.name,model.art.data.sklearn_pipeline.preprocessor.with_mean,model.art.data.sklearn_pipeline.preprocessor.with_std,model.art.initialize.clip_values,model.art.initialize.criterion.name,model.art.initialize.optimizer.lr,model.art.initialize.optimizer.momentum,model.art.initialize.optimizer.name,model.art.library,model.data._target_,model.data.generate._target_,model.data.generate.name,model.data.sample._target_,model.data.sample.random_state,model.data.sample.stratify,model.data.sklearn_pipeline._target_,model.data.sklearn_pipeline.preprocessor.name,model.data.sklearn_pipeline.preprocessor.with_mean,model.data.sklearn_pipeline.preprocessor.with_std,model.init._target_,model.init.name,model.init.num_channels,model.init.num_classes,model.trainer.batch_size,model.trainer.nb_epoch,name.1,scorers._target_,scorers.accuracy._target_,scorers.accuracy.direction,scorers.accuracy.name,scorers.log_loss._target_,scorers.log_loss.direction,scorers.log_loss.name,train_time.1,train_time_per_sample.1,train_start_time.1,train_end_time.1,train_device.1,predict_time.1,predict_time_per_sample.1,predict_start_time.1,predict_end_time.1,predict_device.1,predict_proba_time.1,predict_proba_time_per_sample.1,predict_proba_start_time.1,predict_proba_end_time.1,predict_proba_device.1,predict_log_proba_time.1,predict_log_proba_time_per_sample.1,predict_log_proba_start_time.1,predict_log_proba_end_time.1,predict_log_device.1,adv_fit_time_per_sample.1,adv_fit_time.1,adv_fit_start_time.1,adv_fit_end_time.1,adv_fit_device.1,adv_predict_time_per_sample.1,adv_predict_time.1,adv_predict_start_time.1,adv_predict_end_time.1,adv_predict_device.1,adv_success.1,accuracy.1,log_loss.1,adv_accuracy.1,adv_log_loss.1,predictions.1,adv_predictions.1,exception.1,train_power,peak_power,predict_power,predict_proba_power,predict_log_proba_power,adv_fit_power,adv_predict_power +,stage,_target_,attack,data,device_id,files,model,name,scorers,train_time,train_time_per_sample,train_start_time,train_end_time,train_device,predict_time,predict_time_per_sample,predict_start_time,predict_end_time,predict_device,predict_proba_time,predict_proba_time_per_sample,predict_proba_start_time,predict_proba_end_time,predict_proba_device,predict_log_proba_time,predict_log_proba_time_per_sample,predict_log_proba_start_time,predict_log_proba_end_time,predict_log_device,adv_fit_time_per_sample,adv_fit_time,adv_fit_start_time,adv_fit_end_time,adv_fit_device,adv_predict_time_per_sample,adv_predict_time,adv_predict_start_time,adv_predict_end_time,adv_predict_device,adv_success,accuracy,log_loss,adv_accuracy,adv_log_loss,predictions,adv_predictions,exception,stage.1,_target_.1,attack._target_,attack.attack_size,attack.method,attack.data._target_,attack.data.generate._target_,attack.data.generate.name,attack.data.sample._target_,attack.data.sample.random_state,attack.data.sample.stratify,attack.data.sklearn_pipeline._target_,attack.data.sklearn_pipeline.preprocessor.name,attack.data.sklearn_pipeline.preprocessor.with_mean,attack.data.sklearn_pipeline.preprocessor.with_std,attack.init._target_,attack.init.batch_size,attack.init.eps,attack.init.minimal,attack.init.model._target_,attack.init.model.art._target_,attack.init.model.art.data._target_,attack.init.model.art.data.generate._target_,attack.init.model.art.data.generate.name,attack.init.model.art.data.sample._target_,attack.init.model.art.data.sample.random_state,attack.init.model.art.data.sample.stratify,attack.init.model.art.data.sklearn_pipeline._target_,attack.init.model.art.data.sklearn_pipeline.preprocessor.name,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.art.initialize.clip_values,attack.init.model.art.initialize.criterion.name,attack.init.model.art.initialize.optimizer.lr,attack.init.model.art.initialize.optimizer.momentum,attack.init.model.art.initialize.optimizer.name,attack.init.model.art.library,attack.init.model.data._target_,attack.init.model.data.generate._target_,attack.init.model.data.generate.name,attack.init.model.data.sample._target_,attack.init.model.data.sample.random_state,attack.init.model.data.sample.stratify,attack.init.model.data.sklearn_pipeline._target_,attack.init.model.data.sklearn_pipeline.preprocessor.name,attack.init.model.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.init._target_,attack.init.model.init.name,attack.init.model.init.num_channels,attack.init.model.init.num_classes,attack.init.model.library,attack.init.model.trainer.batch_size,attack.init.model.trainer.np_epochs,attack.init.name,attack.init.targeted,attack.model._target_,attack.model.art._target_,attack.model.art.data._target_,attack.model.art.data.generate._target_,attack.model.art.data.generate.name,attack.model.art.data.sample._target_,attack.model.art.data.sample.random_state,attack.model.art.data.sample.stratify,attack.model.art.data.sklearn_pipeline._target_,attack.model.art.data.sklearn_pipeline.preprocessor.name,attack.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.model.art.initialize.clip_values,attack.model.art.initialize.criterion.name,attack.model.art.initialize.optimizer.lr,attack.model.art.initialize.optimizer.momentum,attack.model.art.initialize.optimizer.name,attack.model.art.library,attack.model.data._target_,attack.model.data.generate._target_,attack.model.data.generate.name,attack.model.data.sample._target_,attack.model.data.sample.random_state,attack.model.data.sample.stratify,attack.model.data.sklearn_pipeline._target_,attack.model.data.sklearn_pipeline.preprocessor.name,attack.model.data.sklearn_pipeline.preprocessor.with_mean,attack.model.data.sklearn_pipeline.preprocessor.with_std,attack.model.init._target_,attack.model.init.name,attack.model.init.num_channels,attack.model.init.num_classes,attack.model.library,attack.model.trainer.batch_size,attack.model.trainer.np_epochs,data._target_,data.generate._target_,data.generate.name,data.sample._target_,data.sample.random_state,data.sample.stratify,data.sklearn_pipeline._target_,data.sklearn_pipeline.preprocessor.name,data.sklearn_pipeline.preprocessor.with_mean,data.sklearn_pipeline.preprocessor.with_std,device_id.1,files._target_,files.adv_predictions_file,files.attack_dir,files.attack_file,files.attack_type,files.data_dir,files.data_file,files.data_type,files.directory,files.name,files.predictions_file,files.reports,files.score_dict_file,files.stage,files.model_dir,files.model_file,files.model_type,files.params_file,model._target_,model.library,model.art._target_,model.art.data._target_,model.art.data.generate._target_,model.art.data.generate.name,model.art.data.sample._target_,model.art.data.sample.random_state,model.art.data.sample.stratify,model.art.data.sklearn_pipeline._target_,model.art.data.sklearn_pipeline.preprocessor.name,model.art.data.sklearn_pipeline.preprocessor.with_mean,model.art.data.sklearn_pipeline.preprocessor.with_std,model.art.initialize.clip_values,model.art.initialize.criterion.name,model.art.initialize.optimizer.lr,model.art.initialize.optimizer.momentum,model.art.initialize.optimizer.name,model.art.library,model.data._target_,model.data.generate._target_,model.data.generate.name,model.data.sample._target_,model.data.sample.random_state,model.data.sample.stratify,model.data.sklearn_pipeline._target_,model.data.sklearn_pipeline.preprocessor.name,model.data.sklearn_pipeline.preprocessor.with_mean,model.data.sklearn_pipeline.preprocessor.with_std,model.init._target_,model.init.name,model.init.num_channels,model.init.num_classes,model.trainer.batch_size,model.trainer.np_epochs,name.1,scorers._target_,scorers.accuracy._target_,scorers.accuracy.direction,scorers.accuracy.name,scorers.log_loss._target_,scorers.log_loss.direction,scorers.log_loss.name,train_time.1,train_time_per_sample.1,train_start_time.1,train_end_time.1,train_device.1,predict_time.1,predict_time_per_sample.1,predict_start_time.1,predict_end_time.1,predict_device.1,predict_proba_time.1,predict_proba_time_per_sample.1,predict_proba_start_time.1,predict_proba_end_time.1,predict_proba_device.1,predict_log_proba_time.1,predict_log_proba_time_per_sample.1,predict_log_proba_start_time.1,predict_log_proba_end_time.1,predict_log_device.1,adv_fit_time_per_sample.1,adv_fit_time.1,adv_fit_start_time.1,adv_fit_end_time.1,adv_fit_device.1,adv_predict_time_per_sample.1,adv_predict_time.1,adv_predict_start_time.1,adv_predict_end_time.1,adv_predict_device.1,adv_success.1,accuracy.1,log_loss.1,adv_accuracy.1,adv_log_loss.1,predictions.1,adv_predictions.1,exception.1,train_power,peak_power,predict_power,predict_proba_power,predict_log_proba_power,adv_fit_power,adv_predict_power 00135b5879d2f88ae748aef45932952d,attack,deckard.base.experiment.Experiment,00135b5879d2f88ae748aef45932952d,00135b5879d2f88ae748aef45932952d,nvidia-tesla-p100,00135b5879d2f88ae748aef45932952d,00135b5879d2f88ae748aef45932952d,00135b5879d2f88ae748aef45932952d,00135b5879d2f88ae748aef45932952d,71.375361253,0.0014869866927708,1701623263.379239,1701623335.9344044,cuda:0,1.592706168,3.31813785e-05,1701623335.9364812,1701623337.547312,cuda:0,0.951278602,1.981830420833333e-05,1701623337.7363863,1701623338.7303908,cuda:0,0.936467429,1.9509738104166663e-05,1701623338.7304602,1701623339.6674187,cuda:0,0.00078040522,0.078040522,1701623343.5360556,1701623343.614123,cuda:0,9.099429e-05,0.009099429,1701623344.8171692,1701623344.8262916,cuda:0,1.0,0.1004166666666666,25.06647256758809,0.08,16.215492764115332,/result/cifar/reports/attack/00135b5879d2f88ae748aef45932952d/predictions.json,/result/cifar/reports/attack/00135b5879d2f88ae748aef45932952d/adv_predictions.json,,attack,deckard.base.experiment.Experiment,deckard.base.attack.Attack,100,evasion,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.attack.AttackInitializer,6182,0.4417486652197132,True,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.6152619702970211,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,10,pytorch,6182,28,art.attacks.evasion.FastGradientMethod,False,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.6152619702970211,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,10,pytorch,6182,28,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,nvidia-tesla-p100,deckard.base.files.FileConfig,adv_predictions.json,attacks,89cb02ce277d3785b172c85c2ae8c362,.pkl,data,ea1a0afb95689dc7b59820b2cd11c56c,.pkl,/result/cifar/,00135b5879d2f88ae748aef45932952d,predictions.json,reports,score_dict.json,attack,,,,,deckard.base.model.Model,pytorch,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.6152619702970211,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,10,6182,28,00135b5879d2f88ae748aef45932952d,deckard.base.scorer.ScorerDict,deckard.base.scorer.ScorerConfig,maximize,sklearn.metrics.accuracy_score,deckard.base.scorer.ScorerConfig,minimize,sklearn.metrics.log_loss,71.375361253,0.0014869866927708,1701623263.379239,1701623335.9344044,cuda:0,1.592706168,3.31813785e-05,1701623335.9364812,1701623337.547312,cuda:0,0.951278602,1.981830420833333e-05,1701623337.7363863,1701623338.7303908,cuda:0,0.936467429,1.9509738104166663e-05,1701623338.7304602,1701623339.6674187,cuda:0,0.00078040522,0.078040522,1701623343.5360556,1701623343.614123,cuda:0,9.099429e-05,0.009099429,1701623344.8171692,1701623344.8262916,cuda:0,1.0,0.1004166666666666,25.06647256758809,0.08,16.215492764115332,/result/cifar/reports/attack/00135b5879d2f88ae748aef45932952d/predictions.json,/result/cifar/reports/attack/00135b5879d2f88ae748aef45932952d/adv_predictions.json,,5.85278333346049,250,0.1320433789756563,0.07503353322545687,0.06266560978194079,0.00467266761759917,0.000557351569334666 001973df8a6e2aa227ed26c519f8f071,attack,deckard.base.experiment.Experiment,001973df8a6e2aa227ed26c519f8f071,001973df8a6e2aa227ed26c519f8f071,nvidia-tesla-v100,001973df8a6e2aa227ed26c519f8f071,001973df8a6e2aa227ed26c519f8f071,001973df8a6e2aa227ed26c519f8f071,001973df8a6e2aa227ed26c519f8f071,44.255178782,0.0009219828912916,1701577683.5071497,1701577727.4155056,cuda:0,0.649264304,1.3526339666666666e-05,1701577727.4171782,1701577728.066254,cuda:0,0.65126136,1.3567945e-05,1701577728.3392982,1701577728.9904,cuda:0,0.630036071,1.3125751479166669e-05,1701577728.9904633,1701577729.6203322,cuda:0,0.00077208022,0.077208022,1701577732.1228857,1701577732.1569226,cuda:0,6.481871e-05,0.006481871,1701577732.7210014,1701577732.727445,cuda:0,0.0,0.675,2.551714852200045,0.15,18.577956516742702,/result/cifar/reports/attack/001973df8a6e2aa227ed26c519f8f071/predictions.json,/result/cifar/reports/attack/001973df8a6e2aa227ed26c519f8f071/adv_predictions.json,,attack,deckard.base.experiment.Experiment,deckard.base.attack.Attack,100,evasion,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.attack.AttackInitializer,406,0.0134828476237641,True,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0223167310356781,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,10,pytorch,406,14,art.attacks.evasion.FastGradientMethod,False,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0223167310356781,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,10,pytorch,406,14,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,nvidia-tesla-v100,deckard.base.files.FileConfig,adv_predictions.json,attacks,c44903f79eea39dbb645a0f2ed47706a,.pkl,data,6e5857478e8d10445dfe508f8632e60d,.pkl,/result/cifar/,001973df8a6e2aa227ed26c519f8f071,predictions.json,reports,score_dict.json,attack,,,,,deckard.base.model.Model,pytorch,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0223167310356781,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,10,406,14,001973df8a6e2aa227ed26c519f8f071,deckard.base.scorer.ScorerDict,deckard.base.scorer.ScorerConfig,maximize,sklearn.metrics.accuracy_score,deckard.base.scorer.ScorerConfig,minimize,sklearn.metrics.log_loss,44.255178782,0.0009219828912916,1701577683.5071497,1701577727.4155056,cuda:0,0.649264304,1.3526339666666666e-05,1701577727.4171782,1701577728.066254,cuda:0,0.65126136,1.3567945e-05,1701577728.3392982,1701577728.9904,cuda:0,0.630036071,1.3125751479166669e-05,1701577728.9904633,1701577729.6203322,cuda:0,0.00077208022,0.077208022,1701577732.1228857,1701577732.1569226,cuda:0,6.481871e-05,0.006481871,1701577732.7210014,1701577732.727445,cuda:0,0.0,0.675,2.551714852200045,0.15,18.577956516742702,/result/cifar/reports/attack/001973df8a6e2aa227ed26c519f8f071/predictions.json,/result/cifar/reports/attack/001973df8a6e2aa227ed26c519f8f071/adv_predictions.json,,3.1568278414160016,250,0.05136262591514323,0.04866986161470414,0.048845464908414425,0.002068921978274981,0.00036258114377657577 0097d00024b7ababad095d91b27a3700,attack,deckard.base.experiment.Experiment,0097d00024b7ababad095d91b27a3700,0097d00024b7ababad095d91b27a3700,nvidia-tesla-v100,0097d00024b7ababad095d91b27a3700,0097d00024b7ababad095d91b27a3700,0097d00024b7ababad095d91b27a3700,0097d00024b7ababad095d91b27a3700,44.443586705,0.0009259080563541,1701553871.1499689,1701553915.2876782,cuda:0,0.641802238,1.3370879958333333e-05,1701553915.290023,1701553915.9316726,cuda:0,0.638468972,1.3301436916666668e-05,1701553916.115003,1701553916.7532885,cuda:0,0.638724204,1.330675425e-05,1701553916.7533565,1701553917.3919475,cuda:0,0.00085138827,0.085138827,1701553919.910931,1701553919.952762,cuda:0,5.626004e-05,0.005626004,1701553920.4987822,1701553920.504411,cuda:0,1.0,0.1,18.01482623844345,0.11,16.60381488442421,/result/cifar/reports/attack/0097d00024b7ababad095d91b27a3700/predictions.json,/result/cifar/reports/attack/0097d00024b7ababad095d91b27a3700/adv_predictions.json,,attack,deckard.base.experiment.Experiment,deckard.base.attack.Attack,100,evasion,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.attack.AttackInitializer,338,0.8293550630015566,True,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.838130072356157,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,10,pytorch,338,42,art.attacks.evasion.FastGradientMethod,False,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.838130072356157,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,10,pytorch,338,42,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,nvidia-tesla-v100,deckard.base.files.FileConfig,adv_predictions.json,attacks,38838db4786d9f7a78c802e9b727f1be,.pkl,data,6e5857478e8d10445dfe508f8632e60d,.pkl,/result/cifar/,0097d00024b7ababad095d91b27a3700,predictions.json,reports,score_dict.json,attack,,,,,deckard.base.model.Model,pytorch,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.838130072356157,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,10,338,42,0097d00024b7ababad095d91b27a3700,deckard.base.scorer.ScorerDict,deckard.base.scorer.ScorerConfig,maximize,sklearn.metrics.accuracy_score,deckard.base.scorer.ScorerConfig,minimize,sklearn.metrics.log_loss,44.443586705,0.0009259080563541,1701553871.1499689,1701553915.2876782,cuda:0,0.641802238,1.3370879958333333e-05,1701553915.290023,1701553915.9316726,cuda:0,0.638468972,1.3301436916666668e-05,1701553916.115003,1701553916.7532885,cuda:0,0.638724204,1.330675425e-05,1701553916.7533565,1701553917.3919475,cuda:0,0.00085138827,0.085138827,1701553919.910931,1701553919.952762,cuda:0,5.626004e-05,0.005626004,1701553920.4987822,1701553920.504411,cuda:0,1.0,0.1,18.01482623844345,0.11,16.60381488442421,/result/cifar/reports/attack/0097d00024b7ababad095d91b27a3700/predictions.json,/result/cifar/reports/attack/0097d00024b7ababad095d91b27a3700/adv_predictions.json,,3.1408271389487714,250,0.04826184422274431,0.045083338805370854,0.0419563189893961,0.0030245976514286462,0.0004225527081224653 diff --git a/examples/power/plots/data/cifar/raw.csv b/examples/power/plots/data/cifar/raw.csv index e373445d..b6250aa8 100644 --- a/examples/power/plots/data/cifar/raw.csv +++ b/examples/power/plots/data/cifar/raw.csv @@ -1,4 +1,4 @@ -,stage,_target_,attack,data,device_id,files,model,name,scorers,train_time,train_time_per_sample,train_start_time,train_end_time,train_device,predict_time,predict_time_per_sample,predict_start_time,predict_end_time,predict_device,predict_proba_time,predict_proba_time_per_sample,predict_proba_start_time,predict_proba_end_time,predict_proba_device,predict_log_proba_time,predict_log_proba_time_per_sample,predict_log_proba_start_time,predict_log_proba_end_time,predict_log_device,adv_fit_time_per_sample,adv_fit_time,adv_fit_start_time,adv_fit_end_time,adv_fit_device,adv_predict_time_per_sample,adv_predict_time,adv_predict_start_time,adv_predict_end_time,adv_predict_device,adv_success,accuracy,log_loss,adv_accuracy,adv_log_loss,predictions,adv_predictions,exception,stage,_target_,attack._target_,attack.attack_size,attack.method,attack.data._target_,attack.data.generate._target_,attack.data.generate.name,attack.data.sample._target_,attack.data.sample.random_state,attack.data.sample.stratify,attack.data.sklearn_pipeline._target_,attack.data.sklearn_pipeline.preprocessor.name,attack.data.sklearn_pipeline.preprocessor.with_mean,attack.data.sklearn_pipeline.preprocessor.with_std,attack.init._target_,attack.init.batch_size,attack.init.eps,attack.init.minimal,attack.init.model._target_,attack.init.model.art._target_,attack.init.model.art.data._target_,attack.init.model.art.data.generate._target_,attack.init.model.art.data.generate.name,attack.init.model.art.data.sample._target_,attack.init.model.art.data.sample.random_state,attack.init.model.art.data.sample.stratify,attack.init.model.art.data.sklearn_pipeline._target_,attack.init.model.art.data.sklearn_pipeline.preprocessor.name,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.art.initialize.clip_values,attack.init.model.art.initialize.criterion.name,attack.init.model.art.initialize.optimizer.lr,attack.init.model.art.initialize.optimizer.momentum,attack.init.model.art.initialize.optimizer.name,attack.init.model.art.library,attack.init.model.data._target_,attack.init.model.data.generate._target_,attack.init.model.data.generate.name,attack.init.model.data.sample._target_,attack.init.model.data.sample.random_state,attack.init.model.data.sample.stratify,attack.init.model.data.sklearn_pipeline._target_,attack.init.model.data.sklearn_pipeline.preprocessor.name,attack.init.model.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.init._target_,attack.init.model.init.name,attack.init.model.init.num_channels,attack.init.model.init.num_classes,attack.init.model.library,attack.init.model.trainer.batch_size,attack.init.model.trainer.nb_epoch,attack.init.name,attack.init.targeted,attack.model._target_,attack.model.art._target_,attack.model.art.data._target_,attack.model.art.data.generate._target_,attack.model.art.data.generate.name,attack.model.art.data.sample._target_,attack.model.art.data.sample.random_state,attack.model.art.data.sample.stratify,attack.model.art.data.sklearn_pipeline._target_,attack.model.art.data.sklearn_pipeline.preprocessor.name,attack.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.model.art.initialize.clip_values,attack.model.art.initialize.criterion.name,attack.model.art.initialize.optimizer.lr,attack.model.art.initialize.optimizer.momentum,attack.model.art.initialize.optimizer.name,attack.model.art.library,attack.model.data._target_,attack.model.data.generate._target_,attack.model.data.generate.name,attack.model.data.sample._target_,attack.model.data.sample.random_state,attack.model.data.sample.stratify,attack.model.data.sklearn_pipeline._target_,attack.model.data.sklearn_pipeline.preprocessor.name,attack.model.data.sklearn_pipeline.preprocessor.with_mean,attack.model.data.sklearn_pipeline.preprocessor.with_std,attack.model.init._target_,attack.model.init.name,attack.model.init.num_channels,attack.model.init.num_classes,attack.model.library,attack.model.trainer.batch_size,attack.model.trainer.nb_epoch,data._target_,data.generate._target_,data.generate.name,data.sample._target_,data.sample.random_state,data.sample.stratify,data.sklearn_pipeline._target_,data.sklearn_pipeline.preprocessor.name,data.sklearn_pipeline.preprocessor.with_mean,data.sklearn_pipeline.preprocessor.with_std,device_id,files._target_,files.adv_predictions_file,files.attack_dir,files.attack_file,files.attack_type,files.data_dir,files.data_file,files.data_type,files.directory,files.name,files.predictions_file,files.reports,files.score_dict_file,files.stage,files.model_dir,files.model_file,files.model_type,files.params_file,model._target_,model.library,model.art._target_,model.art.data._target_,model.art.data.generate._target_,model.art.data.generate.name,model.art.data.sample._target_,model.art.data.sample.random_state,model.art.data.sample.stratify,model.art.data.sklearn_pipeline._target_,model.art.data.sklearn_pipeline.preprocessor.name,model.art.data.sklearn_pipeline.preprocessor.with_mean,model.art.data.sklearn_pipeline.preprocessor.with_std,model.art.initialize.clip_values,model.art.initialize.criterion.name,model.art.initialize.optimizer.lr,model.art.initialize.optimizer.momentum,model.art.initialize.optimizer.name,model.art.library,model.data._target_,model.data.generate._target_,model.data.generate.name,model.data.sample._target_,model.data.sample.random_state,model.data.sample.stratify,model.data.sklearn_pipeline._target_,model.data.sklearn_pipeline.preprocessor.name,model.data.sklearn_pipeline.preprocessor.with_mean,model.data.sklearn_pipeline.preprocessor.with_std,model.init._target_,model.init.name,model.init.num_channels,model.init.num_classes,model.trainer.batch_size,model.trainer.nb_epoch,name,scorers._target_,scorers.accuracy._target_,scorers.accuracy.direction,scorers.accuracy.name,scorers.log_loss._target_,scorers.log_loss.direction,scorers.log_loss.name,train_time,train_time_per_sample,train_start_time,train_end_time,train_device,predict_time,predict_time_per_sample,predict_start_time,predict_end_time,predict_device,predict_proba_time,predict_proba_time_per_sample,predict_proba_start_time,predict_proba_end_time,predict_proba_device,predict_log_proba_time,predict_log_proba_time_per_sample,predict_log_proba_start_time,predict_log_proba_end_time,predict_log_device,adv_fit_time_per_sample,adv_fit_time,adv_fit_start_time,adv_fit_end_time,adv_fit_device,adv_predict_time_per_sample,adv_predict_time,adv_predict_start_time,adv_predict_end_time,adv_predict_device,adv_success,accuracy,log_loss,adv_accuracy,adv_log_loss,predictions,adv_predictions,exception +,stage,_target_,attack,data,device_id,files,model,name,scorers,train_time,train_time_per_sample,train_start_time,train_end_time,train_device,predict_time,predict_time_per_sample,predict_start_time,predict_end_time,predict_device,predict_proba_time,predict_proba_time_per_sample,predict_proba_start_time,predict_proba_end_time,predict_proba_device,predict_log_proba_time,predict_log_proba_time_per_sample,predict_log_proba_start_time,predict_log_proba_end_time,predict_log_device,adv_fit_time_per_sample,adv_fit_time,adv_fit_start_time,adv_fit_end_time,adv_fit_device,adv_predict_time_per_sample,adv_predict_time,adv_predict_start_time,adv_predict_end_time,adv_predict_device,adv_success,accuracy,log_loss,adv_accuracy,adv_log_loss,predictions,adv_predictions,exception,stage,_target_,attack._target_,attack.attack_size,attack.method,attack.data._target_,attack.data.generate._target_,attack.data.generate.name,attack.data.sample._target_,attack.data.sample.random_state,attack.data.sample.stratify,attack.data.sklearn_pipeline._target_,attack.data.sklearn_pipeline.preprocessor.name,attack.data.sklearn_pipeline.preprocessor.with_mean,attack.data.sklearn_pipeline.preprocessor.with_std,attack.init._target_,attack.init.batch_size,attack.init.eps,attack.init.minimal,attack.init.model._target_,attack.init.model.art._target_,attack.init.model.art.data._target_,attack.init.model.art.data.generate._target_,attack.init.model.art.data.generate.name,attack.init.model.art.data.sample._target_,attack.init.model.art.data.sample.random_state,attack.init.model.art.data.sample.stratify,attack.init.model.art.data.sklearn_pipeline._target_,attack.init.model.art.data.sklearn_pipeline.preprocessor.name,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.art.initialize.clip_values,attack.init.model.art.initialize.criterion.name,attack.init.model.art.initialize.optimizer.lr,attack.init.model.art.initialize.optimizer.momentum,attack.init.model.art.initialize.optimizer.name,attack.init.model.art.library,attack.init.model.data._target_,attack.init.model.data.generate._target_,attack.init.model.data.generate.name,attack.init.model.data.sample._target_,attack.init.model.data.sample.random_state,attack.init.model.data.sample.stratify,attack.init.model.data.sklearn_pipeline._target_,attack.init.model.data.sklearn_pipeline.preprocessor.name,attack.init.model.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.init._target_,attack.init.model.init.name,attack.init.model.init.num_channels,attack.init.model.init.num_classes,attack.init.model.library,attack.init.model.trainer.batch_size,attack.init.model.trainer.np_epochs,attack.init.name,attack.init.targeted,attack.model._target_,attack.model.art._target_,attack.model.art.data._target_,attack.model.art.data.generate._target_,attack.model.art.data.generate.name,attack.model.art.data.sample._target_,attack.model.art.data.sample.random_state,attack.model.art.data.sample.stratify,attack.model.art.data.sklearn_pipeline._target_,attack.model.art.data.sklearn_pipeline.preprocessor.name,attack.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.model.art.initialize.clip_values,attack.model.art.initialize.criterion.name,attack.model.art.initialize.optimizer.lr,attack.model.art.initialize.optimizer.momentum,attack.model.art.initialize.optimizer.name,attack.model.art.library,attack.model.data._target_,attack.model.data.generate._target_,attack.model.data.generate.name,attack.model.data.sample._target_,attack.model.data.sample.random_state,attack.model.data.sample.stratify,attack.model.data.sklearn_pipeline._target_,attack.model.data.sklearn_pipeline.preprocessor.name,attack.model.data.sklearn_pipeline.preprocessor.with_mean,attack.model.data.sklearn_pipeline.preprocessor.with_std,attack.model.init._target_,attack.model.init.name,attack.model.init.num_channels,attack.model.init.num_classes,attack.model.library,attack.model.trainer.batch_size,attack.model.trainer.np_epochs,data._target_,data.generate._target_,data.generate.name,data.sample._target_,data.sample.random_state,data.sample.stratify,data.sklearn_pipeline._target_,data.sklearn_pipeline.preprocessor.name,data.sklearn_pipeline.preprocessor.with_mean,data.sklearn_pipeline.preprocessor.with_std,device_id,files._target_,files.adv_predictions_file,files.attack_dir,files.attack_file,files.attack_type,files.data_dir,files.data_file,files.data_type,files.directory,files.name,files.predictions_file,files.reports,files.score_dict_file,files.stage,files.model_dir,files.model_file,files.model_type,files.params_file,model._target_,model.library,model.art._target_,model.art.data._target_,model.art.data.generate._target_,model.art.data.generate.name,model.art.data.sample._target_,model.art.data.sample.random_state,model.art.data.sample.stratify,model.art.data.sklearn_pipeline._target_,model.art.data.sklearn_pipeline.preprocessor.name,model.art.data.sklearn_pipeline.preprocessor.with_mean,model.art.data.sklearn_pipeline.preprocessor.with_std,model.art.initialize.clip_values,model.art.initialize.criterion.name,model.art.initialize.optimizer.lr,model.art.initialize.optimizer.momentum,model.art.initialize.optimizer.name,model.art.library,model.data._target_,model.data.generate._target_,model.data.generate.name,model.data.sample._target_,model.data.sample.random_state,model.data.sample.stratify,model.data.sklearn_pipeline._target_,model.data.sklearn_pipeline.preprocessor.name,model.data.sklearn_pipeline.preprocessor.with_mean,model.data.sklearn_pipeline.preprocessor.with_std,model.init._target_,model.init.name,model.init.num_channels,model.init.num_classes,model.trainer.batch_size,model.trainer.np_epochs,name,scorers._target_,scorers.accuracy._target_,scorers.accuracy.direction,scorers.accuracy.name,scorers.log_loss._target_,scorers.log_loss.direction,scorers.log_loss.name,train_time,train_time_per_sample,train_start_time,train_end_time,train_device,predict_time,predict_time_per_sample,predict_start_time,predict_end_time,predict_device,predict_proba_time,predict_proba_time_per_sample,predict_proba_start_time,predict_proba_end_time,predict_proba_device,predict_log_proba_time,predict_log_proba_time_per_sample,predict_log_proba_start_time,predict_log_proba_end_time,predict_log_device,adv_fit_time_per_sample,adv_fit_time,adv_fit_start_time,adv_fit_end_time,adv_fit_device,adv_predict_time_per_sample,adv_predict_time,adv_predict_start_time,adv_predict_end_time,adv_predict_device,adv_success,accuracy,log_loss,adv_accuracy,adv_log_loss,predictions,adv_predictions,exception 00135b5879d2f88ae748aef45932952d,attack,deckard.base.experiment.Experiment,00135b5879d2f88ae748aef45932952d,00135b5879d2f88ae748aef45932952d,nvidia-tesla-p100,00135b5879d2f88ae748aef45932952d,00135b5879d2f88ae748aef45932952d,00135b5879d2f88ae748aef45932952d,00135b5879d2f88ae748aef45932952d,71.375361253,0.0014869866927708334,1701623263.3792393,1701623335.9344044,cuda:0,1.592706168,3.31813785e-05,1701623335.9364812,1701623337.547312,cuda:0,0.951278602,1.9818304208333334e-05,1701623337.7363865,1701623338.7303908,cuda:0,0.936467429,1.9509738104166666e-05,1701623338.7304602,1701623339.6674187,cuda:0,0.00078040522,0.078040522,1701623343.5360556,1701623343.6141233,cuda:0,9.099429e-05,0.009099429,1701623344.8171692,1701623344.8262916,cuda:0,1.0,0.10041666666666667,25.06647256758809,0.08,16.215492764115332,/result/cifar/reports/attack/00135b5879d2f88ae748aef45932952d/predictions.json,/result/cifar/reports/attack/00135b5879d2f88ae748aef45932952d/adv_predictions.json,,attack,deckard.base.experiment.Experiment,deckard.base.attack.Attack,100,evasion,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.attack.AttackInitializer,6182,0.44174866521971323,True,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.6152619702970211,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,10,pytorch,6182,28,art.attacks.evasion.FastGradientMethod,False,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.6152619702970211,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,10,pytorch,6182,28,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,nvidia-tesla-p100,deckard.base.files.FileConfig,adv_predictions.json,attacks,89cb02ce277d3785b172c85c2ae8c362,.pkl,data,ea1a0afb95689dc7b59820b2cd11c56c,.pkl,/result/cifar/,00135b5879d2f88ae748aef45932952d,predictions.json,reports,score_dict.json,attack,,,,,deckard.base.model.Model,pytorch,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.6152619702970211,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,0,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,10,6182,28,00135b5879d2f88ae748aef45932952d,deckard.base.scorer.ScorerDict,deckard.base.scorer.ScorerConfig,maximize,sklearn.metrics.accuracy_score,deckard.base.scorer.ScorerConfig,minimize,sklearn.metrics.log_loss,71.375361253,0.0014869866927708334,1701623263.3792393,1701623335.9344044,cuda:0,1.592706168,3.31813785e-05,1701623335.9364812,1701623337.547312,cuda:0,0.951278602,1.9818304208333334e-05,1701623337.7363865,1701623338.7303908,cuda:0,0.936467429,1.9509738104166666e-05,1701623338.7304602,1701623339.6674187,cuda:0,0.00078040522,0.078040522,1701623343.5360556,1701623343.6141233,cuda:0,9.099429e-05,0.009099429,1701623344.8171692,1701623344.8262916,cuda:0,1.0,0.10041666666666667,25.06647256758809,0.08,16.215492764115332,/result/cifar/reports/attack/00135b5879d2f88ae748aef45932952d/predictions.json,/result/cifar/reports/attack/00135b5879d2f88ae748aef45932952d/adv_predictions.json, 001973df8a6e2aa227ed26c519f8f071,attack,deckard.base.experiment.Experiment,001973df8a6e2aa227ed26c519f8f071,001973df8a6e2aa227ed26c519f8f071,nvidia-tesla-v100,001973df8a6e2aa227ed26c519f8f071,001973df8a6e2aa227ed26c519f8f071,001973df8a6e2aa227ed26c519f8f071,001973df8a6e2aa227ed26c519f8f071,44.255178782,0.0009219828912916667,1701577683.5071495,1701577727.4155056,cuda:0,0.649264304,1.3526339666666666e-05,1701577727.4171782,1701577728.066254,cuda:0,0.65126136,1.3567945e-05,1701577728.3392982,1701577728.9904,cuda:0,0.630036071,1.3125751479166667e-05,1701577728.9904633,1701577729.6203322,cuda:0,0.00077208022,0.077208022,1701577732.1228857,1701577732.1569226,cuda:0,6.481871e-05,0.006481871,1701577732.7210014,1701577732.727445,cuda:0,0.0,0.675,2.551714852200045,0.15,18.577956516742706,/result/cifar/reports/attack/001973df8a6e2aa227ed26c519f8f071/predictions.json,/result/cifar/reports/attack/001973df8a6e2aa227ed26c519f8f071/adv_predictions.json,,attack,deckard.base.experiment.Experiment,deckard.base.attack.Attack,100,evasion,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.attack.AttackInitializer,406,0.013482847623764137,True,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0223167310356781,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,10,pytorch,406,14,art.attacks.evasion.FastGradientMethod,False,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0223167310356781,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,10,pytorch,406,14,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,nvidia-tesla-v100,deckard.base.files.FileConfig,adv_predictions.json,attacks,c44903f79eea39dbb645a0f2ed47706a,.pkl,data,6e5857478e8d10445dfe508f8632e60d,.pkl,/result/cifar/,001973df8a6e2aa227ed26c519f8f071,predictions.json,reports,score_dict.json,attack,,,,,deckard.base.model.Model,pytorch,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0223167310356781,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,10,406,14,001973df8a6e2aa227ed26c519f8f071,deckard.base.scorer.ScorerDict,deckard.base.scorer.ScorerConfig,maximize,sklearn.metrics.accuracy_score,deckard.base.scorer.ScorerConfig,minimize,sklearn.metrics.log_loss,44.255178782,0.0009219828912916667,1701577683.5071495,1701577727.4155056,cuda:0,0.649264304,1.3526339666666666e-05,1701577727.4171782,1701577728.066254,cuda:0,0.65126136,1.3567945e-05,1701577728.3392982,1701577728.9904,cuda:0,0.630036071,1.3125751479166667e-05,1701577728.9904633,1701577729.6203322,cuda:0,0.00077208022,0.077208022,1701577732.1228857,1701577732.1569226,cuda:0,6.481871e-05,0.006481871,1701577732.7210014,1701577732.727445,cuda:0,0.0,0.675,2.551714852200045,0.15,18.577956516742706,/result/cifar/reports/attack/001973df8a6e2aa227ed26c519f8f071/predictions.json,/result/cifar/reports/attack/001973df8a6e2aa227ed26c519f8f071/adv_predictions.json, 0097d00024b7ababad095d91b27a3700,attack,deckard.base.experiment.Experiment,0097d00024b7ababad095d91b27a3700,0097d00024b7ababad095d91b27a3700,nvidia-tesla-v100,0097d00024b7ababad095d91b27a3700,0097d00024b7ababad095d91b27a3700,0097d00024b7ababad095d91b27a3700,0097d00024b7ababad095d91b27a3700,44.443586705,0.0009259080563541667,1701553871.1499689,1701553915.2876782,cuda:0,0.641802238,1.3370879958333333e-05,1701553915.290023,1701553915.9316726,cuda:0,0.638468972,1.3301436916666667e-05,1701553916.115003,1701553916.7532883,cuda:0,0.638724204,1.330675425e-05,1701553916.7533565,1701553917.3919477,cuda:0,0.00085138827,0.085138827,1701553919.9109313,1701553919.9527621,cuda:0,5.626004e-05,0.005626004,1701553920.4987822,1701553920.504411,cuda:0,1.0,0.1,18.014826238443455,0.11,16.60381488442421,/result/cifar/reports/attack/0097d00024b7ababad095d91b27a3700/predictions.json,/result/cifar/reports/attack/0097d00024b7ababad095d91b27a3700/adv_predictions.json,,attack,deckard.base.experiment.Experiment,deckard.base.attack.Attack,100,evasion,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.attack.AttackInitializer,338,0.8293550630015566,True,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.838130072356157,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,10,pytorch,338,42,art.attacks.evasion.FastGradientMethod,False,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.838130072356157,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,10,pytorch,338,42,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,nvidia-tesla-v100,deckard.base.files.FileConfig,adv_predictions.json,attacks,38838db4786d9f7a78c802e9b727f1be,.pkl,data,6e5857478e8d10445dfe508f8632e60d,.pkl,/result/cifar/,0097d00024b7ababad095d91b27a3700,predictions.json,reports,score_dict.json,attack,,,,,deckard.base.model.Model,pytorch,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.838130072356157,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar10,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,10,338,42,0097d00024b7ababad095d91b27a3700,deckard.base.scorer.ScorerDict,deckard.base.scorer.ScorerConfig,maximize,sklearn.metrics.accuracy_score,deckard.base.scorer.ScorerConfig,minimize,sklearn.metrics.log_loss,44.443586705,0.0009259080563541667,1701553871.1499689,1701553915.2876782,cuda:0,0.641802238,1.3370879958333333e-05,1701553915.290023,1701553915.9316726,cuda:0,0.638468972,1.3301436916666667e-05,1701553916.115003,1701553916.7532883,cuda:0,0.638724204,1.330675425e-05,1701553916.7533565,1701553917.3919477,cuda:0,0.00085138827,0.085138827,1701553919.9109313,1701553919.9527621,cuda:0,5.626004e-05,0.005626004,1701553920.4987822,1701553920.504411,cuda:0,1.0,0.1,18.014826238443455,0.11,16.60381488442421,/result/cifar/reports/attack/0097d00024b7ababad095d91b27a3700/predictions.json,/result/cifar/reports/attack/0097d00024b7ababad095d91b27a3700/adv_predictions.json, diff --git a/examples/power/plots/data/cifar100/power.csv b/examples/power/plots/data/cifar100/power.csv index a1424771..0be1fe96 100644 --- a/examples/power/plots/data/cifar100/power.csv +++ b/examples/power/plots/data/cifar100/power.csv @@ -1,4 +1,4 @@ -,stage,_target_,attack,data,device_id,files,model,name,scorers,train_time,train_time_per_sample,train_start_time,train_end_time,train_device,predict_time,predict_time_per_sample,predict_start_time,predict_end_time,predict_device,predict_proba_time,predict_proba_time_per_sample,predict_proba_start_time,predict_proba_end_time,predict_proba_device,predict_log_proba_time,predict_log_proba_time_per_sample,predict_log_proba_start_time,predict_log_proba_end_time,predict_log_device,adv_fit_time_per_sample,adv_fit_time,adv_fit_start_time,adv_fit_end_time,adv_fit_device,adv_predict_time_per_sample,adv_predict_time,adv_predict_start_time,adv_predict_end_time,adv_predict_device,adv_success,accuracy,log_loss,adv_accuracy,adv_log_loss,predictions,adv_predictions,exception,stage.1,_target_.1,attack._target_,attack.attack_size,attack.method,attack.data._target_,attack.data.generate._target_,attack.data.generate.name,attack.data.generate.path,attack.data.sample._target_,attack.data.sample.random_state,attack.data.sample.stratify,attack.data.sklearn_pipeline._target_,attack.data.sklearn_pipeline.preprocessor.name,attack.data.sklearn_pipeline.preprocessor.with_mean,attack.data.sklearn_pipeline.preprocessor.with_std,attack.init._target_,attack.init.batch_size,attack.init.eps,attack.init.minimal,attack.init.model._target_,attack.init.model.art._target_,attack.init.model.art.data._target_,attack.init.model.art.data.generate._target_,attack.init.model.art.data.generate.name,attack.init.model.art.data.generate.path,attack.init.model.art.data.sample._target_,attack.init.model.art.data.sample.random_state,attack.init.model.art.data.sample.stratify,attack.init.model.art.data.sklearn_pipeline._target_,attack.init.model.art.data.sklearn_pipeline.preprocessor.name,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.art.initialize.clip_values,attack.init.model.art.initialize.criterion.name,attack.init.model.art.initialize.optimizer.lr,attack.init.model.art.initialize.optimizer.momentum,attack.init.model.art.initialize.optimizer.name,attack.init.model.art.library,attack.init.model.data._target_,attack.init.model.data.generate._target_,attack.init.model.data.generate.name,attack.init.model.data.generate.path,attack.init.model.data.sample._target_,attack.init.model.data.sample.random_state,attack.init.model.data.sample.stratify,attack.init.model.data.sklearn_pipeline._target_,attack.init.model.data.sklearn_pipeline.preprocessor.name,attack.init.model.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.init._target_,attack.init.model.init.name,attack.init.model.init.num_channels,attack.init.model.init.num_classes,attack.init.model.library,attack.init.model.trainer.batch_size,attack.init.model.trainer.nb_epoch,attack.init.name,attack.init.targeted,attack.model._target_,attack.model.art._target_,attack.model.art.data._target_,attack.model.art.data.generate._target_,attack.model.art.data.generate.name,attack.model.art.data.generate.path,attack.model.art.data.sample._target_,attack.model.art.data.sample.random_state,attack.model.art.data.sample.stratify,attack.model.art.data.sklearn_pipeline._target_,attack.model.art.data.sklearn_pipeline.preprocessor.name,attack.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.model.art.initialize.clip_values,attack.model.art.initialize.criterion.name,attack.model.art.initialize.optimizer.lr,attack.model.art.initialize.optimizer.momentum,attack.model.art.initialize.optimizer.name,attack.model.art.library,attack.model.data._target_,attack.model.data.generate._target_,attack.model.data.generate.name,attack.model.data.generate.path,attack.model.data.sample._target_,attack.model.data.sample.random_state,attack.model.data.sample.stratify,attack.model.data.sklearn_pipeline._target_,attack.model.data.sklearn_pipeline.preprocessor.name,attack.model.data.sklearn_pipeline.preprocessor.with_mean,attack.model.data.sklearn_pipeline.preprocessor.with_std,attack.model.init._target_,attack.model.init.name,attack.model.init.num_channels,attack.model.init.num_classes,attack.model.library,attack.model.trainer.batch_size,attack.model.trainer.nb_epoch,data._target_,data.generate._target_,data.generate.name,data.generate.path,data.sample._target_,data.sample.random_state,data.sample.stratify,data.sklearn_pipeline._target_,data.sklearn_pipeline.preprocessor.name,data.sklearn_pipeline.preprocessor.with_mean,data.sklearn_pipeline.preprocessor.with_std,device_id.1,files._target_,files.adv_predictions_file,files.attack_dir,files.attack_file,files.attack_type,files.data_dir,files.data_file,files.data_type,files.directory,files.name,files.predictions_file,files.reports,files.score_dict_file,files.stage,files.model_dir,files.model_file,files.model_type,files.params_file,model._target_,model.library,model.art._target_,model.art.data._target_,model.art.data.generate._target_,model.art.data.generate.name,model.art.data.generate.path,model.art.data.sample._target_,model.art.data.sample.random_state,model.art.data.sample.stratify,model.art.data.sklearn_pipeline._target_,model.art.data.sklearn_pipeline.preprocessor.name,model.art.data.sklearn_pipeline.preprocessor.with_mean,model.art.data.sklearn_pipeline.preprocessor.with_std,model.art.initialize.clip_values,model.art.initialize.criterion.name,model.art.initialize.optimizer.lr,model.art.initialize.optimizer.momentum,model.art.initialize.optimizer.name,model.art.library,model.data._target_,model.data.generate._target_,model.data.generate.name,model.data.generate.path,model.data.sample._target_,model.data.sample.random_state,model.data.sample.stratify,model.data.sklearn_pipeline._target_,model.data.sklearn_pipeline.preprocessor.name,model.data.sklearn_pipeline.preprocessor.with_mean,model.data.sklearn_pipeline.preprocessor.with_std,model.init._target_,model.init.name,model.init.num_channels,model.init.num_classes,model.trainer.batch_size,model.trainer.nb_epoch,name.1,scorers._target_,scorers.accuracy._target_,scorers.accuracy.direction,scorers.accuracy.name,scorers.log_loss._target_,scorers.log_loss.direction,scorers.log_loss.name,train_time.1,train_time_per_sample.1,train_start_time.1,train_end_time.1,train_device.1,predict_time.1,predict_time_per_sample.1,predict_start_time.1,predict_end_time.1,predict_device.1,predict_proba_time.1,predict_proba_time_per_sample.1,predict_proba_start_time.1,predict_proba_end_time.1,predict_proba_device.1,predict_log_proba_time.1,predict_log_proba_time_per_sample.1,predict_log_proba_start_time.1,predict_log_proba_end_time.1,predict_log_device.1,adv_fit_time_per_sample.1,adv_fit_time.1,adv_fit_start_time.1,adv_fit_end_time.1,adv_fit_device.1,adv_predict_time_per_sample.1,adv_predict_time.1,adv_predict_start_time.1,adv_predict_end_time.1,adv_predict_device.1,adv_success.1,accuracy.1,log_loss.1,adv_accuracy.1,adv_log_loss.1,predictions.1,adv_predictions.1,exception.1,train_power,peak_power,predict_power,predict_proba_power,predict_log_proba_power,adv_fit_power,adv_predict_power +,stage,_target_,attack,data,device_id,files,model,name,scorers,train_time,train_time_per_sample,train_start_time,train_end_time,train_device,predict_time,predict_time_per_sample,predict_start_time,predict_end_time,predict_device,predict_proba_time,predict_proba_time_per_sample,predict_proba_start_time,predict_proba_end_time,predict_proba_device,predict_log_proba_time,predict_log_proba_time_per_sample,predict_log_proba_start_time,predict_log_proba_end_time,predict_log_device,adv_fit_time_per_sample,adv_fit_time,adv_fit_start_time,adv_fit_end_time,adv_fit_device,adv_predict_time_per_sample,adv_predict_time,adv_predict_start_time,adv_predict_end_time,adv_predict_device,adv_success,accuracy,log_loss,adv_accuracy,adv_log_loss,predictions,adv_predictions,exception,stage.1,_target_.1,attack._target_,attack.attack_size,attack.method,attack.data._target_,attack.data.generate._target_,attack.data.generate.name,attack.data.generate.path,attack.data.sample._target_,attack.data.sample.random_state,attack.data.sample.stratify,attack.data.sklearn_pipeline._target_,attack.data.sklearn_pipeline.preprocessor.name,attack.data.sklearn_pipeline.preprocessor.with_mean,attack.data.sklearn_pipeline.preprocessor.with_std,attack.init._target_,attack.init.batch_size,attack.init.eps,attack.init.minimal,attack.init.model._target_,attack.init.model.art._target_,attack.init.model.art.data._target_,attack.init.model.art.data.generate._target_,attack.init.model.art.data.generate.name,attack.init.model.art.data.generate.path,attack.init.model.art.data.sample._target_,attack.init.model.art.data.sample.random_state,attack.init.model.art.data.sample.stratify,attack.init.model.art.data.sklearn_pipeline._target_,attack.init.model.art.data.sklearn_pipeline.preprocessor.name,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.art.initialize.clip_values,attack.init.model.art.initialize.criterion.name,attack.init.model.art.initialize.optimizer.lr,attack.init.model.art.initialize.optimizer.momentum,attack.init.model.art.initialize.optimizer.name,attack.init.model.art.library,attack.init.model.data._target_,attack.init.model.data.generate._target_,attack.init.model.data.generate.name,attack.init.model.data.generate.path,attack.init.model.data.sample._target_,attack.init.model.data.sample.random_state,attack.init.model.data.sample.stratify,attack.init.model.data.sklearn_pipeline._target_,attack.init.model.data.sklearn_pipeline.preprocessor.name,attack.init.model.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.init._target_,attack.init.model.init.name,attack.init.model.init.num_channels,attack.init.model.init.num_classes,attack.init.model.library,attack.init.model.trainer.batch_size,attack.init.model.trainer.np_epochs,attack.init.name,attack.init.targeted,attack.model._target_,attack.model.art._target_,attack.model.art.data._target_,attack.model.art.data.generate._target_,attack.model.art.data.generate.name,attack.model.art.data.generate.path,attack.model.art.data.sample._target_,attack.model.art.data.sample.random_state,attack.model.art.data.sample.stratify,attack.model.art.data.sklearn_pipeline._target_,attack.model.art.data.sklearn_pipeline.preprocessor.name,attack.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.model.art.initialize.clip_values,attack.model.art.initialize.criterion.name,attack.model.art.initialize.optimizer.lr,attack.model.art.initialize.optimizer.momentum,attack.model.art.initialize.optimizer.name,attack.model.art.library,attack.model.data._target_,attack.model.data.generate._target_,attack.model.data.generate.name,attack.model.data.generate.path,attack.model.data.sample._target_,attack.model.data.sample.random_state,attack.model.data.sample.stratify,attack.model.data.sklearn_pipeline._target_,attack.model.data.sklearn_pipeline.preprocessor.name,attack.model.data.sklearn_pipeline.preprocessor.with_mean,attack.model.data.sklearn_pipeline.preprocessor.with_std,attack.model.init._target_,attack.model.init.name,attack.model.init.num_channels,attack.model.init.num_classes,attack.model.library,attack.model.trainer.batch_size,attack.model.trainer.np_epochs,data._target_,data.generate._target_,data.generate.name,data.generate.path,data.sample._target_,data.sample.random_state,data.sample.stratify,data.sklearn_pipeline._target_,data.sklearn_pipeline.preprocessor.name,data.sklearn_pipeline.preprocessor.with_mean,data.sklearn_pipeline.preprocessor.with_std,device_id.1,files._target_,files.adv_predictions_file,files.attack_dir,files.attack_file,files.attack_type,files.data_dir,files.data_file,files.data_type,files.directory,files.name,files.predictions_file,files.reports,files.score_dict_file,files.stage,files.model_dir,files.model_file,files.model_type,files.params_file,model._target_,model.library,model.art._target_,model.art.data._target_,model.art.data.generate._target_,model.art.data.generate.name,model.art.data.generate.path,model.art.data.sample._target_,model.art.data.sample.random_state,model.art.data.sample.stratify,model.art.data.sklearn_pipeline._target_,model.art.data.sklearn_pipeline.preprocessor.name,model.art.data.sklearn_pipeline.preprocessor.with_mean,model.art.data.sklearn_pipeline.preprocessor.with_std,model.art.initialize.clip_values,model.art.initialize.criterion.name,model.art.initialize.optimizer.lr,model.art.initialize.optimizer.momentum,model.art.initialize.optimizer.name,model.art.library,model.data._target_,model.data.generate._target_,model.data.generate.name,model.data.generate.path,model.data.sample._target_,model.data.sample.random_state,model.data.sample.stratify,model.data.sklearn_pipeline._target_,model.data.sklearn_pipeline.preprocessor.name,model.data.sklearn_pipeline.preprocessor.with_mean,model.data.sklearn_pipeline.preprocessor.with_std,model.init._target_,model.init.name,model.init.num_channels,model.init.num_classes,model.trainer.batch_size,model.trainer.np_epochs,name.1,scorers._target_,scorers.accuracy._target_,scorers.accuracy.direction,scorers.accuracy.name,scorers.log_loss._target_,scorers.log_loss.direction,scorers.log_loss.name,train_time.1,train_time_per_sample.1,train_start_time.1,train_end_time.1,train_device.1,predict_time.1,predict_time_per_sample.1,predict_start_time.1,predict_end_time.1,predict_device.1,predict_proba_time.1,predict_proba_time_per_sample.1,predict_proba_start_time.1,predict_proba_end_time.1,predict_proba_device.1,predict_log_proba_time.1,predict_log_proba_time_per_sample.1,predict_log_proba_start_time.1,predict_log_proba_end_time.1,predict_log_device.1,adv_fit_time_per_sample.1,adv_fit_time.1,adv_fit_start_time.1,adv_fit_end_time.1,adv_fit_device.1,adv_predict_time_per_sample.1,adv_predict_time.1,adv_predict_start_time.1,adv_predict_end_time.1,adv_predict_device.1,adv_success.1,accuracy.1,log_loss.1,adv_accuracy.1,adv_log_loss.1,predictions.1,adv_predictions.1,exception.1,train_power,peak_power,predict_power,predict_proba_power,predict_log_proba_power,adv_fit_power,adv_predict_power 00084e182adc3d1eb46bcb45b533db70,attack,deckard.base.experiment.Experiment,00084e182adc3d1eb46bcb45b533db70,00084e182adc3d1eb46bcb45b533db70,nvidia-tesla-v100,00084e182adc3d1eb46bcb45b533db70,00084e182adc3d1eb46bcb45b533db70,00084e182adc3d1eb46bcb45b533db70,00084e182adc3d1eb46bcb45b533db70,44.563185477,0.0009283996974375,1701844246.83223,1701844291.09493,cuda:0,0.649023392,1.3521320666666666e-05,1701844291.0974412,1701844291.7462597,cuda:0,0.638715124,1.3306565083333334e-05,1701844293.508366,1701844294.146932,cuda:0,0.629740648,1.3119596833333334e-05,1701844294.1470006,1701844294.7765672,cuda:0,0.00094397572,0.094397572,1701844299.6592228,1701844299.710223,cuda:0,5.890148e-05,0.005890148,1701844300.2628386,1701844300.2686725,cuda:0,1.0,0.3310833333333333,5.007507506350676,0.0,18.58668479204178,/result/cifar100/reports/attack/00084e182adc3d1eb46bcb45b533db70/predictions.json,/result/cifar100/reports/attack/00084e182adc3d1eb46bcb45b533db70/adv_predictions.json,,attack,deckard.base.experiment.Experiment,deckard.base.attack.Attack,100,evasion,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.attack.AttackInitializer,337,0.8593273488439556,True,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.1991145545016358,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,100,pytorch,337,29,art.attacks.evasion.FastGradientMethod,False,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.1991145545016358,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,100,pytorch,337,29,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,nvidia-tesla-v100,deckard.base.files.FileConfig,adv_predictions.json,attacks,593f0516312bd8a2ae5e91592037b220,.pkl,data,5e8485acd7f1e1827e21c76e9336ba56,.pkl,/result/cifar100/,00084e182adc3d1eb46bcb45b533db70,predictions.json,reports,score_dict.json,attack,,,,,deckard.base.model.Model,pytorch,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.1991145545016358,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,100,337,29,00084e182adc3d1eb46bcb45b533db70,deckard.base.scorer.ScorerDict,deckard.base.scorer.ScorerConfig,maximize,sklearn.metrics.accuracy_score,deckard.base.scorer.ScorerConfig,minimize,sklearn.metrics.log_loss,44.563185477,0.0009283996974375,1701844246.83223,1701844291.09493,cuda:0,0.649023392,1.3521320666666666e-05,1701844291.0974412,1701844291.7462597,cuda:0,0.638715124,1.3306565083333334e-05,1701844293.508366,1701844294.146932,cuda:0,0.629740648,1.3119596833333334e-05,1701844294.1470006,1701844294.7765672,cuda:0,0.00094397572,0.094397572,1701844299.6592228,1701844299.710223,cuda:0,5.890148e-05,0.005890148,1701844300.2628386,1701844300.2686725,cuda:0,1.0,0.3310833333333333,5.007507506350676,0.0,18.58668479204178,/result/cifar100/reports/attack/00084e182adc3d1eb46bcb45b533db70/predictions.json,/result/cifar100/reports/attack/00084e182adc3d1eb46bcb45b533db70/adv_predictions.json,,2.5263550694800085,250,0.051198087046543765,0.05104535193741322,0.047331171969572705,0.0033263410462273492,0.00035805341601371766 001f6adbe4bf7aa9ffbbcc86b32722ac,attack,deckard.base.experiment.Experiment,001f6adbe4bf7aa9ffbbcc86b32722ac,001f6adbe4bf7aa9ffbbcc86b32722ac,nvidia-tesla-v100,001f6adbe4bf7aa9ffbbcc86b32722ac,001f6adbe4bf7aa9ffbbcc86b32722ac,001f6adbe4bf7aa9ffbbcc86b32722ac,001f6adbe4bf7aa9ffbbcc86b32722ac,89.299512464,0.0018604065096666,1701645552.703049,1701645642.3640127,cuda:0,0.656059879,1.3667914145833334e-05,1701645642.3662217,1701645643.0220573,cuda:0,0.640694344,1.3347798833333334e-05,1701645644.6960628,1701645645.336583,cuda:0,0.638069705,1.3293118854166666e-05,1701645645.33665,1701645645.9747102,cuda:0,0.00171427162,0.171427162,1701645650.8158312,1701645650.9018922,cuda:0,6.267057e-05,0.006267057,1701645651.4656882,1701645651.4719594,cuda:0,0.84,0.41825,4.380509820342064,0.0,18.074019560813905,/result/cifar100/reports/attack/001f6adbe4bf7aa9ffbbcc86b32722ac/predictions.json,/result/cifar100/reports/attack/001f6adbe4bf7aa9ffbbcc86b32722ac/adv_predictions.json,,attack,deckard.base.experiment.Experiment,deckard.base.attack.Attack,100,evasion,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.attack.AttackInitializer,53,0.3843982466173929,True,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.04168708021016,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,100,pytorch,53,20,art.attacks.evasion.FastGradientMethod,False,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.04168708021016,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,100,pytorch,53,20,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,nvidia-tesla-v100,deckard.base.files.FileConfig,adv_predictions.json,attacks,723bc74241e7b6b80d17f0996c308de9,.pkl,data,5e8485acd7f1e1827e21c76e9336ba56,.pkl,/result/cifar100/,001f6adbe4bf7aa9ffbbcc86b32722ac,predictions.json,reports,score_dict.json,attack,,,,,deckard.base.model.Model,pytorch,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.04168708021016,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,100,53,20,001f6adbe4bf7aa9ffbbcc86b32722ac,deckard.base.scorer.ScorerDict,deckard.base.scorer.ScorerConfig,maximize,sklearn.metrics.accuracy_score,deckard.base.scorer.ScorerConfig,minimize,sklearn.metrics.log_loss,89.299512464,0.0018604065096666,1701645552.703049,1701645642.3640127,cuda:0,0.656059879,1.3667914145833334e-05,1701645642.3662217,1701645643.0220573,cuda:0,0.640694344,1.3347798833333334e-05,1701645644.6960628,1701645645.336583,cuda:0,0.638069705,1.3293118854166666e-05,1701645645.33665,1701645645.9747102,cuda:0,0.00171427162,0.171427162,1701645650.8158312,1701645650.9018922,cuda:0,6.267057e-05,0.006267057,1701645651.4656882,1701645651.4719594,cuda:0,0.84,0.41825,4.380509820342064,0.0,18.074019560813905,/result/cifar100/reports/attack/001f6adbe4bf7aa9ffbbcc86b32722ac/predictions.json,/result/cifar100/reports/attack/001f6adbe4bf7aa9ffbbcc86b32722ac/adv_predictions.json,,5.261355165888866,250,0.0450886994600296,0.04755416905879974,0.046768936092654874,0.005512685219446818,0.00044037225511338985 004c81db70edff7701b2e75c90638a3e,attack,deckard.base.experiment.Experiment,004c81db70edff7701b2e75c90638a3e,004c81db70edff7701b2e75c90638a3e,nvidia-tesla-v100,004c81db70edff7701b2e75c90638a3e,004c81db70edff7701b2e75c90638a3e,004c81db70edff7701b2e75c90638a3e,004c81db70edff7701b2e75c90638a3e,52.256856954,0.001088684519875,1701717298.0556853,1701717350.157497,cuda:0,0.651379906,1.3570414708333331e-05,1701717350.1598504,1701717350.8110416,cuda:0,0.645816035,1.3454500729166667e-05,1701717352.7595005,1701717353.405154,cuda:0,0.640197488,1.3337447666666669e-05,1701717353.4052289,1701717354.045172,cuda:0,0.00088138115,0.088138115,1701717358.8750396,1701717358.9200468,cuda:0,6.565136e-05,0.006565136,1701717359.5090797,1701717359.515649,cuda:0,1.0,0.3055833333333333,5.117039807061355,0.0,19.04580527305603,/result/cifar100/reports/attack/004c81db70edff7701b2e75c90638a3e/predictions.json,/result/cifar100/reports/attack/004c81db70edff7701b2e75c90638a3e/adv_predictions.json,,attack,deckard.base.experiment.Experiment,deckard.base.attack.Attack,100,evasion,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.attack.AttackInitializer,169,0.8048944072110157,True,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0258729870222589,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,100,pytorch,169,30,art.attacks.evasion.FastGradientMethod,False,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0258729870222589,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,100,pytorch,169,30,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,nvidia-tesla-v100,deckard.base.files.FileConfig,adv_predictions.json,attacks,b05c547e07610e525cde9b300767fabf,.pkl,data,5e8485acd7f1e1827e21c76e9336ba56,.pkl,/result/cifar100/,004c81db70edff7701b2e75c90638a3e,predictions.json,reports,score_dict.json,attack,,,,,deckard.base.model.Model,pytorch,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0258729870222589,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,100,169,30,004c81db70edff7701b2e75c90638a3e,deckard.base.scorer.ScorerDict,deckard.base.scorer.ScorerConfig,maximize,sklearn.metrics.accuracy_score,deckard.base.scorer.ScorerConfig,minimize,sklearn.metrics.log_loss,52.256856954,0.001088684519875,1701717298.0556853,1701717350.157497,cuda:0,0.651379906,1.3570414708333331e-05,1701717350.1598504,1701717350.8110416,cuda:0,0.645816035,1.3454500729166667e-05,1701717352.7595005,1701717353.405154,cuda:0,0.640197488,1.3337447666666669e-05,1701717353.4052289,1701717354.045172,cuda:0,0.00088138115,0.088138115,1701717358.8750396,1701717358.9200468,cuda:0,6.565136e-05,0.006565136,1701717359.5090797,1701717359.515649,cuda:0,1.0,0.3055833333333333,5.117039807061355,0.0,19.04580527305603,/result/cifar100/reports/attack/004c81db70edff7701b2e75c90638a3e/predictions.json,/result/cifar100/reports/attack/004c81db70edff7701b2e75c90638a3e/adv_predictions.json,,2.924214178711176,250,0.04120141207178434,0.044433514227469766,0.04626255492369334,0.0035736989908748203,0.0003680224517981212 diff --git a/examples/power/plots/data/cifar100/raw.csv b/examples/power/plots/data/cifar100/raw.csv index 7f54be74..36546dfc 100644 --- a/examples/power/plots/data/cifar100/raw.csv +++ b/examples/power/plots/data/cifar100/raw.csv @@ -1,4 +1,4 @@ -,stage,_target_,attack,data,device_id,files,model,name,scorers,train_time,train_time_per_sample,train_start_time,train_end_time,train_device,predict_time,predict_time_per_sample,predict_start_time,predict_end_time,predict_device,predict_proba_time,predict_proba_time_per_sample,predict_proba_start_time,predict_proba_end_time,predict_proba_device,predict_log_proba_time,predict_log_proba_time_per_sample,predict_log_proba_start_time,predict_log_proba_end_time,predict_log_device,adv_fit_time_per_sample,adv_fit_time,adv_fit_start_time,adv_fit_end_time,adv_fit_device,adv_predict_time_per_sample,adv_predict_time,adv_predict_start_time,adv_predict_end_time,adv_predict_device,adv_success,accuracy,log_loss,adv_accuracy,adv_log_loss,predictions,adv_predictions,exception,stage,_target_,attack._target_,attack.attack_size,attack.method,attack.data._target_,attack.data.generate._target_,attack.data.generate.name,attack.data.generate.path,attack.data.sample._target_,attack.data.sample.random_state,attack.data.sample.stratify,attack.data.sklearn_pipeline._target_,attack.data.sklearn_pipeline.preprocessor.name,attack.data.sklearn_pipeline.preprocessor.with_mean,attack.data.sklearn_pipeline.preprocessor.with_std,attack.init._target_,attack.init.batch_size,attack.init.eps,attack.init.minimal,attack.init.model._target_,attack.init.model.art._target_,attack.init.model.art.data._target_,attack.init.model.art.data.generate._target_,attack.init.model.art.data.generate.name,attack.init.model.art.data.generate.path,attack.init.model.art.data.sample._target_,attack.init.model.art.data.sample.random_state,attack.init.model.art.data.sample.stratify,attack.init.model.art.data.sklearn_pipeline._target_,attack.init.model.art.data.sklearn_pipeline.preprocessor.name,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.art.initialize.clip_values,attack.init.model.art.initialize.criterion.name,attack.init.model.art.initialize.optimizer.lr,attack.init.model.art.initialize.optimizer.momentum,attack.init.model.art.initialize.optimizer.name,attack.init.model.art.library,attack.init.model.data._target_,attack.init.model.data.generate._target_,attack.init.model.data.generate.name,attack.init.model.data.generate.path,attack.init.model.data.sample._target_,attack.init.model.data.sample.random_state,attack.init.model.data.sample.stratify,attack.init.model.data.sklearn_pipeline._target_,attack.init.model.data.sklearn_pipeline.preprocessor.name,attack.init.model.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.init._target_,attack.init.model.init.name,attack.init.model.init.num_channels,attack.init.model.init.num_classes,attack.init.model.library,attack.init.model.trainer.batch_size,attack.init.model.trainer.nb_epoch,attack.init.name,attack.init.targeted,attack.model._target_,attack.model.art._target_,attack.model.art.data._target_,attack.model.art.data.generate._target_,attack.model.art.data.generate.name,attack.model.art.data.generate.path,attack.model.art.data.sample._target_,attack.model.art.data.sample.random_state,attack.model.art.data.sample.stratify,attack.model.art.data.sklearn_pipeline._target_,attack.model.art.data.sklearn_pipeline.preprocessor.name,attack.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.model.art.initialize.clip_values,attack.model.art.initialize.criterion.name,attack.model.art.initialize.optimizer.lr,attack.model.art.initialize.optimizer.momentum,attack.model.art.initialize.optimizer.name,attack.model.art.library,attack.model.data._target_,attack.model.data.generate._target_,attack.model.data.generate.name,attack.model.data.generate.path,attack.model.data.sample._target_,attack.model.data.sample.random_state,attack.model.data.sample.stratify,attack.model.data.sklearn_pipeline._target_,attack.model.data.sklearn_pipeline.preprocessor.name,attack.model.data.sklearn_pipeline.preprocessor.with_mean,attack.model.data.sklearn_pipeline.preprocessor.with_std,attack.model.init._target_,attack.model.init.name,attack.model.init.num_channels,attack.model.init.num_classes,attack.model.library,attack.model.trainer.batch_size,attack.model.trainer.nb_epoch,data._target_,data.generate._target_,data.generate.name,data.generate.path,data.sample._target_,data.sample.random_state,data.sample.stratify,data.sklearn_pipeline._target_,data.sklearn_pipeline.preprocessor.name,data.sklearn_pipeline.preprocessor.with_mean,data.sklearn_pipeline.preprocessor.with_std,device_id,files._target_,files.adv_predictions_file,files.attack_dir,files.attack_file,files.attack_type,files.data_dir,files.data_file,files.data_type,files.directory,files.name,files.predictions_file,files.reports,files.score_dict_file,files.stage,files.model_dir,files.model_file,files.model_type,files.params_file,model._target_,model.library,model.art._target_,model.art.data._target_,model.art.data.generate._target_,model.art.data.generate.name,model.art.data.generate.path,model.art.data.sample._target_,model.art.data.sample.random_state,model.art.data.sample.stratify,model.art.data.sklearn_pipeline._target_,model.art.data.sklearn_pipeline.preprocessor.name,model.art.data.sklearn_pipeline.preprocessor.with_mean,model.art.data.sklearn_pipeline.preprocessor.with_std,model.art.initialize.clip_values,model.art.initialize.criterion.name,model.art.initialize.optimizer.lr,model.art.initialize.optimizer.momentum,model.art.initialize.optimizer.name,model.art.library,model.data._target_,model.data.generate._target_,model.data.generate.name,model.data.generate.path,model.data.sample._target_,model.data.sample.random_state,model.data.sample.stratify,model.data.sklearn_pipeline._target_,model.data.sklearn_pipeline.preprocessor.name,model.data.sklearn_pipeline.preprocessor.with_mean,model.data.sklearn_pipeline.preprocessor.with_std,model.init._target_,model.init.name,model.init.num_channels,model.init.num_classes,model.trainer.batch_size,model.trainer.nb_epoch,name,scorers._target_,scorers.accuracy._target_,scorers.accuracy.direction,scorers.accuracy.name,scorers.log_loss._target_,scorers.log_loss.direction,scorers.log_loss.name,train_time,train_time_per_sample,train_start_time,train_end_time,train_device,predict_time,predict_time_per_sample,predict_start_time,predict_end_time,predict_device,predict_proba_time,predict_proba_time_per_sample,predict_proba_start_time,predict_proba_end_time,predict_proba_device,predict_log_proba_time,predict_log_proba_time_per_sample,predict_log_proba_start_time,predict_log_proba_end_time,predict_log_device,adv_fit_time_per_sample,adv_fit_time,adv_fit_start_time,adv_fit_end_time,adv_fit_device,adv_predict_time_per_sample,adv_predict_time,adv_predict_start_time,adv_predict_end_time,adv_predict_device,adv_success,accuracy,log_loss,adv_accuracy,adv_log_loss,predictions,adv_predictions,exception +,stage,_target_,attack,data,device_id,files,model,name,scorers,train_time,train_time_per_sample,train_start_time,train_end_time,train_device,predict_time,predict_time_per_sample,predict_start_time,predict_end_time,predict_device,predict_proba_time,predict_proba_time_per_sample,predict_proba_start_time,predict_proba_end_time,predict_proba_device,predict_log_proba_time,predict_log_proba_time_per_sample,predict_log_proba_start_time,predict_log_proba_end_time,predict_log_device,adv_fit_time_per_sample,adv_fit_time,adv_fit_start_time,adv_fit_end_time,adv_fit_device,adv_predict_time_per_sample,adv_predict_time,adv_predict_start_time,adv_predict_end_time,adv_predict_device,adv_success,accuracy,log_loss,adv_accuracy,adv_log_loss,predictions,adv_predictions,exception,stage,_target_,attack._target_,attack.attack_size,attack.method,attack.data._target_,attack.data.generate._target_,attack.data.generate.name,attack.data.generate.path,attack.data.sample._target_,attack.data.sample.random_state,attack.data.sample.stratify,attack.data.sklearn_pipeline._target_,attack.data.sklearn_pipeline.preprocessor.name,attack.data.sklearn_pipeline.preprocessor.with_mean,attack.data.sklearn_pipeline.preprocessor.with_std,attack.init._target_,attack.init.batch_size,attack.init.eps,attack.init.minimal,attack.init.model._target_,attack.init.model.art._target_,attack.init.model.art.data._target_,attack.init.model.art.data.generate._target_,attack.init.model.art.data.generate.name,attack.init.model.art.data.generate.path,attack.init.model.art.data.sample._target_,attack.init.model.art.data.sample.random_state,attack.init.model.art.data.sample.stratify,attack.init.model.art.data.sklearn_pipeline._target_,attack.init.model.art.data.sklearn_pipeline.preprocessor.name,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.art.initialize.clip_values,attack.init.model.art.initialize.criterion.name,attack.init.model.art.initialize.optimizer.lr,attack.init.model.art.initialize.optimizer.momentum,attack.init.model.art.initialize.optimizer.name,attack.init.model.art.library,attack.init.model.data._target_,attack.init.model.data.generate._target_,attack.init.model.data.generate.name,attack.init.model.data.generate.path,attack.init.model.data.sample._target_,attack.init.model.data.sample.random_state,attack.init.model.data.sample.stratify,attack.init.model.data.sklearn_pipeline._target_,attack.init.model.data.sklearn_pipeline.preprocessor.name,attack.init.model.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.init._target_,attack.init.model.init.name,attack.init.model.init.num_channels,attack.init.model.init.num_classes,attack.init.model.library,attack.init.model.trainer.batch_size,attack.init.model.trainer.np_epochs,attack.init.name,attack.init.targeted,attack.model._target_,attack.model.art._target_,attack.model.art.data._target_,attack.model.art.data.generate._target_,attack.model.art.data.generate.name,attack.model.art.data.generate.path,attack.model.art.data.sample._target_,attack.model.art.data.sample.random_state,attack.model.art.data.sample.stratify,attack.model.art.data.sklearn_pipeline._target_,attack.model.art.data.sklearn_pipeline.preprocessor.name,attack.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.model.art.initialize.clip_values,attack.model.art.initialize.criterion.name,attack.model.art.initialize.optimizer.lr,attack.model.art.initialize.optimizer.momentum,attack.model.art.initialize.optimizer.name,attack.model.art.library,attack.model.data._target_,attack.model.data.generate._target_,attack.model.data.generate.name,attack.model.data.generate.path,attack.model.data.sample._target_,attack.model.data.sample.random_state,attack.model.data.sample.stratify,attack.model.data.sklearn_pipeline._target_,attack.model.data.sklearn_pipeline.preprocessor.name,attack.model.data.sklearn_pipeline.preprocessor.with_mean,attack.model.data.sklearn_pipeline.preprocessor.with_std,attack.model.init._target_,attack.model.init.name,attack.model.init.num_channels,attack.model.init.num_classes,attack.model.library,attack.model.trainer.batch_size,attack.model.trainer.np_epochs,data._target_,data.generate._target_,data.generate.name,data.generate.path,data.sample._target_,data.sample.random_state,data.sample.stratify,data.sklearn_pipeline._target_,data.sklearn_pipeline.preprocessor.name,data.sklearn_pipeline.preprocessor.with_mean,data.sklearn_pipeline.preprocessor.with_std,device_id,files._target_,files.adv_predictions_file,files.attack_dir,files.attack_file,files.attack_type,files.data_dir,files.data_file,files.data_type,files.directory,files.name,files.predictions_file,files.reports,files.score_dict_file,files.stage,files.model_dir,files.model_file,files.model_type,files.params_file,model._target_,model.library,model.art._target_,model.art.data._target_,model.art.data.generate._target_,model.art.data.generate.name,model.art.data.generate.path,model.art.data.sample._target_,model.art.data.sample.random_state,model.art.data.sample.stratify,model.art.data.sklearn_pipeline._target_,model.art.data.sklearn_pipeline.preprocessor.name,model.art.data.sklearn_pipeline.preprocessor.with_mean,model.art.data.sklearn_pipeline.preprocessor.with_std,model.art.initialize.clip_values,model.art.initialize.criterion.name,model.art.initialize.optimizer.lr,model.art.initialize.optimizer.momentum,model.art.initialize.optimizer.name,model.art.library,model.data._target_,model.data.generate._target_,model.data.generate.name,model.data.generate.path,model.data.sample._target_,model.data.sample.random_state,model.data.sample.stratify,model.data.sklearn_pipeline._target_,model.data.sklearn_pipeline.preprocessor.name,model.data.sklearn_pipeline.preprocessor.with_mean,model.data.sklearn_pipeline.preprocessor.with_std,model.init._target_,model.init.name,model.init.num_channels,model.init.num_classes,model.trainer.batch_size,model.trainer.np_epochs,name,scorers._target_,scorers.accuracy._target_,scorers.accuracy.direction,scorers.accuracy.name,scorers.log_loss._target_,scorers.log_loss.direction,scorers.log_loss.name,train_time,train_time_per_sample,train_start_time,train_end_time,train_device,predict_time,predict_time_per_sample,predict_start_time,predict_end_time,predict_device,predict_proba_time,predict_proba_time_per_sample,predict_proba_start_time,predict_proba_end_time,predict_proba_device,predict_log_proba_time,predict_log_proba_time_per_sample,predict_log_proba_start_time,predict_log_proba_end_time,predict_log_device,adv_fit_time_per_sample,adv_fit_time,adv_fit_start_time,adv_fit_end_time,adv_fit_device,adv_predict_time_per_sample,adv_predict_time,adv_predict_start_time,adv_predict_end_time,adv_predict_device,adv_success,accuracy,log_loss,adv_accuracy,adv_log_loss,predictions,adv_predictions,exception 00084e182adc3d1eb46bcb45b533db70,attack,deckard.base.experiment.Experiment,00084e182adc3d1eb46bcb45b533db70,00084e182adc3d1eb46bcb45b533db70,nvidia-tesla-v100,00084e182adc3d1eb46bcb45b533db70,00084e182adc3d1eb46bcb45b533db70,00084e182adc3d1eb46bcb45b533db70,00084e182adc3d1eb46bcb45b533db70,44.563185477,0.0009283996974375,1701844246.83223,1701844291.09493,cuda:0,0.649023392,1.3521320666666666e-05,1701844291.0974412,1701844291.7462595,cuda:0,0.638715124,1.3306565083333334e-05,1701844293.5083659,1701844294.146932,cuda:0,0.629740648,1.3119596833333334e-05,1701844294.1470006,1701844294.7765672,cuda:0,0.00094397572,0.094397572,1701844299.6592228,1701844299.710223,cuda:0,5.890148e-05,0.005890148,1701844300.2628386,1701844300.2686725,cuda:0,1.0,0.33108333333333334,5.007507506350676,0.0,18.58668479204178,/result/cifar100/reports/attack/00084e182adc3d1eb46bcb45b533db70/predictions.json,/result/cifar100/reports/attack/00084e182adc3d1eb46bcb45b533db70/adv_predictions.json,,attack,deckard.base.experiment.Experiment,deckard.base.attack.Attack,100,evasion,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.attack.AttackInitializer,337,0.8593273488439556,True,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.19911455450163584,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,100,pytorch,337,29,art.attacks.evasion.FastGradientMethod,False,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.19911455450163584,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,100,pytorch,337,29,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,nvidia-tesla-v100,deckard.base.files.FileConfig,adv_predictions.json,attacks,593f0516312bd8a2ae5e91592037b220,.pkl,data,5e8485acd7f1e1827e21c76e9336ba56,.pkl,/result/cifar100/,00084e182adc3d1eb46bcb45b533db70,predictions.json,reports,score_dict.json,attack,,,,,deckard.base.model.Model,pytorch,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.19911455450163584,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,100,337,29,00084e182adc3d1eb46bcb45b533db70,deckard.base.scorer.ScorerDict,deckard.base.scorer.ScorerConfig,maximize,sklearn.metrics.accuracy_score,deckard.base.scorer.ScorerConfig,minimize,sklearn.metrics.log_loss,44.563185477,0.0009283996974375,1701844246.83223,1701844291.09493,cuda:0,0.649023392,1.3521320666666666e-05,1701844291.0974412,1701844291.7462595,cuda:0,0.638715124,1.3306565083333334e-05,1701844293.5083659,1701844294.146932,cuda:0,0.629740648,1.3119596833333334e-05,1701844294.1470006,1701844294.7765672,cuda:0,0.00094397572,0.094397572,1701844299.6592228,1701844299.710223,cuda:0,5.890148e-05,0.005890148,1701844300.2628386,1701844300.2686725,cuda:0,1.0,0.33108333333333334,5.007507506350676,0.0,18.58668479204178,/result/cifar100/reports/attack/00084e182adc3d1eb46bcb45b533db70/predictions.json,/result/cifar100/reports/attack/00084e182adc3d1eb46bcb45b533db70/adv_predictions.json, 001f6adbe4bf7aa9ffbbcc86b32722ac,attack,deckard.base.experiment.Experiment,001f6adbe4bf7aa9ffbbcc86b32722ac,001f6adbe4bf7aa9ffbbcc86b32722ac,nvidia-tesla-v100,001f6adbe4bf7aa9ffbbcc86b32722ac,001f6adbe4bf7aa9ffbbcc86b32722ac,001f6adbe4bf7aa9ffbbcc86b32722ac,001f6adbe4bf7aa9ffbbcc86b32722ac,89.299512464,0.0018604065096666667,1701645552.703049,1701645642.3640127,cuda:0,0.656059879,1.3667914145833334e-05,1701645642.3662217,1701645643.0220573,cuda:0,0.640694344,1.3347798833333334e-05,1701645644.6960628,1701645645.336583,cuda:0,0.638069705,1.3293118854166666e-05,1701645645.3366501,1701645645.9747102,cuda:0,0.00171427162,0.171427162,1701645650.8158312,1701645650.9018922,cuda:0,6.267057e-05,0.006267057,1701645651.4656882,1701645651.4719594,cuda:0,0.84,0.41825,4.380509820342064,0.0,18.074019560813905,/result/cifar100/reports/attack/001f6adbe4bf7aa9ffbbcc86b32722ac/predictions.json,/result/cifar100/reports/attack/001f6adbe4bf7aa9ffbbcc86b32722ac/adv_predictions.json,,attack,deckard.base.experiment.Experiment,deckard.base.attack.Attack,100,evasion,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.attack.AttackInitializer,53,0.3843982466173929,True,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.041687080210160044,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,100,pytorch,53,20,art.attacks.evasion.FastGradientMethod,False,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.041687080210160044,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,100,pytorch,53,20,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,nvidia-tesla-v100,deckard.base.files.FileConfig,adv_predictions.json,attacks,723bc74241e7b6b80d17f0996c308de9,.pkl,data,5e8485acd7f1e1827e21c76e9336ba56,.pkl,/result/cifar100/,001f6adbe4bf7aa9ffbbcc86b32722ac,predictions.json,reports,score_dict.json,attack,,,,,deckard.base.model.Model,pytorch,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.041687080210160044,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,100,53,20,001f6adbe4bf7aa9ffbbcc86b32722ac,deckard.base.scorer.ScorerDict,deckard.base.scorer.ScorerConfig,maximize,sklearn.metrics.accuracy_score,deckard.base.scorer.ScorerConfig,minimize,sklearn.metrics.log_loss,89.299512464,0.0018604065096666667,1701645552.703049,1701645642.3640127,cuda:0,0.656059879,1.3667914145833334e-05,1701645642.3662217,1701645643.0220573,cuda:0,0.640694344,1.3347798833333334e-05,1701645644.6960628,1701645645.336583,cuda:0,0.638069705,1.3293118854166666e-05,1701645645.3366501,1701645645.9747102,cuda:0,0.00171427162,0.171427162,1701645650.8158312,1701645650.9018922,cuda:0,6.267057e-05,0.006267057,1701645651.4656882,1701645651.4719594,cuda:0,0.84,0.41825,4.380509820342064,0.0,18.074019560813905,/result/cifar100/reports/attack/001f6adbe4bf7aa9ffbbcc86b32722ac/predictions.json,/result/cifar100/reports/attack/001f6adbe4bf7aa9ffbbcc86b32722ac/adv_predictions.json, 004c81db70edff7701b2e75c90638a3e,attack,deckard.base.experiment.Experiment,004c81db70edff7701b2e75c90638a3e,004c81db70edff7701b2e75c90638a3e,nvidia-tesla-v100,004c81db70edff7701b2e75c90638a3e,004c81db70edff7701b2e75c90638a3e,004c81db70edff7701b2e75c90638a3e,004c81db70edff7701b2e75c90638a3e,52.256856954,0.001088684519875,1701717298.0556853,1701717350.157497,cuda:0,0.651379906,1.3570414708333333e-05,1701717350.1598504,1701717350.8110416,cuda:0,0.645816035,1.3454500729166667e-05,1701717352.7595003,1701717353.405154,cuda:0,0.640197488,1.3337447666666667e-05,1701717353.4052289,1701717354.045172,cuda:0,0.00088138115,0.088138115,1701717358.8750396,1701717358.9200468,cuda:0,6.565136e-05,0.006565136,1701717359.5090797,1701717359.515649,cuda:0,1.0,0.3055833333333333,5.117039807061355,0.0,19.04580527305603,/result/cifar100/reports/attack/004c81db70edff7701b2e75c90638a3e/predictions.json,/result/cifar100/reports/attack/004c81db70edff7701b2e75c90638a3e/adv_predictions.json,,attack,deckard.base.experiment.Experiment,deckard.base.attack.Attack,100,evasion,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.attack.AttackInitializer,169,0.8048944072110157,True,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.025872987022258983,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,100,pytorch,169,30,art.attacks.evasion.FastGradientMethod,False,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.025872987022258983,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,100,pytorch,169,30,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,nvidia-tesla-v100,deckard.base.files.FileConfig,adv_predictions.json,attacks,b05c547e07610e525cde9b300767fabf,.pkl,data,5e8485acd7f1e1827e21c76e9336ba56,.pkl,/result/cifar100/,004c81db70edff7701b2e75c90638a3e,predictions.json,reports,score_dict.json,attack,,,,,deckard.base.model.Model,pytorch,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.025872987022258983,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_cifar100,original_data/,deckard.base.data.sampler.SklearnDataSampler,1,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,3,100,169,30,004c81db70edff7701b2e75c90638a3e,deckard.base.scorer.ScorerDict,deckard.base.scorer.ScorerConfig,maximize,sklearn.metrics.accuracy_score,deckard.base.scorer.ScorerConfig,minimize,sklearn.metrics.log_loss,52.256856954,0.001088684519875,1701717298.0556853,1701717350.157497,cuda:0,0.651379906,1.3570414708333333e-05,1701717350.1598504,1701717350.8110416,cuda:0,0.645816035,1.3454500729166667e-05,1701717352.7595003,1701717353.405154,cuda:0,0.640197488,1.3337447666666667e-05,1701717353.4052289,1701717354.045172,cuda:0,0.00088138115,0.088138115,1701717358.8750396,1701717358.9200468,cuda:0,6.565136e-05,0.006565136,1701717359.5090797,1701717359.515649,cuda:0,1.0,0.3055833333333333,5.117039807061355,0.0,19.04580527305603,/result/cifar100/reports/attack/004c81db70edff7701b2e75c90638a3e/predictions.json,/result/cifar100/reports/attack/004c81db70edff7701b2e75c90638a3e/adv_predictions.json, diff --git a/examples/power/plots/data/mnist/power.csv b/examples/power/plots/data/mnist/power.csv index 3c0388b4..96c79866 100644 --- a/examples/power/plots/data/mnist/power.csv +++ b/examples/power/plots/data/mnist/power.csv @@ -1,4 +1,4 @@ -,stage,_target_,attack,data,device_id,files,model,name,scorers,train_time,train_time_per_sample,train_start_time,train_end_time,train_device,predict_time,predict_time_per_sample,predict_start_time,predict_end_time,predict_device,predict_proba_time,predict_proba_time_per_sample,predict_proba_start_time,predict_proba_end_time,predict_proba_device,predict_log_proba_time,predict_log_proba_time_per_sample,predict_log_proba_start_time,predict_log_proba_end_time,predict_log_device,adv_fit_time_per_sample,adv_fit_time,adv_fit_start_time,adv_fit_end_time,adv_fit_device,adv_predict_time_per_sample,adv_predict_time,adv_predict_start_time,adv_predict_end_time,adv_predict_device,adv_success,accuracy,log_loss,adv_accuracy,adv_log_loss,predictions,adv_predictions,exception,stage.1,_target_.1,attack._target_,attack.attack_size,attack.method,attack.data._target_,attack.data.generate._target_,attack.data.generate.name,attack.data.sample._target_,attack.data.sample.random_state,attack.data.sample.stratify,attack.data.sklearn_pipeline._target_,attack.data.sklearn_pipeline.preprocessor.name,attack.data.sklearn_pipeline.preprocessor.with_mean,attack.data.sklearn_pipeline.preprocessor.with_std,attack.init._target_,attack.init.batch_size,attack.init.eps,attack.init.minimal,attack.init.model._target_,attack.init.model.art._target_,attack.init.model.art.data._target_,attack.init.model.art.data.generate._target_,attack.init.model.art.data.generate.name,attack.init.model.art.data.sample._target_,attack.init.model.art.data.sample.random_state,attack.init.model.art.data.sample.stratify,attack.init.model.art.data.sklearn_pipeline._target_,attack.init.model.art.data.sklearn_pipeline.preprocessor.name,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.art.initialize.clip_values,attack.init.model.art.initialize.criterion.name,attack.init.model.art.initialize.optimizer.lr,attack.init.model.art.initialize.optimizer.momentum,attack.init.model.art.initialize.optimizer.name,attack.init.model.art.library,attack.init.model.data._target_,attack.init.model.data.generate._target_,attack.init.model.data.generate.name,attack.init.model.data.sample._target_,attack.init.model.data.sample.random_state,attack.init.model.data.sample.stratify,attack.init.model.data.sklearn_pipeline._target_,attack.init.model.data.sklearn_pipeline.preprocessor.name,attack.init.model.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.init._target_,attack.init.model.init.name,attack.init.model.init.num_channels,attack.init.model.library,attack.init.model.trainer.batch_size,attack.init.model.trainer.nb_epoch,attack.init.name,attack.init.targeted,attack.model._target_,attack.model.art._target_,attack.model.art.data._target_,attack.model.art.data.generate._target_,attack.model.art.data.generate.name,attack.model.art.data.sample._target_,attack.model.art.data.sample.random_state,attack.model.art.data.sample.stratify,attack.model.art.data.sklearn_pipeline._target_,attack.model.art.data.sklearn_pipeline.preprocessor.name,attack.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.model.art.initialize.clip_values,attack.model.art.initialize.criterion.name,attack.model.art.initialize.optimizer.lr,attack.model.art.initialize.optimizer.momentum,attack.model.art.initialize.optimizer.name,attack.model.art.library,attack.model.data._target_,attack.model.data.generate._target_,attack.model.data.generate.name,attack.model.data.sample._target_,attack.model.data.sample.random_state,attack.model.data.sample.stratify,attack.model.data.sklearn_pipeline._target_,attack.model.data.sklearn_pipeline.preprocessor.name,attack.model.data.sklearn_pipeline.preprocessor.with_mean,attack.model.data.sklearn_pipeline.preprocessor.with_std,attack.model.init._target_,attack.model.init.name,attack.model.init.num_channels,attack.model.library,attack.model.trainer.batch_size,attack.model.trainer.nb_epoch,data._target_,data.generate._target_,data.generate.name,data.sample._target_,data.sample.random_state,data.sample.stratify,data.sklearn_pipeline._target_,data.sklearn_pipeline.preprocessor.name,data.sklearn_pipeline.preprocessor.with_mean,data.sklearn_pipeline.preprocessor.with_std,device_id.1,files._target_,files.adv_predictions_file,files.attack_dir,files.attack_file,files.attack_type,files.data_dir,files.data_file,files.data_type,files.directory,files.name,files.predictions_file,files.reports,files.score_dict_file,files.stage,files.model_dir,files.model_file,files.model_type,files.params_file,model._target_,model.library,model.art._target_,model.art.data._target_,model.art.data.generate._target_,model.art.data.generate.name,model.art.data.sample._target_,model.art.data.sample.random_state,model.art.data.sample.stratify,model.art.data.sklearn_pipeline._target_,model.art.data.sklearn_pipeline.preprocessor.name,model.art.data.sklearn_pipeline.preprocessor.with_mean,model.art.data.sklearn_pipeline.preprocessor.with_std,model.art.initialize.clip_values,model.art.initialize.criterion.name,model.art.initialize.optimizer.lr,model.art.initialize.optimizer.momentum,model.art.initialize.optimizer.name,model.art.library,model.data._target_,model.data.generate._target_,model.data.generate.name,model.data.sample._target_,model.data.sample.random_state,model.data.sample.stratify,model.data.sklearn_pipeline._target_,model.data.sklearn_pipeline.preprocessor.name,model.data.sklearn_pipeline.preprocessor.with_mean,model.data.sklearn_pipeline.preprocessor.with_std,model.init._target_,model.init.name,model.init.num_channels,model.trainer.batch_size,model.trainer.nb_epoch,name.1,scorers._target_,scorers.accuracy._target_,scorers.accuracy.direction,scorers.accuracy.name,scorers.log_loss._target_,scorers.log_loss.direction,scorers.log_loss.name,train_time.1,train_time_per_sample.1,train_start_time.1,train_end_time.1,train_device.1,predict_time.1,predict_time_per_sample.1,predict_start_time.1,predict_end_time.1,predict_device.1,predict_proba_time.1,predict_proba_time_per_sample.1,predict_proba_start_time.1,predict_proba_end_time.1,predict_proba_device.1,predict_log_proba_time.1,predict_log_proba_time_per_sample.1,predict_log_proba_start_time.1,predict_log_proba_end_time.1,predict_log_device.1,adv_fit_time_per_sample.1,adv_fit_time.1,adv_fit_start_time.1,adv_fit_end_time.1,adv_fit_device.1,adv_predict_time_per_sample.1,adv_predict_time.1,adv_predict_start_time.1,adv_predict_end_time.1,adv_predict_device.1,adv_success.1,accuracy.1,log_loss.1,adv_accuracy.1,adv_log_loss.1,predictions.1,adv_predictions.1,exception.1,train_power,peak_power,predict_power,predict_proba_power,predict_log_proba_power,adv_fit_power,adv_predict_power +,stage,_target_,attack,data,device_id,files,model,name,scorers,train_time,train_time_per_sample,train_start_time,train_end_time,train_device,predict_time,predict_time_per_sample,predict_start_time,predict_end_time,predict_device,predict_proba_time,predict_proba_time_per_sample,predict_proba_start_time,predict_proba_end_time,predict_proba_device,predict_log_proba_time,predict_log_proba_time_per_sample,predict_log_proba_start_time,predict_log_proba_end_time,predict_log_device,adv_fit_time_per_sample,adv_fit_time,adv_fit_start_time,adv_fit_end_time,adv_fit_device,adv_predict_time_per_sample,adv_predict_time,adv_predict_start_time,adv_predict_end_time,adv_predict_device,adv_success,accuracy,log_loss,adv_accuracy,adv_log_loss,predictions,adv_predictions,exception,stage.1,_target_.1,attack._target_,attack.attack_size,attack.method,attack.data._target_,attack.data.generate._target_,attack.data.generate.name,attack.data.sample._target_,attack.data.sample.random_state,attack.data.sample.stratify,attack.data.sklearn_pipeline._target_,attack.data.sklearn_pipeline.preprocessor.name,attack.data.sklearn_pipeline.preprocessor.with_mean,attack.data.sklearn_pipeline.preprocessor.with_std,attack.init._target_,attack.init.batch_size,attack.init.eps,attack.init.minimal,attack.init.model._target_,attack.init.model.art._target_,attack.init.model.art.data._target_,attack.init.model.art.data.generate._target_,attack.init.model.art.data.generate.name,attack.init.model.art.data.sample._target_,attack.init.model.art.data.sample.random_state,attack.init.model.art.data.sample.stratify,attack.init.model.art.data.sklearn_pipeline._target_,attack.init.model.art.data.sklearn_pipeline.preprocessor.name,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.art.initialize.clip_values,attack.init.model.art.initialize.criterion.name,attack.init.model.art.initialize.optimizer.lr,attack.init.model.art.initialize.optimizer.momentum,attack.init.model.art.initialize.optimizer.name,attack.init.model.art.library,attack.init.model.data._target_,attack.init.model.data.generate._target_,attack.init.model.data.generate.name,attack.init.model.data.sample._target_,attack.init.model.data.sample.random_state,attack.init.model.data.sample.stratify,attack.init.model.data.sklearn_pipeline._target_,attack.init.model.data.sklearn_pipeline.preprocessor.name,attack.init.model.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.init._target_,attack.init.model.init.name,attack.init.model.init.num_channels,attack.init.model.library,attack.init.model.trainer.batch_size,attack.init.model.trainer.np_epochs,attack.init.name,attack.init.targeted,attack.model._target_,attack.model.art._target_,attack.model.art.data._target_,attack.model.art.data.generate._target_,attack.model.art.data.generate.name,attack.model.art.data.sample._target_,attack.model.art.data.sample.random_state,attack.model.art.data.sample.stratify,attack.model.art.data.sklearn_pipeline._target_,attack.model.art.data.sklearn_pipeline.preprocessor.name,attack.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.model.art.initialize.clip_values,attack.model.art.initialize.criterion.name,attack.model.art.initialize.optimizer.lr,attack.model.art.initialize.optimizer.momentum,attack.model.art.initialize.optimizer.name,attack.model.art.library,attack.model.data._target_,attack.model.data.generate._target_,attack.model.data.generate.name,attack.model.data.sample._target_,attack.model.data.sample.random_state,attack.model.data.sample.stratify,attack.model.data.sklearn_pipeline._target_,attack.model.data.sklearn_pipeline.preprocessor.name,attack.model.data.sklearn_pipeline.preprocessor.with_mean,attack.model.data.sklearn_pipeline.preprocessor.with_std,attack.model.init._target_,attack.model.init.name,attack.model.init.num_channels,attack.model.library,attack.model.trainer.batch_size,attack.model.trainer.np_epochs,data._target_,data.generate._target_,data.generate.name,data.sample._target_,data.sample.random_state,data.sample.stratify,data.sklearn_pipeline._target_,data.sklearn_pipeline.preprocessor.name,data.sklearn_pipeline.preprocessor.with_mean,data.sklearn_pipeline.preprocessor.with_std,device_id.1,files._target_,files.adv_predictions_file,files.attack_dir,files.attack_file,files.attack_type,files.data_dir,files.data_file,files.data_type,files.directory,files.name,files.predictions_file,files.reports,files.score_dict_file,files.stage,files.model_dir,files.model_file,files.model_type,files.params_file,model._target_,model.library,model.art._target_,model.art.data._target_,model.art.data.generate._target_,model.art.data.generate.name,model.art.data.sample._target_,model.art.data.sample.random_state,model.art.data.sample.stratify,model.art.data.sklearn_pipeline._target_,model.art.data.sklearn_pipeline.preprocessor.name,model.art.data.sklearn_pipeline.preprocessor.with_mean,model.art.data.sklearn_pipeline.preprocessor.with_std,model.art.initialize.clip_values,model.art.initialize.criterion.name,model.art.initialize.optimizer.lr,model.art.initialize.optimizer.momentum,model.art.initialize.optimizer.name,model.art.library,model.data._target_,model.data.generate._target_,model.data.generate.name,model.data.sample._target_,model.data.sample.random_state,model.data.sample.stratify,model.data.sklearn_pipeline._target_,model.data.sklearn_pipeline.preprocessor.name,model.data.sklearn_pipeline.preprocessor.with_mean,model.data.sklearn_pipeline.preprocessor.with_std,model.init._target_,model.init.name,model.init.num_channels,model.trainer.batch_size,model.trainer.np_epochs,name.1,scorers._target_,scorers.accuracy._target_,scorers.accuracy.direction,scorers.accuracy.name,scorers.log_loss._target_,scorers.log_loss.direction,scorers.log_loss.name,train_time.1,train_time_per_sample.1,train_start_time.1,train_end_time.1,train_device.1,predict_time.1,predict_time_per_sample.1,predict_start_time.1,predict_end_time.1,predict_device.1,predict_proba_time.1,predict_proba_time_per_sample.1,predict_proba_start_time.1,predict_proba_end_time.1,predict_proba_device.1,predict_log_proba_time.1,predict_log_proba_time_per_sample.1,predict_log_proba_start_time.1,predict_log_proba_end_time.1,predict_log_device.1,adv_fit_time_per_sample.1,adv_fit_time.1,adv_fit_start_time.1,adv_fit_end_time.1,adv_fit_device.1,adv_predict_time_per_sample.1,adv_predict_time.1,adv_predict_start_time.1,adv_predict_end_time.1,adv_predict_device.1,adv_success.1,accuracy.1,log_loss.1,adv_accuracy.1,adv_log_loss.1,predictions.1,adv_predictions.1,exception.1,train_power,peak_power,predict_power,predict_proba_power,predict_log_proba_power,adv_fit_power,adv_predict_power 001b18a7f37374c8917d3c5f1c28c52e,attack,deckard.base.experiment.Experiment,001b18a7f37374c8917d3c5f1c28c52e,001b18a7f37374c8917d3c5f1c28c52e,nvidia-tesla-v100,001b18a7f37374c8917d3c5f1c28c52e,001b18a7f37374c8917d3c5f1c28c52e,001b18a7f37374c8917d3c5f1c28c52e,001b18a7f37374c8917d3c5f1c28c52e,39.751303719,0.0007098447092678,1701459440.4891484,1701459481.362319,cuda:0,0.715353228,1.2774164785714286e-05,1701459481.3645277,1701459482.083008,cuda:0,0.605021746,1.080395975e-05,1701459482.3989463,1701459483.082725,cuda:0,0.574894972,1.0265981642857142e-05,1701459483.0827985,1701459483.676509,cuda:0,0.00036139836,0.036139836,1701459485.5705557,1701459485.6067028,cuda:0,5.690071e-05,0.005690071,1701459486.1954916,1701459486.2011943,cuda:0,0.86,0.9298571428571428,1.1525793640744106,0.12,15.620485631227494,/result/mnist/reports/attack/001b18a7f37374c8917d3c5f1c28c52e/predictions.json,/result/mnist/reports/attack/001b18a7f37374c8917d3c5f1c28c52e/adv_predictions.json,,attack,deckard.base.experiment.Experiment,deckard.base.attack.Attack,100,evasion,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,8,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.attack.AttackInitializer,5373,0.1188387972670761,True,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,8,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0006187426178555,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,8,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,pytorch,5373,35,art.attacks.evasion.FastGradientMethod,False,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,8,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0006187426178555,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,8,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,pytorch,5373,35,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,8,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,nvidia-tesla-v100,deckard.base.files.FileConfig,adv_predictions.json,attacks,263f052edd0c22b868c151e6a1483e5f,.pkl,data,0eb1f091c8209a68cd8bbbed01a9d22e,.pkl,/result/mnist/,001b18a7f37374c8917d3c5f1c28c52e,predictions.json,reports,score_dict.json,attack,,,,,deckard.base.model.Model,pytorch,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,8,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0006187426178555,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,8,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,5373,35,001b18a7f37374c8917d3c5f1c28c52e,deckard.base.scorer.ScorerDict,deckard.base.scorer.ScorerConfig,maximize,sklearn.metrics.accuracy_score,deckard.base.scorer.ScorerConfig,minimize,sklearn.metrics.log_loss,39.751303719,0.0007098447092678,1701459440.4891484,1701459481.362319,cuda:0,0.715353228,1.2774164785714286e-05,1701459481.3645277,1701459482.083008,cuda:0,0.605021746,1.080395975e-05,1701459482.3989463,1701459483.082725,cuda:0,0.574894972,1.0265981642857142e-05,1701459483.0827985,1701459483.676509,cuda:0,0.00036139836,0.036139836,1701459485.5705557,1701459485.6067028,cuda:0,5.690071e-05,0.005690071,1701459486.1954916,1701459486.2011943,cuda:0,0.86,0.9298571428571428,1.1525793640744106,0.12,15.620485631227494,/result/mnist/reports/attack/001b18a7f37374c8917d3c5f1c28c52e/predictions.json,/result/mnist/reports/attack/001b18a7f37374c8917d3c5f1c28c52e/adv_predictions.json,,3.657126940709353,250,0.05709523836771647,0.05205265831947327,0.0576047537446022,0.0035382003625233967,0.00042456854581832885 0026ee702647cb53c5eb77aa83919db4,attack,deckard.base.experiment.Experiment,0026ee702647cb53c5eb77aa83919db4,0026ee702647cb53c5eb77aa83919db4,nvidia-tesla-v100,0026ee702647cb53c5eb77aa83919db4,0026ee702647cb53c5eb77aa83919db4,0026ee702647cb53c5eb77aa83919db4,0026ee702647cb53c5eb77aa83919db4,40.402154356,0.0007214670420714,1701465755.3175354,1701465796.7700725,cuda:0,0.698124667,1.2466511910714284e-05,1701465796.7722094,1701465797.485137,cuda:0,0.534780805,9.549657232142856e-06,1701465797.776939,1701465798.3114192,cuda:0,0.538273669,9.612029803571427e-06,1701465798.3115156,1701465798.850526,cuda:0,0.0003136345,0.03136345,1701465800.6785083,1701465800.709914,cuda:0,5.490154e-05,0.005490154,1701465801.3149922,1701465801.3204854,cuda:0,0.0,0.2681428571428571,11.801480109670331,0.13,9.312280102968217,/result/mnist/reports/attack/0026ee702647cb53c5eb77aa83919db4/predictions.json,/result/mnist/reports/attack/0026ee702647cb53c5eb77aa83919db4/adv_predictions.json,,attack,deckard.base.experiment.Experiment,deckard.base.attack.Attack,100,evasion,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.attack.AttackInitializer,7708,0.0189697598276657,True,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.4324736587434206,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,pytorch,7708,37,art.attacks.evasion.FastGradientMethod,False,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.4324736587434206,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,pytorch,7708,37,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,nvidia-tesla-v100,deckard.base.files.FileConfig,adv_predictions.json,attacks,1437dfe71829257f8ffbcdce21a10119,.pkl,data,20761683b94eca729b58b63071f98327,.pkl,/result/mnist/,0026ee702647cb53c5eb77aa83919db4,predictions.json,reports,score_dict.json,attack,,,,,deckard.base.model.Model,pytorch,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.4324736587434206,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,7708,37,0026ee702647cb53c5eb77aa83919db4,deckard.base.scorer.ScorerDict,deckard.base.scorer.ScorerConfig,maximize,sklearn.metrics.accuracy_score,deckard.base.scorer.ScorerConfig,minimize,sklearn.metrics.log_loss,40.402154356,0.0007214670420714,1701465755.3175354,1701465796.7700725,cuda:0,0.698124667,1.2466511910714284e-05,1701465796.7722094,1701465797.485137,cuda:0,0.534780805,9.549657232142856e-06,1701465797.776939,1701465798.3114192,cuda:0,0.538273669,9.612029803571427e-06,1701465798.3115156,1701465798.850526,cuda:0,0.0003136345,0.03136345,1701465800.6785083,1701465800.709914,cuda:0,5.490154e-05,0.005490154,1701465801.3149922,1701465801.3204854,cuda:0,0.0,0.2681428571428571,11.801480109670331,0.13,9.312280102968217,/result/mnist/reports/attack/0026ee702647cb53c5eb77aa83919db4/predictions.json,/result/mnist/reports/attack/0026ee702647cb53c5eb77aa83919db4/adv_predictions.json,,3.4160344915350276,250,0.06105036509037018,0.04122179570794106,0.04891520512104034,0.0026723622779051466,0.0005074768066406249 003beec36d3939eb1fad3c9088be8655,attack,deckard.base.experiment.Experiment,003beec36d3939eb1fad3c9088be8655,003beec36d3939eb1fad3c9088be8655,nvidia-tesla-v100,003beec36d3939eb1fad3c9088be8655,003beec36d3939eb1fad3c9088be8655,003beec36d3939eb1fad3c9088be8655,003beec36d3939eb1fad3c9088be8655,39.910459009,0.0007126867680178,1701475813.4550047,1701475854.3889067,cuda:0,0.837386706,1.4953334035714286e-05,1701475854.391328,1701475855.2755497,cuda:0,0.576092868,1.0287372642857144e-05,1701475855.5957618,1701475856.174563,cuda:0,0.58573969,1.0459637321428572e-05,1701475856.1746368,1701475856.7726789,cuda:0,0.00031571891,0.031571891,1701475858.4797752,1701475858.511399,cuda:0,5.679119e-05,0.005679119,1701475859.1243455,1701475859.1300282,cuda:0,0.0,0.9829285714285714,1.142575143955228,0.1,20.5655116045475,/result/mnist/reports/attack/003beec36d3939eb1fad3c9088be8655/predictions.json,/result/mnist/reports/attack/003beec36d3939eb1fad3c9088be8655/adv_predictions.json,,attack,deckard.base.experiment.Experiment,deckard.base.attack.Attack,100,evasion,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.attack.AttackInitializer,6492,0.0697717436905115,True,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0176054598714787,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,pytorch,6492,35,art.attacks.evasion.FastGradientMethod,False,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0176054598714787,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,pytorch,6492,35,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,nvidia-tesla-v100,deckard.base.files.FileConfig,adv_predictions.json,attacks,7e2322c60ed43df53de685b137ed42ba,.pkl,data,20761683b94eca729b58b63071f98327,.pkl,/result/mnist/,003beec36d3939eb1fad3c9088be8655,predictions.json,reports,score_dict.json,attack,,,,,deckard.base.model.Model,pytorch,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0176054598714787,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,6492,35,003beec36d3939eb1fad3c9088be8655,deckard.base.scorer.ScorerDict,deckard.base.scorer.ScorerConfig,maximize,sklearn.metrics.accuracy_score,deckard.base.scorer.ScorerConfig,minimize,sklearn.metrics.log_loss,39.910459009,0.0007126867680178,1701475813.4550047,1701475854.3889067,cuda:0,0.837386706,1.4953334035714286e-05,1701475854.391328,1701475855.2755497,cuda:0,0.576092868,1.0287372642857144e-05,1701475855.5957618,1701475856.174563,cuda:0,0.58573969,1.0459637321428572e-05,1701475856.1746368,1701475856.7726789,cuda:0,0.00031571891,0.031571891,1701475858.4797752,1701475858.511399,cuda:0,5.679119e-05,0.005679119,1701475859.1243455,1701475859.1300282,cuda:0,0.0,0.9829285714285714,1.142575143955228,0.1,20.5655116045475,/result/mnist/reports/attack/003beec36d3939eb1fad3c9088be8655/predictions.json,/result/mnist/reports/attack/003beec36d3939eb1fad3c9088be8655/adv_predictions.json,,4.080427800214291,250,0.07420829390287399,0.043366676545143125,0.04081636726856232,0.0024526923497517907,0.0005623985528945923 diff --git a/examples/power/plots/data/mnist/raw.csv b/examples/power/plots/data/mnist/raw.csv index 157c0b5c..034d634f 100644 --- a/examples/power/plots/data/mnist/raw.csv +++ b/examples/power/plots/data/mnist/raw.csv @@ -1,4 +1,4 @@ -,stage,_target_,attack,data,device_id,files,model,name,scorers,train_time,train_time_per_sample,train_start_time,train_end_time,train_device,predict_time,predict_time_per_sample,predict_start_time,predict_end_time,predict_device,predict_proba_time,predict_proba_time_per_sample,predict_proba_start_time,predict_proba_end_time,predict_proba_device,predict_log_proba_time,predict_log_proba_time_per_sample,predict_log_proba_start_time,predict_log_proba_end_time,predict_log_device,adv_fit_time_per_sample,adv_fit_time,adv_fit_start_time,adv_fit_end_time,adv_fit_device,adv_predict_time_per_sample,adv_predict_time,adv_predict_start_time,adv_predict_end_time,adv_predict_device,adv_success,accuracy,log_loss,adv_accuracy,adv_log_loss,predictions,adv_predictions,exception,stage,_target_,attack._target_,attack.attack_size,attack.method,attack.data._target_,attack.data.generate._target_,attack.data.generate.name,attack.data.sample._target_,attack.data.sample.random_state,attack.data.sample.stratify,attack.data.sklearn_pipeline._target_,attack.data.sklearn_pipeline.preprocessor.name,attack.data.sklearn_pipeline.preprocessor.with_mean,attack.data.sklearn_pipeline.preprocessor.with_std,attack.init._target_,attack.init.batch_size,attack.init.eps,attack.init.minimal,attack.init.model._target_,attack.init.model.art._target_,attack.init.model.art.data._target_,attack.init.model.art.data.generate._target_,attack.init.model.art.data.generate.name,attack.init.model.art.data.sample._target_,attack.init.model.art.data.sample.random_state,attack.init.model.art.data.sample.stratify,attack.init.model.art.data.sklearn_pipeline._target_,attack.init.model.art.data.sklearn_pipeline.preprocessor.name,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.art.initialize.clip_values,attack.init.model.art.initialize.criterion.name,attack.init.model.art.initialize.optimizer.lr,attack.init.model.art.initialize.optimizer.momentum,attack.init.model.art.initialize.optimizer.name,attack.init.model.art.library,attack.init.model.data._target_,attack.init.model.data.generate._target_,attack.init.model.data.generate.name,attack.init.model.data.sample._target_,attack.init.model.data.sample.random_state,attack.init.model.data.sample.stratify,attack.init.model.data.sklearn_pipeline._target_,attack.init.model.data.sklearn_pipeline.preprocessor.name,attack.init.model.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.init._target_,attack.init.model.init.name,attack.init.model.init.num_channels,attack.init.model.library,attack.init.model.trainer.batch_size,attack.init.model.trainer.nb_epoch,attack.init.name,attack.init.targeted,attack.model._target_,attack.model.art._target_,attack.model.art.data._target_,attack.model.art.data.generate._target_,attack.model.art.data.generate.name,attack.model.art.data.sample._target_,attack.model.art.data.sample.random_state,attack.model.art.data.sample.stratify,attack.model.art.data.sklearn_pipeline._target_,attack.model.art.data.sklearn_pipeline.preprocessor.name,attack.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.model.art.initialize.clip_values,attack.model.art.initialize.criterion.name,attack.model.art.initialize.optimizer.lr,attack.model.art.initialize.optimizer.momentum,attack.model.art.initialize.optimizer.name,attack.model.art.library,attack.model.data._target_,attack.model.data.generate._target_,attack.model.data.generate.name,attack.model.data.sample._target_,attack.model.data.sample.random_state,attack.model.data.sample.stratify,attack.model.data.sklearn_pipeline._target_,attack.model.data.sklearn_pipeline.preprocessor.name,attack.model.data.sklearn_pipeline.preprocessor.with_mean,attack.model.data.sklearn_pipeline.preprocessor.with_std,attack.model.init._target_,attack.model.init.name,attack.model.init.num_channels,attack.model.library,attack.model.trainer.batch_size,attack.model.trainer.nb_epoch,data._target_,data.generate._target_,data.generate.name,data.sample._target_,data.sample.random_state,data.sample.stratify,data.sklearn_pipeline._target_,data.sklearn_pipeline.preprocessor.name,data.sklearn_pipeline.preprocessor.with_mean,data.sklearn_pipeline.preprocessor.with_std,device_id,files._target_,files.adv_predictions_file,files.attack_dir,files.attack_file,files.attack_type,files.data_dir,files.data_file,files.data_type,files.directory,files.name,files.predictions_file,files.reports,files.score_dict_file,files.stage,files.model_dir,files.model_file,files.model_type,files.params_file,model._target_,model.library,model.art._target_,model.art.data._target_,model.art.data.generate._target_,model.art.data.generate.name,model.art.data.sample._target_,model.art.data.sample.random_state,model.art.data.sample.stratify,model.art.data.sklearn_pipeline._target_,model.art.data.sklearn_pipeline.preprocessor.name,model.art.data.sklearn_pipeline.preprocessor.with_mean,model.art.data.sklearn_pipeline.preprocessor.with_std,model.art.initialize.clip_values,model.art.initialize.criterion.name,model.art.initialize.optimizer.lr,model.art.initialize.optimizer.momentum,model.art.initialize.optimizer.name,model.art.library,model.data._target_,model.data.generate._target_,model.data.generate.name,model.data.sample._target_,model.data.sample.random_state,model.data.sample.stratify,model.data.sklearn_pipeline._target_,model.data.sklearn_pipeline.preprocessor.name,model.data.sklearn_pipeline.preprocessor.with_mean,model.data.sklearn_pipeline.preprocessor.with_std,model.init._target_,model.init.name,model.init.num_channels,model.trainer.batch_size,model.trainer.nb_epoch,name,scorers._target_,scorers.accuracy._target_,scorers.accuracy.direction,scorers.accuracy.name,scorers.log_loss._target_,scorers.log_loss.direction,scorers.log_loss.name,train_time,train_time_per_sample,train_start_time,train_end_time,train_device,predict_time,predict_time_per_sample,predict_start_time,predict_end_time,predict_device,predict_proba_time,predict_proba_time_per_sample,predict_proba_start_time,predict_proba_end_time,predict_proba_device,predict_log_proba_time,predict_log_proba_time_per_sample,predict_log_proba_start_time,predict_log_proba_end_time,predict_log_device,adv_fit_time_per_sample,adv_fit_time,adv_fit_start_time,adv_fit_end_time,adv_fit_device,adv_predict_time_per_sample,adv_predict_time,adv_predict_start_time,adv_predict_end_time,adv_predict_device,adv_success,accuracy,log_loss,adv_accuracy,adv_log_loss,predictions,adv_predictions,exception +,stage,_target_,attack,data,device_id,files,model,name,scorers,train_time,train_time_per_sample,train_start_time,train_end_time,train_device,predict_time,predict_time_per_sample,predict_start_time,predict_end_time,predict_device,predict_proba_time,predict_proba_time_per_sample,predict_proba_start_time,predict_proba_end_time,predict_proba_device,predict_log_proba_time,predict_log_proba_time_per_sample,predict_log_proba_start_time,predict_log_proba_end_time,predict_log_device,adv_fit_time_per_sample,adv_fit_time,adv_fit_start_time,adv_fit_end_time,adv_fit_device,adv_predict_time_per_sample,adv_predict_time,adv_predict_start_time,adv_predict_end_time,adv_predict_device,adv_success,accuracy,log_loss,adv_accuracy,adv_log_loss,predictions,adv_predictions,exception,stage,_target_,attack._target_,attack.attack_size,attack.method,attack.data._target_,attack.data.generate._target_,attack.data.generate.name,attack.data.sample._target_,attack.data.sample.random_state,attack.data.sample.stratify,attack.data.sklearn_pipeline._target_,attack.data.sklearn_pipeline.preprocessor.name,attack.data.sklearn_pipeline.preprocessor.with_mean,attack.data.sklearn_pipeline.preprocessor.with_std,attack.init._target_,attack.init.batch_size,attack.init.eps,attack.init.minimal,attack.init.model._target_,attack.init.model.art._target_,attack.init.model.art.data._target_,attack.init.model.art.data.generate._target_,attack.init.model.art.data.generate.name,attack.init.model.art.data.sample._target_,attack.init.model.art.data.sample.random_state,attack.init.model.art.data.sample.stratify,attack.init.model.art.data.sklearn_pipeline._target_,attack.init.model.art.data.sklearn_pipeline.preprocessor.name,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.art.initialize.clip_values,attack.init.model.art.initialize.criterion.name,attack.init.model.art.initialize.optimizer.lr,attack.init.model.art.initialize.optimizer.momentum,attack.init.model.art.initialize.optimizer.name,attack.init.model.art.library,attack.init.model.data._target_,attack.init.model.data.generate._target_,attack.init.model.data.generate.name,attack.init.model.data.sample._target_,attack.init.model.data.sample.random_state,attack.init.model.data.sample.stratify,attack.init.model.data.sklearn_pipeline._target_,attack.init.model.data.sklearn_pipeline.preprocessor.name,attack.init.model.data.sklearn_pipeline.preprocessor.with_mean,attack.init.model.data.sklearn_pipeline.preprocessor.with_std,attack.init.model.init._target_,attack.init.model.init.name,attack.init.model.init.num_channels,attack.init.model.library,attack.init.model.trainer.batch_size,attack.init.model.trainer.np_epochs,attack.init.name,attack.init.targeted,attack.model._target_,attack.model.art._target_,attack.model.art.data._target_,attack.model.art.data.generate._target_,attack.model.art.data.generate.name,attack.model.art.data.sample._target_,attack.model.art.data.sample.random_state,attack.model.art.data.sample.stratify,attack.model.art.data.sklearn_pipeline._target_,attack.model.art.data.sklearn_pipeline.preprocessor.name,attack.model.art.data.sklearn_pipeline.preprocessor.with_mean,attack.model.art.data.sklearn_pipeline.preprocessor.with_std,attack.model.art.initialize.clip_values,attack.model.art.initialize.criterion.name,attack.model.art.initialize.optimizer.lr,attack.model.art.initialize.optimizer.momentum,attack.model.art.initialize.optimizer.name,attack.model.art.library,attack.model.data._target_,attack.model.data.generate._target_,attack.model.data.generate.name,attack.model.data.sample._target_,attack.model.data.sample.random_state,attack.model.data.sample.stratify,attack.model.data.sklearn_pipeline._target_,attack.model.data.sklearn_pipeline.preprocessor.name,attack.model.data.sklearn_pipeline.preprocessor.with_mean,attack.model.data.sklearn_pipeline.preprocessor.with_std,attack.model.init._target_,attack.model.init.name,attack.model.init.num_channels,attack.model.library,attack.model.trainer.batch_size,attack.model.trainer.np_epochs,data._target_,data.generate._target_,data.generate.name,data.sample._target_,data.sample.random_state,data.sample.stratify,data.sklearn_pipeline._target_,data.sklearn_pipeline.preprocessor.name,data.sklearn_pipeline.preprocessor.with_mean,data.sklearn_pipeline.preprocessor.with_std,device_id,files._target_,files.adv_predictions_file,files.attack_dir,files.attack_file,files.attack_type,files.data_dir,files.data_file,files.data_type,files.directory,files.name,files.predictions_file,files.reports,files.score_dict_file,files.stage,files.model_dir,files.model_file,files.model_type,files.params_file,model._target_,model.library,model.art._target_,model.art.data._target_,model.art.data.generate._target_,model.art.data.generate.name,model.art.data.sample._target_,model.art.data.sample.random_state,model.art.data.sample.stratify,model.art.data.sklearn_pipeline._target_,model.art.data.sklearn_pipeline.preprocessor.name,model.art.data.sklearn_pipeline.preprocessor.with_mean,model.art.data.sklearn_pipeline.preprocessor.with_std,model.art.initialize.clip_values,model.art.initialize.criterion.name,model.art.initialize.optimizer.lr,model.art.initialize.optimizer.momentum,model.art.initialize.optimizer.name,model.art.library,model.data._target_,model.data.generate._target_,model.data.generate.name,model.data.sample._target_,model.data.sample.random_state,model.data.sample.stratify,model.data.sklearn_pipeline._target_,model.data.sklearn_pipeline.preprocessor.name,model.data.sklearn_pipeline.preprocessor.with_mean,model.data.sklearn_pipeline.preprocessor.with_std,model.init._target_,model.init.name,model.init.num_channels,model.trainer.batch_size,model.trainer.np_epochs,name,scorers._target_,scorers.accuracy._target_,scorers.accuracy.direction,scorers.accuracy.name,scorers.log_loss._target_,scorers.log_loss.direction,scorers.log_loss.name,train_time,train_time_per_sample,train_start_time,train_end_time,train_device,predict_time,predict_time_per_sample,predict_start_time,predict_end_time,predict_device,predict_proba_time,predict_proba_time_per_sample,predict_proba_start_time,predict_proba_end_time,predict_proba_device,predict_log_proba_time,predict_log_proba_time_per_sample,predict_log_proba_start_time,predict_log_proba_end_time,predict_log_device,adv_fit_time_per_sample,adv_fit_time,adv_fit_start_time,adv_fit_end_time,adv_fit_device,adv_predict_time_per_sample,adv_predict_time,adv_predict_start_time,adv_predict_end_time,adv_predict_device,adv_success,accuracy,log_loss,adv_accuracy,adv_log_loss,predictions,adv_predictions,exception 001b18a7f37374c8917d3c5f1c28c52e,attack,deckard.base.experiment.Experiment,001b18a7f37374c8917d3c5f1c28c52e,001b18a7f37374c8917d3c5f1c28c52e,nvidia-tesla-v100,001b18a7f37374c8917d3c5f1c28c52e,001b18a7f37374c8917d3c5f1c28c52e,001b18a7f37374c8917d3c5f1c28c52e,001b18a7f37374c8917d3c5f1c28c52e,39.751303719,0.0007098447092678572,1701459440.4891484,1701459481.362319,cuda:0,0.715353228,1.2774164785714286e-05,1701459481.3645275,1701459482.083008,cuda:0,0.605021746,1.080395975e-05,1701459482.3989463,1701459483.082725,cuda:0,0.574894972,1.0265981642857142e-05,1701459483.0827985,1701459483.676509,cuda:0,0.00036139836,0.036139836,1701459485.5705557,1701459485.6067028,cuda:0,5.690071e-05,0.005690071,1701459486.1954916,1701459486.2011943,cuda:0,0.86,0.9298571428571428,1.1525793640744106,0.12,15.620485631227494,/result/mnist/reports/attack/001b18a7f37374c8917d3c5f1c28c52e/predictions.json,/result/mnist/reports/attack/001b18a7f37374c8917d3c5f1c28c52e/adv_predictions.json,,attack,deckard.base.experiment.Experiment,deckard.base.attack.Attack,100,evasion,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,8,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.attack.AttackInitializer,5373,0.11883879726707616,True,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,8,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0006187426178555001,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,8,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,pytorch,5373,35,art.attacks.evasion.FastGradientMethod,False,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,8,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0006187426178555001,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,8,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,pytorch,5373,35,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,8,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,nvidia-tesla-v100,deckard.base.files.FileConfig,adv_predictions.json,attacks,263f052edd0c22b868c151e6a1483e5f,.pkl,data,0eb1f091c8209a68cd8bbbed01a9d22e,.pkl,/result/mnist/,001b18a7f37374c8917d3c5f1c28c52e,predictions.json,reports,score_dict.json,attack,,,,,deckard.base.model.Model,pytorch,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,8,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.0006187426178555001,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,8,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,5373,35,001b18a7f37374c8917d3c5f1c28c52e,deckard.base.scorer.ScorerDict,deckard.base.scorer.ScorerConfig,maximize,sklearn.metrics.accuracy_score,deckard.base.scorer.ScorerConfig,minimize,sklearn.metrics.log_loss,39.751303719,0.0007098447092678572,1701459440.4891484,1701459481.362319,cuda:0,0.715353228,1.2774164785714286e-05,1701459481.3645275,1701459482.083008,cuda:0,0.605021746,1.080395975e-05,1701459482.3989463,1701459483.082725,cuda:0,0.574894972,1.0265981642857142e-05,1701459483.0827985,1701459483.676509,cuda:0,0.00036139836,0.036139836,1701459485.5705557,1701459485.6067028,cuda:0,5.690071e-05,0.005690071,1701459486.1954916,1701459486.2011943,cuda:0,0.86,0.9298571428571428,1.1525793640744106,0.12,15.620485631227494,/result/mnist/reports/attack/001b18a7f37374c8917d3c5f1c28c52e/predictions.json,/result/mnist/reports/attack/001b18a7f37374c8917d3c5f1c28c52e/adv_predictions.json, 0026ee702647cb53c5eb77aa83919db4,attack,deckard.base.experiment.Experiment,0026ee702647cb53c5eb77aa83919db4,0026ee702647cb53c5eb77aa83919db4,nvidia-tesla-v100,0026ee702647cb53c5eb77aa83919db4,0026ee702647cb53c5eb77aa83919db4,0026ee702647cb53c5eb77aa83919db4,0026ee702647cb53c5eb77aa83919db4,40.402154356,0.0007214670420714285,1701465755.3175354,1701465796.7700725,cuda:0,0.698124667,1.2466511910714285e-05,1701465796.7722094,1701465797.485137,cuda:0,0.534780805,9.549657232142857e-06,1701465797.776939,1701465798.3114192,cuda:0,0.538273669,9.612029803571429e-06,1701465798.3115156,1701465798.8505259,cuda:0,0.0003136345,0.03136345,1701465800.6785083,1701465800.709914,cuda:0,5.490154e-05,0.005490154,1701465801.3149922,1701465801.3204854,cuda:0,0.0,0.2681428571428571,11.801480109670333,0.13,9.312280102968217,/result/mnist/reports/attack/0026ee702647cb53c5eb77aa83919db4/predictions.json,/result/mnist/reports/attack/0026ee702647cb53c5eb77aa83919db4/adv_predictions.json,,attack,deckard.base.experiment.Experiment,deckard.base.attack.Attack,100,evasion,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.attack.AttackInitializer,7708,0.018969759827665737,True,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.4324736587434206,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,pytorch,7708,37,art.attacks.evasion.FastGradientMethod,False,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.4324736587434206,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,pytorch,7708,37,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,nvidia-tesla-v100,deckard.base.files.FileConfig,adv_predictions.json,attacks,1437dfe71829257f8ffbcdce21a10119,.pkl,data,20761683b94eca729b58b63071f98327,.pkl,/result/mnist/,0026ee702647cb53c5eb77aa83919db4,predictions.json,reports,score_dict.json,attack,,,,,deckard.base.model.Model,pytorch,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.4324736587434206,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,7708,37,0026ee702647cb53c5eb77aa83919db4,deckard.base.scorer.ScorerDict,deckard.base.scorer.ScorerConfig,maximize,sklearn.metrics.accuracy_score,deckard.base.scorer.ScorerConfig,minimize,sklearn.metrics.log_loss,40.402154356,0.0007214670420714285,1701465755.3175354,1701465796.7700725,cuda:0,0.698124667,1.2466511910714285e-05,1701465796.7722094,1701465797.485137,cuda:0,0.534780805,9.549657232142857e-06,1701465797.776939,1701465798.3114192,cuda:0,0.538273669,9.612029803571429e-06,1701465798.3115156,1701465798.8505259,cuda:0,0.0003136345,0.03136345,1701465800.6785083,1701465800.709914,cuda:0,5.490154e-05,0.005490154,1701465801.3149922,1701465801.3204854,cuda:0,0.0,0.2681428571428571,11.801480109670333,0.13,9.312280102968217,/result/mnist/reports/attack/0026ee702647cb53c5eb77aa83919db4/predictions.json,/result/mnist/reports/attack/0026ee702647cb53c5eb77aa83919db4/adv_predictions.json, 003beec36d3939eb1fad3c9088be8655,attack,deckard.base.experiment.Experiment,003beec36d3939eb1fad3c9088be8655,003beec36d3939eb1fad3c9088be8655,nvidia-tesla-v100,003beec36d3939eb1fad3c9088be8655,003beec36d3939eb1fad3c9088be8655,003beec36d3939eb1fad3c9088be8655,003beec36d3939eb1fad3c9088be8655,39.910459009,0.0007126867680178571,1701475813.4550047,1701475854.3889067,cuda:0,0.837386706,1.4953334035714286e-05,1701475854.3913279,1701475855.2755497,cuda:0,0.576092868,1.0287372642857144e-05,1701475855.5957618,1701475856.174563,cuda:0,0.58573969,1.0459637321428572e-05,1701475856.1746368,1701475856.7726789,cuda:0,0.00031571891,0.031571891,1701475858.4797752,1701475858.511399,cuda:0,5.679119e-05,0.005679119,1701475859.1243455,1701475859.1300282,cuda:0,0.0,0.9829285714285714,1.142575143955228,0.1,20.5655116045475,/result/mnist/reports/attack/003beec36d3939eb1fad3c9088be8655/predictions.json,/result/mnist/reports/attack/003beec36d3939eb1fad3c9088be8655/adv_predictions.json,,attack,deckard.base.experiment.Experiment,deckard.base.attack.Attack,100,evasion,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.attack.AttackInitializer,6492,0.06977174369051158,True,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.017605459871478782,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,pytorch,6492,35,art.attacks.evasion.FastGradientMethod,False,deckard.base.model.Model,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.017605459871478782,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,pytorch,6492,35,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,nvidia-tesla-v100,deckard.base.files.FileConfig,adv_predictions.json,attacks,7e2322c60ed43df53de685b137ed42ba,.pkl,data,20761683b94eca729b58b63071f98327,.pkl,/result/mnist/,003beec36d3939eb1fad3c9088be8655,predictions.json,reports,score_dict.json,attack,,,,,deckard.base.model.Model,pytorch,deckard.base.model.art_pipeline.ArtPipeline,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,"[0, 255]",torch.nn.CrossEntropyLoss,0.017605459871478782,0.9,torch.optim.SGD,pytorch,deckard.base.data.Data,deckard.base.data.generator.DataGenerator,torch_mnist,deckard.base.data.sampler.SklearnDataSampler,7,True,deckard.base.data.sklearn_pipeline.SklearnDataPipeline,sklearn.preprocessing.StandardScaler,True,True,deckard.base.model.ModelInitializer,torch_example.ResNet18,1,6492,35,003beec36d3939eb1fad3c9088be8655,deckard.base.scorer.ScorerDict,deckard.base.scorer.ScorerConfig,maximize,sklearn.metrics.accuracy_score,deckard.base.scorer.ScorerConfig,minimize,sklearn.metrics.log_loss,39.910459009,0.0007126867680178571,1701475813.4550047,1701475854.3889067,cuda:0,0.837386706,1.4953334035714286e-05,1701475854.3913279,1701475855.2755497,cuda:0,0.576092868,1.0287372642857144e-05,1701475855.5957618,1701475856.174563,cuda:0,0.58573969,1.0459637321428572e-05,1701475856.1746368,1701475856.7726789,cuda:0,0.00031571891,0.031571891,1701475858.4797752,1701475858.511399,cuda:0,5.679119e-05,0.005679119,1701475859.1243455,1701475859.1300282,cuda:0,0.0,0.9829285714285714,1.142575143955228,0.1,20.5655116045475,/result/mnist/reports/attack/003beec36d3939eb1fad3c9088be8655/predictions.json,/result/mnist/reports/attack/003beec36d3939eb1fad3c9088be8655/adv_predictions.json, diff --git a/examples/power/plots/dvc.lock b/examples/power/plots/dvc.lock index 97aaf48f..b12e97df 100644 --- a/examples/power/plots/dvc.lock +++ b/examples/power/plots/dvc.lock @@ -127,7 +127,7 @@ stages: DeepFool: attack.init.eps FSQ: model.art.preprocessor.bit_depth Control: model_layers - Epochs: model.trainer.nb_epoch + Epochs: model.trainer.np_epochs Batch_Size: model.trainer.batch_size outs: - path: mnist/clean.csv @@ -215,7 +215,7 @@ stages: DeepFool: attack.init.eps FSQ: model.art.preprocessor.bit_depth Control: model_layers - Epochs: model.trainer.nb_epoch + Epochs: model.trainer.np_epochs Batch_Size: model.trainer.batch_size outs: - path: cifar100/clean.csv @@ -461,7 +461,7 @@ stages: DeepFool: attack.init.eps FSQ: model.art.preprocessor.bit_depth Control: model_layers - Epochs: model.trainer.nb_epoch + Epochs: model.trainer.np_epochs Batch_Size: model.trainer.batch_size outs: - path: combined/clean.csv diff --git a/examples/pytorch/.gitignore b/examples/pytorch/.gitignore index e2755d59..3c4e3e25 100644 --- a/examples/pytorch/.gitignore +++ b/examples/pytorch/.gitignore @@ -6,3 +6,4 @@ cifar100/ /mnist.yaml /cifar.yaml original_data/ +/cifar100.yaml diff --git a/examples/pytorch/attacks.sh b/examples/pytorch/attacks.sh index f31abe05..bba66e7c 100644 --- a/examples/pytorch/attacks.sh +++ b/examples/pytorch/attacks.sh @@ -3,64 +3,63 @@ # # This script is used to generate the attacks for the example. # Fast Gradient Method -bash models.sh \ - stage=attack \ - attack=default \ - ++attack.init.name=art.attacks.evasion.FastGradientMethod \ - ++attack.init.eps=.001,.01,.1,.5,1 \ - ++attack.init.norm=2 \ - atk_name=FGM $@ -# ##################################################### -# Projected Gradient Descent -bash models.sh \ - stage=attack \ - attack=default \ - ++attack.init.name=art.attacks.evasion.ProjectedGradientDescent \ - ++attack.init.eps=.001,.01,.1,.5,1 \ - ++attack.init.norm=2 \ - ++attack.init.eps_step=.001,.003,.01 \ - atk_name=PGD \ - ++attack.init.max_iter=1,5,10,50,100 $@ -# ##################################################### -# DeepFool -bash models.sh \ - stage=attack \ - attack=default \ - ++attack.init.name=art.attacks.evasion.DeepFool \ - ++attack.init.max_iter=10 \ - ++attack.init.batch_size=4096 \ - ++attack.init.nb_grads=1,3,5,8,10 \ - atk_name=Deep $@ +# bash models.sh \ +# stage=attack \ +# attack=default \ +# ++attack.init.name=art.attacks.evasion.FastGradientMethod \ +# ++attack.init.eps=.001,.01,.1,.5,1 \ +# ++attack.init.norm=2 \ +# atk_name=FGM $@ # # ##################################################### -# # HopSkipJump +# # Projected Gradient Descent # bash models.sh \ # stage=attack \ # attack=default \ -# ++attack.init.name=art.attacks.evasion.HopSkipJump \ -# ++attack.init.max_iter=1,3,5,10,15 \ -# ++attack.init.init_eval=10 \ -# ++attack.init.batch_size=4096 \ -# ++attack.init.max_eval=100 \ +# ++attack.init.name=art.attacks.evasion.ProjectedGradientDescent \ +# ++attack.init.eps=.001,.01,.1,.5,1 \ # ++attack.init.norm=2 \ -# atk_name=HSJ $@ +# ++attack.init.eps_step=.001,.003,.01 \ +# atk_name=PGD \ +# ++attack.init.max_iter=1,5,10,50,100 $@ # # ##################################################### -# # PixelAttack +# # DeepFool # bash models.sh \ # stage=attack \ -# attack=default \ -# ++attack.init.name=art.attacks.evasion.PixelAttack \ -# ~attack.init.batch_size \ -# ++attack.init.th=1,4,16,64,256 \ -# atk_name=Pixel $@ +# attack=default \ +# ++attack.init.name=art.attacks.evasion.DeepFool \ +# ++attack.init.max_iter=10 \ +# ++attack.init.batch_size=4096 \ +# ++attack.init.nb_grads=1,3,5,8,10 \ +# atk_name=Deep $@ # # ##################################################### -# # ThresholdAttack +# # HopSkipJump # bash models.sh \ # stage=attack \ # attack=default \ -# ++attack.init.name=art.attacks.evasion.ThresholdAttack \ -# ~attack.init.batch_size \ -# ++attack.init.th=1,4,16,64,256 \ -# atk_name=Thresh $@ +# ++attack.init.name=art.attacks.evasion.HopSkipJump \ +# ++attack.init.max_iter=1,3,5,10,15 \ +# ++attack.init.init_eval=3 \ +# ++attack.init.max_eval=10 \ +# ++attack.init.norm=2 \ +# atk_name=HSJ $@ +# ##################################################### +# PixelAttack +bash models.sh \ + stage=attack \ + attack=default \ + ++attack.init.name=art.attacks.evasion.PixelAttack \ + ~attack.init.batch_size \ + ++attack.init.th=1,4,16,64,256 \ + atk_name=Pixel $@ +# ##################################################### +# ThresholdAttack +bash models.sh \ + stage=attack \ + attack=default \ + ++attack.init.name=art.attacks.evasion.ThresholdAttack \ + ~attack.init.batch_size \ + ++attack.init.th=1,4,16,64,256 \ + atk_name=Thresh $@ # ##################################################### # # ZooAttack # bash models.sh \ diff --git a/examples/pytorch/cifar100.yaml b/examples/pytorch/cifar100.yaml index 471c78a7..e69de29b 100644 --- a/examples/pytorch/cifar100.yaml +++ b/examples/pytorch/cifar100.yaml @@ -1,226 +0,0 @@ -_target_: deckard.base.experiment.Experiment -atk_name: hsj -attack: - _target_: deckard.base.attack.Attack - attack_size: 100 - data: - _target_: deckard.base.data.Data - generate: - name: torch_cifar100 - path: original_data/ - sample: - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - init: - _target_: deckard.base.attack.AttackInitializer - batch_size: 128 - model: - _target_: deckard.base.model.Model - art: - _target_: deckard.base.model.art_pipeline.ArtPipeline - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - data: - _target_: deckard.base.data.Data - generate: - name: torch_cifar100 - path: original_data/ - sample: - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - initialize: - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - library: pytorch - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - data: - _target_: deckard.base.data.Data - generate: - name: torch_cifar100 - path: original_data/ - sample: - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - init: - _target_: deckard.base.model.ModelInitializer - name: torch_example.ResNet18 - num_channels: 3 - num_classes: 100 - library: pytorch - trainer: - batch_size: 128 - nb_epochs: 1 - verbose: true - name: art.attacks.evasion.HopSkipJump - method: evasion - model: - _target_: deckard.base.model.Model - art: - _target_: deckard.base.model.art_pipeline.ArtPipeline - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - data: - _target_: deckard.base.data.Data - generate: - name: torch_cifar100 - path: original_data/ - sample: - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - initialize: - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - library: pytorch - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - data: - _target_: deckard.base.data.Data - generate: - name: torch_cifar100 - path: original_data/ - sample: - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - init: - _target_: deckard.base.model.ModelInitializer - name: torch_example.ResNet18 - num_channels: 3 - num_classes: 100 - library: pytorch - trainer: - batch_size: 128 - nb_epochs: 1 - verbose: true -data: - _target_: deckard.base.data.Data - generate: - name: torch_cifar100 - path: original_data/ - sample: - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 -dataset: cifar100 -def_name: control -device_id: gpu -direction: -- maximize -files: - _target_: deckard.base.files.FileConfig - adv_predictions_file: adv_predictions.json - attack_dir: attacks - attack_file: attack - attack_type: .pkl - directory: cifar100 - model_dir: models - model_file: model - model_type: .pt - name: default - params_file: params.yaml - predictions_file: predictions.json - reports: reports - score_dict_file: score_dict.json -model: - _target_: deckard.base.model.Model - art: - _target_: deckard.base.model.art_pipeline.ArtPipeline - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - data: - _target_: deckard.base.data.Data - generate: - name: torch_cifar100 - path: original_data/ - sample: - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - initialize: - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - library: pytorch - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - data: - _target_: deckard.base.data.Data - generate: - name: torch_cifar100 - path: original_data/ - sample: - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - init: - _target_: deckard.base.model.ModelInitializer - name: torch_example.ResNet18 - num_channels: 3 - num_classes: 100 - library: pytorch - trainer: - batch_size: 128 - nb_epochs: 1 - verbose: true -model_name: ResNet18 -optimizers: -- accuracy -scorers: - _target_: deckard.base.scorer.ScorerDict - accuracy: - _target_: deckard.base.scorer.ScorerConfig - direction: maximize - name: sklearn.metrics.accuracy_score - log_loss: - _target_: deckard.base.scorer.ScorerConfig - direction: minimize - name: sklearn.metrics.log_loss -stage: ??? diff --git a/examples/pytorch/conf/afr.yaml b/examples/pytorch/conf/afr.yaml new file mode 100644 index 00000000..751c5eda --- /dev/null +++ b/examples/pytorch/conf/afr.yaml @@ -0,0 +1,177 @@ +covariates: + - adv_fit_time + - accuracy + - train_time + - atk_value + - def_value + - Epochs + - model_layers + - id + - atk_gen + - def_gen + - adv_failures + - adv_accuracy + - predict_time +fillna: + Epochs: 20 + data.sample.train_size : 48000 + data.sample.test_size: 12000 +cox: + plot: + file : cox_aft.pdf + title : Cox Model + qq_title : Cox QQ Plot + t0: .3 + model: + penalizer: .4 + labels: + "data.sample.random_state": "Random State" + "atk_value": "Attack Strength" + "train_time": "$T_{train}$" + "predict_time": "$T_{predict}$" + "adv_accuracy": "Adv. Accuracy" + "def_value": "Defence Strength" + "accuracy": "Ben. Accuracy" + "model_layers": "Layers" + "Epochs": "No. of Epochs" + "def_gen": "Defence" +# aalen: +# plot: +# file : aalen_aft.pdf +# title : Aalen Model +# qq_title : Aalen QQ Plot +# t0: 1 +# model: +# alpha: 1 +# coef_penalizer: .1 +# smoothing_penalizer: .1 +# labels: +# "data.sample.random_state": "Random State" +# "atk_value": "Attack Strength" +# "train_time": "$T_{train}$" +# "predict_time": "$T_{predict}$" +# "adv_accuracy": "Adv. Accuracy" +# "accuracy": "Ben. Accuracy" +# "adv_fit_time_per_sample": "$T_{attack}$" +# "adv_failure_rate": "$f_{adv.}(t;\\theta)$" +# "failure_rate": "$f_{ben.}(t;\\theta)$" +# "Epochs": "No. of Epochs" +# "model.trainer.batch_size": "Batch Size" +# "def_gen": "Defence" +gamma: + plot: + file : gamma_aft.pdf + title : Generalized Gamma Model + qq_title : Gamma QQ Plot + t0: .3 + model: + penalizer : .5 + labels: + "Intercept: alpha_": "$\\alpha$" + "Intercept: beta_": "$\\beta$" + "data.sample.random_state: beta_": "Random State" + "def_value: beta_": "Defence Strength" + "atk_value: beta_": "Attack Strength" + "train_time: beta_": "$T_{train}$" + "model_layers: beta_": "Layers" + "predict_time: beta_": "$T_{predict}$" + "adv_accuracy: beta_": "Adv. Accuracy" + "accuracy: beta_": "Ben. Accuracy" + "Epochs: beta_": "No. of Epochs" + "def_gen": "Defence" + "attack.init.eps: beta_": "$\\varepsilon$" +weibull: + plot: + file : weibull_aft.pdf + title : Weibull AFT Model + qq_title : Weibull QQ Plot + t0: .3 + model: + penalizer: .1 + labels: + "Intercept: rho_": "$\\rho$" + "Intercept: lambda_": "$\\lambda$" + "data.sample.random_state: lambda_": "Random State" + "atk_value: lambda_": "Attack Strength" + "model_layers: lambda_": "Layers" + "train_time: lambda_": "$T_{train}$" + "predict_time: lambda_": "$T_{predict}$" + "adv_accuracy: lambda_": "Adv. Accuracy" + "accuracy: lambda_": "Ben. Accuracy" + "Epochs: lambda_": "No. of Epochs" + "model.trainer.batch_size: lambda_": "Batch Size" + "def_gen": "Defence" + "def_value: lambda_" : "Defence Strength" + ": lambda_" : "" +exponential: + plot: + file : exponential_aft.pdf + title : Exponential Model + qq_title : Exponential QQ Plot + t0: .1 + model: + breakpoints: + - .1 + labels: + "Intercept: rho_": "$\\rho$" + "Intercept: lambda_": "$\\lambda$" + "atk_value: lambda_": "Attack Strength" + "def_value: lambda_": "Defence Strength" + "model_layers: lambda_": "Layers" + "train_time: lambda_": "$T_{train}$" + "predict_time: lambda_": "$T_{predict}$" + "adv_accuracy: lambda_": "Adv. Accuracy" + "accuracy: lambda_": "Ben. Accuracy" + "Epochs: lambda_": "No. of Epochs" + "def_gen": "Defence" + ": lambda_" : "" +log_logistic: + plot: + file : log_logistic_aft.pdf + title : Log logistic AFT Model + qq_title : Log Logistic QQ Plot + t0: 1 + model: + penalizer: .2 + labels: + "Intercept: beta_": "$\\beta$" + "Intercept: alpha_": "$\\alpha$" + "data.sample.random_state: alpha_": "Random State" + "atk_value: alpha_": "Attack Strength" + "def_value: alpha_": "Defence Strength" + "model_layers: alpha_": "Layers" + "train_time: alpha_": "$T_{train}$" + "predict_time: alpha_": "$T_{predict}$" + "adv_accuracy: alpha_": "Adv. Accuracy" + "accuracy: alpha_": "Ben. Accuracy" + "Epochs: alpha_": "No. of Epochs" + "def_gen": "Defence" +log_normal: + plot: + file : log_normal_aft.pdf + title : Log Normal AFT Model + qq_title : Log Normal QQ Plot + t0: 2 + model: + penalizer: .5 + labels: + "Intercept: sigma_": "$\\sigma$" + "Intercept: mu_": "$\\mu$" + "atk_value: mu_": "Attack Strength" + "def_value: mu_": "Defence Strength" + "model_layers: mu_": "Layers" + "train_time: mu_": "$T_{train}$" + "predict_time: mu_": "$T_{predict}$" + "adv_accuracy: mu_": "Adv. Accuracy" + "accuracy: mu_": "Ben. Accuracy" + "adv_failure_rate: mu_": "$h_{adv.}(t;\\theta)$" + "failure_rate: mu_": "$h_{ben.}(t;\\theta)$" + "Epochs: mu_": "No. of Epochs" + "model.trainer.batch_size: mu_": "Batch Size" + "def_gen": "Defence" + "attack.init.eps: mu_": "$\\varepsilon$" + "data.sample.random_state: mu_": "Random State" +dummies: + "atk_gen": "Atk:" + "def_gen": "Def:" + "id" : "Data:" diff --git a/examples/pytorch/conf/cifar.yaml b/examples/pytorch/conf/cifar.yaml index 3a4dff94..1c960bb6 100644 --- a/examples/pytorch/conf/cifar.yaml +++ b/examples/pytorch/conf/cifar.yaml @@ -49,3 +49,57 @@ hydra: temp_folder: /tmp/deckard max_nbytes: 100000 mmap_mode: r + +defaults: + - _self_ + - data: torch_cifar + - model: torch_cifar + - attack: default + - files: cifar + - scorers: default + - override hydra/sweeper : optuna + - override hydra/sweeper/sampler : grid + - override hydra/launcher : joblib +def_name : control +atk_name : hsj +dataset : cifar +model_name : ResNet18 +device_id : gpu +stage : '???' +direction : + - "maximize" +_target_ : deckard.base.experiment.Experiment +optimizers : + - accuracy +hydra: + run: + dir: ${files.directory}/logs/${stage}/ + sweep: + dir: ${files.directory}/logs/${stage}/${model_name}/${model.trainer.nb_epochs} + subdir : ${def_name}/${atk_name}/${hydra.job.num} + sweeper: + sampler: + _target_: optuna.samplers.GridSampler + direction: ${direction} + study_name: ${model_name}_${def_name}_${atk_name} + storage: sqlite:///${dataset}.db + n_jobs: ${oc.env:HYDRA_SWEEPER_N_JOBS, 32} + n_trials: ${oc.env:HYDRA_SWEEPER_N_TRIALS, 128} + max_failure_rate: 1.0 + params: + ++model.art.initialize.optimizer.lr: shuffle(choice( 0.1, 0.01, 0.001, .0001, .00001, 0.000001)) + ++model.art.initialize.optimizer.momentum: choice(0.1, 0.9, 0.95, 0.99) + # ++model.trainer.nb_epochs: choice(1, 10, 30, 50, 100) + ++model.trainer.batch_size: choice(128, 256, 512, 1024) + _target_: hydra_plugins.hydra_optuna_sweeper.optuna_sweeper.OptunaSweeper + launcher: + _target_: hydra_plugins.hydra_joblib_launcher.joblib_launcher.JoblibLauncher + n_jobs: ${oc.env:HYDRA_SWEEPER_N_JOBS, 8} + prefer : threads + verbose: 10 + timeout: null + pre_dispatch: n_jobs + batch_size: auto + temp_folder: /tmp/deckard + max_nbytes: 100000 + mmap_mode: r diff --git a/examples/pytorch/conf/cifar100.yaml b/examples/pytorch/conf/cifar100.yaml index 1189c84d..a595d9e7 100644 --- a/examples/pytorch/conf/cifar100.yaml +++ b/examples/pytorch/conf/cifar100.yaml @@ -23,7 +23,7 @@ hydra: run: dir: ${files.directory}/logs/${stage}/ sweep: - dir: ${files.directory}/logs/${stage}/${model_name} + dir: ${files.directory}/logs/${stage}/${model_name}/${model.trainer.nb_epochs} subdir : ${def_name}/${atk_name}/${hydra.job.num} sweeper: sampler: @@ -31,13 +31,14 @@ hydra: direction: ${direction} study_name: ${model_name}_${def_name}_${atk_name} storage: sqlite:///${dataset}.db - n_jobs: ${oc.env:HYDRA_SWEEPER_N_JOBS, 8} + n_jobs: ${oc.env:HYDRA_SWEEPER_N_JOBS, 16} n_trials: ${oc.env:HYDRA_SWEEPER_N_TRIALS, 128} max_failure_rate: 1.0 params: ++model.art.initialize.optimizer.lr: choice( 0.1, 0.01, 0.001, .0001, .00001, 0.000001) - ++model.trainer.nb_epoch: choice(1, 10, 30, 50, 100) ++model.art.initialize.optimizer.momentum: choice(0.1, 0.9, 0.95, 0.99) + # ++model.trainer.nb_epochs: choice(1, 10, 30, 50, 100) + ++model.trainer.batch_size: choice(128, 256, 512, 1024) _target_: hydra_plugins.hydra_optuna_sweeper.optuna_sweeper.OptunaSweeper launcher: _target_: hydra_plugins.hydra_joblib_launcher.joblib_launcher.JoblibLauncher diff --git a/examples/pytorch/conf/mnist.yaml b/examples/pytorch/conf/mnist.yaml index 1b144ff7..ab0ebc7e 100644 --- a/examples/pytorch/conf/mnist.yaml +++ b/examples/pytorch/conf/mnist.yaml @@ -23,7 +23,7 @@ hydra: run: dir: ${files.directory}/logs/${stage}/ sweep: - dir: ${files.directory}/logs/${stage}/${model_name} + dir: ${files.directory}/logs/${stage}/${model_name}/${model.trainer.nb_epochs} subdir : ${def_name}/${atk_name}/${hydra.job.num} sweeper: sampler: @@ -31,12 +31,14 @@ hydra: direction: ${direction} study_name: ${model_name}_${def_name}_${atk_name} storage: sqlite:///${dataset}.db - n_jobs: ${oc.env:HYDRA_SWEEPER_N_JOBS, 8} + n_jobs: ${oc.env:HYDRA_SWEEPER_N_JOBS, 32} n_trials: ${oc.env:HYDRA_SWEEPER_N_TRIALS, 128} max_failure_rate: 1.0 params: ++model.art.initialize.optimizer.lr: choice( 0.1, 0.01, 0.001, .0001, .00001, 0.000001) - ++model.trainer.nb_epoch: choice(1, 10, 30, 50, 100) + ++model.art.initialize.optimizer.momentum: choice(0.1, 0.9, 0.95, 0.99) + # ++model.trainer.nb_epochs: choice(1, 10, 30, 50, 100) + ++model.trainer.batch_size: choice(128, 256, 512, 1024) _target_: hydra_plugins.hydra_optuna_sweeper.optuna_sweeper.OptunaSweeper launcher: _target_: hydra_plugins.hydra_joblib_launcher.joblib_launcher.JoblibLauncher diff --git a/examples/pytorch/conf/plots.yaml b/examples/pytorch/conf/plots.yaml index b684aec3..7bc30e78 100644 --- a/examples/pytorch/conf/plots.yaml +++ b/examples/pytorch/conf/plots.yaml @@ -1,10 +1,9 @@ cat_plot: -- file: adv_accuracy_vs_defence_type.eps +- file: adv_accuracy_vs_defence_type.pdf hue: model_name kind: boxen set: yscale: linear - titles: Adv. Accuracy vs Defence Type x: def_gen xlabels: Defence Type y: adv_accuracy @@ -16,10 +15,12 @@ cat_plot: - ResNet50 - ResNet101 - ResNet152 -- file: ben_accuracy_vs_defence_type.eps + # titles: "{col_name} dataset" + legend_title: "Model" + # col: 'id' +- file: ben_accuracy_vs_defence_type.pdf hue: model_name kind: boxen - titles: Ben. Accuracy vs Defence Type x: def_gen xlabels: Defence Type y: accuracy @@ -31,32 +32,21 @@ cat_plot: - ResNet50 - ResNet101 - ResNet152 -- file: ben_failures_per_train_time_vs_defence_type.eps + # titles: "{col_name} dataset" + legend_title: "Model" + # col: 'id' + # col_order: + # - mnist + # - cifar + # - cifar100 +- file: trash_score_vs_defence_type.pdf hue: model_name kind: boxen set: yscale: log - titles: $\bar{C}_{ben.}$ vs Defence Type x: def_gen xlabels: Defence Type - y: training_time_per_failure - ylabels: $\bar{C}_{ben.}$ - rotation : 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 -- file: adv_failures_per_train_time_vs_defence_type.eps - hue: model_name - kind: boxen - set: - yscale: log - titles: $\bar{C}_{adv.}$ vs Defence Type - x: def_gen - xlabels: Defence Type - y: training_time_per_adv_failure + y: c_adv ylabels: $\bar{C}_{adv.}$ rotation : 90 hue_order: @@ -65,16 +55,21 @@ cat_plot: - ResNet50 - ResNet101 - ResNet152 -- file: adv_failures_per_train_time_vs_attack_type.eps + # titles: "{col_name} dataset" + legend_title: "Model" + # col: 'id' + # col_order: + # - mnist + # - cifar + # - cifar100 +- file: trash_score_vs_attack_type.pdf hue: model_name kind: boxen - legend_title: Model Name set: yscale: log - titles: $\bar{C}_{adv.}$ vs Attack Type x: atk_gen xlabels: Attack Type - y: training_time_per_adv_failure + y: c_adv ylabels: $\bar{C}_{adv.}$ rotation : 90 hue_order: @@ -83,28 +78,16 @@ cat_plot: - ResNet50 - ResNet101 - ResNet152 -- file: adv_failures_per_test_time_vs_defence_type.eps + # titles: "{col_name} dataset" + legend_title: "Model" + # col: 'id' + # col_order: + # - mnist + # - cifar + # - cifar100 +- file: adv_accuracy_vs_attack_type.pdf hue: model_name kind: boxen - legend_title: Model Name - titles: $f_{adv}$ vs Defence Type - x: def_gen - xlabels: Defence Type - y: adv_failure_rate - ylabels: $f_{adv.}$ - rotation : 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - -- file: adv_accuracy_vs_attack_type.eps - hue: model_name - kind: boxen - legend_title: Model Name - titles: Adv. Accuracy vs Attack Type x: atk_gen xlabels: Attack Type y: adv_accuracy @@ -116,26 +99,15 @@ cat_plot: - ResNet50 - ResNet101 - ResNet152 -- file: ben_failure_rate_vs_defence_type.eps - hue: model_name - kind: boxen - legend_title: Model Name - set: - yscale: log - titles: $f_{ben}(t; \theta)$ vs Defence Type - x: def_gen - xlabels: Defence Type - y: failure_rate - ylabels: $f_{ben}(t; \theta)$ - rotation : 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 + # titles: "{col_name} dataset" + legend_title: "Model" + # col: 'id' + # col_order: + # - mnist + # - cifar + # - cifar100 line_plot: -- file: def_param_vs_accuracy.eps +- file: def_param_vs_accuracy.pdf hue: def_gen legend: {"bbox_to_anchor": [1.05, 1], "title": "Defence"} title: Ben. Accuracy vs Defence Strength @@ -148,14 +120,12 @@ line_plot: hue_order: - Control - Conf - - Epochs - Gauss-in - Gauss-out - - Conf - FSQ errorbar: se err_style: bars -- file: def_param_vs_adv_accuracy.eps +- file: def_param_vs_adv_accuracy.pdf hue: def_gen legend: {"bbox_to_anchor": [1.05, 1], "title": "Defence"} title: Adv. Accuracy vs Defence Strength @@ -168,14 +138,13 @@ line_plot: hue_order: - Control - Conf - - Epochs - Gauss-in - Gauss-out - - Conf - FSQ + # style : id errorbar: se err_style: bars -- file: def_param_vs_adv_failure_rate.eps +- file: def_param_vs_adv_failure_rate.pdf hue: def_gen legend: {"bbox_to_anchor": [1.05, 1], "title": "Defence"} title: $f_{adv}$ vs Defence Strength @@ -185,17 +154,16 @@ line_plot: y: adv_failure_rate y_scale: linear ylabel: $f_{adv.}$ + # style: id hue_order: - Control - Conf - - Epochs - Gauss-in - Gauss-out - - Conf - FSQ errorbar: se err_style: bars -- file: atk_param_vs_accuracy.eps +- file: atk_param_vs_accuracy.pdf hue: atk_gen legend: {bbox_to_anchor: [1.05, 1]} title: Adv. Accuracy vs Attack Strength @@ -221,7 +189,7 @@ scatter_plot: xlabel: $t_{train}$ ylabel: $f_{adv}$ title: $f_{adv}$ vs $t_{train}$ - file: adv_failure_rate_vs_train_time.eps + file: adv_failure_rate_vs_train_time.pdf # y_scale: log x_scale: log legend: diff --git a/examples/pytorch/dvc.lock b/examples/pytorch/dvc.lock index d17b377a..8bbbfc23 100644 --- a/examples/pytorch/dvc.lock +++ b/examples/pytorch/dvc.lock @@ -108,16 +108,16 @@ stages: outs: - path: mnist/models/model.optimizer.pt hash: md5 - md5: e16de4ab0b1f1a5808aaeb8cdd76553c + md5: ba33d30facc5b91aba2d608d37ba771f size: 44781294 - path: mnist/models/model.pt hash: md5 - md5: ae8a80a475c5ccfbf50eaa62adb24e1e + md5: a6ae9e91474793326c6a3c139866292f size: 44787138 - path: mnist/reports/train/default/score_dict.json hash: md5 - md5: f55673763c20d36336fc1a23bd057a5f - size: 515 + md5: 1f33102e4833438d6a2151c8eb49d4e7 + size: 525 train@cifar: cmd: python -m deckard.layers.experiment train@cifar --config_file cifar.yaml --params_file cifar.yaml @@ -226,16 +226,16 @@ stages: outs: - path: cifar/models/model.optimizer.pt hash: md5 - md5: 325f6119d397c468be7d7fe01b9b04ea + md5: 48311974bdb089e3912a018165e588f1 size: 44781294 - path: cifar/models/model.pt hash: md5 - md5: 64271d0a795a3bdf5a46951ea1e3ded9 + md5: 65ee5487dd484d5bb26fde29dd7a6311 size: 44787138 - path: cifar/reports/train/default/score_dict.json hash: md5 - md5: 73c3315c8f9def00a519bdf6bb9998e2 - size: 508 + md5: 5f47af0c6e48142318d161e0dae6eb06 + size: 526 train@cifar100: cmd: python -m deckard.layers.experiment train@cifar100 --config_file cifar100.yaml --params_file cifar100.yaml @@ -344,27 +344,27 @@ stages: outs: - path: cifar100/models/model.optimizer.pt hash: md5 - md5: b469db4312d99c3ccf8d198c9936f7ab + md5: 1cc5f1335ddd96097060130f3508c04f size: 44781294 - path: cifar100/models/model.pt hash: md5 - md5: 76593e090e9815dfac73c6f9d1d84d57 + md5: 926a0e9bfe98ceef42fb9bc060c54218 size: 44787138 - path: cifar100/reports/train/default/score_dict.json hash: md5 - md5: 41191be9e2760b49f21bfe27ccef654c - size: 514 + md5: 8871d4ed73592450b4ceaa09fdfb4c0f + size: 513 attack@mnist: cmd: python -m deckard.layers.experiment attack@mnist --config_file mnist.yaml --params_file mnist.yaml deps: - path: mnist/models/model.optimizer.pt hash: md5 - md5: e16de4ab0b1f1a5808aaeb8cdd76553c + md5: ba33d30facc5b91aba2d608d37ba771f size: 44781294 - path: mnist/models/model.pt hash: md5 - md5: ae8a80a475c5ccfbf50eaa62adb24e1e + md5: a6ae9e91474793326c6a3c139866292f size: 44787138 params: params.yaml: @@ -598,12 +598,12 @@ stages: outs: - path: mnist/attacks/attack.pkl hash: md5 - md5: bb64f427b6df9067d9c59d5021937164 + md5: 0d9b4dcaaffbc077dec93a4ae8c69171 size: 313766 - path: mnist/reports/attack/default/score_dict.json hash: md5 - md5: b63b8d925ab6db88e3b707ec025800d3 - size: 834 + md5: d38a998a2b08eff46f3bab06e7c77d64 + size: 851 parse@mnist: cmd: python -m deckard.layers.parse --config_file mnist.yaml --params_file mnist.yaml deps: @@ -614,8 +614,8 @@ stages: nfiles: 1 - path: conf/data hash: md5 - md5: ab878db613853b9479b012b228a00d50.dir - size: 643 + md5: eafea1e119e2f54e9788ee46101afc79.dir + size: 645 nfiles: 4 - path: conf/files hash: md5 @@ -624,12 +624,12 @@ stages: nfiles: 3 - path: conf/mnist.yaml hash: md5 - md5: ed726950d032472aedbb267ae88c0f9a - size: 1501 + md5: 73ed848330388ef5d98a54d03e3cdcf9 + size: 1671 - path: conf/model hash: md5 - md5: 5a1ef81c82a950e745c930067d8c870d.dir - size: 2093 + md5: eb0d17ad4826334ac8488c1683106580.dir + size: 2095 nfiles: 10 - path: conf/scorers hash: md5 @@ -651,12 +651,12 @@ stages: nfiles: 1 - path: conf/cifar.yaml hash: md5 - md5: 20703e37d826c71d762491158f302bfe - size: 1501 + md5: 5171900fd679f57392fa59f160301911 + size: 1681 - path: conf/data hash: md5 - md5: ab878db613853b9479b012b228a00d50.dir - size: 643 + md5: eafea1e119e2f54e9788ee46101afc79.dir + size: 645 nfiles: 4 - path: conf/files hash: md5 @@ -665,8 +665,8 @@ stages: nfiles: 3 - path: conf/model hash: md5 - md5: 5a1ef81c82a950e745c930067d8c870d.dir - size: 2093 + md5: eb0d17ad4826334ac8488c1683106580.dir + size: 2095 nfiles: 10 - path: conf/scorers hash: md5 @@ -689,12 +689,12 @@ stages: nfiles: 1 - path: conf/cifar100.yaml hash: md5 - md5: 008f14b0f28ad1abdb09f56b702ae0ab - size: 1591 + md5: 38d9e68b647acbca29a89ef6a699b4bf + size: 1684 - path: conf/data hash: md5 - md5: ab878db613853b9479b012b228a00d50.dir - size: 643 + md5: eafea1e119e2f54e9788ee46101afc79.dir + size: 645 nfiles: 4 - path: conf/files hash: md5 @@ -703,8 +703,8 @@ stages: nfiles: 3 - path: conf/model hash: md5 - md5: 5a1ef81c82a950e745c930067d8c870d.dir - size: 2093 + md5: eb0d17ad4826334ac8488c1683106580.dir + size: 2095 nfiles: 10 - path: conf/scorers hash: md5 @@ -722,11 +722,11 @@ stages: deps: - path: cifar/models/model.optimizer.pt hash: md5 - md5: 325f6119d397c468be7d7fe01b9b04ea + md5: 48311974bdb089e3912a018165e588f1 size: 44781294 - path: cifar/models/model.pt hash: md5 - md5: 64271d0a795a3bdf5a46951ea1e3ded9 + md5: 65ee5487dd484d5bb26fde29dd7a6311 size: 44787138 params: params.yaml: @@ -960,23 +960,23 @@ stages: outs: - path: cifar/attacks/attack.pkl hash: md5 - md5: c3e171e067df56e31ee5d4d968aad9d5 + md5: 974de8139d9ce86187cf824f00e80d95 size: 313766 - path: cifar/reports/attack/default/score_dict.json hash: md5 - md5: 6aae48892e49e75742b1d67a5e1ec276 - size: 838 + md5: 46802563046219bac6f958866dda96f5 + size: 852 attack@cifar100: cmd: python -m deckard.layers.experiment attack@cifar100 --config_file cifar100.yaml --params_file cifar100.yaml deps: - path: cifar100/models/model.optimizer.pt hash: md5 - md5: b469db4312d99c3ccf8d198c9936f7ab + md5: 1cc5f1335ddd96097060130f3508c04f size: 44781294 - path: cifar100/models/model.pt hash: md5 - md5: 76593e090e9815dfac73c6f9d1d84d57 + md5: 926a0e9bfe98ceef42fb9bc060c54218 size: 44787138 params: params.yaml: @@ -1210,12 +1210,12 @@ stages: outs: - path: cifar100/attacks/attack.pkl hash: md5 - md5: fa9665c5d8fc6a1118123c25a30e8978 + md5: 7b3384f27a7edde0d3813ffb297e4123 size: 313766 - path: cifar100/reports/attack/default/score_dict.json hash: md5 - md5: 7396ec38b31f7ff9ae1fcab467a98c6c - size: 854 + md5: c2f9e1e3db07148a510a2dd237d22218 + size: 828 attacks@mnist-ResNet18: cmd: bash attacks.sh ++model.init.name=torch_example.ResNet18 stage=attack model_name=ResNet18 attack.attack_size=100 data=torch_mnist model=torch_mnist +direction="[maximize,maximize,minimize]" @@ -1223,18 +1223,18 @@ stages: deps: - path: mnist/reports/attack/default/score_dict.json hash: md5 - md5: b63b8d925ab6db88e3b707ec025800d3 - size: 834 + md5: d38a998a2b08eff46f3bab06e7c77d64 + size: 851 - path: mnist/reports/train/default/score_dict.json hash: md5 - md5: f55673763c20d36336fc1a23bd057a5f - size: 515 + md5: 1f33102e4833438d6a2151c8eb49d4e7 + size: 525 outs: - path: mnist/logs/attack/ResNet18/ hash: md5 - md5: 240762a55b7e67e04f4b337e2aa58e50.dir - size: 67964097 - nfiles: 7809 + md5: 21ccd321fb271ef569c8e2844d3a580c.dir + size: 539921753 + nfiles: 61569 attacks@mnist-ResNet50: cmd: bash attacks.sh ++model.init.name=torch_example.ResNet50 stage=attack model_name=ResNet50 attack.attack_size=100 data=torch_mnist model=torch_mnist +direction="[maximize,maximize,minimize]" @@ -1242,18 +1242,18 @@ stages: deps: - path: mnist/reports/attack/default/score_dict.json hash: md5 - md5: b63b8d925ab6db88e3b707ec025800d3 - size: 834 + md5: d38a998a2b08eff46f3bab06e7c77d64 + size: 851 - path: mnist/reports/train/default/score_dict.json hash: md5 - md5: f55673763c20d36336fc1a23bd057a5f - size: 515 + md5: 1f33102e4833438d6a2151c8eb49d4e7 + size: 525 outs: - path: mnist/logs/attack/ResNet50/ hash: md5 - md5: 481f3de00d4fe02fde5534cdcc1c2b30.dir - size: 78656482 - nfiles: 7681 + md5: 31a2f23ef4d25e0fb50579ddc5b6a111.dir + size: 238399439 + nfiles: 28729 attacks@mnist-ResNet152: cmd: bash attacks.sh ++model.init.name=torch_example.ResNet152 stage=attack model_name=ResNet152 attack.attack_size=100 data=torch_mnist model=torch_mnist @@ -1282,18 +1282,18 @@ stages: deps: - path: mnist/reports/attack/default/score_dict.json hash: md5 - md5: b63b8d925ab6db88e3b707ec025800d3 - size: 834 + md5: d38a998a2b08eff46f3bab06e7c77d64 + size: 851 - path: mnist/reports/train/default/score_dict.json hash: md5 - md5: f55673763c20d36336fc1a23bd057a5f - size: 515 + md5: 1f33102e4833438d6a2151c8eb49d4e7 + size: 525 outs: - path: mnist/logs/attack/ResNet101/ hash: md5 - md5: 92951fdd84c6ad7dfa6e724499196111.dir - size: 90289426 - nfiles: 7681 + md5: 14ed49640893e98063aa73c996db5652.dir + size: 259212024 + nfiles: 25349 attacks@mnist-ResNet34: cmd: bash attacks.sh ++model.init.name=torch_example.ResNet34 stage=attack model_name=ResNet34 attack.attack_size=100 data=torch_mnist model=torch_mnist +direction="[maximize,maximize,minimize]" @@ -1301,18 +1301,18 @@ stages: deps: - path: mnist/reports/attack/default/score_dict.json hash: md5 - md5: b63b8d925ab6db88e3b707ec025800d3 - size: 834 + md5: d38a998a2b08eff46f3bab06e7c77d64 + size: 851 - path: mnist/reports/train/default/score_dict.json hash: md5 - md5: f55673763c20d36336fc1a23bd057a5f - size: 515 + md5: 1f33102e4833438d6a2151c8eb49d4e7 + size: 525 outs: - path: mnist/logs/attack/ResNet34/ hash: md5 - md5: 60ec86982d5cf4693ffd7e5420ce2863.dir - size: 110646599 - nfiles: 9985 + md5: f18222865d4ced43dff21f5086e83c3c.dir + size: 258848998 + nfiles: 31817 attacks@cifar-ResNet18: cmd: bash attacks.sh ++model.init.name=torch_example.ResNet18 stage=attack model_name=ResNet18 attack.attack_size=100 data=torch_cifar model=torch_cifar +direction="[maximize,maximize,minimize]" @@ -1487,3 +1487,436 @@ stages: md5: 57911c3eaeeb3888a220a1fcf12e8442.dir size: 63892972 nfiles: 7681 + attacks@cifar100-ResNet152: + cmd: bash attacks.sh ++model.init.name=torch_example.ResNet152 stage=attack + model_name=ResNet152 attack.attack_size=100 data=torch_cifar100 model=torch_cifar100 + +direction="[maximize,maximize,minimize]" +optimizers="[accuracy,adv_accuracy,adv_success]" + --config-name cifar100.yaml + deps: + - path: cifar100/reports/attack/default/score_dict.json + hash: md5 + md5: 7396ec38b31f7ff9ae1fcab467a98c6c + size: 854 + - path: cifar100/reports/train/default/score_dict.json + hash: md5 + md5: 41191be9e2760b49f21bfe27ccef654c + size: 514 + outs: + - path: cifar100/logs/attack/ResNet152/ + hash: md5 + md5: bef4f0ac68bc29e2b41bd35ee86e5def.dir + size: 80996700 + nfiles: 7681 + compile@cifar-attack: + cmd: python -m deckard.layers.compile --report_folder cifar/reports/attack --results_file + cifar/reports/attack.csv + deps: + - path: cifar/logs/attack/ + hash: md5 + md5: cd234596a07f78c2d3bfc852a38af7ad.dir + size: 265902793 + nfiles: 38405 + - path: cifar/reports/attack/ + hash: md5 + md5: d655cde61b88f9e0d9e014f634ccf84b.dir + size: 76673481 + nfiles: 21300 + outs: + - path: cifar/reports/attack.csv + hash: md5 + md5: 70bcdb22a2abc085664a59873902a508 + size: 31567491 + compile@cifar100-attack: + cmd: python -m deckard.layers.compile --report_folder cifar100/reports/attack + --results_file cifar100/reports/attack.csv + deps: + - path: cifar100/logs/attack/ + hash: md5 + md5: 1f01af02a4a7d87769f4eddcedda6fb0.dir + size: 250371493 + nfiles: 38405 + - path: cifar100/reports/attack/ + hash: md5 + md5: ed1b20a41054766cb993c99a7d2cbf84.dir + size: 68252477 + nfiles: 19208 + outs: + - path: cifar100/reports/attack.csv + hash: md5 + md5: 6416228973971f0719f394c31e0e639d + size: 29836704 + prepare_plot_folder@cifar100-attack: + cmd: cp cifar100/reports/attack.csv plots/data/attack_cifar100.csv + deps: + - path: cifar100/reports/attack.csv + hash: md5 + md5: 6416228973971f0719f394c31e0e639d + size: 29836704 + outs: + - path: plots/data/attack_cifar100.csv + hash: md5 + md5: 6416228973971f0719f394c31e0e639d + size: 29836704 + prepare_plot_folder@cifar-attack: + cmd: cp cifar/reports/attack.csv plots/data/attack_cifar.csv + deps: + - path: cifar/reports/attack.csv + hash: md5 + md5: 70bcdb22a2abc085664a59873902a508 + size: 31567491 + outs: + - path: plots/data/attack_cifar.csv + hash: md5 + md5: 70bcdb22a2abc085664a59873902a508 + size: 31567491 + attacks@mnist-ResNet18-1: + cmd: bash attacks.sh ++model.init.name=torch_example.ResNet18 stage=attack model_name=ResNet18 + attack.attack_size=100 model.trainer.nb_epochs=1 data=torch_mnist model=torch_mnist + +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" + --config-name mnist.yaml + deps: + - path: mnist/reports/attack/default/score_dict.json + hash: md5 + md5: d38a998a2b08eff46f3bab06e7c77d64 + size: 851 + - path: mnist/reports/train/default/score_dict.json + hash: md5 + md5: 1f33102e4833438d6a2151c8eb49d4e7 + size: 525 + outs: + - path: mnist/logs/attack/ResNet18/1/ + hash: md5 + md5: c4e3a8b5197420f5f54553617bc20873.dir + size: 36422730 + nfiles: 7681 + attacks@mnist-ResNet18-10: + cmd: bash attacks.sh ++model.init.name=torch_example.ResNet18 stage=attack model_name=ResNet18 + attack.attack_size=100 model.trainer.nb_epochs=10 data=torch_mnist model=torch_mnist + +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" + --config-name mnist.yaml + deps: + - path: mnist/reports/attack/default/score_dict.json + hash: md5 + md5: d38a998a2b08eff46f3bab06e7c77d64 + size: 851 + - path: mnist/reports/train/default/score_dict.json + hash: md5 + md5: 1f33102e4833438d6a2151c8eb49d4e7 + size: 525 + outs: + - path: mnist/logs/attack/ResNet18/10/ + hash: md5 + md5: d624fd4cefda6b891493b29e876bdc52.dir + size: 32664620 + nfiles: 6037 + attacks@mnist-ResNet34-1: + cmd: bash attacks.sh ++model.init.name=torch_example.ResNet34 stage=attack model_name=ResNet34 + attack.attack_size=100 model.trainer.nb_epochs=1 data=torch_mnist model=torch_mnist + +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" + --config-name mnist.yaml + deps: + - path: mnist/reports/attack/default/score_dict.json + hash: md5 + md5: d38a998a2b08eff46f3bab06e7c77d64 + size: 851 + - path: mnist/reports/train/default/score_dict.json + hash: md5 + md5: 1f33102e4833438d6a2151c8eb49d4e7 + size: 525 + outs: + - path: mnist/logs/attack/ResNet34/1/ + hash: md5 + md5: e497d20650c616bd9a458e3ef5781f10.dir + size: 82396544 + nfiles: 9345 + attacks@mnist-ResNet50-1: + cmd: bash attacks.sh ++model.init.name=torch_example.ResNet50 stage=attack model_name=ResNet50 + attack.attack_size=100 model.trainer.nb_epochs=1 data=torch_mnist model=torch_mnist + +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" + --config-name mnist.yaml + deps: + - path: mnist/reports/attack/default/score_dict.json + hash: md5 + md5: d38a998a2b08eff46f3bab06e7c77d64 + size: 851 + - path: mnist/reports/train/default/score_dict.json + hash: md5 + md5: 1f33102e4833438d6a2151c8eb49d4e7 + size: 525 + outs: + - path: mnist/logs/attack/ResNet50/1/ + hash: md5 + md5: 4cff9becc018ddf89b6a912553fe237e.dir + size: 47146814 + nfiles: 7681 + attacks@mnist-ResNet101-1: + cmd: bash attacks.sh ++model.init.name=torch_example.ResNet101 stage=attack + model_name=ResNet101 attack.attack_size=100 model.trainer.nb_epochs=1 data=torch_mnist + model=torch_mnist +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" + --config-name mnist.yaml + deps: + - path: mnist/reports/attack/default/score_dict.json + hash: md5 + md5: d38a998a2b08eff46f3bab06e7c77d64 + size: 851 + - path: mnist/reports/train/default/score_dict.json + hash: md5 + md5: 1f33102e4833438d6a2151c8eb49d4e7 + size: 525 + outs: + - path: mnist/logs/attack/ResNet101/1/ + hash: md5 + md5: 8459e065681999a3e82830141acddd36.dir + size: 58634505 + nfiles: 7681 + attacks@mnist-ResNet152-1: + cmd: bash attacks.sh ++model.init.name=torch_example.ResNet152 stage=attack + model_name=ResNet152 attack.attack_size=100 model.trainer.nb_epochs=1 data=torch_mnist + model=torch_mnist +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" + --config-name mnist.yaml + deps: + - path: mnist/reports/attack/default/score_dict.json + hash: md5 + md5: d38a998a2b08eff46f3bab06e7c77d64 + size: 851 + - path: mnist/reports/train/default/score_dict.json + hash: md5 + md5: 1f33102e4833438d6a2151c8eb49d4e7 + size: 525 + outs: + - path: mnist/logs/attack/ResNet152/1/ + hash: md5 + md5: 3169ac92a9461de72740ec83d01dd0e1.dir + size: 76579893 + nfiles: 7681 + attacks@cifar-ResNet18-1: + cmd: bash attacks.sh ++model.init.name=torch_example.ResNet18 stage=attack model_name=ResNet18 + attack.attack_size=100 model.trainer.nb_epochs=1 data=torch_cifar model=torch_cifar + +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" + --config-name cifar.yaml + deps: + - path: cifar/reports/attack/default/score_dict.json + hash: md5 + md5: 46802563046219bac6f958866dda96f5 + size: 852 + - path: cifar/reports/train/default/score_dict.json + hash: md5 + md5: 5f47af0c6e48142318d161e0dae6eb06 + size: 526 + outs: + - path: cifar/logs/attack/ResNet18/1/ + hash: md5 + md5: 3b9bae7e98835391bd8683b8e138596c.dir + size: 36146151 + nfiles: 7681 + attacks@cifar-ResNet34-1: + cmd: bash attacks.sh ++model.init.name=torch_example.ResNet34 stage=attack model_name=ResNet34 + attack.attack_size=100 model.trainer.nb_epochs=1 data=torch_cifar model=torch_cifar + +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" + --config-name cifar.yaml + deps: + - path: cifar/reports/attack/default/score_dict.json + hash: md5 + md5: 46802563046219bac6f958866dda96f5 + size: 852 + - path: cifar/reports/train/default/score_dict.json + hash: md5 + md5: 5f47af0c6e48142318d161e0dae6eb06 + size: 526 + outs: + - path: cifar/logs/attack/ResNet34/1/ + hash: md5 + md5: 52ac24dec18dffe1eb8f019548d81a7c.dir + size: 42447682 + nfiles: 7681 + attacks@cifar-ResNet50-1: + cmd: bash attacks.sh ++model.init.name=torch_example.ResNet50 stage=attack model_name=ResNet50 + attack.attack_size=100 model.trainer.nb_epochs=1 data=torch_cifar model=torch_cifar + +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" + --config-name cifar.yaml + deps: + - path: cifar/reports/attack/default/score_dict.json + hash: md5 + md5: 46802563046219bac6f958866dda96f5 + size: 852 + - path: cifar/reports/train/default/score_dict.json + hash: md5 + md5: 5f47af0c6e48142318d161e0dae6eb06 + size: 526 + outs: + - path: cifar/logs/attack/ResNet50/1/ + hash: md5 + md5: 41bdebb342f790e4174c95841b1e514b.dir + size: 61996057 + nfiles: 7681 + attacks@cifar-ResNet101-1: + cmd: bash attacks.sh ++model.init.name=torch_example.ResNet101 stage=attack + model_name=ResNet101 attack.attack_size=100 model.trainer.nb_epochs=1 data=torch_cifar + model=torch_cifar +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" + --config-name cifar.yaml + deps: + - path: cifar/reports/attack/default/score_dict.json + hash: md5 + md5: 46802563046219bac6f958866dda96f5 + size: 852 + - path: cifar/reports/train/default/score_dict.json + hash: md5 + md5: 5f47af0c6e48142318d161e0dae6eb06 + size: 526 + outs: + - path: cifar/logs/attack/ResNet101/1/ + hash: md5 + md5: b7bfc568067aca4ce163b1163e5b5404.dir + size: 56855189 + nfiles: 7681 + attacks@cifar-ResNet152-1: + cmd: bash attacks.sh ++model.init.name=torch_example.ResNet152 stage=attack + model_name=ResNet152 attack.attack_size=100 model.trainer.nb_epochs=1 data=torch_cifar + model=torch_cifar +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" + --config-name cifar.yaml + deps: + - path: cifar/reports/attack/default/score_dict.json + hash: md5 + md5: 46802563046219bac6f958866dda96f5 + size: 852 + - path: cifar/reports/train/default/score_dict.json + hash: md5 + md5: 5f47af0c6e48142318d161e0dae6eb06 + size: 526 + outs: + - path: cifar/logs/attack/ResNet152/1/ + hash: md5 + md5: d68c082a5525f00982df9b769be8e2f8.dir + size: 68457714 + nfiles: 7681 + attacks@cifar100-ResNet18-1: + cmd: bash attacks.sh ++model.init.name=torch_example.ResNet18 stage=attack model_name=ResNet18 + attack.attack_size=100 model.trainer.nb_epochs=1 data=torch_cifar100 model=torch_cifar100 + +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" + --config-name cifar100.yaml + deps: + - path: cifar100/reports/attack/default/score_dict.json + hash: md5 + md5: c2f9e1e3db07148a510a2dd237d22218 + size: 828 + - path: cifar100/reports/train/default/score_dict.json + hash: md5 + md5: 8871d4ed73592450b4ceaa09fdfb4c0f + size: 513 + outs: + - path: cifar100/logs/attack/ResNet18/1/ + hash: md5 + md5: ac985ee1b0c7d97477205ac1b0be64db.dir + size: 35832419 + nfiles: 7681 + attacks@cifar100-ResNet34-1: + cmd: bash attacks.sh ++model.init.name=torch_example.ResNet34 stage=attack model_name=ResNet34 + attack.attack_size=100 model.trainer.nb_epochs=1 data=torch_cifar100 model=torch_cifar100 + +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" + --config-name cifar100.yaml + deps: + - path: cifar100/reports/attack/default/score_dict.json + hash: md5 + md5: c2f9e1e3db07148a510a2dd237d22218 + size: 828 + - path: cifar100/reports/train/default/score_dict.json + hash: md5 + md5: 8871d4ed73592450b4ceaa09fdfb4c0f + size: 513 + outs: + - path: cifar100/logs/attack/ResNet34/1/ + hash: md5 + md5: c3ce673126d443de9434667307930a6c.dir + size: 41411277 + nfiles: 7681 + attacks@cifar100-ResNet50-1: + cmd: bash attacks.sh ++model.init.name=torch_example.ResNet50 stage=attack model_name=ResNet50 + attack.attack_size=100 model.trainer.nb_epochs=1 data=torch_cifar100 model=torch_cifar100 + +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" + --config-name cifar100.yaml + deps: + - path: cifar100/reports/attack/default/score_dict.json + hash: md5 + md5: c2f9e1e3db07148a510a2dd237d22218 + size: 828 + - path: cifar100/reports/train/default/score_dict.json + hash: md5 + md5: 8871d4ed73592450b4ceaa09fdfb4c0f + size: 513 + outs: + - path: cifar100/logs/attack/ResNet50/1/ + hash: md5 + md5: 5206d77e526ee19a768ef63731e4656e.dir + size: 44976271 + nfiles: 7681 + attacks@cifar100-ResNet101-1: + cmd: bash attacks.sh ++model.init.name=torch_example.ResNet101 stage=attack + model_name=ResNet101 attack.attack_size=100 model.trainer.nb_epochs=1 data=torch_cifar100 + model=torch_cifar100 +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" + --config-name cifar100.yaml + deps: + - path: cifar100/reports/attack/default/score_dict.json + hash: md5 + md5: c2f9e1e3db07148a510a2dd237d22218 + size: 828 + - path: cifar100/reports/train/default/score_dict.json + hash: md5 + md5: 8871d4ed73592450b4ceaa09fdfb4c0f + size: 513 + outs: + - path: cifar100/logs/attack/ResNet101/1/ + hash: md5 + md5: 63708391dbb0828d6270a5d78e6ad76c.dir + size: 57235553 + nfiles: 7681 + attacks@cifar100-ResNet152-1: + cmd: bash attacks.sh ++model.init.name=torch_example.ResNet152 stage=attack + model_name=ResNet152 attack.attack_size=100 model.trainer.nb_epochs=1 data=torch_cifar100 + model=torch_cifar100 +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" + --config-name cifar100.yaml + deps: + - path: cifar100/reports/attack/default/score_dict.json + hash: md5 + md5: c2f9e1e3db07148a510a2dd237d22218 + size: 828 + - path: cifar100/reports/train/default/score_dict.json + hash: md5 + md5: 8871d4ed73592450b4ceaa09fdfb4c0f + size: 513 + outs: + - path: cifar100/logs/attack/ResNet152/1/ + hash: md5 + md5: 9d1a7833ce57f0b5825c1114bc88ce36.dir + size: 70915973 + nfiles: 7681 + compile@mnist-attack: + cmd: python -m deckard.layers.compile --report_folder mnist/reports/attack --results_file + mnist/reports/attack.csv + deps: + - path: mnist/logs/attack/ + hash: md5 + md5: 4d6bdfb3d27cfd532c69a05b2e7cf2d0.dir + size: 343311550 + nfiles: 48119 + - path: mnist/reports/attack/ + hash: md5 + md5: 90555fff902b52c29e799cfac76c2d5b.dir + size: 108999231 + nfiles: 28416 + outs: + - path: mnist/reports/attack.csv + hash: md5 + md5: e50857f937b52c64d262302b4aa9740e + size: 53174519 + prepare_plot_folder@mnist-attack: + cmd: cp mnist/reports/attack.csv plots/data/attack_mnist.csv + deps: + - path: mnist/reports/attack.csv + hash: md5 + md5: e50857f937b52c64d262302b4aa9740e + size: 53174519 + outs: + - path: plots/data/attack_mnist.csv + hash: md5 + md5: e50857f937b52c64d262302b4aa9740e + size: 53174519 diff --git a/examples/pytorch/dvc.yaml b/examples/pytorch/dvc.yaml index d1ddd790..f249f80b 100644 --- a/examples/pytorch/dvc.yaml +++ b/examples/pytorch/dvc.yaml @@ -88,22 +88,25 @@ stages: matrix: dataset : [mnist, cifar, cifar100] model : [ResNet18, ResNet34, ResNet50, ResNet101, ResNet152] + epochs : [1] #10, 20, 30, 50, 100 cmd: >- bash attacks.sh ++model.init.name=torch_example.${item.model} stage=attack model_name=${item.model} attack.attack_size=100 + model.trainer.nb_epochs=${item.epochs} data=torch_${item.dataset} model=torch_${item.dataset} +direction="[maximize,maximize,minimize]" - +optimizers="[accuracy,adv_accuracy,adv_success]" + ++optimizers="[accuracy,adv_accuracy,adv_success]" --config-name ${item.dataset}.yaml deps: - ${item.dataset}/${files.reports}/attack/${files.name}/${files.score_dict_file} # This is here just to ensure it runs after the attack stage - ${item.dataset}/${files.reports}/train/${files.name}/${files.score_dict_file} + - attacks.sh outs: - - ${item.dataset}/logs/attack/${item.model}/: + - ${item.dataset}/logs/attack/${item.model}/${item.epochs}/: cache: True persist: True compile: diff --git a/examples/pytorch/old_cifar100.yaml b/examples/pytorch/old_cifar100.yaml new file mode 100644 index 00000000..471c78a7 --- /dev/null +++ b/examples/pytorch/old_cifar100.yaml @@ -0,0 +1,226 @@ +_target_: deckard.base.experiment.Experiment +atk_name: hsj +attack: + _target_: deckard.base.attack.Attack + attack_size: 100 + data: + _target_: deckard.base.data.Data + generate: + name: torch_cifar100 + path: original_data/ + sample: + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + init: + _target_: deckard.base.attack.AttackInitializer + batch_size: 128 + model: + _target_: deckard.base.model.Model + art: + _target_: deckard.base.model.art_pipeline.ArtPipeline + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + data: + _target_: deckard.base.data.Data + generate: + name: torch_cifar100 + path: original_data/ + sample: + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + initialize: + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + library: pytorch + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + data: + _target_: deckard.base.data.Data + generate: + name: torch_cifar100 + path: original_data/ + sample: + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + init: + _target_: deckard.base.model.ModelInitializer + name: torch_example.ResNet18 + num_channels: 3 + num_classes: 100 + library: pytorch + trainer: + batch_size: 128 + nb_epochs: 1 + verbose: true + name: art.attacks.evasion.HopSkipJump + method: evasion + model: + _target_: deckard.base.model.Model + art: + _target_: deckard.base.model.art_pipeline.ArtPipeline + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + data: + _target_: deckard.base.data.Data + generate: + name: torch_cifar100 + path: original_data/ + sample: + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + initialize: + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + library: pytorch + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + data: + _target_: deckard.base.data.Data + generate: + name: torch_cifar100 + path: original_data/ + sample: + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + init: + _target_: deckard.base.model.ModelInitializer + name: torch_example.ResNet18 + num_channels: 3 + num_classes: 100 + library: pytorch + trainer: + batch_size: 128 + nb_epochs: 1 + verbose: true +data: + _target_: deckard.base.data.Data + generate: + name: torch_cifar100 + path: original_data/ + sample: + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 +dataset: cifar100 +def_name: control +device_id: gpu +direction: +- maximize +files: + _target_: deckard.base.files.FileConfig + adv_predictions_file: adv_predictions.json + attack_dir: attacks + attack_file: attack + attack_type: .pkl + directory: cifar100 + model_dir: models + model_file: model + model_type: .pt + name: default + params_file: params.yaml + predictions_file: predictions.json + reports: reports + score_dict_file: score_dict.json +model: + _target_: deckard.base.model.Model + art: + _target_: deckard.base.model.art_pipeline.ArtPipeline + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + data: + _target_: deckard.base.data.Data + generate: + name: torch_cifar100 + path: original_data/ + sample: + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + initialize: + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + library: pytorch + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + data: + _target_: deckard.base.data.Data + generate: + name: torch_cifar100 + path: original_data/ + sample: + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + init: + _target_: deckard.base.model.ModelInitializer + name: torch_example.ResNet18 + num_channels: 3 + num_classes: 100 + library: pytorch + trainer: + batch_size: 128 + nb_epochs: 1 + verbose: true +model_name: ResNet18 +optimizers: +- accuracy +scorers: + _target_: deckard.base.scorer.ScorerDict + accuracy: + _target_: deckard.base.scorer.ScorerConfig + direction: maximize + name: sklearn.metrics.accuracy_score + log_loss: + _target_: deckard.base.scorer.ScorerConfig + direction: minimize + name: sklearn.metrics.log_loss +stage: ??? diff --git a/examples/pytorch/plots/dvc.lock b/examples/pytorch/plots/dvc.lock index 62c8ff3c..792bf29b 100644 --- a/examples/pytorch/plots/dvc.lock +++ b/examples/pytorch/plots/dvc.lock @@ -1,450 +1,39 @@ schema: '2.0' stages: - clean@cifar: - cmd: python -m deckard.layers.clean_data -i data/attack_cifar.csv -o data/cifar.csv -c - ../conf/clean_cifar.yaml --drop_if_empty adv_fit_time accuracy train_time predict_time - adv_accuracy - deps: - - path: ../conf/clean_cifar.yaml - hash: md5 - md5: 64150a388252f623fd67838ed36ca670 - size: 873 - - path: data/attack_cifar.csv - hash: md5 - md5: 9fb240676d1ba7faea366fad55931a01 - size: 26090605 - outs: - - path: data/cifar.csv - hash: md5 - md5: 93572521018214433674bcc3256bba0c - size: 20644834 - clean@mnist: - cmd: python -m deckard.layers.clean_data -i data/attack_mnist.csv -o data/mnist.csv -c - ../conf/clean_mnist.yaml --drop_if_empty adv_fit_time accuracy train_time predict_time - adv_accuracy - deps: - - path: ../conf/clean_mnist.yaml - hash: md5 - md5: b5ee8305cb0c9daa6ff09d6a096c5a34 - size: 869 - - path: data/attack_mnist.csv - hash: md5 - md5: 5b422f124272c3b012f023a354ba2737 - size: 189644858 - outs: - - path: data/mnist.csv - hash: md5 - md5: 994d1bcab57756d6d7aa112c464cd13d - size: 23238114 - clean@cifar100: - cmd: python -m deckard.layers.clean_data -i data/attack_cifar100.csv -o data/cifar100.csv -c - ../conf/clean_cifar100.yaml --drop_if_empty adv_fit_time accuracy train_time - predict_time adv_accuracy - deps: - - path: ../conf/clean_cifar100.yaml - hash: md5 - md5: c5e8689321f99fd2f691ef7f9d8b5e4f - size: 897 - - path: data/attack_cifar100.csv - hash: md5 - md5: 8f508ea00d964ebb838f5ad4e678dc44 - size: 36167717 - outs: - - path: data/cifar100.csv - hash: md5 - md5: 898bb3154def96e0ee20fe3041b2bbff - size: 30522799 merge: cmd: python -m deckard.layers.merge --smaller_file data/cifar.csv data/cifar100.csv data/mnist.csv --output_folder data --output_file merged.csv deps: - path: data/cifar.csv hash: md5 - md5: 93572521018214433674bcc3256bba0c - size: 20644834 + md5: 4c6e158a421999f4eae6c238395f63e4 + size: 20578565 - path: data/cifar100.csv hash: md5 - md5: 898bb3154def96e0ee20fe3041b2bbff - size: 30522799 + md5: c7bbce2ed692f4697b9ab4cbbe36f9c9 + size: 30226661 - path: data/mnist.csv hash: md5 - md5: 994d1bcab57756d6d7aa112c464cd13d - size: 23238114 + md5: 083e65989c7b8a5f103d407eb675aaf8 + size: 70077338 outs: - path: data/merged.csv hash: md5 - md5: 8d1038b925e7229ed5756e5f8df972a2 - size: 75922957 - afr: - cmd: python -m deckard.layers.afr --data_file data/merged.csv --target adv_failures - --duration_col adv_fit_time_per_sample --config_file afr.yaml --plots_folder - plots/ - deps: - - path: data/merged.csv - hash: md5 - md5: ebca17142f7fbfc920b37824ea1c480e - size: 85713970 - params: - afr.yaml: - covariates: - - adv_fit_time_per_sample - - accuracy - - train_time_per_sample - - atk_value - - def_value - - data.sample.random_state - - Epochs - - model_layers - - id - - atk_gen - - def_gen - - adv_failures - - adv_accuracy - - predict_time_per_sample - cox: - plot: - file: cox_aft.pdf - title: Cox Model - qq_title: Cox QQ Plot - t0: 0.3 - model: - penalizer: 0.1 - labels: - data.sample.random_state: Random State - atk_value: Attack Strength - train_time_per_sample: $t_{train}$ - predict_time_per_sample: $t_{predict}$ - adv_accuracy: Adv. Accuracy - def_value: Defence Strength - accuracy: Ben. Accuracy - model_layers: Layers - adv_fit_time_per_sample: $t_{attack}$ - adv_failure_rate: $f_{adv.}(t;\theta)$ - failure_rate: $f_{ben.}(t;\theta)$ - Epochs: No. of Epochs - model.trainer.batch_size: Batch Size - def_gen: Defence - dummies: - atk_gen: 'Atk:' - def_gen: 'Def:' - id: 'Data:' - exponential: - plot: - file: exponential_aft.pdf - title: Exponential Model - qq_title: Exponential QQ Plot - t0: 0.1 - model: - breakpoints: - - 0.1 - labels: - 'Intercept: rho_': $\rho$ - 'Intercept: lambda_': $\lambda$ - 'data.sample.random_state: lambda_': Random State - 'atk_value: lambda_': Attack Strength - 'def_value: lambda_': Defence Strength - 'model_layers: lambda_': Layers - 'train_time_per_sample: lambda_': $t_{train}$ - 'predict_time_per_sample: lambda_': $t_{predict}$ - 'adv_accuracy: lambda_': Adv. Accuracy - 'accuracy: lambda_': Ben. Accuracy - 'adv_fit_time_per_sample: lambda_': $t_{attack}$ - 'adv_failure_rate: lambda_': $f_{adv.}(t;\theta)$ - 'failure_rate: lambda_': $f_{ben.}(t;\theta)$ - 'Epochs: lambda_': No. of Epochs - 'model.trainer.batch_size: lambda_': Batch Size - def_gen: Defence - ': lambda_': '' - gamma: - plot: - file: gamma_aft.pdf - title: Generalized Gamma Model - qq_title: Gamma QQ Plot - t0: 0.3 - model: - penalizer: 0.4 - labels: - 'Intercept: alpha_': $\alpha$ - 'Intercept: beta_': $\beta$ - 'data.sample.random_state: beta_': Random State - 'def_value: beta_': Defence Strength - 'atk_value: beta_': Attack Strength - 'train_time_per_sample: beta_': $t_{train}$ - 'model_layers: beta_': Layers - 'predict_time_per_sample: beta_': $t_{predict}$ - 'adv_accuracy: beta_': Adv. Accuracy - 'accuracy: beta_': Ben. Accuracy - 'adv_fit_time_per_sample: beta_': $t_{attack}$ - 'adv_failure_rate: beta_': $h_{adv.}(t;\theta)$ - 'failure_rate: beta_': $h_{ben.}(t;\theta)$ - 'Epochs: beta_': No. of Epochs - 'model.trainer.batch_size: beta_': Batch Size - def_gen: Defence - 'attack.init.eps: beta_': $\varepsilon$ - log_logistic: - plot: - file: log_logistic_aft.pdf - title: Log logistic AFR Model - qq_title: Log Logistic QQ Plot - t0: 1 - model: - penalizer: 0.2 - labels: - 'Intercept: beta_': $\beta$ - 'Intercept: alpha_': $\alpha$ - 'data.sample.random_state: alpha_': Random State - 'atk_value: alpha_': Attack Strength - 'def_value: alpha_': Defence Strength - 'model_layers: alpha_': Layers - 'train_time_per_sample: alpha_': $t_{train}$ - 'predict_time_per_sample: alpha_': $t_{predict}$ - 'adv_accuracy: alpha_': Adv. Accuracy - 'accuracy: alpha_': Ben. Accuracy - 'adv_fit_time_per_sample: alpha_': $t_{attack}$ - 'adv_failure_rate: alpha_': $h_{adv.}(t;\theta)$ - 'failure_rate: alpha_': $h_{ben.}(t;\theta)$ - 'Epochs: alpha_': No. of Epochs - 'model.trainer.batch_size: alpha_': Batch Size - def_gen: Defence - 'attack.init.eps: alpha_': $\varepsilon$ - log_normal: - plot: - file: log_normal_aft.pdf - title: Log Normal AFR Model - qq_title: Log Normal QQ Plot - t0: 2 - model: - penalizer: 0.5 - labels: - 'Intercept: sigma_': $\sigma$ - 'Intercept: mu_': $\mu$ - 'atk_value: mu_': Attack Strength - 'def_value: mu_': Defence Strength - 'model_layers: mu_': Layers - 'train_time_per_sample: mu_': $t_{train}$ - 'predict_time_per_sample: mu_': $t_{predict}$ - 'adv_accuracy: mu_': Adv. Accuracy - 'accuracy: mu_': Ben. Accuracy - 'adv_fit_time_per_sample: mu_': $t_{attack}$ - 'adv_failure_rate: mu_': $h_{adv.}(t;\theta)$ - 'failure_rate: mu_': $h_{ben.}(t;\theta)$ - 'Epochs: mu_': No. of Epochs - 'model.trainer.batch_size: mu_': Batch Size - def_gen: Defence - 'attack.init.eps: mu_': $\varepsilon$ - 'data.sample.random_state: mu_': Random State - weibull: - plot: - file: weibull_aft.pdf - title: Weibull AFR Model - qq_title: Weibull QQ Plot - t0: 0.3 - model: - penalizer: 0.1 - labels: - 'Intercept: rho_': $\rho$ - 'Intercept: lambda_': $\lambda$ - 'data.sample.random_state: lambda_': Random State - 'atk_value: lambda_': Attack Strength - 'model_layers: lambda_': Layers - 'train_time_per_sample: lambda_': $t_{train}$ - 'predict_time_per_sample: lambda_': $t_{predict}$ - 'adv_accuracy: lambda_': Adv. Accuracy - 'accuracy: lambda_': Ben. Accuracy - 'adv_fit_time_per_sample: lambda_': $t_{attack}$ - 'adv_failure_rate: lambda_': $f_{adv.}(t;\theta)$ - 'failure_rate: lambda_': $f_{ben.}(t;\theta)$ - 'Epochs: lambda_': No. of Epochs - 'model.trainer.batch_size: lambda_': Batch Size - def_gen: Defence - 'def_value: lambda_': Defence Strength - ': lambda_': '' - outs: - - path: plots/aft_comparison.csv - hash: md5 - md5: d73ecadc5fa94fef2549e63280343bcd - size: 339 - - path: plots/aft_comparison.tex - hash: md5 - md5: 91fb7cb7394fd71f7e817a34af87315a - size: 631 - - path: plots/cox_aft.pdf - hash: md5 - md5: ed27d1978e06ae05dd5d35ddecdd6dc4 - size: 30876 - - path: plots/cox_aft_dummies.pdf - hash: md5 - md5: 3124eb6beac82d84772d5bd49ba9e503 - size: 28856 - - path: plots/cox_qq.pdf - hash: md5 - md5: 5d1a38624404554f04677a952d03a839 - size: 20181 - - path: plots/cox_summary.csv - hash: md5 - md5: 39903e136f766f9840cbe1cc0154a39b - size: 4529 - - path: plots/exponential_aft.pdf - hash: md5 - md5: 66a13486cd114e4cc8267b69a4dd2f04 - size: 33155 - - path: plots/exponential_aft_dummies.pdf - hash: md5 - md5: 6b506006f830ce108dde579d640e1d37 - size: 32185 - - path: plots/exponential_qq.pdf - hash: md5 - md5: 4093a29d709ad92aedf846b50dcab965 - size: 20240 - - path: plots/exponential_summary.csv - hash: md5 - md5: 90b7e50b371d5f54a5fa7b0c4c5743b3 - size: 9022 - - path: plots/gamma_aft.pdf - hash: md5 - md5: 4c0368cb06e211c7d0c2905481701901 - size: 29572 - - path: plots/gamma_aft_dummies.pdf - hash: md5 - md5: 48381e11d6eb87065cccc2179d0e19e6 - size: 35152 - - path: plots/gamma_qq.pdf - hash: md5 - md5: 2c7a027149f292d78f7d4886751413e0 - size: 11343 - - path: plots/gamma_summary.csv - hash: md5 - md5: 1150bbfaebb81f5ea9651766a1f868f7 - size: 13975 - - path: plots/log_logistic_aft.pdf - hash: md5 - md5: 31260174c6af90e4b748d7e06a36f81e - size: 32733 - - path: plots/log_logistic_aft_dummies.pdf - hash: md5 - md5: dd120a0385f2d4341c54f2a8476b7d2e - size: 31885 - - path: plots/log_logistic_qq.pdf - hash: md5 - md5: cd2f80c1444b3420c1c7dfdc066e4fab - size: 18830 - - path: plots/log_logistic_summary.csv - hash: md5 - md5: 55ff092ab00e5cc45860c11caee59024 - size: 4898 - - path: plots/log_normal_aft.pdf - hash: md5 - md5: 11fa430feb2cf96730b85d930541d0ab - size: 32716 - - path: plots/log_normal_aft_dummies.pdf - hash: md5 - md5: 9031ebb8918e71be9c68e7db68c59497 - size: 32241 - - path: plots/log_normal_qq.pdf - hash: md5 - md5: 4410d657b6fe927cb0ca47909f6504b1 - size: 19875 - - path: plots/log_normal_summary.csv - hash: md5 - md5: 6d49d9f09a9aeca15846bcc74a0906fc - size: 5002 - - path: plots/weibull_aft.pdf - hash: md5 - md5: 91ec661dd0ef73b03dbac81292522c92 - size: 32493 - - path: plots/weibull_aft_dummies.pdf - hash: md5 - md5: 54de898f6abcd858bfa0da07d8db002a - size: 31439 - - path: plots/weibull_qq.pdf - hash: md5 - md5: 07b89dc4f873a61bd45a9f4ac580f276 - size: 18007 - - path: plots/weibull_summary.csv - hash: md5 - md5: c77df236abf393dbf50fc58629886dbd - size: 4952 - predict_survival_time: - cmd: python predict_with_best.py --data data/merged.csv --config_file afr.yaml --model - weibull --target adv_failures --duration_col adv_fit_time_per_sample --output - data/merged_afr.csv - deps: - - path: afr.yaml - hash: md5 - md5: 1e39b01abdaf8c823a25379f6ce391d4 - size: 6349 - - path: data/merged.csv - hash: md5 - md5: ebca17142f7fbfc920b37824ea1c480e - size: 85713970 - - path: plots/aft_comparison.tex - hash: md5 - md5: 91fb7cb7394fd71f7e817a34af87315a - size: 631 - - path: predict_with_best.py - hash: md5 - md5: e60a437ab37d2ee22256268a207f2431 - size: 2571 - outs: - - path: data/merged_afr.csv - hash: md5 - md5: f366cd694109920b498c8cf876440050 - size: 86609777 + md5: f98b901fa3ee817a55a36f78d0c95233 + size: 124219079 plot: cmd: python -m deckard.layers.plots --path plots/ --file data/merged_afr.csv -c plots.yaml deps: - path: data/merged_afr.csv hash: md5 - md5: f366cd694109920b498c8cf876440050 - size: 86609777 + md5: e8c06cf834297c5ee44950ea4d640cc8 + size: 125347398 - path: plots.yaml hash: md5 - md5: a0c5c100248543bb5f0de8949b459bc5 - size: 3815 + md5: 0f349596bdb4c2cee3c462325dffe4ba + size: 5070 params: - afr.yaml: - covariates: - - adv_fit_time_per_sample - - accuracy - - train_time_per_sample - - atk_value - - def_value - - data.sample.random_state - - Epochs - - model_layers - - id - - atk_gen - - def_gen - - adv_failures - - adv_accuracy - - predict_time_per_sample - weibull: - plot: - file: weibull_aft.pdf - title: Weibull AFR Model - qq_title: Weibull QQ Plot - t0: 0.3 - model: - penalizer: 0.1 - labels: - 'Intercept: rho_': $\rho$ - 'Intercept: lambda_': $\lambda$ - 'data.sample.random_state: lambda_': Random State - 'atk_value: lambda_': Attack Strength - 'model_layers: lambda_': Layers - 'train_time_per_sample: lambda_': $t_{train}$ - 'predict_time_per_sample: lambda_': $t_{predict}$ - 'adv_accuracy: lambda_': Adv. Accuracy - 'accuracy: lambda_': Ben. Accuracy - 'adv_fit_time_per_sample: lambda_': $t_{attack}$ - 'adv_failure_rate: lambda_': $f_{adv.}(t;\theta)$ - 'failure_rate: lambda_': $f_{ben.}(t;\theta)$ - 'Epochs: lambda_': No. of Epochs - 'model.trainer.batch_size: lambda_': Batch Size - def_gen: Defence - 'def_value: lambda_': Defence Strength - ': lambda_': '' plots.yaml: cat_plot: - file: adv_accuracy_vs_defence_type.pdf @@ -479,7 +68,24 @@ stages: - ResNet101 - ResNet152 legend_title: Model - - file: trash_score_vs_defence_type.pdf + - file: ben_failures_per_train_time_vs_defence_type.pdf + hue: model_name + kind: boxen + set: + yscale: log + x: def_gen + xlabels: Defence Type + y: c_ben + ylabels: $\bar{C}_{ben.}$ + rotation: 90 + hue_order: + - ResNet18 + - ResNet34 + - ResNet50 + - ResNet101 + - ResNet152 + legend_title: Model + - file: adv_failures_per_train_time_vs_defence_type.pdf hue: model_name kind: boxen set: @@ -496,7 +102,7 @@ stages: - ResNet101 - ResNet152 legend_title: Model - - file: trash_score_vs_attack_type.pdf + - file: adv_failures_per_train_time_vs_attack_type.pdf hue: model_name kind: boxen set: @@ -513,6 +119,21 @@ stages: - ResNet101 - ResNet152 legend_title: Model + - file: adv_failures_per_test_time_vs_defence_type.pdf + hue: model_name + kind: boxen + x: def_gen + xlabels: Defence Type + y: adv_failure_rate + ylabels: $f_{adv.}$ + rotation: 90 + hue_order: + - ResNet18 + - ResNet34 + - ResNet50 + - ResNet101 + - ResNet152 + legend_title: Model - file: adv_accuracy_vs_attack_type.pdf hue: model_name kind: boxen @@ -528,6 +149,23 @@ stages: - ResNet101 - ResNet152 legend_title: Model + - file: ben_failure_rate_vs_defence_type.pdf + hue: model_name + kind: boxen + set: + yscale: log + x: def_gen + xlabels: Defence Type + y: failure_rate + ylabels: $f_{ben}(t; \theta)$ + rotation: 90 + hue_order: + - ResNet18 + - ResNet34 + - ResNet50 + - ResNet101 + - ResNet152 + legend_title: Model line_plot: - file: def_param_vs_accuracy.pdf hue: def_gen @@ -640,49 +278,430 @@ stages: outs: - path: plots/adv_accuracy_vs_attack_type.pdf hash: md5 - md5: d98e39efcf123c924edb8d76afd9d9ad + md5: 589df119d033ae26afd171bb532b70b0 size: 36004 - path: plots/adv_accuracy_vs_defence_type.pdf hash: md5 - md5: 0eb6d612c6bc0c7dd9843922b7a85cf5 - size: 32374 + md5: 234b264b494a6e396dbaff15ba1c06b7 + size: 33276 - path: plots/adv_failure_rate_vs_train_time.pdf hash: md5 - md5: b4045f90c0849d5f28cd3519872c636e - size: 211056 + md5: 409d1d5afc7f53a0bbd6235ea40f3670 + size: 276434 + - path: plots/adv_failures_per_test_time_vs_defence_type.pdf + hash: md5 + md5: bc209f40e4dc19e9b64aa120907f6029 + size: 41070 + - path: plots/adv_failures_per_train_time_vs_attack_type.pdf + hash: md5 + md5: 18eb7fb28635190e81bb9bd6e670a43f + size: 45248 + - path: plots/adv_failures_per_train_time_vs_defence_type.pdf + hash: md5 + md5: 164dd701541215cd35a31a5506a851b9 + size: 41334 - path: plots/atk_param_vs_accuracy.pdf hash: md5 - md5: e27401d8b9ade96e516c954cc5ca916c - size: 20341 + md5: 2f65cc39685356bb1173b3c84a930d65 + size: 21456 - path: plots/ben_accuracy_vs_defence_type.pdf hash: md5 - md5: 323231eb1f498228dbd22caf2efc4c86 - size: 35108 + md5: b582dadd56d56ff587411b17a1435fa2 + size: 34713 + - path: plots/ben_failure_rate_vs_defence_type.pdf + hash: md5 + md5: 1c800cde38bdaa41d2758d75e977d35c + size: 43727 + - path: plots/ben_failures_per_train_time_vs_defence_type.pdf + hash: md5 + md5: 77c8a489ec159b88afeac48ee0aeeb18 + size: 41312 - path: plots/def_param_vs_accuracy.pdf hash: md5 - md5: 5640f125ea5212ed7018cd2535c20206 - size: 18670 + md5: 8247cc225c005509e2fe9ece9d4c88af + size: 19260 - path: plots/def_param_vs_adv_accuracy.pdf hash: md5 - md5: 64404302bc5acf53d54a6e1ec50a8085 - size: 18499 + md5: 3146258daf3dc7d7cea85f957a77557d + size: 18821 - path: plots/def_param_vs_adv_failure_rate.pdf hash: md5 - md5: 4a9e2f937c5a0c5fada95d8e5eb0144c - size: 22357 - - path: plots/trash_score_vs_attack_type.pdf + md5: 600022c5a83acf5532e84f7f0daa635c + size: 23053 + afr: + cmd: python -m deckard.layers.afr --data_file data/merged.csv --target adv_failures + --duration_col adv_fit_time --config_file afr.yaml --plots_folder plots/ + deps: + - path: data/merged.csv + hash: md5 + md5: f98b901fa3ee817a55a36f78d0c95233 + size: 124219079 + params: + afr.yaml: + covariates: + - adv_fit_time + - accuracy + - train_time + - atk_value + - def_value + - data.sample.random_state + - Epochs + - model_layers + - id + - atk_gen + - def_gen + - adv_failures + - adv_accuracy + - predict_time + cox: + plot: + file: cox_aft.pdf + title: Cox Model + title: Cox Model + qq_title: Cox QQ Plot + t0: 0.3 + model: + penalizer: 0.2 + labels: + data.sample.random_state: Random State + atk_value: Attack Strength + train_time: $t_{train}$ + predict_proba_time: $t_{predict}$ + adv_accuracy: Adv. Accuracy + accuracy: Ben. Accuracy + adv_fit_time: $t_{attack}$ + adv_failure_rate: $f_{adv.}(t;\theta)$ + failure_rate: $f_{ben.}(t;\theta)$ + Epochs: No. of Epochs + model.trainer.batch_size: Batch Size + def_gen: Defence + exponential: + plot: + file: exponential_aft.pdf + title: Exponential Model + qq_title: Exponential QQ Plot + t0: 0.1 + model: + breakpoints: + - 0.1 + labels: + 'Intercept: rho_': $\rho$ + 'Intercept: lambda_': $\lambda$ + 'data.sample.random_state: lambda_': Random State + 'atk_value: lambda_': Attack Strength + 'train_time: lambda_': $t_{train}$ + 'predict_proba_time: lambda_': $t_{predict}$ + 'adv_accuracy: lambda_': Adv. Accuracy + 'accuracy: lambda_': Ben. Accuracy + 'adv_fit_time: lambda_': $t_{attack}$ + 'adv_failure_rate: lambda_': $f_{adv.}(t;\theta)$ + 'failure_rate: lambda_': $f_{ben.}(t;\theta)$ + 'Epochs: lambda_': No. of Epochs + 'model.trainer.batch_size: lambda_': Batch Size + def_gen: Defence + ': lambda_': '' + gamma: + plot: + file: gamma_aft.pdf + title: Generalized Gamma Model + qq_title: Gamma QQ Plot + t0: 0.3 + model: + penalizer: 0.3 + labels: + 'Intercept: alpha_': $\alpha$ + 'Intercept: beta_': $\beta$ + 'data.sample.random_state: beta_': Random State + 'atk_value: beta_': Attack Strength + 'train_time: beta_': $t_{train}$ + 'predict_proba_time: beta_': $t_{predict}$ + 'adv_accuracy: beta_': Adv. Accuracy + 'accuracy: beta_': Ben. Accuracy + 'adv_fit_time: beta_': $t_{attack}$ + 'adv_failure_rate: beta_': $h_{adv.}(t;\theta)$ + 'failure_rate: beta_': $h_{ben.}(t;\theta)$ + 'Epochs: beta_': No. of Epochs + 'model.trainer.batch_size: beta_': Batch Size + def_gen: Defence + 'attack.init.eps: beta_': $\varepsilon$ + log_logistic: + plot: + file: log_logistic_aft.pdf + title: Log logistic AFR Model + qq_title: Log Logistic QQ Plot + t0: 0.1 + model: + penalizer: 0.1 + labels: + 'Intercept: beta_': $\beta$ + 'Intercept: alpha_': $\alpha$ + 'data.sample.random_state: alpha_': Random State + 'atk_value: alpha_': Attack Strength + 'train_time: alpha_': $t_{train}$ + 'predict_proba_time: alpha_': $t_{predict}$ + 'adv_accuracy: alpha_': Adv. Accuracy + 'accuracy: alpha_': Ben. Accuracy + 'adv_fit_time: alpha_': $t_{attack}$ + 'adv_failure_rate: alpha_': $h_{adv.}(t;\theta)$ + 'failure_rate: alpha_': $h_{ben.}(t;\theta)$ + 'Epochs: alpha_': No. of Epochs + 'model.trainer.batch_size: alpha_': Batch Size + def_gen: Defence + 'attack.init.eps: alpha_': $\varepsilon$ + log_normal: + plot: + file: log_normal_aft.pdf + title: Log Normal AFR Model + qq_title: Log Normal QQ Plot + t0: 2 + t0: 2 + model: + penalizer: 0.5 + labels: + 'Intercept: sigma_': $\sigma$ + 'Intercept: mu_': $\mu$ + 'atk_value: mu_': Attack Strength + 'train_time: mu_': $t_{train}$ + 'predict_proba_time: mu_': $t_{predict}$ + 'adv_accuracy: mu_': Adv. Accuracy + 'accuracy: mu_': Ben. Accuracy + 'adv_fit_time: mu_': $t_{attack}$ + 'adv_failure_rate: mu_': $h_{adv.}(t;\theta)$ + 'failure_rate: mu_': $h_{ben.}(t;\theta)$ + 'Epochs: mu_': No. of Epochs + 'model.trainer.batch_size: mu_': Batch Size + def_gen: Defence + 'attack.init.eps: mu_': $\varepsilon$ + 'data.sample.random_state: mu_': Random State + weibull: + plot: + file: weibull_aft.pdf + title: Weibull AFR Model + qq_title: Weibull QQ Plot + t0: 1 + model: + penalizer: 0.1 + labels: + 'Intercept: rho_': $\rho$ + 'Intercept: lambda_': $\lambda$ + 'data.sample.random_state: lambda_': Random State + 'atk_value: lambda_': Attack Strength + 'train_time: lambda_': $t_{train}$ + 'predict_proba_time: lambda_': $t_{predict}$ + 'adv_accuracy: lambda_': Adv. Accuracy + 'accuracy: lambda_': Ben. Accuracy + 'adv_fit_time: lambda_': $t_{attack}$ + 'adv_failure_rate: lambda_': $f_{adv.}(t;\theta)$ + 'failure_rate: lambda_': $f_{ben.}(t;\theta)$ + 'Epochs: lambda_': No. of Epochs + 'model.trainer.batch_size: lambda_': Batch Size + def_gen: Defence + 'model_layers: lambda_': Layers + 'def_value: lambda_': Defence Strength + 'predict_time: lambda_': $t_{predict}$ + ': lambda_': '' + outs: + - path: plots/aft_comparison.csv + hash: md5 + md5: c2baa8f717cdc1da5c66c45b6fbe15ed + size: 484 + - path: plots/aft_comparison.tex + hash: md5 + md5: 36326edf164e3e50eb1fa9cdfe779883 + size: 771 + - path: plots/cox_aft.pdf + hash: md5 + md5: 49cc31dfa8deb7a973efa79aa5661023 + size: 29060 + - path: plots/cox_aft_dummies.pdf + hash: md5 + md5: 2886a507102de701086572d45da2dfa8 + size: 29294 + - path: plots/cox_qq.pdf + hash: md5 + md5: 1871952097e3a7ced7597800a08864a1 + size: 19783 + - path: plots/cox_summary.csv + hash: md5 + md5: 5c2afaed99bd7a6459ffd1417b085386 + size: 4704 + - path: plots/exponential_aft.pdf + hash: md5 + md5: bc07de53e57a4b8ac93621e31afce321 + size: 31719 + - path: plots/exponential_aft_dummies.pdf + hash: md5 + md5: 1590788252530487ef7c12a2412533c5 + size: 31689 + - path: plots/exponential_qq.pdf + hash: md5 + md5: 5eb9206940c0556752308da1e30db16e + size: 20360 + - path: plots/exponential_summary.csv + hash: md5 + md5: 14692228f8862cdcf958ca6519f3eeb8 + size: 6115 + - path: plots/gamma_aft.pdf + hash: md5 + md5: c14f76b8a748a81eb652df11c4168430 + size: 28179 + - path: plots/gamma_aft_dummies.pdf + hash: md5 + md5: 0a64f85cf0c0dbf042c2af6ac3f1e514 + size: 34487 + - path: plots/gamma_qq.pdf + hash: md5 + md5: 86b107aed5da777f5e11c7148cece4f6 + size: 19114 + - path: plots/gamma_summary.csv + hash: md5 + md5: eae68c47d2098fd5dcb6a4456e50909d + size: 14596 + - path: plots/log_logistic_aft.pdf + hash: md5 + md5: c7927f6d138223b3d0814e08ae70cf50 + size: 29805 + - path: plots/log_logistic_aft_dummies.pdf + hash: md5 + md5: 3451396aab89eda953b67b7013043df5 + size: 30815 + - path: plots/log_logistic_qq.pdf + hash: md5 + md5: 6db594338475a53d925e72d9a5be69e6 + size: 21187 + - path: plots/log_logistic_summary.csv + hash: md5 + md5: bc339874fa9cb8aa52471140047bab73 + size: 5167 + - path: plots/log_normal_aft.pdf + hash: md5 + md5: 60147688508ead164e45910baace6fd9 + size: 30291 + - path: plots/log_normal_aft_dummies.pdf + hash: md5 + md5: 267b2a890ffd2a034d9825cefec12a45 + size: 31496 + - path: plots/log_normal_qq.pdf + hash: md5 + md5: ff928a6fbd9d7277df3f88c99d22c096 + size: 22085 + - path: plots/log_normal_summary.csv + hash: md5 + md5: d48ff310a60a8ad40313cf3b69ab5e58 + size: 5149 + - path: plots/weibull_aft.pdf + hash: md5 + md5: 799bddf708e9c4a48784783c629e722a + size: 31698 + - path: plots/weibull_aft_dummies.pdf + hash: md5 + md5: 99fef4eb8ec5dbf7844ba6e76c904b73 + size: 31043 + - path: plots/weibull_qq.pdf hash: md5 - md5: 7c976e25527e965c383d8cbe1379e151 - size: 43833 - - path: plots/trash_score_vs_defence_type.pdf + md5: 258bebd08669196cf738183ecd05dc14 + size: 19374 + - path: plots/weibull_summary.csv hash: md5 - md5: 9043e42e1a1af3b4823197c69e67625b - size: 39732 + md5: 8ee7e7d66bcf24eb8424460826ab7421 + size: 5217 copy_results: cmd: mkdir -p ~/ml_afr/plots && cp -r plots/* ~/ml_afr/plots/ deps: - path: plots/ hash: md5 - md5: e533f85afb78e6228b2e11037932a9c5.dir - size: 1013121 - nfiles: 37 + md5: 5858e5f1ee079e8c99ac53c7834601da.dir + size: 5938101 + nfiles: 43 + clean: + cmd: python -m deckard.layers.clean_data -i data/merged.csv -o data/clean.csv + -c clean.yaml + deps: + - path: data/merged.csv + hash: md5 + md5: 1b4802747f91a7001c43401ad21d997a + size: 50761113 + params: + clean.yaml: + fillna: + Epochs: 10 + outs: + - path: data/clean.csv + hash: md5 + md5: 2913117ef7997065dafc860c9d3080f5 + size: 54972304 + clean@mnist: + cmd: python -m deckard.layers.clean_data -i data/attack_mnist.csv -o data/mnist.csv -c + ../conf/clean_mnist.yaml --drop_if_empty adv_fit_time accuracy train_time predict_time + adv_accuracy Epochs + deps: + - path: ../conf/clean_mnist.yaml + hash: md5 + md5: bb112947b87ca42a244135a52cc5e7d5 + size: 1003 + - path: data/attack_mnist.csv + hash: md5 + md5: c4db49ae4fc1a0e6fa4c3d52b03d650a + size: 93006845 + outs: + - path: data/mnist.csv + hash: md5 + md5: 083e65989c7b8a5f103d407eb675aaf8 + size: 70077338 + clean@cifar: + cmd: python -m deckard.layers.clean_data -i data/attack_cifar.csv -o data/cifar.csv -c + ../conf/clean_cifar.yaml --drop_if_empty adv_fit_time accuracy train_time predict_time + adv_accuracy Epochs + deps: + - path: ../conf/clean_cifar.yaml + hash: md5 + md5: 4e2abc093db66b77b424854549b80497 + size: 961 + - path: data/attack_cifar.csv + hash: md5 + md5: 5a430aa13b88dff6a8fdf0277c9cd53d + size: 23267442 + outs: + - path: data/cifar.csv + hash: md5 + md5: 4c6e158a421999f4eae6c238395f63e4 + size: 20578565 + clean@cifar100: + cmd: python -m deckard.layers.clean_data -i data/attack_cifar100.csv -o data/cifar100.csv -c + ../conf/clean_cifar100.yaml --drop_if_empty adv_fit_time accuracy train_time + predict_time adv_accuracy Epochs + deps: + - path: ../conf/clean_cifar100.yaml + hash: md5 + md5: 5a8cc7e71f7036e2f590bb882acd34fb + size: 897 + - path: data/attack_cifar100.csv + hash: md5 + md5: ea55355b5b530f5751aea19d13067099 + size: 36035453 + outs: + - path: data/cifar100.csv + hash: md5 + md5: c7bbce2ed692f4697b9ab4cbbe36f9c9 + size: 30226661 + predict_survival_time: + cmd: python predict_with_best.py --data data/merged.csv --config_file afr.yaml --model + weibull --target adv_failures --duration_col adv_fit_time --output data/merged_afr.csv + deps: + - path: afr.yaml + hash: md5 + md5: 82c7a6430d38320263db502d57cbd5c6 + size: 5728 + - path: data/merged.csv + hash: md5 + md5: f98b901fa3ee817a55a36f78d0c95233 + size: 124219079 + - path: plots/aft_comparison.tex + hash: md5 + md5: 36326edf164e3e50eb1fa9cdfe779883 + size: 771 + outs: + - path: data/merged_afr.csv + hash: md5 + md5: e8c06cf834297c5ee44950ea4d640cc8 + size: 125347398 diff --git a/examples/pytorch/plots/dvc.yaml b/examples/pytorch/plots/dvc.yaml index fe1588b0..8ed37f99 100644 --- a/examples/pytorch/plots/dvc.yaml +++ b/examples/pytorch/plots/dvc.yaml @@ -1,15 +1,15 @@ vars: - - plots.yaml:cat_plot - - plots.yaml:line_plot - - plots.yaml:scatter_plot - - afr.yaml:covariates - - afr.yaml:weibull - - afr.yaml:log_logistic - - afr.yaml:log_normal - - afr.yaml:gamma - - afr.yaml:exponential - - afr.yaml:cox - - afr.yaml:dummies + - ../conf/plots.yaml:cat_plot + - ../conf/plots.yaml:line_plot + - ../conf/plots.yaml:scatter_plot + - ../conf/afr.yaml:covariates + - ../conf/afr.yaml:weibull + - ../conf/afr.yaml:log_logistic + - ../conf/afr.yaml:log_normal + - ../conf/afr.yaml:gamma + - ../conf/afr.yaml:exponential + - ../conf/afr.yaml:cox + - ../conf/afr.yaml:dummies stages: clean: foreach: @@ -21,12 +21,12 @@ stages: python -m deckard.layers.clean_data -i data/raw_${item}.csv -o data/${item}.csv - -c ../conf/clean_${item}.yaml + -c ../conf/clean.yaml --drop_if_empty adv_fit_time accuracy train_time predict_time adv_accuracy def_value atk_value model_layers deps: - data/raw_${item}.csv - - ../conf/clean_${item}.yaml + - ../conf/clean.yaml outs: - data/${item}.csv merge: @@ -41,9 +41,8 @@ stages: --output_file merged.csv outs: - data/merged.csv - afr: - cmd: python -m deckard.layers.afr --data_file data/merged.csv --target adv_accuracy --duration_col adv_fit_time_per_sample --config_file afr.yaml --plots_folder plots/ + cmd: python -m deckard.layers.afr --data_file data/merged.csv --target adv_failures --duration_col adv_fit_time --config_file ../conf/afr.yaml --plots_folder plots/ deps: - data/merged.csv plots: @@ -66,7 +65,7 @@ stages: - plots/cox_aft_dummies.pdf - plots/cox_qq.pdf params: - - afr.yaml: + - ../conf/afr.yaml: - dummies - covariates - weibull @@ -89,23 +88,23 @@ stages: cmd: >- python predict_with_best.py --data data/merged.csv - --config_file afr.yaml + --config_file ../conf/afr.yaml --model weibull --target adv_failures - --duration_col adv_fit_time_per_sample + --duration_col adv_fit_time --output data/merged_afr.csv deps: - data/merged.csv - - afr.yaml + - ../conf/afr.yaml - plots/aft_comparison.tex - predict_with_best.py outs: - data/merged_afr.csv plot: - cmd : python -m deckard.layers.plots --path plots/ --file data/merged_afr.csv -c plots.yaml + cmd : python -m deckard.layers.plots --path plots/ --file data/merged_afr.csv -c ../conf/plots.yaml deps: - data/merged_afr.csv - - plots.yaml + - ../conf/plots.yaml plots: - plots/${cat_plot[0].file} - plots/${cat_plot[1].file} @@ -118,11 +117,11 @@ stages: - plots/${line_plot[3].file} - plots/${scatter_plot[0].file} params: - - plots.yaml: + - ../conf/plots.yaml: - line_plot - scatter_plot - cat_plot - - afr.yaml: + - ../conf/afr.yaml: - covariates - weibull copy_results: diff --git a/examples/pytorch/torch_example.py b/examples/pytorch/torch_example.py index 3fff2080..34089bdc 100644 --- a/examples/pytorch/torch_example.py +++ b/examples/pytorch/torch_example.py @@ -11,7 +11,7 @@ def ResNet18(num_channels=1, num_classes=10): - model = models.resnet18() + model = models.resnet18(pretrained=True) model.conv1 = nn.Conv2d( num_channels, 64, @@ -25,7 +25,7 @@ def ResNet18(num_channels=1, num_classes=10): def ResNet34(num_channels=1, num_classes=10): - model = models.resnet34() + model = models.resnet34(pretrained=True) model.conv1 = nn.Conv2d( num_channels, 64, @@ -39,7 +39,7 @@ def ResNet34(num_channels=1, num_classes=10): def ResNet50(num_channels=1, num_classes=10): - model = models.resnet50() + model = models.resnet50(pretrained=True) model.conv1 = nn.Conv2d( num_channels, 64, @@ -53,7 +53,7 @@ def ResNet50(num_channels=1, num_classes=10): def ResNet101(num_channels=1, num_classes=10): - model = models.resnet101() + model = models.resnet101(pretrained=True) model.conv1 = nn.Conv2d( num_channels, 64, @@ -67,7 +67,7 @@ def ResNet101(num_channels=1, num_classes=10): def ResNet152(num_channels=1, num_classes=10): - model = models.resnet152() + model = models.resnet152(pretrained=True) model.conv1 = nn.Conv2d( num_channels, 64, diff --git a/hydra b/hydra new file mode 160000 index 00000000..6baf91e3 --- /dev/null +++ b/hydra @@ -0,0 +1 @@ +Subproject commit 6baf91e3e9a985503e68abcab9b86c3999d023c7