diff --git a/examples/pytorch/cifar10/.dvc/tmp/rwlock b/examples/pytorch/cifar10/.dvc/tmp/rwlock deleted file mode 100644 index 0967ef42..00000000 --- a/examples/pytorch/cifar10/.dvc/tmp/rwlock +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/examples/pytorch/conf/cifar.yaml b/examples/pytorch/conf/cifar.yaml index 1c960bb6..72de8749 100644 --- a/examples/pytorch/conf/cifar.yaml +++ b/examples/pytorch/conf/cifar.yaml @@ -1,55 +1,3 @@ -defaults: - - _self_ - - data: torch_cifar - - model: torch_cifar - - attack: default - - files: cifar - - scorers: default - - override hydra/sweeper : optuna - - override hydra/sweeper/sampler : grid - - override hydra/launcher : joblib -def_name : control -atk_name : hsj -dataset : cifar -model_name : ResNet18 -device_id : gpu -stage : '???' -direction : - - "maximize" -_target_ : deckard.base.experiment.Experiment -optimizers : - - accuracy -hydra: - run: - dir: ${files.directory}/logs/${stage}/ - sweep: - dir: ${files.directory}/logs/${stage}/${model_name} - subdir : ${def_name}/${atk_name}/${hydra.job.num} - sweeper: - sampler: - _target_: optuna.samplers.GridSampler - direction: ${direction} - study_name: ${model_name}_${def_name}_${atk_name} - storage: sqlite:///${dataset}.db - n_jobs: ${oc.env:HYDRA_SWEEPER_N_JOBS, 8} - n_trials: ${oc.env:HYDRA_SWEEPER_N_TRIALS, 128} - max_failure_rate: 1.0 - params: - ++model.art.initialize.optimizer.lr: choice( 0.1, 0.01, 0.001, .0001, .00001, 0.000001) - ++model.trainer.nb_epoch: choice(1, 10, 30, 50, 100) - _target_: hydra_plugins.hydra_optuna_sweeper.optuna_sweeper.OptunaSweeper - launcher: - _target_: hydra_plugins.hydra_joblib_launcher.joblib_launcher.JoblibLauncher - n_jobs: ${oc.env:HYDRA_SWEEPER_N_JOBS, 8} - prefer : threads - verbose: 10 - timeout: null - pre_dispatch: n_jobs - batch_size: auto - temp_folder: /tmp/deckard - max_nbytes: 100000 - mmap_mode: r - defaults: - _self_ - data: torch_cifar @@ -81,8 +29,8 @@ hydra: sampler: _target_: optuna.samplers.GridSampler direction: ${direction} - study_name: ${model_name}_${def_name}_${atk_name} - storage: sqlite:///${dataset}.db + study_name: ${dataset}_${model_name}_${def_name}_${atk_name} + storage: sqlite:///optuna.db n_jobs: ${oc.env:HYDRA_SWEEPER_N_JOBS, 32} n_trials: ${oc.env:HYDRA_SWEEPER_N_TRIALS, 128} max_failure_rate: 1.0 diff --git a/examples/pytorch/conf/cifar100.yaml b/examples/pytorch/conf/cifar100.yaml index a595d9e7..dadd4d07 100644 --- a/examples/pytorch/conf/cifar100.yaml +++ b/examples/pytorch/conf/cifar100.yaml @@ -23,14 +23,14 @@ hydra: run: dir: ${files.directory}/logs/${stage}/ sweep: - dir: ${files.directory}/logs/${stage}/${model_name}/${model.trainer.nb_epochs} - subdir : ${def_name}/${atk_name}/${hydra.job.num} + dir: ${files.directory}/logs/${stage}/${model_name}/${atk_name}/${model.trainer.nb_epochs}/ + subdir : ${def_name}/${hydra.job.num} sweeper: sampler: _target_: optuna.samplers.GridSampler direction: ${direction} - study_name: ${model_name}_${def_name}_${atk_name} - storage: sqlite:///${dataset}.db + study_name: ${dataset}_${model_name}_${def_name}_${atk_name} + storage: sqlite:///optuna.db n_jobs: ${oc.env:HYDRA_SWEEPER_N_JOBS, 16} n_trials: ${oc.env:HYDRA_SWEEPER_N_TRIALS, 128} max_failure_rate: 1.0 diff --git a/examples/pytorch/conf/mnist.yaml b/examples/pytorch/conf/mnist.yaml index ab0ebc7e..1ab43006 100644 --- a/examples/pytorch/conf/mnist.yaml +++ b/examples/pytorch/conf/mnist.yaml @@ -23,14 +23,14 @@ hydra: run: dir: ${files.directory}/logs/${stage}/ sweep: - dir: ${files.directory}/logs/${stage}/${model_name}/${model.trainer.nb_epochs} - subdir : ${def_name}/${atk_name}/${hydra.job.num} + dir: ${files.directory}/logs/${stage}/${model_name}/${atk_name}/${model.trainer.nb_epochs}/ + subdir : ${def_name}/${hydra.job.num} sweeper: sampler: _target_: optuna.samplers.GridSampler direction: ${direction} - study_name: ${model_name}_${def_name}_${atk_name} - storage: sqlite:///${dataset}.db + study_name: ${dataset}_${model_name}_${def_name}_${atk_name} + storage: sqlite:///optuna.db n_jobs: ${oc.env:HYDRA_SWEEPER_N_JOBS, 32} n_trials: ${oc.env:HYDRA_SWEEPER_N_TRIALS, 128} max_failure_rate: 1.0 diff --git a/examples/pytorch/dvc.lock b/examples/pytorch/dvc.lock index 8bbbfc23..95222e30 100644 --- a/examples/pytorch/dvc.lock +++ b/examples/pytorch/dvc.lock @@ -624,8 +624,8 @@ stages: nfiles: 3 - path: conf/mnist.yaml hash: md5 - md5: 73ed848330388ef5d98a54d03e3cdcf9 - size: 1671 + md5: 95cb45a0526e83ffb7777edebe4fa9ff + size: 1680 - path: conf/model hash: md5 md5: eb0d17ad4826334ac8488c1683106580.dir @@ -651,8 +651,8 @@ stages: nfiles: 1 - path: conf/cifar.yaml hash: md5 - md5: 5171900fd679f57392fa59f160301911 - size: 1681 + md5: 8088e4f61386676df9560a65c6a2697f + size: 1688 - path: conf/data hash: md5 md5: eafea1e119e2f54e9788ee46101afc79.dir @@ -689,8 +689,8 @@ stages: nfiles: 1 - path: conf/cifar100.yaml hash: md5 - md5: 38d9e68b647acbca29a89ef6a699b4bf - size: 1684 + md5: 11aa3a14e9d39196a1f46f46ee434493 + size: 1692 - path: conf/data hash: md5 md5: eafea1e119e2f54e9788ee46101afc79.dir @@ -1920,3 +1920,87 @@ stages: hash: md5 md5: e50857f937b52c64d262302b4aa9740e size: 53174519 + attacks@FGM-mnist-ResNet18-1: + cmd: bash scripts/FGM.sh ++model.init.name=torch_example.ResNet18 stage=attack + model_name=ResNet18 attack.attack_size=100 model.trainer.nb_epochs=1 data=torch_mnist + model=torch_mnist +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" + --config-name mnist.yaml + deps: + - path: mnist/reports/attack/default/score_dict.json + hash: md5 + md5: d38a998a2b08eff46f3bab06e7c77d64 + size: 851 + - path: mnist/reports/train/default/score_dict.json + hash: md5 + md5: 1f33102e4833438d6a2151c8eb49d4e7 + size: 525 + - path: scripts/FGM.sh + hash: md5 + md5: 8605a0b6717cb85022a60ee362adfd37 + size: 217 + - path: scripts/models.sh + hash: md5 + md5: 295e4d857ee4514c8014e95bf87012a4 + size: 1364 + outs: + - path: mnist/logs/attack/ResNet18/FGM/1/ + hash: md5 + md5: 32c0f1cdc0dbb1912343d66ec1486140.dir + size: 11926866 + nfiles: 2561 + attacks@FGM-mnist-ResNet18-10: + cmd: bash scripts/FGM.sh ++model.init.name=torch_example.ResNet18 stage=attack + model_name=ResNet18 attack.attack_size=100 model.trainer.nb_epochs=10 data=torch_mnist + model=torch_mnist +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" + --config-name mnist.yaml + deps: + - path: mnist/reports/attack/default/score_dict.json + hash: md5 + md5: d38a998a2b08eff46f3bab06e7c77d64 + size: 851 + - path: mnist/reports/train/default/score_dict.json + hash: md5 + md5: 1f33102e4833438d6a2151c8eb49d4e7 + size: 525 + - path: scripts/FGM.sh + hash: md5 + md5: 8605a0b6717cb85022a60ee362adfd37 + size: 217 + - path: scripts/models.sh + hash: md5 + md5: 295e4d857ee4514c8014e95bf87012a4 + size: 1364 + outs: + - path: mnist/logs/attack/ResNet18/FGM/10/ + hash: md5 + md5: 92e511213bf942f55e7f924657dde7d4.dir + size: 9303027 + nfiles: 2561 + attacks@FGM-mnist-ResNet18-20: + cmd: bash scripts/FGM.sh ++model.init.name=torch_example.ResNet18 stage=attack + model_name=ResNet18 attack.attack_size=100 model.trainer.nb_epochs=20 data=torch_mnist + model=torch_mnist +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" + --config-name mnist.yaml + deps: + - path: mnist/reports/attack/default/score_dict.json + hash: md5 + md5: d38a998a2b08eff46f3bab06e7c77d64 + size: 851 + - path: mnist/reports/train/default/score_dict.json + hash: md5 + md5: 1f33102e4833438d6a2151c8eb49d4e7 + size: 525 + - path: scripts/FGM.sh + hash: md5 + md5: 8605a0b6717cb85022a60ee362adfd37 + size: 217 + - path: scripts/models.sh + hash: md5 + md5: 295e4d857ee4514c8014e95bf87012a4 + size: 1364 + outs: + - path: mnist/logs/attack/ResNet18/FGM/20/ + hash: md5 + md5: c3294e1c13e61cb2d3094f565edab466.dir + size: 8189626 + nfiles: 2561 diff --git a/examples/pytorch/dvc.yaml b/examples/pytorch/dvc.yaml index f249f80b..41043e5f 100644 --- a/examples/pytorch/dvc.yaml +++ b/examples/pytorch/dvc.yaml @@ -86,11 +86,12 @@ stages: # - ${item.dataset}/${files.reports}/train/${item.model}/ attacks: matrix: + atk_name : [FGM, PGD, Deep, Pixel, Thresh, HSJ] dataset : [mnist, cifar, cifar100] model : [ResNet18, ResNet34, ResNet50, ResNet101, ResNet152] - epochs : [1] #10, 20, 30, 50, 100 + epochs : [1, 10, 20, 30, 50, 100] # cmd: >- - bash attacks.sh + bash scripts/${item.atk_name}.sh ++model.init.name=torch_example.${item.model} stage=attack model_name=${item.model} @@ -104,9 +105,10 @@ stages: deps: - ${item.dataset}/${files.reports}/attack/${files.name}/${files.score_dict_file} # This is here just to ensure it runs after the attack stage - ${item.dataset}/${files.reports}/train/${files.name}/${files.score_dict_file} - - attacks.sh + - scripts/${item.atk_name}.sh + - scripts/models.sh outs: - - ${item.dataset}/logs/attack/${item.model}/${item.epochs}/: + - ${item.dataset}/logs/attack/${item.model}/${item.atk_name}/${item.epochs}/: cache: True persist: True compile: diff --git a/examples/pytorch/main.sh b/examples/pytorch/main.sh deleted file mode 100644 index 34e724ac..00000000 --- a/examples/pytorch/main.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -paper_dir=~/ml_afr/ -# set downstream to 2 or nothing -for d in */ ; do - cd $d - # run command and write to log file - dvc repro --downstream clean -f >| dvc_repro.log - # dvc push - cd - -done -# change to paper directory -cd $paper_dir -# run dvc repro and dvc push -dvc repro -dvc push -# change back to original directory -cd - diff --git a/examples/pytorch/old_cifar100.yaml b/examples/pytorch/old_cifar100.yaml deleted file mode 100644 index 471c78a7..00000000 --- a/examples/pytorch/old_cifar100.yaml +++ /dev/null @@ -1,226 +0,0 @@ -_target_: deckard.base.experiment.Experiment -atk_name: hsj -attack: - _target_: deckard.base.attack.Attack - attack_size: 100 - data: - _target_: deckard.base.data.Data - generate: - name: torch_cifar100 - path: original_data/ - sample: - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - init: - _target_: deckard.base.attack.AttackInitializer - batch_size: 128 - model: - _target_: deckard.base.model.Model - art: - _target_: deckard.base.model.art_pipeline.ArtPipeline - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - data: - _target_: deckard.base.data.Data - generate: - name: torch_cifar100 - path: original_data/ - sample: - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - initialize: - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - library: pytorch - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - data: - _target_: deckard.base.data.Data - generate: - name: torch_cifar100 - path: original_data/ - sample: - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - init: - _target_: deckard.base.model.ModelInitializer - name: torch_example.ResNet18 - num_channels: 3 - num_classes: 100 - library: pytorch - trainer: - batch_size: 128 - nb_epochs: 1 - verbose: true - name: art.attacks.evasion.HopSkipJump - method: evasion - model: - _target_: deckard.base.model.Model - art: - _target_: deckard.base.model.art_pipeline.ArtPipeline - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - data: - _target_: deckard.base.data.Data - generate: - name: torch_cifar100 - path: original_data/ - sample: - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - initialize: - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - library: pytorch - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - data: - _target_: deckard.base.data.Data - generate: - name: torch_cifar100 - path: original_data/ - sample: - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - init: - _target_: deckard.base.model.ModelInitializer - name: torch_example.ResNet18 - num_channels: 3 - num_classes: 100 - library: pytorch - trainer: - batch_size: 128 - nb_epochs: 1 - verbose: true -data: - _target_: deckard.base.data.Data - generate: - name: torch_cifar100 - path: original_data/ - sample: - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 -dataset: cifar100 -def_name: control -device_id: gpu -direction: -- maximize -files: - _target_: deckard.base.files.FileConfig - adv_predictions_file: adv_predictions.json - attack_dir: attacks - attack_file: attack - attack_type: .pkl - directory: cifar100 - model_dir: models - model_file: model - model_type: .pt - name: default - params_file: params.yaml - predictions_file: predictions.json - reports: reports - score_dict_file: score_dict.json -model: - _target_: deckard.base.model.Model - art: - _target_: deckard.base.model.art_pipeline.ArtPipeline - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - data: - _target_: deckard.base.data.Data - generate: - name: torch_cifar100 - path: original_data/ - sample: - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - initialize: - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - library: pytorch - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - data: - _target_: deckard.base.data.Data - generate: - name: torch_cifar100 - path: original_data/ - sample: - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - init: - _target_: deckard.base.model.ModelInitializer - name: torch_example.ResNet18 - num_channels: 3 - num_classes: 100 - library: pytorch - trainer: - batch_size: 128 - nb_epochs: 1 - verbose: true -model_name: ResNet18 -optimizers: -- accuracy -scorers: - _target_: deckard.base.scorer.ScorerDict - accuracy: - _target_: deckard.base.scorer.ScorerConfig - direction: maximize - name: sklearn.metrics.accuracy_score - log_loss: - _target_: deckard.base.scorer.ScorerConfig - direction: minimize - name: sklearn.metrics.log_loss -stage: ??? diff --git a/examples/pytorch/plots/dvc.lock b/examples/pytorch/plots/dvc.lock deleted file mode 100644 index 792bf29b..00000000 --- a/examples/pytorch/plots/dvc.lock +++ /dev/null @@ -1,707 +0,0 @@ -schema: '2.0' -stages: - merge: - cmd: python -m deckard.layers.merge --smaller_file data/cifar.csv data/cifar100.csv - data/mnist.csv --output_folder data --output_file merged.csv - deps: - - path: data/cifar.csv - hash: md5 - md5: 4c6e158a421999f4eae6c238395f63e4 - size: 20578565 - - path: data/cifar100.csv - hash: md5 - md5: c7bbce2ed692f4697b9ab4cbbe36f9c9 - size: 30226661 - - path: data/mnist.csv - hash: md5 - md5: 083e65989c7b8a5f103d407eb675aaf8 - size: 70077338 - outs: - - path: data/merged.csv - hash: md5 - md5: f98b901fa3ee817a55a36f78d0c95233 - size: 124219079 - plot: - cmd: python -m deckard.layers.plots --path plots/ --file data/merged_afr.csv -c - plots.yaml - deps: - - path: data/merged_afr.csv - hash: md5 - md5: e8c06cf834297c5ee44950ea4d640cc8 - size: 125347398 - - path: plots.yaml - hash: md5 - md5: 0f349596bdb4c2cee3c462325dffe4ba - size: 5070 - params: - plots.yaml: - cat_plot: - - file: adv_accuracy_vs_defence_type.pdf - hue: model_name - kind: boxen - set: - yscale: linear - x: def_gen - xlabels: Defence Type - y: adv_accuracy - ylabels: Adv. Accuracy - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - legend_title: Model - - file: ben_accuracy_vs_defence_type.pdf - hue: model_name - kind: boxen - x: def_gen - xlabels: Defence Type - y: accuracy - ylabels: Ben. Accuracy - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - legend_title: Model - - file: ben_failures_per_train_time_vs_defence_type.pdf - hue: model_name - kind: boxen - set: - yscale: log - x: def_gen - xlabels: Defence Type - y: c_ben - ylabels: $\bar{C}_{ben.}$ - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - legend_title: Model - - file: adv_failures_per_train_time_vs_defence_type.pdf - hue: model_name - kind: boxen - set: - yscale: log - x: def_gen - xlabels: Defence Type - y: c_adv - ylabels: $\bar{C}_{adv.}$ - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - legend_title: Model - - file: adv_failures_per_train_time_vs_attack_type.pdf - hue: model_name - kind: boxen - set: - yscale: log - x: atk_gen - xlabels: Attack Type - y: c_adv - ylabels: $\bar{C}_{adv.}$ - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - legend_title: Model - - file: adv_failures_per_test_time_vs_defence_type.pdf - hue: model_name - kind: boxen - x: def_gen - xlabels: Defence Type - y: adv_failure_rate - ylabels: $f_{adv.}$ - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - legend_title: Model - - file: adv_accuracy_vs_attack_type.pdf - hue: model_name - kind: boxen - x: atk_gen - xlabels: Attack Type - y: adv_accuracy - ylabels: Adv. Accuracy - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - legend_title: Model - - file: ben_failure_rate_vs_defence_type.pdf - hue: model_name - kind: boxen - set: - yscale: log - x: def_gen - xlabels: Defence Type - y: failure_rate - ylabels: $f_{ben}(t; \theta)$ - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - legend_title: Model - line_plot: - - file: def_param_vs_accuracy.pdf - hue: def_gen - legend: - bbox_to_anchor: - - 1.05 - - 1 - title: Defence - title: Ben. Accuracy vs Defence Strength - x: def_value - x_scale: linear - xlabel: Defence Control Parameter - y: accuracy - y_scale: - ylabel: Ben. Accuracy - hue_order: - - Control - - Conf - - Gauss-in - - Gauss-out - - FSQ - errorbar: se - err_style: bars - - file: def_param_vs_adv_accuracy.pdf - hue: def_gen - legend: - bbox_to_anchor: - - 1.05 - - 1 - title: Defence - title: Adv. Accuracy vs Defence Strength - x: def_value - x_scale: linear - xlabel: Defence Control Parameter - y: adv_accuracy - y_scale: - ylabel: Adv. Accuracy - hue_order: - - Control - - Conf - - Gauss-in - - Gauss-out - - FSQ - errorbar: se - err_style: bars - - file: def_param_vs_adv_failure_rate.pdf - hue: def_gen - legend: - bbox_to_anchor: - - 1.05 - - 1 - title: Defence - title: $f_{adv}$ vs Defence Strength - x: def_value - x_scale: linear - xlabel: Defence Control Parameter - y: adv_failure_rate - y_scale: linear - ylabel: $f_{adv.}$ - hue_order: - - Control - - Conf - - Gauss-in - - Gauss-out - - FSQ - errorbar: se - err_style: bars - - file: atk_param_vs_accuracy.pdf - hue: atk_gen - legend: - bbox_to_anchor: - - 1.05 - - 1 - title: Adv. Accuracy vs Attack Strength - x: atk_value - x_scale: linear - xlabel: Attack Control Parameter - y: adv_accuracy - y_scale: - ylabel: Adv. Accuracy - hue_order: - - FGM - - PGD - - Deep - - HSJ - - Pixel - - Thresh - errorbar: se - err_style: bars - scatter_plot: - - x: train_time_per_sample - y: adv_failure_rate - hue: model_name - xlabel: $t_{train}$ - ylabel: $f_{adv}$ - title: $f_{adv}$ vs $t_{train}$ - file: adv_failure_rate_vs_train_time.pdf - x_scale: log - legend: - title: Model Name - bbox_to_anchor: - - 1.05 - - 1 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - outs: - - path: plots/adv_accuracy_vs_attack_type.pdf - hash: md5 - md5: 589df119d033ae26afd171bb532b70b0 - size: 36004 - - path: plots/adv_accuracy_vs_defence_type.pdf - hash: md5 - md5: 234b264b494a6e396dbaff15ba1c06b7 - size: 33276 - - path: plots/adv_failure_rate_vs_train_time.pdf - hash: md5 - md5: 409d1d5afc7f53a0bbd6235ea40f3670 - size: 276434 - - path: plots/adv_failures_per_test_time_vs_defence_type.pdf - hash: md5 - md5: bc209f40e4dc19e9b64aa120907f6029 - size: 41070 - - path: plots/adv_failures_per_train_time_vs_attack_type.pdf - hash: md5 - md5: 18eb7fb28635190e81bb9bd6e670a43f - size: 45248 - - path: plots/adv_failures_per_train_time_vs_defence_type.pdf - hash: md5 - md5: 164dd701541215cd35a31a5506a851b9 - size: 41334 - - path: plots/atk_param_vs_accuracy.pdf - hash: md5 - md5: 2f65cc39685356bb1173b3c84a930d65 - size: 21456 - - path: plots/ben_accuracy_vs_defence_type.pdf - hash: md5 - md5: b582dadd56d56ff587411b17a1435fa2 - size: 34713 - - path: plots/ben_failure_rate_vs_defence_type.pdf - hash: md5 - md5: 1c800cde38bdaa41d2758d75e977d35c - size: 43727 - - path: plots/ben_failures_per_train_time_vs_defence_type.pdf - hash: md5 - md5: 77c8a489ec159b88afeac48ee0aeeb18 - size: 41312 - - path: plots/def_param_vs_accuracy.pdf - hash: md5 - md5: 8247cc225c005509e2fe9ece9d4c88af - size: 19260 - - path: plots/def_param_vs_adv_accuracy.pdf - hash: md5 - md5: 3146258daf3dc7d7cea85f957a77557d - size: 18821 - - path: plots/def_param_vs_adv_failure_rate.pdf - hash: md5 - md5: 600022c5a83acf5532e84f7f0daa635c - size: 23053 - afr: - cmd: python -m deckard.layers.afr --data_file data/merged.csv --target adv_failures - --duration_col adv_fit_time --config_file afr.yaml --plots_folder plots/ - deps: - - path: data/merged.csv - hash: md5 - md5: f98b901fa3ee817a55a36f78d0c95233 - size: 124219079 - params: - afr.yaml: - covariates: - - adv_fit_time - - accuracy - - train_time - - atk_value - - def_value - - data.sample.random_state - - Epochs - - model_layers - - id - - atk_gen - - def_gen - - adv_failures - - adv_accuracy - - predict_time - cox: - plot: - file: cox_aft.pdf - title: Cox Model - title: Cox Model - qq_title: Cox QQ Plot - t0: 0.3 - model: - penalizer: 0.2 - labels: - data.sample.random_state: Random State - atk_value: Attack Strength - train_time: $t_{train}$ - predict_proba_time: $t_{predict}$ - adv_accuracy: Adv. Accuracy - accuracy: Ben. Accuracy - adv_fit_time: $t_{attack}$ - adv_failure_rate: $f_{adv.}(t;\theta)$ - failure_rate: $f_{ben.}(t;\theta)$ - Epochs: No. of Epochs - model.trainer.batch_size: Batch Size - def_gen: Defence - exponential: - plot: - file: exponential_aft.pdf - title: Exponential Model - qq_title: Exponential QQ Plot - t0: 0.1 - model: - breakpoints: - - 0.1 - labels: - 'Intercept: rho_': $\rho$ - 'Intercept: lambda_': $\lambda$ - 'data.sample.random_state: lambda_': Random State - 'atk_value: lambda_': Attack Strength - 'train_time: lambda_': $t_{train}$ - 'predict_proba_time: lambda_': $t_{predict}$ - 'adv_accuracy: lambda_': Adv. Accuracy - 'accuracy: lambda_': Ben. Accuracy - 'adv_fit_time: lambda_': $t_{attack}$ - 'adv_failure_rate: lambda_': $f_{adv.}(t;\theta)$ - 'failure_rate: lambda_': $f_{ben.}(t;\theta)$ - 'Epochs: lambda_': No. of Epochs - 'model.trainer.batch_size: lambda_': Batch Size - def_gen: Defence - ': lambda_': '' - gamma: - plot: - file: gamma_aft.pdf - title: Generalized Gamma Model - qq_title: Gamma QQ Plot - t0: 0.3 - model: - penalizer: 0.3 - labels: - 'Intercept: alpha_': $\alpha$ - 'Intercept: beta_': $\beta$ - 'data.sample.random_state: beta_': Random State - 'atk_value: beta_': Attack Strength - 'train_time: beta_': $t_{train}$ - 'predict_proba_time: beta_': $t_{predict}$ - 'adv_accuracy: beta_': Adv. Accuracy - 'accuracy: beta_': Ben. Accuracy - 'adv_fit_time: beta_': $t_{attack}$ - 'adv_failure_rate: beta_': $h_{adv.}(t;\theta)$ - 'failure_rate: beta_': $h_{ben.}(t;\theta)$ - 'Epochs: beta_': No. of Epochs - 'model.trainer.batch_size: beta_': Batch Size - def_gen: Defence - 'attack.init.eps: beta_': $\varepsilon$ - log_logistic: - plot: - file: log_logistic_aft.pdf - title: Log logistic AFR Model - qq_title: Log Logistic QQ Plot - t0: 0.1 - model: - penalizer: 0.1 - labels: - 'Intercept: beta_': $\beta$ - 'Intercept: alpha_': $\alpha$ - 'data.sample.random_state: alpha_': Random State - 'atk_value: alpha_': Attack Strength - 'train_time: alpha_': $t_{train}$ - 'predict_proba_time: alpha_': $t_{predict}$ - 'adv_accuracy: alpha_': Adv. Accuracy - 'accuracy: alpha_': Ben. Accuracy - 'adv_fit_time: alpha_': $t_{attack}$ - 'adv_failure_rate: alpha_': $h_{adv.}(t;\theta)$ - 'failure_rate: alpha_': $h_{ben.}(t;\theta)$ - 'Epochs: alpha_': No. of Epochs - 'model.trainer.batch_size: alpha_': Batch Size - def_gen: Defence - 'attack.init.eps: alpha_': $\varepsilon$ - log_normal: - plot: - file: log_normal_aft.pdf - title: Log Normal AFR Model - qq_title: Log Normal QQ Plot - t0: 2 - t0: 2 - model: - penalizer: 0.5 - labels: - 'Intercept: sigma_': $\sigma$ - 'Intercept: mu_': $\mu$ - 'atk_value: mu_': Attack Strength - 'train_time: mu_': $t_{train}$ - 'predict_proba_time: mu_': $t_{predict}$ - 'adv_accuracy: mu_': Adv. Accuracy - 'accuracy: mu_': Ben. Accuracy - 'adv_fit_time: mu_': $t_{attack}$ - 'adv_failure_rate: mu_': $h_{adv.}(t;\theta)$ - 'failure_rate: mu_': $h_{ben.}(t;\theta)$ - 'Epochs: mu_': No. of Epochs - 'model.trainer.batch_size: mu_': Batch Size - def_gen: Defence - 'attack.init.eps: mu_': $\varepsilon$ - 'data.sample.random_state: mu_': Random State - weibull: - plot: - file: weibull_aft.pdf - title: Weibull AFR Model - qq_title: Weibull QQ Plot - t0: 1 - model: - penalizer: 0.1 - labels: - 'Intercept: rho_': $\rho$ - 'Intercept: lambda_': $\lambda$ - 'data.sample.random_state: lambda_': Random State - 'atk_value: lambda_': Attack Strength - 'train_time: lambda_': $t_{train}$ - 'predict_proba_time: lambda_': $t_{predict}$ - 'adv_accuracy: lambda_': Adv. Accuracy - 'accuracy: lambda_': Ben. Accuracy - 'adv_fit_time: lambda_': $t_{attack}$ - 'adv_failure_rate: lambda_': $f_{adv.}(t;\theta)$ - 'failure_rate: lambda_': $f_{ben.}(t;\theta)$ - 'Epochs: lambda_': No. of Epochs - 'model.trainer.batch_size: lambda_': Batch Size - def_gen: Defence - 'model_layers: lambda_': Layers - 'def_value: lambda_': Defence Strength - 'predict_time: lambda_': $t_{predict}$ - ': lambda_': '' - outs: - - path: plots/aft_comparison.csv - hash: md5 - md5: c2baa8f717cdc1da5c66c45b6fbe15ed - size: 484 - - path: plots/aft_comparison.tex - hash: md5 - md5: 36326edf164e3e50eb1fa9cdfe779883 - size: 771 - - path: plots/cox_aft.pdf - hash: md5 - md5: 49cc31dfa8deb7a973efa79aa5661023 - size: 29060 - - path: plots/cox_aft_dummies.pdf - hash: md5 - md5: 2886a507102de701086572d45da2dfa8 - size: 29294 - - path: plots/cox_qq.pdf - hash: md5 - md5: 1871952097e3a7ced7597800a08864a1 - size: 19783 - - path: plots/cox_summary.csv - hash: md5 - md5: 5c2afaed99bd7a6459ffd1417b085386 - size: 4704 - - path: plots/exponential_aft.pdf - hash: md5 - md5: bc07de53e57a4b8ac93621e31afce321 - size: 31719 - - path: plots/exponential_aft_dummies.pdf - hash: md5 - md5: 1590788252530487ef7c12a2412533c5 - size: 31689 - - path: plots/exponential_qq.pdf - hash: md5 - md5: 5eb9206940c0556752308da1e30db16e - size: 20360 - - path: plots/exponential_summary.csv - hash: md5 - md5: 14692228f8862cdcf958ca6519f3eeb8 - size: 6115 - - path: plots/gamma_aft.pdf - hash: md5 - md5: c14f76b8a748a81eb652df11c4168430 - size: 28179 - - path: plots/gamma_aft_dummies.pdf - hash: md5 - md5: 0a64f85cf0c0dbf042c2af6ac3f1e514 - size: 34487 - - path: plots/gamma_qq.pdf - hash: md5 - md5: 86b107aed5da777f5e11c7148cece4f6 - size: 19114 - - path: plots/gamma_summary.csv - hash: md5 - md5: eae68c47d2098fd5dcb6a4456e50909d - size: 14596 - - path: plots/log_logistic_aft.pdf - hash: md5 - md5: c7927f6d138223b3d0814e08ae70cf50 - size: 29805 - - path: plots/log_logistic_aft_dummies.pdf - hash: md5 - md5: 3451396aab89eda953b67b7013043df5 - size: 30815 - - path: plots/log_logistic_qq.pdf - hash: md5 - md5: 6db594338475a53d925e72d9a5be69e6 - size: 21187 - - path: plots/log_logistic_summary.csv - hash: md5 - md5: bc339874fa9cb8aa52471140047bab73 - size: 5167 - - path: plots/log_normal_aft.pdf - hash: md5 - md5: 60147688508ead164e45910baace6fd9 - size: 30291 - - path: plots/log_normal_aft_dummies.pdf - hash: md5 - md5: 267b2a890ffd2a034d9825cefec12a45 - size: 31496 - - path: plots/log_normal_qq.pdf - hash: md5 - md5: ff928a6fbd9d7277df3f88c99d22c096 - size: 22085 - - path: plots/log_normal_summary.csv - hash: md5 - md5: d48ff310a60a8ad40313cf3b69ab5e58 - size: 5149 - - path: plots/weibull_aft.pdf - hash: md5 - md5: 799bddf708e9c4a48784783c629e722a - size: 31698 - - path: plots/weibull_aft_dummies.pdf - hash: md5 - md5: 99fef4eb8ec5dbf7844ba6e76c904b73 - size: 31043 - - path: plots/weibull_qq.pdf - hash: md5 - md5: 258bebd08669196cf738183ecd05dc14 - size: 19374 - - path: plots/weibull_summary.csv - hash: md5 - md5: 8ee7e7d66bcf24eb8424460826ab7421 - size: 5217 - copy_results: - cmd: mkdir -p ~/ml_afr/plots && cp -r plots/* ~/ml_afr/plots/ - deps: - - path: plots/ - hash: md5 - md5: 5858e5f1ee079e8c99ac53c7834601da.dir - size: 5938101 - nfiles: 43 - clean: - cmd: python -m deckard.layers.clean_data -i data/merged.csv -o data/clean.csv - -c clean.yaml - deps: - - path: data/merged.csv - hash: md5 - md5: 1b4802747f91a7001c43401ad21d997a - size: 50761113 - params: - clean.yaml: - fillna: - Epochs: 10 - outs: - - path: data/clean.csv - hash: md5 - md5: 2913117ef7997065dafc860c9d3080f5 - size: 54972304 - clean@mnist: - cmd: python -m deckard.layers.clean_data -i data/attack_mnist.csv -o data/mnist.csv -c - ../conf/clean_mnist.yaml --drop_if_empty adv_fit_time accuracy train_time predict_time - adv_accuracy Epochs - deps: - - path: ../conf/clean_mnist.yaml - hash: md5 - md5: bb112947b87ca42a244135a52cc5e7d5 - size: 1003 - - path: data/attack_mnist.csv - hash: md5 - md5: c4db49ae4fc1a0e6fa4c3d52b03d650a - size: 93006845 - outs: - - path: data/mnist.csv - hash: md5 - md5: 083e65989c7b8a5f103d407eb675aaf8 - size: 70077338 - clean@cifar: - cmd: python -m deckard.layers.clean_data -i data/attack_cifar.csv -o data/cifar.csv -c - ../conf/clean_cifar.yaml --drop_if_empty adv_fit_time accuracy train_time predict_time - adv_accuracy Epochs - deps: - - path: ../conf/clean_cifar.yaml - hash: md5 - md5: 4e2abc093db66b77b424854549b80497 - size: 961 - - path: data/attack_cifar.csv - hash: md5 - md5: 5a430aa13b88dff6a8fdf0277c9cd53d - size: 23267442 - outs: - - path: data/cifar.csv - hash: md5 - md5: 4c6e158a421999f4eae6c238395f63e4 - size: 20578565 - clean@cifar100: - cmd: python -m deckard.layers.clean_data -i data/attack_cifar100.csv -o data/cifar100.csv -c - ../conf/clean_cifar100.yaml --drop_if_empty adv_fit_time accuracy train_time - predict_time adv_accuracy Epochs - deps: - - path: ../conf/clean_cifar100.yaml - hash: md5 - md5: 5a8cc7e71f7036e2f590bb882acd34fb - size: 897 - - path: data/attack_cifar100.csv - hash: md5 - md5: ea55355b5b530f5751aea19d13067099 - size: 36035453 - outs: - - path: data/cifar100.csv - hash: md5 - md5: c7bbce2ed692f4697b9ab4cbbe36f9c9 - size: 30226661 - predict_survival_time: - cmd: python predict_with_best.py --data data/merged.csv --config_file afr.yaml --model - weibull --target adv_failures --duration_col adv_fit_time --output data/merged_afr.csv - deps: - - path: afr.yaml - hash: md5 - md5: 82c7a6430d38320263db502d57cbd5c6 - size: 5728 - - path: data/merged.csv - hash: md5 - md5: f98b901fa3ee817a55a36f78d0c95233 - size: 124219079 - - path: plots/aft_comparison.tex - hash: md5 - md5: 36326edf164e3e50eb1fa9cdfe779883 - size: 771 - outs: - - path: data/merged_afr.csv - hash: md5 - md5: e8c06cf834297c5ee44950ea4d640cc8 - size: 125347398 diff --git a/examples/pytorch/scripts/Deep.sh b/examples/pytorch/scripts/Deep.sh new file mode 100644 index 00000000..0e7293e3 --- /dev/null +++ b/examples/pytorch/scripts/Deep.sh @@ -0,0 +1,8 @@ +bash scripts/models.sh \ + stage=attack \ + attack=default \ + ++attack.init.name=art.attacks.evasion.DeepFool \ + ++attack.init.max_iter=10 \ + ++attack.init.batch_size=4096 \ + ++attack.init.nb_grads=1,3,5,8,10 \ + atk_name=Deep $@ \ No newline at end of file diff --git a/examples/pytorch/scripts/FGM.sh b/examples/pytorch/scripts/FGM.sh new file mode 100644 index 00000000..7b66169f --- /dev/null +++ b/examples/pytorch/scripts/FGM.sh @@ -0,0 +1,7 @@ +bash scripts/models.sh \ + stage=attack \ + attack=default \ + ++attack.init.name=art.attacks.evasion.FastGradientMethod \ + ++attack.init.eps=.001,.01,.1,.5,1 \ + ++attack.init.norm=2 \ + atk_name=FGM $@ \ No newline at end of file diff --git a/examples/pytorch/scripts/HSJ.sh b/examples/pytorch/scripts/HSJ.sh new file mode 100644 index 00000000..fffa757e --- /dev/null +++ b/examples/pytorch/scripts/HSJ.sh @@ -0,0 +1,9 @@ +bash scripts/models.sh \ + stage=attack \ + attack=default \ + ++attack.init.name=art.attacks.evasion.HopSkipJump \ + ++attack.init.max_iter=1,3,5,10,15 \ + ++attack.init.init_eval=3 \ + ++attack.init.max_eval=10 \ + ++attack.init.norm=2 \ + atk_name=HSJ $@ \ No newline at end of file diff --git a/examples/pytorch/scripts/PGD.sh b/examples/pytorch/scripts/PGD.sh new file mode 100644 index 00000000..8c45315d --- /dev/null +++ b/examples/pytorch/scripts/PGD.sh @@ -0,0 +1,9 @@ +bash scripts/models.sh \ + stage=attack \ + attack=default \ + ++attack.init.name=art.attacks.evasion.ProjectedGradientDescent \ + ++attack.init.eps=.001,.01,.1,.5,1 \ + ++attack.init.norm=2 \ + ++attack.init.eps_step=.001,.003,.01 \ + atk_name=PGD \ + ++attack.init.max_iter=1,5,10,50,100 $@ \ No newline at end of file diff --git a/examples/pytorch/scripts/Pixel.sh b/examples/pytorch/scripts/Pixel.sh new file mode 100644 index 00000000..f684b572 --- /dev/null +++ b/examples/pytorch/scripts/Pixel.sh @@ -0,0 +1,7 @@ +bash scripts/models.sh \ + stage=attack \ + attack=default \ + ++attack.init.name=art.attacks.evasion.PixelAttack \ + ~attack.init.batch_size \ + ++attack.init.th=1,4,16,64,256 \ + atk_name=Pixel $@ \ No newline at end of file diff --git a/examples/pytorch/scripts/Thresh.sh b/examples/pytorch/scripts/Thresh.sh new file mode 100644 index 00000000..7f2e774c --- /dev/null +++ b/examples/pytorch/scripts/Thresh.sh @@ -0,0 +1,7 @@ +bash scripts/models.sh \ + stage=attack \ + attack=default \ + ++attack.init.name=art.attacks.evasion.ThresholdAttack \ + ~attack.init.batch_size \ + ++attack.init.th=1,4,16,64,256 \ + atk_name=Thresh $@ \ No newline at end of file diff --git a/examples/pytorch/attacks.sh b/examples/pytorch/scripts/attacks.sh similarity index 84% rename from examples/pytorch/attacks.sh rename to examples/pytorch/scripts/attacks.sh index bba66e7c..0ed0d0da 100644 --- a/examples/pytorch/attacks.sh +++ b/examples/pytorch/scripts/attacks.sh @@ -10,7 +10,7 @@ # ++attack.init.eps=.001,.01,.1,.5,1 \ # ++attack.init.norm=2 \ # atk_name=FGM $@ -# # ##################################################### +# ##################################################### # # Projected Gradient Descent # bash models.sh \ # stage=attack \ @@ -43,23 +43,23 @@ # ++attack.init.norm=2 \ # atk_name=HSJ $@ # ##################################################### -# PixelAttack -bash models.sh \ - stage=attack \ - attack=default \ - ++attack.init.name=art.attacks.evasion.PixelAttack \ - ~attack.init.batch_size \ - ++attack.init.th=1,4,16,64,256 \ - atk_name=Pixel $@ -# ##################################################### -# ThresholdAttack -bash models.sh \ - stage=attack \ - attack=default \ - ++attack.init.name=art.attacks.evasion.ThresholdAttack \ - ~attack.init.batch_size \ - ++attack.init.th=1,4,16,64,256 \ - atk_name=Thresh $@ +# # # PixelAttack +# bash models.sh \ +# stage=attack \ +# attack=default \ +# ++attack.init.name=art.attacks.evasion.PixelAttack \ +# ~attack.init.batch_size \ +# ++attack.init.th=1,4,16,64,256 \ +# atk_name=Pixel $@ +# # ##################################################### +# # ThresholdAttack +# bash models.sh \ +# stage=attack \ +# attack=default \ +# ++attack.init.name=art.attacks.evasion.ThresholdAttack \ +# ~attack.init.batch_size \ +# ++attack.init.th=1,4,16,64,256 \ +# atk_name=Thresh $@ # ##################################################### # # ZooAttack # bash models.sh \ diff --git a/examples/pytorch/models.sh b/examples/pytorch/scripts/models.sh similarity index 84% rename from examples/pytorch/models.sh rename to examples/pytorch/scripts/models.sh index 2a54e307..8a3864b8 100644 --- a/examples/pytorch/models.sh +++ b/examples/pytorch/scripts/models.sh @@ -14,9 +14,6 @@ python -m deckard.layers.optimise \ def_name=FSQ \ $@ --multirun -rm -rf mnist/models -rm -rf cifar/models -rm -rf cifar100/models # Gaussian Augmentation (Input) python -m deckard.layers.optimise \ @@ -27,9 +24,6 @@ python -m deckard.layers.optimise \ def_name=Gauss-In \ $@ --multirun -rm -rf mnist/models -rm -rf cifar/models -rm -rf cifar100/model # Gaussian Noise (Output) python -m deckard.layers.optimise \ @@ -39,9 +33,6 @@ python -m deckard.layers.optimise \ def_name=Gauss-Out \ $@ --multirun -rm -rf mnist/models -rm -rf cifar/models -rm -rf cifar100/models # High Confidence python -m deckard.layers.optimise \ @@ -50,6 +41,3 @@ python -m deckard.layers.optimise \ def_name=Conf \ $@ --multirun -rm -rf mnist/models -rm -rf cifar/models -rm -rf cifar100/models diff --git a/examples/pytorch/wait.sh b/examples/pytorch/scripts/wait.sh similarity index 100% rename from examples/pytorch/wait.sh rename to examples/pytorch/scripts/wait.sh