broke up attack scripts for better dvc tracking

simplymathematics · Jul 31, 2024 · ac972da · ac972da
1 parent 59b777f
commit ac972da
Show file tree

Hide file tree

Showing 18 changed files with 171 additions and 1,053 deletions.
diff --git a/examples/pytorch/cifar10/.dvc/tmp/rwlock b/examples/pytorch/cifar10/.dvc/tmp/rwlock
diff --git a/examples/pytorch/conf/cifar.yaml b/examples/pytorch/conf/cifar.yaml
@@ -1,55 +1,3 @@
-defaults:
-    - _self_
-    - data: torch_cifar
-    - model: torch_cifar
-    - attack: default
-    - files: cifar
-    - scorers: default
-    - override hydra/sweeper : optuna
-    - override hydra/sweeper/sampler : grid
-    - override hydra/launcher : joblib
-def_name : control
-atk_name : hsj
-dataset : cifar
-model_name : ResNet18
-device_id : gpu
-stage : '???'
-direction : 
-  -  "maximize"
-_target_ : deckard.base.experiment.Experiment
-optimizers : 
-  - accuracy
-hydra:
-  run:
-    dir: ${files.directory}/logs/${stage}/
-  sweep:
-    dir: ${files.directory}/logs/${stage}/${model_name}
-    subdir : ${def_name}/${atk_name}/${hydra.job.num}
-  sweeper:
-    sampler:
-      _target_: optuna.samplers.GridSampler
-    direction: ${direction}
-    study_name: ${model_name}_${def_name}_${atk_name}
-    storage: sqlite:///${dataset}.db
-    n_jobs: ${oc.env:HYDRA_SWEEPER_N_JOBS, 8}
-    n_trials: ${oc.env:HYDRA_SWEEPER_N_TRIALS, 128}
-    max_failure_rate: 1.0
-    params:
-      ++model.art.initialize.optimizer.lr: choice( 0.1, 0.01, 0.001, .0001, .00001, 0.000001)
-      ++model.trainer.nb_epoch: choice(1, 10, 30, 50, 100)
-    _target_: hydra_plugins.hydra_optuna_sweeper.optuna_sweeper.OptunaSweeper
-  launcher:
-    _target_: hydra_plugins.hydra_joblib_launcher.joblib_launcher.JoblibLauncher
-    n_jobs: ${oc.env:HYDRA_SWEEPER_N_JOBS, 8}
-    prefer : threads
-    verbose: 10
-    timeout: null
-    pre_dispatch: n_jobs
-    batch_size: auto
-    temp_folder: /tmp/deckard
-    max_nbytes: 100000
-    mmap_mode: r
-
 defaults:
     - _self_
     - data: torch_cifar
@@ -81,8 +29,8 @@ hydra:
     sampler:
       _target_: optuna.samplers.GridSampler
     direction: ${direction}
-    study_name: ${model_name}_${def_name}_${atk_name}
-    storage: sqlite:///${dataset}.db
+    study_name: ${dataset}_${model_name}_${def_name}_${atk_name}
+    storage: sqlite:///optuna.db
     n_jobs: ${oc.env:HYDRA_SWEEPER_N_JOBS, 32}
     n_trials: ${oc.env:HYDRA_SWEEPER_N_TRIALS, 128}
     max_failure_rate: 1.0

diff --git a/examples/pytorch/conf/cifar100.yaml b/examples/pytorch/conf/cifar100.yaml
@@ -23,14 +23,14 @@ hydra:
   run:
     dir: ${files.directory}/logs/${stage}/
   sweep:
-    dir: ${files.directory}/logs/${stage}/${model_name}/${model.trainer.nb_epochs}
-    subdir : ${def_name}/${atk_name}/${hydra.job.num}
+    dir: ${files.directory}/logs/${stage}/${model_name}/${atk_name}/${model.trainer.nb_epochs}/
+    subdir : ${def_name}/${hydra.job.num}
   sweeper:
     sampler:
       _target_: optuna.samplers.GridSampler
     direction: ${direction}
-    study_name: ${model_name}_${def_name}_${atk_name}
-    storage: sqlite:///${dataset}.db
+    study_name: ${dataset}_${model_name}_${def_name}_${atk_name}
+    storage: sqlite:///optuna.db
     n_jobs: ${oc.env:HYDRA_SWEEPER_N_JOBS, 16}
     n_trials: ${oc.env:HYDRA_SWEEPER_N_TRIALS, 128}
     max_failure_rate: 1.0

diff --git a/examples/pytorch/conf/mnist.yaml b/examples/pytorch/conf/mnist.yaml
@@ -23,14 +23,14 @@ hydra:
   run:
     dir: ${files.directory}/logs/${stage}/
   sweep:
-    dir: ${files.directory}/logs/${stage}/${model_name}/${model.trainer.nb_epochs}
-    subdir : ${def_name}/${atk_name}/${hydra.job.num}
+    dir: ${files.directory}/logs/${stage}/${model_name}/${atk_name}/${model.trainer.nb_epochs}/
+    subdir : ${def_name}/${hydra.job.num}
   sweeper:
     sampler:
       _target_: optuna.samplers.GridSampler
     direction: ${direction}
-    study_name: ${model_name}_${def_name}_${atk_name}
-    storage: sqlite:///${dataset}.db
+    study_name: ${dataset}_${model_name}_${def_name}_${atk_name}
+    storage: sqlite:///optuna.db
     n_jobs: ${oc.env:HYDRA_SWEEPER_N_JOBS, 32}
     n_trials: ${oc.env:HYDRA_SWEEPER_N_TRIALS, 128}
     max_failure_rate: 1.0

diff --git a/examples/pytorch/dvc.lock b/examples/pytorch/dvc.lock
@@ -624,8 +624,8 @@ stages:
       nfiles: 3
     - path: conf/mnist.yaml
       hash: md5
-      md5: 73ed848330388ef5d98a54d03e3cdcf9
-      size: 1671
+      md5: 95cb45a0526e83ffb7777edebe4fa9ff
+      size: 1680
     - path: conf/model
       hash: md5
       md5: eb0d17ad4826334ac8488c1683106580.dir
@@ -651,8 +651,8 @@ stages:
       nfiles: 1
     - path: conf/cifar.yaml
       hash: md5
-      md5: 5171900fd679f57392fa59f160301911
-      size: 1681
+      md5: 8088e4f61386676df9560a65c6a2697f
+      size: 1688
     - path: conf/data
       hash: md5
       md5: eafea1e119e2f54e9788ee46101afc79.dir
@@ -689,8 +689,8 @@ stages:
       nfiles: 1
     - path: conf/cifar100.yaml
       hash: md5
-      md5: 38d9e68b647acbca29a89ef6a699b4bf
-      size: 1684
+      md5: 11aa3a14e9d39196a1f46f46ee434493
+      size: 1692
     - path: conf/data
       hash: md5
       md5: eafea1e119e2f54e9788ee46101afc79.dir
@@ -1920,3 +1920,87 @@ stages:
       hash: md5
       md5: e50857f937b52c64d262302b4aa9740e
       size: 53174519
+  attacks@FGM-mnist-ResNet18-1:
+    cmd: bash scripts/FGM.sh  ++model.init.name=torch_example.ResNet18  stage=attack
+      model_name=ResNet18 attack.attack_size=100 model.trainer.nb_epochs=1 data=torch_mnist
+      model=torch_mnist +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]"
+      --config-name mnist.yaml
+    deps:
+    - path: mnist/reports/attack/default/score_dict.json
+      hash: md5
+      md5: d38a998a2b08eff46f3bab06e7c77d64
+      size: 851
+    - path: mnist/reports/train/default/score_dict.json
+      hash: md5
+      md5: 1f33102e4833438d6a2151c8eb49d4e7
+      size: 525
+    - path: scripts/FGM.sh
+      hash: md5
+      md5: 8605a0b6717cb85022a60ee362adfd37
+      size: 217
+    - path: scripts/models.sh
+      hash: md5
+      md5: 295e4d857ee4514c8014e95bf87012a4
+      size: 1364
+    outs:
+    - path: mnist/logs/attack/ResNet18/FGM/1/
+      hash: md5
+      md5: 32c0f1cdc0dbb1912343d66ec1486140.dir
+      size: 11926866
+      nfiles: 2561
+  attacks@FGM-mnist-ResNet18-10:
+    cmd: bash scripts/FGM.sh  ++model.init.name=torch_example.ResNet18  stage=attack
+      model_name=ResNet18 attack.attack_size=100 model.trainer.nb_epochs=10 data=torch_mnist
+      model=torch_mnist +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]"
+      --config-name mnist.yaml
+    deps:
+    - path: mnist/reports/attack/default/score_dict.json
+      hash: md5
+      md5: d38a998a2b08eff46f3bab06e7c77d64
+      size: 851
+    - path: mnist/reports/train/default/score_dict.json
+      hash: md5
+      md5: 1f33102e4833438d6a2151c8eb49d4e7
+      size: 525
+    - path: scripts/FGM.sh
+      hash: md5
+      md5: 8605a0b6717cb85022a60ee362adfd37
+      size: 217
+    - path: scripts/models.sh
+      hash: md5
+      md5: 295e4d857ee4514c8014e95bf87012a4
+      size: 1364
+    outs:
+    - path: mnist/logs/attack/ResNet18/FGM/10/
+      hash: md5
+      md5: 92e511213bf942f55e7f924657dde7d4.dir
+      size: 9303027
+      nfiles: 2561
+  attacks@FGM-mnist-ResNet18-20:
+    cmd: bash scripts/FGM.sh  ++model.init.name=torch_example.ResNet18  stage=attack
+      model_name=ResNet18 attack.attack_size=100 model.trainer.nb_epochs=20 data=torch_mnist
+      model=torch_mnist +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]"
+      --config-name mnist.yaml
+    deps:
+    - path: mnist/reports/attack/default/score_dict.json
+      hash: md5
+      md5: d38a998a2b08eff46f3bab06e7c77d64
+      size: 851
+    - path: mnist/reports/train/default/score_dict.json
+      hash: md5
+      md5: 1f33102e4833438d6a2151c8eb49d4e7
+      size: 525
+    - path: scripts/FGM.sh
+      hash: md5
+      md5: 8605a0b6717cb85022a60ee362adfd37
+      size: 217
+    - path: scripts/models.sh
+      hash: md5
+      md5: 295e4d857ee4514c8014e95bf87012a4
+      size: 1364
+    outs:
+    - path: mnist/logs/attack/ResNet18/FGM/20/
+      hash: md5
+      md5: c3294e1c13e61cb2d3094f565edab466.dir
+      size: 8189626
+      nfiles: 2561
diff --git a/examples/pytorch/dvc.yaml b/examples/pytorch/dvc.yaml
@@ -86,11 +86,12 @@ stages:
   #     - ${item.dataset}/${files.reports}/train/${item.model}/
   attacks:
     matrix:
+      atk_name : [FGM, PGD, Deep, Pixel, Thresh, HSJ]
       dataset : [mnist, cifar, cifar100]
       model : [ResNet18, ResNet34, ResNet50, ResNet101, ResNet152]
-      epochs : [1] #10, 20, 30, 50, 100
+      epochs : [1, 10, 20, 30, 50, 100] #
     cmd: >- 
-      bash attacks.sh 
+      bash scripts/${item.atk_name}.sh 
       ++model.init.name=torch_example.${item.model} 
       stage=attack
       model_name=${item.model}
@@ -104,9 +105,10 @@ stages:
     deps:
       - ${item.dataset}/${files.reports}/attack/${files.name}/${files.score_dict_file} # This is here just to ensure it runs after the attack stage
       - ${item.dataset}/${files.reports}/train/${files.name}/${files.score_dict_file}
-      - attacks.sh
+      - scripts/${item.atk_name}.sh
+      - scripts/models.sh
     outs:
-      - ${item.dataset}/logs/attack/${item.model}/${item.epochs}/:
+      - ${item.dataset}/logs/attack/${item.model}/${item.atk_name}/${item.epochs}/:
           cache: True
           persist: True
   compile:

diff --git a/examples/pytorch/main.sh b/examples/pytorch/main.sh