diff --git a/examples/power/conf/attack/default.yaml b/examples/power/conf/attack/default.yaml index 6cc5ec60..7d528793 100644 --- a/examples/power/conf/attack/default.yaml +++ b/examples/power/conf/attack/default.yaml @@ -6,6 +6,9 @@ init: _target_: deckard.base.attack.AttackInitializer name: art.attacks.evasion.FastGradientMethod eps: .99 - eps_step : ${eval:'(.1)*${.eps}'} + # eps_step : ${eval:'(.1)*${.eps}'} + batch_size : ${model.trainer.batch_size} + targeted : false + minimal : true attack_size : 10 method : evasion diff --git a/examples/power/conf/torch_cifar10.yaml b/examples/power/conf/torch_cifar10.yaml index 6e69ab4a..1b862654 100644 --- a/examples/power/conf/torch_cifar10.yaml +++ b/examples/power/conf/torch_cifar10.yaml @@ -7,28 +7,25 @@ defaults: - scorers: default - override hydra/sweeper : optuna - override hydra/sweeper/sampler : tpe + # - override hydra/launcher : rq device_id : ${oc.env:DECKARD_DEVICE,cpu} stage : '???' direction : - maximize - minimize - # - minimize - maximize - minimize - # - minimize optimizers: - accuracy - train_time - # - predict_proba_time - adv_accuracy - adv_fit_time - # - adv_predict_time _target_ : deckard.base.experiment.Experiment hydra: run: - dir: ${files.directory}/${files.reports}/${stage}/logs/${data.generate.name} + dir: ${files.directory}/${files.reports}/logs/${stage}/${data.generate.name} sweep: - dir: ${files.directory}/${stage}/${data.generate.name} + dir: ${files.directory}/logs/${stage}/${data.generate.name}/ subdir : ${hydra.sweeper.study_name}/${hydra.job.num} sweeper: study_name: ${model.init.name} @@ -50,7 +47,28 @@ hydra: ++model.trainer.nb_epoch: int(range(1, 50)) ++model.trainer.batch_size: int(range(10, 10000)) ++attack.init.eps : range(0.01, 1.0) - ++attack.init.eps_step : ${eval:'(.01)*${attack.init.eps}'} - ++attack.init.batch_size: ${model.trainer.batch_size} _target_: hydra_plugins.hydra_optuna_sweeper.optuna_sweeper.OptunaSweeper direction: ${direction} +# launcher: REDIS QUEUE LAUNCHER +# # @package hydra.launcher +# _target_: hydra_plugins.hydra_rq_launcher.rq_launcher.RQLauncher +# enqueue: +# job_timeout: 3600s +# ttl: null +# result_ttl: 1s +# failure_ttl: 1h +# at_front: false +# job_id: null +# description: null +# queue: default +# redis: +# host: localhost +# port: ${oc.env:REDIS_PORT,6379} +# db: ${oc.env:REDIS_DB,0} +# password: ${oc.env:REDIS_PASSWORD,D3vAQY9y4YdwhV} +# ssl: ${oc.env:REDIS_SSL,False} +# ssl_ca_certs: ${oc.env:REDIS_SSL_CA_CERTS,null} +# mock: ${oc.env:REDIS_MOCK,False} +# stop_after_enqueue: false +# wait_polling: 1.0 +# raise_exception: True diff --git a/examples/power/conf/torch_cifar100.yaml b/examples/power/conf/torch_cifar100.yaml index a249b2af..07ef3690 100644 --- a/examples/power/conf/torch_cifar100.yaml +++ b/examples/power/conf/torch_cifar100.yaml @@ -13,23 +13,19 @@ stage : '???' direction : - maximize - minimize - # - minimize - maximize - minimize - # - minimize optimizers: - accuracy - train_time - # - predict_proba_time - adv_accuracy - adv_fit_time - # - adv_predict_time _target_ : deckard.base.experiment.Experiment hydra: run: - dir: ${files.directory}/${files.reports}/${stage}/logs/${data.generate.name} + dir: ${files.directory}/${files.reports}/logs/${stage}/${data.generate.name} sweep: - dir: ${files.directory}/${stage}//${data.generate.name} + dir: ${files.directory}/logs/${stage}/${data.generate.name}/ subdir : ${hydra.sweeper.study_name}/${hydra.job.num} sweeper: study_name: ${model.init.name} @@ -44,14 +40,35 @@ hydra: n_ei_candidates: 24 # default: 24 multivariate: true warn_independent_sampling: true - n_trials : 100 + n_trials : 100 params: ++data.sample.random_state: int(range(0, 1)) ++model.art.initialize.optimizer.lr: range(0.0001, 1) ++model.trainer.nb_epoch: int(range(1, 50)) ++model.trainer.batch_size: int(range(10, 10000)) ++attack.init.eps : range(0.01, 1.0) - ++attack.init.eps_step : ${eval:'(.01)*${attack.init.eps}'} - ++attack.init.batch_size: ${model.trainer.batch_size} _target_: hydra_plugins.hydra_optuna_sweeper.optuna_sweeper.OptunaSweeper direction: ${direction} +# launcher: REDIS QUEUE LAUNCHER +# # @package hydra.launcher +# _target_: hydra_plugins.hydra_rq_launcher.rq_launcher.RQLauncher +# enqueue: +# job_timeout: 3600s +# ttl: null +# result_ttl: 1s +# failure_ttl: 1h +# at_front: false +# job_id: null +# description: null +# queue: default +# redis: +# host: localhost +# port: ${oc.env:REDIS_PORT,6379} +# db: ${oc.env:REDIS_DB,0} +# password: ${oc.env:REDIS_PASSWORD,D3vAQY9y4YdwhV} +# ssl: ${oc.env:REDIS_SSL,False} +# ssl_ca_certs: ${oc.env:REDIS_SSL_CA_CERTS,null} +# mock: ${oc.env:REDIS_MOCK,False} +# stop_after_enqueue: false +# wait_polling: 1.0 +# raise_exception: True diff --git a/examples/power/conf/torch_mnist.yaml b/examples/power/conf/torch_mnist.yaml index a2a34c5b..3fae4e57 100644 --- a/examples/power/conf/torch_mnist.yaml +++ b/examples/power/conf/torch_mnist.yaml @@ -13,23 +13,19 @@ stage : '???' direction : - maximize - minimize - # - minimize - maximize - minimize - # - minimize optimizers: - accuracy - train_time - # - predict_proba_time - adv_accuracy - adv_fit_time - # - adv_predict_time _target_ : deckard.base.experiment.Experiment hydra: run: - dir: ${files.directory}/${files.reports}/${stage}/logs/${data.generate.name} + dir: ${files.directory}/${files.reports}/logs/${stage}/${data.generate.name} sweep: - dir: ${files.directory}/${stage}/${data.generate.name} + dir: ${files.directory}/logs/${stage}/${data.generate.name}/ subdir : ${hydra.sweeper.study_name}/${hydra.job.num} sweeper: study_name: ${model.init.name} @@ -51,8 +47,6 @@ hydra: ++model.trainer.nb_epoch: int(range(1, 50)) ++model.trainer.batch_size: int(range(10, 10000)) ++attack.init.eps : range(0.01, 1.0) - ++attack.init.eps_step : ${eval:'(.01)*${attack.init.eps}'} - ++attack.init.batch_size: ${model.trainer.batch_size} _target_: hydra_plugins.hydra_optuna_sweeper.optuna_sweeper.OptunaSweeper direction: ${direction} # launcher: REDIS QUEUE LAUNCHER diff --git a/examples/pytorch/cifar100/conf/cifar100.yaml b/examples/pytorch/cifar100/conf/cifar100.yaml index a0727d43..1cfee53b 100644 --- a/examples/pytorch/cifar100/conf/cifar100.yaml +++ b/examples/pytorch/cifar100/conf/cifar100.yaml @@ -24,7 +24,7 @@ hydra: direction: ${direction} study_name: control storage: sqlite:///model.db - n_jobs: 4 + n_jobs: 8 n_trials : 32 params: ++data.sample.random_state: choice(0, 1, 2, 3, 4, 5, 6, 7, 8, 9) diff --git a/examples/pytorch/cifar100/dvc.lock b/examples/pytorch/cifar100/dvc.lock index ebda37d0..3a6310c7 100644 --- a/examples/pytorch/cifar100/dvc.lock +++ b/examples/pytorch/cifar100/dvc.lock @@ -100,11 +100,11 @@ stages: md5: 1070854e6c00fc787bc0fdfc82792fd6 size: 761280311 - path: cifar100/reports/train/default/predictions.json - md5: 406b5670c6fc9741adf82a3922f5d5f2 - size: 24401460 + md5: 59e3f3a1d965ce83bce8b10bfb568142 + size: 24418063 - path: cifar100/reports/train/default/score_dict.json - md5: 39cfcc1fc0d5bef2dafef2eb0295d934 - size: 885 + md5: 9cb64ff83dede1bd4adb4fbef7bb0d9c + size: 900 attack: cmd: python -m deckard.layers.experiment attack --config_file cifar100.yaml deps: @@ -326,14 +326,14 @@ stages: name: sklearn.metrics.log_loss outs: - path: cifar100/attacks/attack.pkl - md5: a80777574854b5cefe05d26ca2a8ba7c + md5: 408c383e6708d42e2271cc33956af948 size: 123046 - path: cifar100/reports/attack/default/adv_predictions.json - md5: 321ea49f7061832db3c5142f0ad7b042 - size: 21395 + md5: f4effd039275cf68f3978bd17091b8dd + size: 21441 - path: cifar100/reports/attack/default/score_dict.json - md5: e0afe623a549e81ca7d44a65ce362ee2 - size: 1356 + md5: 5571912697a68b76528e517c58159245 + size: 1359 attacks@ResNet18: cmd: bash attacks.sh ++attack.attack_size=100 ++model.init.name=torch_example.ResNet18 stage=attack ++hydra.sweeper.storage=sqlite:///cifar100/reports/attack/ResNet18.db