diff --git a/examples/power/.dvc/.gitignore b/examples/power/.dvc/.gitignore deleted file mode 100644 index 528f30c7..00000000 --- a/examples/power/.dvc/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/config.local -/tmp -/cache diff --git a/examples/power/conf/clean.yaml b/examples/power/conf/clean.yaml new file mode 100644 index 00000000..47de8dad --- /dev/null +++ b/examples/power/conf/clean.yaml @@ -0,0 +1,13 @@ +attacks: + FastGradientMethod: FGM +defences: + Control: Control + FeatureSqueezing: FSQ +params: + FGM: attack.init.eps + Control: model.trainer.nb_epoch + FSQ: model.art.pipeline.preprocessor.bit_depth +fillna: + FGM: 0.0 + Control: 20 + FSQ: 32 diff --git a/examples/power/conf/compile.yaml b/examples/power/conf/compile.yaml deleted file mode 100644 index 8514e6df..00000000 --- a/examples/power/conf/compile.yaml +++ /dev/null @@ -1,8 +0,0 @@ -attacks: - FastGradientMethod: FGM -defences: - Control: Control - # FeatureSqueezing: FSQ -params: - FGM: attack.init.eps - Control: epochs diff --git a/examples/power/params.yaml b/examples/power/params.yaml new file mode 100644 index 00000000..81856502 --- /dev/null +++ b/examples/power/params.yaml @@ -0,0 +1,241 @@ +_target_: deckard.base.experiment.Experiment +attack: + _target_: deckard.base.attack.Attack + attack_size: 10 + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + sklearn_pipeline: + _target_: deckard.base.data.sklearn_pipeline.SklearnDataPipeline + preprocessor: + name: sklearn.preprocessing.StandardScaler + with_mean: true + with_std: true + init: + _target_: deckard.base.attack.AttackInitializer + batch_size: 1024 + eps: 0.99 + minimal: true + model: + _target_: deckard.base.model.Model + art: + _target_: deckard.base.model.art_pipeline.ArtPipeline + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + sklearn_pipeline: + _target_: deckard.base.data.sklearn_pipeline.SklearnDataPipeline + preprocessor: + name: sklearn.preprocessing.StandardScaler + with_mean: true + with_std: true + initialize: + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + library: pytorch + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + sklearn_pipeline: + _target_: deckard.base.data.sklearn_pipeline.SklearnDataPipeline + preprocessor: + name: sklearn.preprocessing.StandardScaler + with_mean: true + with_std: true + init: + _target_: deckard.base.model.ModelInitializer + name: torch_example.ResNet18 + num_channels: 1 + library: pytorch + trainer: + batch_size: 1024 + nb_epoch: 1 + name: art.attacks.evasion.FastGradientMethod + targeted: false + method: evasion + model: + _target_: deckard.base.model.Model + art: + _target_: deckard.base.model.art_pipeline.ArtPipeline + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + sklearn_pipeline: + _target_: deckard.base.data.sklearn_pipeline.SklearnDataPipeline + preprocessor: + name: sklearn.preprocessing.StandardScaler + with_mean: true + with_std: true + initialize: + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + library: pytorch + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + sklearn_pipeline: + _target_: deckard.base.data.sklearn_pipeline.SklearnDataPipeline + preprocessor: + name: sklearn.preprocessing.StandardScaler + with_mean: true + with_std: true + init: + _target_: deckard.base.model.ModelInitializer + name: torch_example.ResNet18 + num_channels: 1 + library: pytorch + trainer: + batch_size: 1024 + nb_epoch: 1 +data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + sklearn_pipeline: + _target_: deckard.base.data.sklearn_pipeline.SklearnDataPipeline + preprocessor: + name: sklearn.preprocessing.StandardScaler + with_mean: true + with_std: true +device_id: cpu +direction: +- maximize +- minimize +- minimize +- minimize +files: + _target_: deckard.base.files.FileConfig + adv_predictions_file: adv_predictions.json + attack_dir: attacks + attack_file: attack + attack_type: .pkl + data_dir: data + data_file: data + data_type: .pkl + directory: /result/mnist/ + model_dir: null + model_file: null + model_type: null + name: default + params_file: params.yaml + predictions_file: predictions.json + reports: reports + score_dict_file: score_dict.json +model: + _target_: deckard.base.model.Model + art: + _target_: deckard.base.model.art_pipeline.ArtPipeline + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + sklearn_pipeline: + _target_: deckard.base.data.sklearn_pipeline.SklearnDataPipeline + preprocessor: + name: sklearn.preprocessing.StandardScaler + with_mean: true + with_std: true + initialize: + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + library: pytorch + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + sklearn_pipeline: + _target_: deckard.base.data.sklearn_pipeline.SklearnDataPipeline + preprocessor: + name: sklearn.preprocessing.StandardScaler + with_mean: true + with_std: true + init: + _target_: deckard.base.model.ModelInitializer + name: torch_example.ResNet18 + num_channels: 1 + library: pytorch + trainer: + batch_size: 1024 + nb_epoch: 1 +optimizers: +- accuracy +- train_time +- adv_accuracy +- adv_fit_time +scorers: + _target_: deckard.base.scorer.ScorerDict + accuracy: + _target_: deckard.base.scorer.ScorerConfig + direction: maximize + name: sklearn.metrics.accuracy_score + log_loss: + _target_: deckard.base.scorer.ScorerConfig + direction: minimize + name: sklearn.metrics.log_loss +stage: ??? diff --git a/examples/power/plots/dvc.lock b/examples/power/plots/dvc.lock deleted file mode 100644 index ad996cd0..00000000 --- a/examples/power/plots/dvc.lock +++ /dev/null @@ -1,502 +0,0 @@ -schema: '2.0' -stages: - compile@mnist: - cmd: python -m deckard.layers.compile --report_folder /result/mnist/reports/attack/ - --results_file attack.csv --results_folder mnist --config "../conf/compile.yaml" - deps: - - path: ../conf/compile.yaml - hash: md5 - md5: 61054aeaa58015252df89c20db9ec595 - size: 139 - - path: /result/mnist/reports/attack/ - hash: md5 - md5: 2fd9e6eb909c65078ded91f85c5938d1.dir - size: 5846593866 - nfiles: 8024 - outs: - - path: mnist/attack.csv - hash: md5 - md5: e58a980dc2e0da18144d8a7d55e41ac4 - size: 9759136 - compile@cifar: - cmd: python -m deckard.layers.compile --report_folder /result/cifar/reports/attack/ - --results_file attack.csv --results_folder cifar --config "../conf/compile.yaml" - deps: - - path: ../conf/compile.yaml - hash: md5 - md5: 61054aeaa58015252df89c20db9ec595 - size: 139 - - path: /result/cifar/reports/attack/ - hash: md5 - md5: ed8b007df2788a65643c2c0b99a87a4d.dir - size: 5035287644 - nfiles: 8031 - outs: - - path: cifar/attack.csv - hash: md5 - md5: 1adead42bdfa4a1cf7721c563f08fb17 - size: 9787000 - compile@cifar100: - cmd: python -m deckard.layers.compile --report_folder /result/cifar100/reports/attack/ - --results_file attack.csv --results_folder cifar100 --config "../conf/compile.yaml" - deps: - - path: ../conf/compile.yaml - hash: md5 - md5: 61054aeaa58015252df89c20db9ec595 - size: 139 - - path: /result/cifar100/reports/attack/ - hash: md5 - md5: d70914292c9d7d2d6703ce8c5179f334.dir - size: 66900144277 - nfiles: 11419 - outs: - - path: cifar100/attack.csv - hash: md5 - md5: d710860394e7af9976b96f399dcc9e8a - size: 14439946 - plot@mnist: - cmd: python -m deckard.layers.plots --path mnist --file mnist/attack.csv -o plot_data.csv - -c "../conf/plots.yaml" --subset train_time,predict_proba_time,accuracy,adv_accuracy,adv_fit_time - deps: - - path: ../conf/plots.yaml - hash: md5 - md5: 6b372706e37d0d371d245d9d927e7577 - size: 1321 - - path: mnist/attack.csv - hash: md5 - md5: e58a980dc2e0da18144d8a7d55e41ac4 - size: 9759136 - outs: - - path: mnist/adv_failure_rate_vs_train_time.pdf - hash: md5 - md5: 4d67d03038f6910890a9b3de6c388a5d - size: 32913 - - path: mnist/ben_failure_rate_vs_train_time.pdf - hash: md5 - md5: cf4423c2e57d02639ac7ad3f73e7923f - size: 34114 - - path: mnist/plot_data.csv - hash: md5 - md5: ef9fa5261715b08c90deec69c6e6831b - size: 4773500 - - path: mnist/train_time_vs_accuracy.pdf - hash: md5 - md5: 15597e7f04132220b4263a5c0d92625e - size: 21898 - plot@cifar: - cmd: python -m deckard.layers.plots --path cifar --file cifar/attack.csv -o plot_data.csv - -c "../conf/plots.yaml" --subset train_time,predict_proba_time,accuracy,adv_accuracy,adv_fit_time - deps: - - path: ../conf/plots.yaml - hash: md5 - md5: 6b372706e37d0d371d245d9d927e7577 - size: 1321 - - path: cifar/attack.csv - hash: md5 - md5: 1adead42bdfa4a1cf7721c563f08fb17 - size: 9787000 - outs: - - path: cifar/adv_failure_rate_vs_train_time.pdf - hash: md5 - md5: 68daee0b3952983c42c57e8ec1b52e73 - size: 28362 - - path: cifar/ben_failure_rate_vs_train_time.pdf - hash: md5 - md5: 01496b89613804e691a7ed15a375d302 - size: 29341 - - path: cifar/plot_data.csv - hash: md5 - md5: 71f94277b1c3e8f50136a11a3d563f8d - size: 3311418 - - path: cifar/train_time_vs_accuracy.pdf - hash: md5 - md5: fa158ac5977c5acca89eefcde397ae11 - size: 22760 - plot@cifar100: - cmd: python -m deckard.layers.plots --path cifar100 --file cifar100/attack.csv - -o plot_data.csv -c "../conf/plots.yaml" --subset train_time,predict_proba_time,accuracy,adv_accuracy,adv_fit_time - deps: - - path: ../conf/plots.yaml - hash: md5 - md5: 6b372706e37d0d371d245d9d927e7577 - size: 1321 - - path: cifar100/attack.csv - hash: md5 - md5: d710860394e7af9976b96f399dcc9e8a - size: 14439946 - outs: - - path: cifar100/adv_failure_rate_vs_train_time.pdf - hash: md5 - md5: e63a303612b5c5e5b71bdb8efc4fead9 - size: 30309 - - path: cifar100/ben_failure_rate_vs_train_time.pdf - hash: md5 - md5: 2a1e64977699d108f9a6ceb14a44fc89 - size: 32121 - - path: cifar100/plot_data.csv - hash: md5 - md5: a8a813a9d3a8a4171730c3ef09ece6ee - size: 4016181 - - path: cifar100/train_time_vs_accuracy.pdf - hash: md5 - md5: df6751797da220c51e14f1424522dcd5 - size: 23971 - afr@mnist: - cmd: python -m deckard.layers.afr --dataset mnist --data_file mnist/plot_data.csv - --config_file "../conf/afr.yaml" - deps: - - path: ../conf/afr.yaml - hash: md5 - md5: 2f7ff61ce1517a4320ab990cbf5f2b16 - size: 8084 - - path: mnist/plot_data.csv - hash: md5 - md5: ef9fa5261715b08c90deec69c6e6831b - size: 4773500 - outs: - - path: mnist/aft_comparison.csv - hash: md5 - md5: 17d4ef4926c0725a02c16d16a83098fc - size: 213 - - path: mnist/aft_comparison.tex - hash: md5 - md5: 106a151a290681609de5d9073ad83cd7 - size: 451 - - path: mnist/cox_aft.pdf - hash: md5 - md5: 738d989e4e913ec2a4a0aa770108ecf9 - size: 20071 - - path: mnist/cox_epochs_partial_effect.pdf - hash: md5 - md5: 7b2e73f00570d8ea48993e85aae4a462 - size: 27938 - - path: mnist/log_logistic_aft.pdf - hash: md5 - md5: f8ec551175e747f38da9fae23a76250d - size: 22634 - - path: mnist/log_logistic_epochs_partial_effect.pdf - hash: md5 - md5: b1f1c283e0ccb15c6a0946cd5bcad1c6 - size: 28146 - - path: mnist/log_normal_aft.pdf - hash: md5 - md5: c8f0312afa35ff2a92958666581831a1 - size: 23628 - - path: mnist/log_normal_epochs_partial_effect.pdf - hash: md5 - md5: 3b4dedcbb7c7b41b362c4f6b41648a2f - size: 28548 - - path: mnist/weibull_aft.pdf - hash: md5 - md5: 8538a93dcc9267ec07bf554a7d703d02 - size: 31596 - - path: mnist/weibull_epochs_partial_effect.pdf - hash: md5 - md5: f6d46d7e580e3834abd9cef71f0f03b3 - size: 28446 - afr@cifar: - cmd: python -m deckard.layers.afr --dataset cifar --data_file cifar/plot_data.csv - --config_file "../conf/afr.yaml" - deps: - - path: ../conf/afr.yaml - hash: md5 - md5: 2f7ff61ce1517a4320ab990cbf5f2b16 - size: 8084 - - path: cifar/plot_data.csv - hash: md5 - md5: 71f94277b1c3e8f50136a11a3d563f8d - size: 3311418 - outs: - - path: cifar/aft_comparison.csv - hash: md5 - md5: 7ecfb10f7474b4fbebb530fb50fe164e - size: 207 - - path: cifar/aft_comparison.tex - hash: md5 - md5: 8904c5c2bc9a04ec544cede940da5819 - size: 449 - - path: cifar/cox_aft.pdf - hash: md5 - md5: 268bc658b99003fc31bc2f0cbc474c21 - size: 19797 - - path: cifar/cox_epochs_partial_effect.pdf - hash: md5 - md5: 0ff0825bae278d55cb35e795b7daa236 - size: 27647 - - path: cifar/log_logistic_aft.pdf - hash: md5 - md5: ff26a0a9eea8d3ae73a671c058aa477b - size: 22632 - - path: cifar/log_logistic_epochs_partial_effect.pdf - hash: md5 - md5: 0fd26ac4269fc6a36179a7724aa42664 - size: 27417 - - path: cifar/log_normal_aft.pdf - hash: md5 - md5: 0c8ad4f39d0c1df902b4764c667e9006 - size: 23630 - - path: cifar/log_normal_epochs_partial_effect.pdf - hash: md5 - md5: f46d273316733b83d06639fa017f53c6 - size: 28173 - - path: cifar/weibull_aft.pdf - hash: md5 - md5: bea79bd48e84d4f314852e108d3f460a - size: 31594 - - path: cifar/weibull_epochs_partial_effect.pdf - hash: md5 - md5: 9d8a75464641a821cbac19af2df585c6 - size: 27820 - afr@cifar100: - cmd: python -m deckard.layers.afr --dataset cifar100 --data_file cifar100/plot_data.csv - --config_file "../conf/afr.yaml" - deps: - - path: ../conf/afr.yaml - hash: md5 - md5: 2f7ff61ce1517a4320ab990cbf5f2b16 - size: 8084 - - path: cifar100/plot_data.csv - hash: md5 - md5: a8a813a9d3a8a4171730c3ef09ece6ee - size: 4016181 - outs: - - path: cifar100/aft_comparison.csv - hash: md5 - md5: 398a9aad117e230afa2ca484581b37bc - size: 212 - - path: cifar100/aft_comparison.tex - hash: md5 - md5: ab03a9867122740b0409c6de0da49ef3 - size: 457 - - path: cifar100/cox_aft.pdf - hash: md5 - md5: b648636cea335b8b003ff9aae39c265b - size: 19562 - - path: cifar100/cox_epochs_partial_effect.pdf - hash: md5 - md5: e62c782ffcbec52ff8f1378bbb85ddf6 - size: 27029 - - path: cifar100/log_logistic_aft.pdf - hash: md5 - md5: 6ee44957cd08349936dffacafe8bd7c9 - size: 22629 - - path: cifar100/log_logistic_epochs_partial_effect.pdf - hash: md5 - md5: a8ed3b1d0a9a804220ef7e6e0c7839bc - size: 27149 - - path: cifar100/log_normal_aft.pdf - hash: md5 - md5: f6cddf8b1053801067bdb5dba610fd32 - size: 23633 - - path: cifar100/log_normal_epochs_partial_effect.pdf - hash: md5 - md5: 34ee6b72c4570faf0da1e0023146b16b - size: 27783 - - path: cifar100/weibull_aft.pdf - hash: md5 - md5: 444e08b12dd3332a2dc3c90a6458a527 - size: 32232 - - path: cifar100/weibull_epochs_partial_effect.pdf - hash: md5 - md5: 4f34911f42f42218ff08c13c0e35ba18 - size: 26687 - compile@bit_depth/mnist/: - cmd: python -m deckard.layers.compile --report_folder /result/bit_depth/mnist//reports/attack/ - --results_file attack.csv --results_folder bit_depth/mnist/ --config "../conf/compile.yaml" - deps: - - path: ../conf/compile.yaml - hash: md5 - md5: 61054aeaa58015252df89c20db9ec595 - size: 139 - - path: /result/bit_depth/mnist//reports/attack/ - hash: md5 - md5: 0e98e6497ec11efe7d15d6d26d48e373.dir - size: 3045516686 - nfiles: 4182 - outs: - - path: bit_depth/mnist//attack.csv - hash: md5 - md5: 0a82d226a00f0e72266122a5dfcc063e - size: 5312357 - compile@bit_depth/cifar/: - cmd: python -m deckard.layers.compile --report_folder /result/bit_depth/cifar//reports/attack/ - --results_file attack.csv --results_folder bit_depth/cifar/ --config "../conf/compile.yaml" - deps: - - path: ../conf/compile.yaml - hash: md5 - md5: 61054aeaa58015252df89c20db9ec595 - size: 139 - - path: /result/bit_depth/cifar//reports/attack/ - hash: md5 - md5: 2ccefb9124bfd7288cb342cfa615b814.dir - size: 1706177501 - nfiles: 2719 - outs: - - path: bit_depth/cifar//attack.csv - hash: md5 - md5: d31c178c71be151864b2867130b25834 - size: 3470593 - plot@bit_depth/mnist/: - cmd: python -m deckard.layers.plots --path bit_depth/mnist/ --file bit_depth/mnist//attack.csv - -o plot_data.csv -c "../conf/plots.yaml" --subset train_time,predict_proba_time,accuracy,adv_accuracy,adv_fit_time - deps: - - path: ../conf/plots.yaml - hash: md5 - md5: 6b372706e37d0d371d245d9d927e7577 - size: 1321 - - path: bit_depth/mnist//attack.csv - hash: md5 - md5: 0a82d226a00f0e72266122a5dfcc063e - size: 5312357 - outs: - - path: bit_depth/mnist//adv_failure_rate_vs_train_time.pdf - hash: md5 - md5: 1a12d0b27a77579d682185449bad34ba - size: 26881 - - path: bit_depth/mnist//ben_failure_rate_vs_train_time.pdf - hash: md5 - md5: dc2a0e22088ecdf8da8a416c3c9983b0 - size: 27978 - - path: bit_depth/mnist//plot_data.csv - hash: md5 - md5: 33ba83a5d6f73875448ab74a2f18916d - size: 2982894 - - path: bit_depth/mnist//train_time_vs_accuracy.pdf - hash: md5 - md5: 1c220c2e8566b0a63846b1868c4d1f20 - size: 20902 - plot@bit_depth/cifar/: - cmd: python -m deckard.layers.plots --path bit_depth/cifar/ --file bit_depth/cifar//attack.csv - -o plot_data.csv -c "../conf/plots.yaml" --subset train_time,predict_proba_time,accuracy,adv_accuracy,adv_fit_time - deps: - - path: ../conf/plots.yaml - hash: md5 - md5: 6b372706e37d0d371d245d9d927e7577 - size: 1321 - - path: bit_depth/cifar//attack.csv - hash: md5 - md5: d31c178c71be151864b2867130b25834 - size: 3470593 - outs: - - path: bit_depth/cifar//adv_failure_rate_vs_train_time.pdf - hash: md5 - md5: 6ff4890ddbfeb078d62dfb5707fe82e2 - size: 22234 - - path: bit_depth/cifar//ben_failure_rate_vs_train_time.pdf - hash: md5 - md5: c9f9e2bab55d1b0683de55a3217cadbf - size: 23164 - - path: bit_depth/cifar//plot_data.csv - hash: md5 - md5: 08ca910134d6e1806f05c1d298b0f664 - size: 1561678 - - path: bit_depth/cifar//train_time_vs_accuracy.pdf - hash: md5 - md5: 82c370b0db144053df5370102026435d - size: 20902 - afr@bit_depth/mnist/: - cmd: python -m deckard.layers.afr --dataset bit_depth/mnist/ --data_file bit_depth/mnist//plot_data.csv - --config_file "../conf/afr.yaml" - deps: - - path: ../conf/afr.yaml - hash: md5 - md5: 2f7ff61ce1517a4320ab990cbf5f2b16 - size: 8084 - - path: bit_depth/mnist//plot_data.csv - hash: md5 - md5: 33ba83a5d6f73875448ab74a2f18916d - size: 2982894 - outs: - - path: bit_depth/mnist//aft_comparison.csv - hash: md5 - md5: 0985db77aaa78ff65efb3d981ac19a7d - size: 207 - - path: bit_depth/mnist//aft_comparison.tex - hash: md5 - md5: b66410cbc16c7287655066e624cc4a64 - size: 467 - - path: bit_depth/mnist//cox_aft.pdf - hash: md5 - md5: 81ab402581dfeb80094b02e96ccbebd6 - size: 19564 - - path: bit_depth/mnist//cox_epochs_partial_effect.pdf - hash: md5 - md5: c8bf7109789922c17a281a87e7115c05 - size: 28240 - - path: bit_depth/mnist//log_logistic_aft.pdf - hash: md5 - md5: 981244878b659ce0978bf393da0e6ef0 - size: 22635 - - path: bit_depth/mnist//log_logistic_epochs_partial_effect.pdf - hash: md5 - md5: 9293c5362f3e21d03723a8a9fb979efb - size: 27817 - - path: bit_depth/mnist//log_normal_aft.pdf - hash: md5 - md5: 33fcc4485293fa4704c033191f910578 - size: 23621 - - path: bit_depth/mnist//log_normal_epochs_partial_effect.pdf - hash: md5 - md5: 6977db1ba0125ef9d4151ddafdd8529e - size: 28250 - - path: bit_depth/mnist//weibull_aft.pdf - hash: md5 - md5: 261fb580d43eaffff310d6e3c6dde80b - size: 32224 - - path: bit_depth/mnist//weibull_epochs_partial_effect.pdf - hash: md5 - md5: b4780c1de1b658896c1044183f1b6fb6 - size: 28039 - afr@bit_depth/cifar/: - cmd: python -m deckard.layers.afr --dataset bit_depth/cifar/ --data_file bit_depth/cifar//plot_data.csv - --config_file "../conf/afr.yaml" - deps: - - path: ../conf/afr.yaml - hash: md5 - md5: 2f7ff61ce1517a4320ab990cbf5f2b16 - size: 8084 - - path: bit_depth/cifar//plot_data.csv - hash: md5 - md5: 08ca910134d6e1806f05c1d298b0f664 - size: 1561678 - outs: - - path: bit_depth/cifar//aft_comparison.csv - hash: md5 - md5: e2ad4bbeaceffac83f2669d1fc7aaf6d - size: 207 - - path: bit_depth/cifar//aft_comparison.tex - hash: md5 - md5: b84efba153648578fba3167e642c566f - size: 467 - - path: bit_depth/cifar//cox_aft.pdf - hash: md5 - md5: 4c361cb01e604398080ac26b712ec931 - size: 19972 - - path: bit_depth/cifar//cox_epochs_partial_effect.pdf - hash: md5 - md5: 8e150547d47eb745d4be47974cb1029c - size: 27127 - - path: bit_depth/cifar//log_logistic_aft.pdf - hash: md5 - md5: 10cc72619e730733e3e975834aaec15e - size: 22642 - - path: bit_depth/cifar//log_logistic_epochs_partial_effect.pdf - hash: md5 - md5: de33aa8a707b76b647f1eac3f71ed444 - size: 28988 - - path: bit_depth/cifar//log_normal_aft.pdf - hash: md5 - md5: 6ef63cefaea6cbb2bdf6d3d92635d99e - size: 23631 - - path: bit_depth/cifar//log_normal_epochs_partial_effect.pdf - hash: md5 - md5: 19153bae3e9b778fccee3cda459832c5 - size: 29285 - - path: bit_depth/cifar//weibull_aft.pdf - hash: md5 - md5: 8518326aecf02b25f3b69d5d2e2a582e - size: 32225 - - path: bit_depth/cifar//weibull_epochs_partial_effect.pdf - hash: md5 - md5: 8ccc2f82015de53960a131d92b8d2653 - size: 29270 diff --git a/examples/power/plots/dvc.yaml b/examples/power/plots/dvc.yaml index 7a8f988a..02e75d1d 100644 --- a/examples/power/plots/dvc.yaml +++ b/examples/power/plots/dvc.yaml @@ -1,6 +1,10 @@ vars: - ../conf/plots.yaml:line_plot - ../conf/plots.yaml:scatter_plot + - ../conf/clean.yaml:attacks + - ../conf/clean.yaml:defences + - ../conf/clean.yaml:params + - ../conf/clean.yaml:fillna stages: compile: foreach: # iterates through each stage @@ -9,53 +13,74 @@ stages: - cifar100 - bit_depth/mnist/ - bit_depth/cifar/ + - bit_depth/cifar100/ do: cmd: python -m deckard.layers.compile --report_folder /result/${item}/reports/attack/ --results_file attack.csv --results_folder ${item} --config "../conf/compile.yaml" - deps: - - "../conf/compile.yaml" - - /result/${item}/reports/attack/ + # deps: + # - /result/${item}/reports/attack/ outs: - ${item}/attack.csv - plot: + clean: foreach: - - mnist - - cifar - - cifar100 - - bit_depth/mnist/ - - bit_depth/cifar/ + - mnist + - cifar + - cifar100 + - bit_depth/mnist/ + - bit_depth/cifar/ + - bit_depth/cifar100/ do: - cmd: python -m deckard.layers.plots --path ${item} --file ${item}/attack.csv -o plot_data.csv -c "../conf/plots.yaml" --subset train_time,predict_proba_time,accuracy,adv_accuracy,adv_fit_time + cmd: python -m deckard.layers.clean_data --path ${item} --file attack.csv --config "../conf/clean.yaml" deps: - ${item}/attack.csv - - "../conf/plots.yaml" - outs: - - ${item}/plot_data.csv - plots: - - ${item}/${scatter_plot[0].file} # You can use the file specified in each list entry from the parameters specified above in vars - - ${item}/${scatter_plot[1].file} - - ${item}/${line_plot[0].file} - afr: - foreach: - - mnist - - cifar - - cifar100 - - bit_depth/mnist/ - - bit_depth/cifar/ - do: - cmd: python -m deckard.layers.afr --dataset ${item} --data_file ${item}/plot_data.csv --config_file "../conf/afr.yaml" - deps: - - ${item}/plot_data.csv - - ../conf/afr.yaml - plots: - - ${item}/weibull_aft.pdf # You can also define them manually - - ${item}/weibull_epochs_partial_effect.pdf - - ${item}/cox_epochs_partial_effect.pdf - - ${item}/cox_aft.pdf - - ${item}/log_logistic_aft.pdf - - ${item}/log_logistic_epochs_partial_effect.pdf - - ${item}/log_normal_aft.pdf - - ${item}/log_normal_epochs_partial_effect.pdf - metrics: - - ${item}/aft_comparison.csv + - ../conf/clean.yaml outs: - - ${item}/aft_comparison.tex + - ${item}/clean.csv + params: + - ../conf/clean.yaml: + - attacks + - defences + - params + - fillna + # plot: + # foreach: + # - mnist + # - cifar + # - cifar100 + # - bit_depth/mnist/ + # - bit_depth/cifar/ + # do: + # cmd: python -m deckard.layers.plots --path ${item} --file ${item}/attack.csv -o plot_data.csv -c "../conf/plots.yaml" --subset train_time,predict_proba_time,accuracy,adv_accuracy,adv_fit_time + # deps: + # # - ${item}/attack.csv # This should be uncommented, but I don't have enough disk space to cache the hashes of the results. + # - "../conf/plots.yaml" + # outs: + # - ${item}/plot_data.csv + # plots: + # - ${item}/${scatter_plot[0].file} # You can use the file specified in each list entry from the parameters specified above in vars + # - ${item}/${scatter_plot[1].file} + # - ${item}/${line_plot[0].file} + # afr: + # foreach: + # - mnist + # - cifar + # - cifar100 + # - bit_depth/mnist/ + # - bit_depth/cifar/ + # do: + # cmd: python -m deckard.layers.afr --dataset ${item} --data_file ${item}/plot_data.csv --config_file "../conf/afr.yaml" + # deps: + # - ${item}/plot_data.csv + # - ../conf/afr.yaml + # plots: + # - ${item}/weibull_aft.pdf # You can also define them manually + # - ${item}/weibull_epochs_partial_effect.pdf + # - ${item}/cox_epochs_partial_effect.pdf + # - ${item}/cox_aft.pdf + # - ${item}/log_logistic_aft.pdf + # - ${item}/log_logistic_epochs_partial_effect.pdf + # - ${item}/log_normal_aft.pdf + # - ${item}/log_normal_epochs_partial_effect.pdf + # metrics: + # - ${item}/aft_comparison.csv + # outs: + # - ${item}/aft_comparison.tex