diff --git a/deckard/base/attack/attack.py b/deckard/base/attack/attack.py index 5df0748c..22f3f2b7 100644 --- a/deckard/base/attack/attack.py +++ b/deckard/base/attack/attack.py @@ -134,7 +134,6 @@ def __init__( self.attack_size = attack_size self.init = AttackInitializer(model, name, **init) self.kwargs = kwargs - logger.info("Instantiating Attack with id: {}".format(self.__hash__())) def __hash__(self): return int(my_hash(self), 16) @@ -300,7 +299,6 @@ def __init__( self.attack_size = attack_size self.init = AttackInitializer(model, name, **init) self.kwargs = kwargs - logger.info("Instantiating Attack with id: {}".format(self.__hash__())) def __hash__(self): return int(my_hash(self), 16) @@ -493,7 +491,6 @@ def __init__( self.attack_size = attack_size self.init = AttackInitializer(model, name, **init) self.kwargs = kwargs - logger.info("Instantiating Attack with id: {}".format(self.__hash__())) def __hash__(self): return int(my_hash(self), 16) @@ -618,7 +615,6 @@ def __init__( f"kwargs must be of type DictConfig or dict. Got {type(kwargs)}", ) self.kwargs = kwargs - logger.info("Instantiating Attack with id: {}".format(self.__hash__())) def __hash__(self): return int(my_hash(self), 16) @@ -813,7 +809,6 @@ def __init__( kwargs.update(**kwargs.pop("kwargs")) self.kwargs = kwargs self.name = name if name is not None else my_hash(self) - logger.info("Instantiating Attack with id: {}".format(self.name)) def __call__( self, diff --git a/deckard/base/data/data.py b/deckard/base/data/data.py index 7b1859ea..1f1b6028 100644 --- a/deckard/base/data/data.py +++ b/deckard/base/data/data.py @@ -148,7 +148,6 @@ def save(self, data, filename): :param filename: str """ if filename is not None: - logger.info(f"Saving data to {filename}") suffix = Path(filename).suffix Path(filename).parent.mkdir(parents=True, exist_ok=True) if isinstance(data, dict): diff --git a/deckard/base/data/generator.py b/deckard/base/data/generator.py index 5a6f69bb..f2c81978 100644 --- a/deckard/base/data/generator.py +++ b/deckard/base/data/generator.py @@ -51,9 +51,6 @@ class SklearnDataGenerator: kwargs: dict = field(default_factory=dict) def __init__(self, name, **kwargs): - logger.info( - f"Instantiating {self.__class__.__name__} with name={name} and kwargs={kwargs}", - ) self.name = name self.kwargs = {k: v for k, v in kwargs.items() if v is not None} @@ -91,9 +88,6 @@ class TorchDataGenerator: kwargs: dict = field(default_factory=dict) def __init__(self, name, path=None, **kwargs): - logger.info( - f"Instantiating {self.__class__.__name__} with name={name} and kwargs={kwargs}", - ) self.name = name self.path = path self.kwargs = {k: v for k, v in kwargs.items() if v is not None} @@ -179,9 +173,6 @@ class KerasDataGenerator: kwargs: dict = field(default_factory=dict) def __init__(self, name, **kwargs): - logger.info( - f"Instantiating {self.__class__.__name__} with name={name} and kwargs={kwargs}", - ) self.name = name self.kwargs = {k: v for k, v in kwargs.items() if v is not None} diff --git a/deckard/base/model/model.py b/deckard/base/model/model.py index 06082735..21abd239 100644 --- a/deckard/base/model/model.py +++ b/deckard/base/model/model.py @@ -70,7 +70,6 @@ def __init__(self, **kwargs): self.kwargs = kwargs def __call__(self, data: list, model: object, library=None): - logger.info(f"Training model {model} with fit params: {self.kwargs}") device = str(model.device) if hasattr(model, "device") else "cpu" trainer = self.kwargs if library in sklearn_dict.keys(): @@ -91,7 +90,6 @@ def __call__(self, data: list, model: object, library=None): try: start = process_time_ns() start_timestamp = time() - logger.info(f"Fitting type(model): {type(model)} with kwargs {trainer}") model.fit(data[0], data[2], **trainer) end = process_time_ns() end_timestamp = time() diff --git a/deckard/base/model/sklearn_pipeline.py b/deckard/base/model/sklearn_pipeline.py index 03318512..60c9af7f 100644 --- a/deckard/base/model/sklearn_pipeline.py +++ b/deckard/base/model/sklearn_pipeline.py @@ -57,16 +57,10 @@ class SklearnModelPipelineStage: kwargs: dict = field(default_factory=dict) def __init__(self, name, stage_name, **kwargs): - logger.debug( - f"Instantiating {self.__class__.__name__} with name={name} and kwargs={kwargs}", - ) self.name = name self.kwargs = kwargs self.stage_name = stage_name - def __hash__(self): - return int(my_hash(self), 16) - def __call__(self, model): logger.debug( f"Calling SklearnModelPipelineStage with name={self.name} and kwargs={self.kwargs}", @@ -76,7 +70,7 @@ def __call__(self, model): stage_name = self.stage_name if self.stage_name is not None else name while "kwargs" in kwargs: kwargs.update(**kwargs.pop("kwargs")) - if "art." in str(type(model)): + if str(type(model)).startswith("art."): assert isinstance( model.model, BaseEstimator, @@ -102,7 +96,6 @@ class SklearnModelPipeline: pipeline: Dict[str, SklearnModelPipelineStage] = field(default_factory=dict) def __init__(self, **kwargs): - logger.debug(f"Instantiating {self.__class__.__name__} with kwargs={kwargs}") pipe = {} while "kwargs" in kwargs: pipe.update(**kwargs.pop("kwargs")) @@ -145,12 +138,12 @@ def __len__(self): else: return 0 - def __hash__(self): - return int(my_hash(self), 16) - def __iter__(self): return iter(self.pipeline) + def __hash__(self): + return int(my_hash(self), 16) + def __call__(self, model): params = deepcopy(asdict(self)) pipeline = params.pop("pipeline") @@ -172,7 +165,7 @@ def __call__(self, model): elif isinstance(stage, SklearnModelPipelineStage): model = stage(model=model) elif hasattr(stage, "fit"): - if "art." in str(type(model)): + if str(type(model)).startswith("art."): assert isinstance( model.model, BaseEstimator, @@ -184,12 +177,15 @@ def __call__(self, model): ), f"model must be a sklearn estimator. Got {type(model)}" if not isinstance(model, Pipeline) and "art." not in str(type(model)): model = Pipeline([("model", model)]) - elif "art." in str(type(model)) and not isinstance( + elif str(type(model)).startswith("art.") and not isinstance( model.model, Pipeline, ): model.model = Pipeline([("model", model.model)]) - elif "art." in str(type(model)) and isinstance(model.model, Pipeline): + elif str(type(model)).startswith("art.") and isinstance( + model.model, + Pipeline, + ): model.model.steps.insert(-2, [stage, model.model]) else: model.steps.insert(-2, [stage, model]) @@ -213,6 +209,9 @@ class SklearnModelInitializer: pipeline: SklearnModelPipeline = field(default_factory=None) kwargs: Union[dict, None] = field(default_factory=dict) + def __hash__(self): + return int(my_hash(self), 16) + def __init__(self, data, model=None, library="sklearn", pipeline={}, **kwargs): self.data = data self.model = model @@ -267,6 +266,3 @@ def __call__(self): "fit", ), f"model must have a fit method. Got type {type(model)}" return model - - def __hash__(self): - return int(my_hash(self), 16) diff --git a/deckard/base/utils/hashing.py b/deckard/base/utils/hashing.py index 4ee35d15..a8e7b1b2 100644 --- a/deckard/base/utils/hashing.py +++ b/deckard/base/utils/hashing.py @@ -1,7 +1,7 @@ from hashlib import md5 from collections import OrderedDict from typing import NamedTuple, Union -from dataclasses import asdict, is_dataclass +from dataclasses import asdict, is_dataclass, dataclass from omegaconf import DictConfig, OmegaConf, SCMode, ListConfig from copy import deepcopy import logging @@ -71,3 +71,9 @@ def to_dict(obj: Union[dict, OrderedDict, NamedTuple]) -> dict: def my_hash(obj: Union[dict, OrderedDict, NamedTuple]) -> str: return md5(str(to_dict(obj)).encode("utf-8")).hexdigest() + + +@dataclass +class Hashable: + def __hash__(self): + return int(my_hash(self), 16) diff --git a/deckard/layers/afr.py b/deckard/layers/afr.py index 449ae744..6984c62f 100644 --- a/deckard/layers/afr.py +++ b/deckard/layers/afr.py @@ -75,7 +75,11 @@ def ccl(p): ax = plt.gca() T = model.duration_col E = model.event_col - + # Cast df to numeric DataFrame + for col in df.columns: + df[col] = pd.to_numeric(df[col], errors="raise") + # Drop NaNs + df = df.dropna() predictions_at_t0 = np.clip( 1 - model.predict_survival_function(df, times=[t0]).T.squeeze(), 1e-10, @@ -347,8 +351,6 @@ def plot_aft( ax.set_xlabel(xlabel) ax.set_ylabel(ylabel) ax.set_title(title) - # symlog-scale the x-axis - # ax.set_xscale("linear") ax.get_figure().tight_layout() ax.get_figure().savefig(file) plt.gcf().clear() @@ -624,7 +626,7 @@ def make_afr_table( pretty_dataset = dataset.upper() aft_data = aft_data.round(2) aft_data.to_csv(folder / "aft_comparison.csv") - logger.info(f"Saved AFR comparison to {folder / 'aft_comparison.csv'}") + logger.info(f"Saved AFT comparison to {folder / 'aft_comparison.csv'}") aft_data = aft_data.round(2) aft_data.fillna("--", inplace=True) aft_data.to_latex( diff --git a/deckard/layers/clean_data.py b/deckard/layers/clean_data.py index 69a272ef..4cf0c63e 100644 --- a/deckard/layers/clean_data.py +++ b/deckard/layers/clean_data.py @@ -81,7 +81,7 @@ def drop_rows_without_results( logger.info(f"Shape of data before data before dropping na: {data.shape}") data.dropna(axis=0, subset=[col], inplace=True) after = data.shape[0] - logger.info(f"Shape of data before data after dropping na: {data.shape}") + logger.info(f"Shape of data after data after dropping na: {data.shape}") percent_change = (before - after) / before * 100 if percent_change > 5: # input(f"{percent_change:.2f}% of data dropped for {col}. Press any key to continue.") @@ -593,7 +593,6 @@ def clean_data_for_plotting( data = fill_na(data, fillna) data = replace_strings_in_data(data, replace_dict) data = replace_strings_in_columns(data, col_replace_dict) - if len(pareto_dict) > 0: data = find_pareto_set(data, pareto_dict) return data diff --git a/deckard/layers/plots.py b/deckard/layers/plots.py index 6e37ce7f..d12d76f0 100644 --- a/deckard/layers/plots.py +++ b/deckard/layers/plots.py @@ -103,6 +103,7 @@ def cat_plot( file = Path(file).with_suffix(filetype) logger.info(f"Rendering graph {file}") data = digitize_cols(data, digitize) + set_ = kwargs.pop("set", {}) if hue is not None: data = data.sort_values(by=[hue, x, y]) logger.debug( @@ -162,6 +163,8 @@ def cat_plot( graph.set(xlim=x_lim) if y_lim is not None: graph.set(ylim=y_lim) + if len(set_) > 0: + graph.set(**set_) graph.tight_layout() graph.savefig(folder / file) plt.gcf().clear() diff --git a/examples/power/conf/afr.yaml b/examples/power/conf/afr.yaml index 386cf4b1..59cfdb13 100644 --- a/examples/power/conf/afr.yaml +++ b/examples/power/conf/afr.yaml @@ -15,7 +15,7 @@ fillna: weibull: plot: file : weibull_aft.pdf - title : Weibull AFR Model + title : Weibull AFT Model labels: "Intercept: rho_": "$\\rho$" "Intercept: lambda_": "$\\lambda$" @@ -36,7 +36,7 @@ weibull: - "file": "weibull_epochs_partial_effect.pdf" "covariate_array": "model.trainer.np_epochs" "values_array": [1,10,25,50] - "title": "$S(t)$ for Weibull AFR" + "title": "$S(t)$ for Weibull AFT" "ylabel": "$\\mathbb{P}~(T>t)$" "xlabel": "Time $t$ (seconds)" "legend_kwargs": { @@ -46,7 +46,7 @@ weibull: cox: plot: file : cox_aft.pdf - title : Cox AFR Model + title : Cox AFT Model labels: "data.sample.random_state": "Random State" "atk_value": "Attack Strength" @@ -65,7 +65,7 @@ cox: - "file": "cox_epochs_partial_effect.pdf" "covariate_array": "model.trainer.np_epochs" "values_array": [1,10,25,50] - "title": "$S(t)$ for Cox AFR" + "title": "$S(t)$ for Cox AFT" "ylabel": "$\\mathbb{P}~(T>t)$" "xlabel": "Time $t$ (seconds)" "legend_kwargs": { @@ -75,7 +75,7 @@ cox: log_logistic: plot: file : log_logistic_aft.pdf - title : Log logistic AFR Model + title : Log logistic AFT Model labels: "Intercept: beta_": "$\\beta$" "Intercept: alpha_": "$\\alpha$" @@ -96,7 +96,7 @@ log_logistic: - "file": "log_logistic_epochs_partial_effect.pdf" "covariate_array": "model.trainer.np_epochs" "values_array": [1,10,25,50] - "title": "$S(t)$ for Log-Logistic AFR" + "title": "$S(t)$ for Log-Logistic AFT" "ylabel": "$\\mathbb{P}~(T>t)$" "xlabel": "Time $t$ (seconds)" "legend_kwargs": { @@ -106,7 +106,7 @@ log_logistic: log_normal: plot: file : log_normal_aft.pdf - title : Log Normal AFR Model + title : Log Normal AFT Model labels: "Intercept: sigma_": "$\\sigma$" "Intercept: mu_": "$\\mu$" @@ -127,7 +127,7 @@ log_normal: - "file": "log_normal_epochs_partial_effect.pdf" "covariate_array": "model.trainer.np_epochs" "values_array": [1,10,25,50] - "title": "$S(t)$ for Log-Normal AFR" + "title": "$S(t)$ for Log-Normal AFT" "ylabel": "$\\mathbb{P}~(T>t)$" "xlabel": "Time $t$ (seconds)" "legend_kwargs": { diff --git a/examples/power/conf/combined_afr.yaml b/examples/power/conf/combined_afr.yaml index 2e8709d8..7da3c435 100644 --- a/examples/power/conf/combined_afr.yaml +++ b/examples/power/conf/combined_afr.yaml @@ -17,7 +17,7 @@ fillna: weibull: plot: file : weibull_aft.pdf - title : Weibull AFR Model + title : Weibull AFT Model labels: "Intercept: rho_": "$\\rho$" "Intercept: lambda_": "$\\lambda$" @@ -39,7 +39,7 @@ weibull: # cox: # plot: # file : cox_aft.pdf -# title : Cox AFR Model +# title : Cox AFT Model # labels: # "data.sample.random_state": "Random State" # "atk_value": "Attack Strength" @@ -58,7 +58,7 @@ weibull: # - "file": "cox_epochs_partial_effect.pdf" # "covariate_array": "model.trainer.np_epochs" # "values_array": [1,10,25,50] -# "title": "$S(t)$ for Cox AFR" +# "title": "$S(t)$ for Cox AFT" # "ylabel": "$\\mathbb{P}~(T>t)$" # "xlabel": "Time $t$ (seconds)" # "legend_kwargs": { @@ -68,7 +68,7 @@ weibull: # - "file": "cox_batch_size_partial_effect.pdf" # "covariate_array": "model.trainer.batch_size" # "values_array": [1,10,25,50] -# "title": "$S(t)$ for Cox AFR" +# "title": "$S(t)$ for Cox AFT" # "ylabel": "$\\mathbb{P}~(T>t)$" # "xlabel": "Time $t$ (seconds)" # "legend_kwargs": { @@ -78,7 +78,7 @@ weibull: # - "file" : cox_attack_eps_partial_effect.pdf # "covariate_array" : attack.init.eps # "values_array" : [1e-4, 1e-3, 1e-2, 1e-1, 1] -# "title": "$S(t)$ for Cox AFR" +# "title": "$S(t)$ for Cox AFT" # "ylabel": "$\\mathbb{P}~(T>t)$" # "xlabel": "Time $t$ (seconds)" # "legend_kwargs": { @@ -88,7 +88,7 @@ weibull: log_logistic: plot: file : log_logistic_aft.pdf - title : Log logistic AFR Model + title : Log logistic AFT Model labels: "Intercept: beta_": "$\\beta$" "Intercept: alpha_": "$\\alpha$" @@ -110,7 +110,7 @@ log_logistic: log_normal: plot: file : log_normal_aft.pdf - title : Log-Normal AFR Model + title : Log-Normal AFT Model labels: "Intercept: sigma_": "$\\sigma$" "Intercept: mu_": "$\\mu$" diff --git a/examples/pytorch/cifar10/.dvc/tmp/rwlock b/examples/pytorch/cifar10/.dvc/tmp/rwlock deleted file mode 100644 index 0967ef42..00000000 --- a/examples/pytorch/cifar10/.dvc/tmp/rwlock +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/examples/pytorch/cifar100.yaml b/examples/pytorch/cifar100.yaml deleted file mode 100644 index e69de29b..00000000 diff --git a/examples/pytorch/conf/cifar.yaml b/examples/pytorch/conf/cifar.yaml index 1c960bb6..b1854777 100644 --- a/examples/pytorch/conf/cifar.yaml +++ b/examples/pytorch/conf/cifar.yaml @@ -23,20 +23,22 @@ hydra: run: dir: ${files.directory}/logs/${stage}/ sweep: - dir: ${files.directory}/logs/${stage}/${model_name} + dir: ${files.directory}/logs/${stage}/${model_name}/${model.trainer.nb_epochs} subdir : ${def_name}/${atk_name}/${hydra.job.num} sweeper: sampler: _target_: optuna.samplers.GridSampler direction: ${direction} - study_name: ${model_name}_${def_name}_${atk_name} - storage: sqlite:///${dataset}.db - n_jobs: ${oc.env:HYDRA_SWEEPER_N_JOBS, 8} + study_name: ${dataset}_${model_name}_${def_name}_${atk_name} + storage: sqlite:///optuna.db + n_jobs: ${oc.env:HYDRA_SWEEPER_N_JOBS, 32} n_trials: ${oc.env:HYDRA_SWEEPER_N_TRIALS, 128} max_failure_rate: 1.0 params: - ++model.art.initialize.optimizer.lr: choice( 0.1, 0.01, 0.001, .0001, .00001, 0.000001) - ++model.trainer.nb_epoch: choice(1, 10, 30, 50, 100) + ++model.art.initialize.optimizer.lr: shuffle(choice( 0.1, 0.01, 0.001, .0001, .00001, 0.000001)) + ++model.art.initialize.optimizer.momentum: choice(0.1, 0.9, 0.95, 0.99) + ++model.trainer.nb_epochs: choice(1, 10, 30, 50, 100) + ++model.trainer.batch_size: choice(128, 256, 512, 1024) _target_: hydra_plugins.hydra_optuna_sweeper.optuna_sweeper.OptunaSweeper launcher: _target_: hydra_plugins.hydra_joblib_launcher.joblib_launcher.JoblibLauncher diff --git a/examples/pytorch/conf/cifar100.yaml b/examples/pytorch/conf/cifar100.yaml index a595d9e7..be555e9c 100644 --- a/examples/pytorch/conf/cifar100.yaml +++ b/examples/pytorch/conf/cifar100.yaml @@ -23,21 +23,21 @@ hydra: run: dir: ${files.directory}/logs/${stage}/ sweep: - dir: ${files.directory}/logs/${stage}/${model_name}/${model.trainer.nb_epochs} - subdir : ${def_name}/${atk_name}/${hydra.job.num} + dir: ${files.directory}/logs/${stage}/${model_name}/${atk_name}/${model.trainer.nb_epochs}/ + subdir : ${def_name}/${hydra.job.num} sweeper: sampler: _target_: optuna.samplers.GridSampler direction: ${direction} - study_name: ${model_name}_${def_name}_${atk_name} - storage: sqlite:///${dataset}.db + study_name: ${dataset}_${model_name}_${def_name}_${atk_name} + storage: sqlite:///optuna.db n_jobs: ${oc.env:HYDRA_SWEEPER_N_JOBS, 16} n_trials: ${oc.env:HYDRA_SWEEPER_N_TRIALS, 128} max_failure_rate: 1.0 params: ++model.art.initialize.optimizer.lr: choice( 0.1, 0.01, 0.001, .0001, .00001, 0.000001) ++model.art.initialize.optimizer.momentum: choice(0.1, 0.9, 0.95, 0.99) - # ++model.trainer.nb_epochs: choice(1, 10, 30, 50, 100) + ++model.trainer.nb_epochs: choice(1, 10, 30, 50, 100) ++model.trainer.batch_size: choice(128, 256, 512, 1024) _target_: hydra_plugins.hydra_optuna_sweeper.optuna_sweeper.OptunaSweeper launcher: diff --git a/examples/pytorch/conf/mnist.yaml b/examples/pytorch/conf/mnist.yaml index ab0ebc7e..1686123b 100644 --- a/examples/pytorch/conf/mnist.yaml +++ b/examples/pytorch/conf/mnist.yaml @@ -23,21 +23,21 @@ hydra: run: dir: ${files.directory}/logs/${stage}/ sweep: - dir: ${files.directory}/logs/${stage}/${model_name}/${model.trainer.nb_epochs} - subdir : ${def_name}/${atk_name}/${hydra.job.num} + dir: ${files.directory}/logs/${stage}/${model_name}/${atk_name}/${model.trainer.nb_epochs}/ + subdir : ${def_name}/${hydra.job.num} sweeper: sampler: _target_: optuna.samplers.GridSampler direction: ${direction} - study_name: ${model_name}_${def_name}_${atk_name} - storage: sqlite:///${dataset}.db + study_name: ${dataset}_${model_name}_${def_name}_${atk_name} + storage: sqlite:///optuna.db n_jobs: ${oc.env:HYDRA_SWEEPER_N_JOBS, 32} n_trials: ${oc.env:HYDRA_SWEEPER_N_TRIALS, 128} max_failure_rate: 1.0 params: ++model.art.initialize.optimizer.lr: choice( 0.1, 0.01, 0.001, .0001, .00001, 0.000001) ++model.art.initialize.optimizer.momentum: choice(0.1, 0.9, 0.95, 0.99) - # ++model.trainer.nb_epochs: choice(1, 10, 30, 50, 100) + ++model.trainer.nb_epochs: choice(1, 10, 30, 50, 100) ++model.trainer.batch_size: choice(128, 256, 512, 1024) _target_: hydra_plugins.hydra_optuna_sweeper.optuna_sweeper.OptunaSweeper launcher: diff --git a/examples/pytorch/dvc.lock b/examples/pytorch/dvc.lock deleted file mode 100644 index 8bbbfc23..00000000 --- a/examples/pytorch/dvc.lock +++ /dev/null @@ -1,1922 +0,0 @@ -schema: '2.0' -stages: - train@mnist: - cmd: python -m deckard.layers.experiment train@mnist --config_file mnist.yaml - --params_file mnist.yaml - deps: - - path: mnist.yaml - hash: md5 - md5: 494df904ccf4eb077c0642efde302fee - size: 6005 - params: - params.yaml: - atk_name: hsj - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - def_name: control - device_id: gpu - files: - _target_: deckard.base.files.FileConfig - adv_predictions_file: adv_predictions.json - attack_dir: attacks - attack_file: attack - attack_type: .pkl - directory: mnist - model_dir: models - model_file: model - model_type: .pt - name: default - params_file: params.yaml - predictions_file: predictions.json - reports: reports - score_dict_file: score_dict.json - model: - _target_: deckard.base.model.Model - art: - _target_: deckard.base.model.art_pipeline.ArtPipeline - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - initialize: - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - library: pytorch - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - init: - _target_: deckard.base.model.ModelInitializer - name: torch_example.ResNet18 - num_channels: 1 - num_classes: 10 - library: pytorch - trainer: - batch_size: 1024 - nb_epochs: 1 - verbose: true - scorers: - _target_: deckard.base.scorer.ScorerDict - accuracy: - _target_: deckard.base.scorer.ScorerConfig - direction: maximize - name: sklearn.metrics.accuracy_score - log_loss: - _target_: deckard.base.scorer.ScorerConfig - direction: minimize - name: sklearn.metrics.log_loss - outs: - - path: mnist/models/model.optimizer.pt - hash: md5 - md5: ba33d30facc5b91aba2d608d37ba771f - size: 44781294 - - path: mnist/models/model.pt - hash: md5 - md5: a6ae9e91474793326c6a3c139866292f - size: 44787138 - - path: mnist/reports/train/default/score_dict.json - hash: md5 - md5: 1f33102e4833438d6a2151c8eb49d4e7 - size: 525 - train@cifar: - cmd: python -m deckard.layers.experiment train@cifar --config_file cifar.yaml - --params_file cifar.yaml - deps: - - path: cifar.yaml - hash: md5 - md5: 15fc7cec034cfea8e390c77f0ebd2d26 - size: 5037 - params: - params.yaml: - atk_name: hsj - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - def_name: control - device_id: gpu - files: - _target_: deckard.base.files.FileConfig - adv_predictions_file: adv_predictions.json - attack_dir: attacks - attack_file: attack - attack_type: .pkl - directory: mnist - model_dir: models - model_file: model - model_type: .pt - name: default - params_file: params.yaml - predictions_file: predictions.json - reports: reports - score_dict_file: score_dict.json - model: - _target_: deckard.base.model.Model - art: - _target_: deckard.base.model.art_pipeline.ArtPipeline - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - initialize: - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - library: pytorch - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - init: - _target_: deckard.base.model.ModelInitializer - name: torch_example.ResNet18 - num_channels: 1 - num_classes: 10 - library: pytorch - trainer: - batch_size: 1024 - nb_epochs: 1 - verbose: true - scorers: - _target_: deckard.base.scorer.ScorerDict - accuracy: - _target_: deckard.base.scorer.ScorerConfig - direction: maximize - name: sklearn.metrics.accuracy_score - log_loss: - _target_: deckard.base.scorer.ScorerConfig - direction: minimize - name: sklearn.metrics.log_loss - outs: - - path: cifar/models/model.optimizer.pt - hash: md5 - md5: 48311974bdb089e3912a018165e588f1 - size: 44781294 - - path: cifar/models/model.pt - hash: md5 - md5: 65ee5487dd484d5bb26fde29dd7a6311 - size: 44787138 - - path: cifar/reports/train/default/score_dict.json - hash: md5 - md5: 5f47af0c6e48142318d161e0dae6eb06 - size: 526 - train@cifar100: - cmd: python -m deckard.layers.experiment train@cifar100 --config_file cifar100.yaml - --params_file cifar100.yaml - deps: - - path: cifar100.yaml - hash: md5 - md5: deb1b04848c71b5819cf21880b4368f0 - size: 5282 - params: - params.yaml: - atk_name: hsj - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - def_name: control - device_id: gpu - files: - _target_: deckard.base.files.FileConfig - adv_predictions_file: adv_predictions.json - attack_dir: attacks - attack_file: attack - attack_type: .pkl - directory: mnist - model_dir: models - model_file: model - model_type: .pt - name: default - params_file: params.yaml - predictions_file: predictions.json - reports: reports - score_dict_file: score_dict.json - model: - _target_: deckard.base.model.Model - art: - _target_: deckard.base.model.art_pipeline.ArtPipeline - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - initialize: - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - library: pytorch - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - init: - _target_: deckard.base.model.ModelInitializer - name: torch_example.ResNet18 - num_channels: 1 - num_classes: 10 - library: pytorch - trainer: - batch_size: 1024 - nb_epochs: 1 - verbose: true - scorers: - _target_: deckard.base.scorer.ScorerDict - accuracy: - _target_: deckard.base.scorer.ScorerConfig - direction: maximize - name: sklearn.metrics.accuracy_score - log_loss: - _target_: deckard.base.scorer.ScorerConfig - direction: minimize - name: sklearn.metrics.log_loss - outs: - - path: cifar100/models/model.optimizer.pt - hash: md5 - md5: 1cc5f1335ddd96097060130f3508c04f - size: 44781294 - - path: cifar100/models/model.pt - hash: md5 - md5: 926a0e9bfe98ceef42fb9bc060c54218 - size: 44787138 - - path: cifar100/reports/train/default/score_dict.json - hash: md5 - md5: 8871d4ed73592450b4ceaa09fdfb4c0f - size: 513 - attack@mnist: - cmd: python -m deckard.layers.experiment attack@mnist --config_file mnist.yaml - --params_file mnist.yaml - deps: - - path: mnist/models/model.optimizer.pt - hash: md5 - md5: ba33d30facc5b91aba2d608d37ba771f - size: 44781294 - - path: mnist/models/model.pt - hash: md5 - md5: a6ae9e91474793326c6a3c139866292f - size: 44787138 - params: - params.yaml: - atk_name: hsj - attack: - _target_: deckard.base.attack.Attack - attack_size: 100 - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - init: - _target_: deckard.base.attack.AttackInitializer - batch_size: 1024 - model: - _target_: deckard.base.model.Model - art: - _target_: deckard.base.model.art_pipeline.ArtPipeline - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - initialize: - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - library: pytorch - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - init: - _target_: deckard.base.model.ModelInitializer - name: torch_example.ResNet18 - num_channels: 1 - num_classes: 10 - library: pytorch - trainer: - batch_size: 1024 - nb_epochs: 1 - verbose: true - name: art.attacks.evasion.HopSkipJump - method: evasion - model: - _target_: deckard.base.model.Model - art: - _target_: deckard.base.model.art_pipeline.ArtPipeline - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - initialize: - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - library: pytorch - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - init: - _target_: deckard.base.model.ModelInitializer - name: torch_example.ResNet18 - num_channels: 1 - num_classes: 10 - library: pytorch - trainer: - batch_size: 1024 - nb_epochs: 1 - verbose: true - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - dataset: mnist - def_name: control - device_id: gpu - files: - _target_: deckard.base.files.FileConfig - adv_predictions_file: adv_predictions.json - attack_dir: attacks - attack_file: attack - attack_type: .pkl - directory: mnist - model_dir: models - model_file: model - model_type: .pt - name: default - params_file: params.yaml - predictions_file: predictions.json - reports: reports - score_dict_file: score_dict.json - model: - _target_: deckard.base.model.Model - art: - _target_: deckard.base.model.art_pipeline.ArtPipeline - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - initialize: - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - library: pytorch - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - init: - _target_: deckard.base.model.ModelInitializer - name: torch_example.ResNet18 - num_channels: 1 - num_classes: 10 - library: pytorch - trainer: - batch_size: 1024 - nb_epochs: 1 - verbose: true - scorers: - _target_: deckard.base.scorer.ScorerDict - accuracy: - _target_: deckard.base.scorer.ScorerConfig - direction: maximize - name: sklearn.metrics.accuracy_score - log_loss: - _target_: deckard.base.scorer.ScorerConfig - direction: minimize - name: sklearn.metrics.log_loss - outs: - - path: mnist/attacks/attack.pkl - hash: md5 - md5: 0d9b4dcaaffbc077dec93a4ae8c69171 - size: 313766 - - path: mnist/reports/attack/default/score_dict.json - hash: md5 - md5: d38a998a2b08eff46f3bab06e7c77d64 - size: 851 - parse@mnist: - cmd: python -m deckard.layers.parse --config_file mnist.yaml --params_file mnist.yaml - deps: - - path: conf/attack - hash: md5 - md5: f5637e0106257966dd59bca49db76523.dir - size: 268 - nfiles: 1 - - path: conf/data - hash: md5 - md5: eafea1e119e2f54e9788ee46101afc79.dir - size: 645 - nfiles: 4 - - path: conf/files - hash: md5 - md5: b10c002322e63c5ed04e88dda4f5b9fa.dir - size: 1530 - nfiles: 3 - - path: conf/mnist.yaml - hash: md5 - md5: 73ed848330388ef5d98a54d03e3cdcf9 - size: 1671 - - path: conf/model - hash: md5 - md5: eb0d17ad4826334ac8488c1683106580.dir - size: 2095 - nfiles: 10 - - path: conf/scorers - hash: md5 - md5: 1316006b6b9fbc05fd8ed56d46e15718.dir - size: 279 - nfiles: 1 - outs: - - path: mnist.yaml - hash: md5 - md5: 494df904ccf4eb077c0642efde302fee - size: 6005 - parse@cifar: - cmd: python -m deckard.layers.parse --config_file cifar.yaml --params_file cifar.yaml - deps: - - path: conf/attack - hash: md5 - md5: f5637e0106257966dd59bca49db76523.dir - size: 268 - nfiles: 1 - - path: conf/cifar.yaml - hash: md5 - md5: 5171900fd679f57392fa59f160301911 - size: 1681 - - path: conf/data - hash: md5 - md5: eafea1e119e2f54e9788ee46101afc79.dir - size: 645 - nfiles: 4 - - path: conf/files - hash: md5 - md5: b10c002322e63c5ed04e88dda4f5b9fa.dir - size: 1530 - nfiles: 3 - - path: conf/model - hash: md5 - md5: eb0d17ad4826334ac8488c1683106580.dir - size: 2095 - nfiles: 10 - - path: conf/scorers - hash: md5 - md5: 1316006b6b9fbc05fd8ed56d46e15718.dir - size: 279 - nfiles: 1 - outs: - - path: cifar.yaml - hash: md5 - md5: 15fc7cec034cfea8e390c77f0ebd2d26 - size: 5037 - parse@cifar100: - cmd: python -m deckard.layers.parse --config_file cifar100.yaml --params_file - cifar100.yaml - deps: - - path: conf/attack - hash: md5 - md5: f5637e0106257966dd59bca49db76523.dir - size: 268 - nfiles: 1 - - path: conf/cifar100.yaml - hash: md5 - md5: 38d9e68b647acbca29a89ef6a699b4bf - size: 1684 - - path: conf/data - hash: md5 - md5: eafea1e119e2f54e9788ee46101afc79.dir - size: 645 - nfiles: 4 - - path: conf/files - hash: md5 - md5: b10c002322e63c5ed04e88dda4f5b9fa.dir - size: 1530 - nfiles: 3 - - path: conf/model - hash: md5 - md5: eb0d17ad4826334ac8488c1683106580.dir - size: 2095 - nfiles: 10 - - path: conf/scorers - hash: md5 - md5: 1316006b6b9fbc05fd8ed56d46e15718.dir - size: 279 - nfiles: 1 - outs: - - path: cifar100.yaml - hash: md5 - md5: deb1b04848c71b5819cf21880b4368f0 - size: 5282 - attack@cifar: - cmd: python -m deckard.layers.experiment attack@cifar --config_file cifar.yaml - --params_file cifar.yaml - deps: - - path: cifar/models/model.optimizer.pt - hash: md5 - md5: 48311974bdb089e3912a018165e588f1 - size: 44781294 - - path: cifar/models/model.pt - hash: md5 - md5: 65ee5487dd484d5bb26fde29dd7a6311 - size: 44787138 - params: - params.yaml: - atk_name: hsj - attack: - _target_: deckard.base.attack.Attack - attack_size: 100 - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - init: - _target_: deckard.base.attack.AttackInitializer - batch_size: 1024 - model: - _target_: deckard.base.model.Model - art: - _target_: deckard.base.model.art_pipeline.ArtPipeline - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - initialize: - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - library: pytorch - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - init: - _target_: deckard.base.model.ModelInitializer - name: torch_example.ResNet18 - num_channels: 1 - num_classes: 10 - library: pytorch - trainer: - batch_size: 1024 - nb_epochs: 1 - verbose: true - name: art.attacks.evasion.HopSkipJump - method: evasion - model: - _target_: deckard.base.model.Model - art: - _target_: deckard.base.model.art_pipeline.ArtPipeline - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - initialize: - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - library: pytorch - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - init: - _target_: deckard.base.model.ModelInitializer - name: torch_example.ResNet18 - num_channels: 1 - num_classes: 10 - library: pytorch - trainer: - batch_size: 1024 - nb_epochs: 1 - verbose: true - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - dataset: mnist - def_name: control - device_id: gpu - files: - _target_: deckard.base.files.FileConfig - adv_predictions_file: adv_predictions.json - attack_dir: attacks - attack_file: attack - attack_type: .pkl - directory: mnist - model_dir: models - model_file: model - model_type: .pt - name: default - params_file: params.yaml - predictions_file: predictions.json - reports: reports - score_dict_file: score_dict.json - model: - _target_: deckard.base.model.Model - art: - _target_: deckard.base.model.art_pipeline.ArtPipeline - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - initialize: - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - library: pytorch - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - init: - _target_: deckard.base.model.ModelInitializer - name: torch_example.ResNet18 - num_channels: 1 - num_classes: 10 - library: pytorch - trainer: - batch_size: 1024 - nb_epochs: 1 - verbose: true - scorers: - _target_: deckard.base.scorer.ScorerDict - accuracy: - _target_: deckard.base.scorer.ScorerConfig - direction: maximize - name: sklearn.metrics.accuracy_score - log_loss: - _target_: deckard.base.scorer.ScorerConfig - direction: minimize - name: sklearn.metrics.log_loss - outs: - - path: cifar/attacks/attack.pkl - hash: md5 - md5: 974de8139d9ce86187cf824f00e80d95 - size: 313766 - - path: cifar/reports/attack/default/score_dict.json - hash: md5 - md5: 46802563046219bac6f958866dda96f5 - size: 852 - attack@cifar100: - cmd: python -m deckard.layers.experiment attack@cifar100 --config_file cifar100.yaml - --params_file cifar100.yaml - deps: - - path: cifar100/models/model.optimizer.pt - hash: md5 - md5: 1cc5f1335ddd96097060130f3508c04f - size: 44781294 - - path: cifar100/models/model.pt - hash: md5 - md5: 926a0e9bfe98ceef42fb9bc060c54218 - size: 44787138 - params: - params.yaml: - atk_name: hsj - attack: - _target_: deckard.base.attack.Attack - attack_size: 100 - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - init: - _target_: deckard.base.attack.AttackInitializer - batch_size: 1024 - model: - _target_: deckard.base.model.Model - art: - _target_: deckard.base.model.art_pipeline.ArtPipeline - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - initialize: - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - library: pytorch - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - init: - _target_: deckard.base.model.ModelInitializer - name: torch_example.ResNet18 - num_channels: 1 - num_classes: 10 - library: pytorch - trainer: - batch_size: 1024 - nb_epochs: 1 - verbose: true - name: art.attacks.evasion.HopSkipJump - method: evasion - model: - _target_: deckard.base.model.Model - art: - _target_: deckard.base.model.art_pipeline.ArtPipeline - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - initialize: - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - library: pytorch - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - init: - _target_: deckard.base.model.ModelInitializer - name: torch_example.ResNet18 - num_channels: 1 - num_classes: 10 - library: pytorch - trainer: - batch_size: 1024 - nb_epochs: 1 - verbose: true - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - dataset: mnist - def_name: control - device_id: gpu - files: - _target_: deckard.base.files.FileConfig - adv_predictions_file: adv_predictions.json - attack_dir: attacks - attack_file: attack - attack_type: .pkl - directory: mnist - model_dir: models - model_file: model - model_type: .pt - name: default - params_file: params.yaml - predictions_file: predictions.json - reports: reports - score_dict_file: score_dict.json - model: - _target_: deckard.base.model.Model - art: - _target_: deckard.base.model.art_pipeline.ArtPipeline - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - initialize: - clip_values: - - 0 - - 255 - criterion: - name: torch.nn.CrossEntropyLoss - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - library: pytorch - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - data: - _target_: deckard.base.data.Data - generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist - sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state: 0 - stratify: true - test_size: 12000 - train_size: 48000 - init: - _target_: deckard.base.model.ModelInitializer - name: torch_example.ResNet18 - num_channels: 1 - num_classes: 10 - library: pytorch - trainer: - batch_size: 1024 - nb_epochs: 1 - verbose: true - scorers: - _target_: deckard.base.scorer.ScorerDict - accuracy: - _target_: deckard.base.scorer.ScorerConfig - direction: maximize - name: sklearn.metrics.accuracy_score - log_loss: - _target_: deckard.base.scorer.ScorerConfig - direction: minimize - name: sklearn.metrics.log_loss - outs: - - path: cifar100/attacks/attack.pkl - hash: md5 - md5: 7b3384f27a7edde0d3813ffb297e4123 - size: 313766 - - path: cifar100/reports/attack/default/score_dict.json - hash: md5 - md5: c2f9e1e3db07148a510a2dd237d22218 - size: 828 - attacks@mnist-ResNet18: - cmd: bash attacks.sh ++model.init.name=torch_example.ResNet18 stage=attack model_name=ResNet18 - attack.attack_size=100 data=torch_mnist model=torch_mnist +direction="[maximize,maximize,minimize]" - +optimizers="[accuracy,adv_accuracy,adv_success]" --config-name mnist.yaml - deps: - - path: mnist/reports/attack/default/score_dict.json - hash: md5 - md5: d38a998a2b08eff46f3bab06e7c77d64 - size: 851 - - path: mnist/reports/train/default/score_dict.json - hash: md5 - md5: 1f33102e4833438d6a2151c8eb49d4e7 - size: 525 - outs: - - path: mnist/logs/attack/ResNet18/ - hash: md5 - md5: 21ccd321fb271ef569c8e2844d3a580c.dir - size: 539921753 - nfiles: 61569 - attacks@mnist-ResNet50: - cmd: bash attacks.sh ++model.init.name=torch_example.ResNet50 stage=attack model_name=ResNet50 - attack.attack_size=100 data=torch_mnist model=torch_mnist +direction="[maximize,maximize,minimize]" - +optimizers="[accuracy,adv_accuracy,adv_success]" --config-name mnist.yaml - deps: - - path: mnist/reports/attack/default/score_dict.json - hash: md5 - md5: d38a998a2b08eff46f3bab06e7c77d64 - size: 851 - - path: mnist/reports/train/default/score_dict.json - hash: md5 - md5: 1f33102e4833438d6a2151c8eb49d4e7 - size: 525 - outs: - - path: mnist/logs/attack/ResNet50/ - hash: md5 - md5: 31a2f23ef4d25e0fb50579ddc5b6a111.dir - size: 238399439 - nfiles: 28729 - attacks@mnist-ResNet152: - cmd: bash attacks.sh ++model.init.name=torch_example.ResNet152 stage=attack - model_name=ResNet152 attack.attack_size=100 data=torch_mnist model=torch_mnist - +direction="[maximize,maximize,minimize]" +optimizers="[accuracy,adv_accuracy,adv_success]" - --config-name mnist.yaml - deps: - - path: mnist/reports/attack/default/score_dict.json - hash: md5 - md5: b63b8d925ab6db88e3b707ec025800d3 - size: 834 - - path: mnist/reports/train/default/score_dict.json - hash: md5 - md5: f55673763c20d36336fc1a23bd057a5f - size: 515 - outs: - - path: mnist/logs/attack/ResNet152/ - hash: md5 - md5: caf83ba317d2b12ecc75f638dc07e7fb.dir - size: 152486967 - nfiles: 7681 - attacks@mnist-ResNet101: - cmd: bash attacks.sh ++model.init.name=torch_example.ResNet101 stage=attack - model_name=ResNet101 attack.attack_size=100 data=torch_mnist model=torch_mnist - +direction="[maximize,maximize,minimize]" +optimizers="[accuracy,adv_accuracy,adv_success]" - --config-name mnist.yaml - deps: - - path: mnist/reports/attack/default/score_dict.json - hash: md5 - md5: d38a998a2b08eff46f3bab06e7c77d64 - size: 851 - - path: mnist/reports/train/default/score_dict.json - hash: md5 - md5: 1f33102e4833438d6a2151c8eb49d4e7 - size: 525 - outs: - - path: mnist/logs/attack/ResNet101/ - hash: md5 - md5: 14ed49640893e98063aa73c996db5652.dir - size: 259212024 - nfiles: 25349 - attacks@mnist-ResNet34: - cmd: bash attacks.sh ++model.init.name=torch_example.ResNet34 stage=attack model_name=ResNet34 - attack.attack_size=100 data=torch_mnist model=torch_mnist +direction="[maximize,maximize,minimize]" - +optimizers="[accuracy,adv_accuracy,adv_success]" --config-name mnist.yaml - deps: - - path: mnist/reports/attack/default/score_dict.json - hash: md5 - md5: d38a998a2b08eff46f3bab06e7c77d64 - size: 851 - - path: mnist/reports/train/default/score_dict.json - hash: md5 - md5: 1f33102e4833438d6a2151c8eb49d4e7 - size: 525 - outs: - - path: mnist/logs/attack/ResNet34/ - hash: md5 - md5: f18222865d4ced43dff21f5086e83c3c.dir - size: 258848998 - nfiles: 31817 - attacks@cifar-ResNet18: - cmd: bash attacks.sh ++model.init.name=torch_example.ResNet18 stage=attack model_name=ResNet18 - attack.attack_size=100 data=torch_cifar model=torch_cifar +direction="[maximize,maximize,minimize]" - +optimizers="[accuracy,adv_accuracy,adv_success]" --config-name cifar.yaml - deps: - - path: cifar/reports/attack/default/score_dict.json - hash: md5 - md5: 6aae48892e49e75742b1d67a5e1ec276 - size: 838 - - path: cifar/reports/train/default/score_dict.json - hash: md5 - md5: 73c3315c8f9def00a519bdf6bb9998e2 - size: 508 - outs: - - path: cifar/logs/attack/ResNet18/ - hash: md5 - md5: 1f8b0fd2b4e94d6e67d017d9ae9d4ea0.dir - size: 66807197 - nfiles: 9345 - attacks@cifar-ResNet34: - cmd: bash attacks.sh ++model.init.name=torch_example.ResNet34 stage=attack model_name=ResNet34 - attack.attack_size=100 data=torch_cifar model=torch_cifar +direction="[maximize,maximize,minimize]" - +optimizers="[accuracy,adv_accuracy,adv_success]" --config-name cifar.yaml - deps: - - path: cifar/reports/attack/default/score_dict.json - hash: md5 - md5: 6aae48892e49e75742b1d67a5e1ec276 - size: 838 - - path: cifar/reports/train/default/score_dict.json - hash: md5 - md5: 73c3315c8f9def00a519bdf6bb9998e2 - size: 508 - outs: - - path: cifar/logs/attack/ResNet34/ - hash: md5 - md5: ca652fb37adbb2bc48c785d3db8cb90e.dir - size: 57831382 - nfiles: 7681 - attacks@cifar-ResNet50: - cmd: bash attacks.sh ++model.init.name=torch_example.ResNet50 stage=attack model_name=ResNet50 - attack.attack_size=100 data=torch_cifar model=torch_cifar +direction="[maximize,maximize,minimize]" - +optimizers="[accuracy,adv_accuracy,adv_success]" --config-name cifar.yaml - deps: - - path: cifar/reports/attack/default/score_dict.json - hash: md5 - md5: 6aae48892e49e75742b1d67a5e1ec276 - size: 838 - - path: cifar/reports/train/default/score_dict.json - hash: md5 - md5: 73c3315c8f9def00a519bdf6bb9998e2 - size: 508 - outs: - - path: cifar/logs/attack/ResNet50/ - hash: md5 - md5: 63b0e940b64d9b941c4877ff05ceb4e6.dir - size: 61565958 - nfiles: 7681 - attacks@cifar-ResNet101: - cmd: bash attacks.sh ++model.init.name=torch_example.ResNet101 stage=attack - model_name=ResNet101 attack.attack_size=100 data=torch_cifar model=torch_cifar - +direction="[maximize,maximize,minimize]" +optimizers="[accuracy,adv_accuracy,adv_success]" - --config-name cifar.yaml - deps: - - path: cifar/reports/attack/default/score_dict.json - hash: md5 - md5: 6aae48892e49e75742b1d67a5e1ec276 - size: 838 - - path: cifar/reports/train/default/score_dict.json - hash: md5 - md5: 73c3315c8f9def00a519bdf6bb9998e2 - size: 508 - outs: - - path: cifar/logs/attack/ResNet101/ - hash: md5 - md5: 6899712a5e067be74511fd4b85c5531b.dir - size: 80647437 - nfiles: 7681 - attacks@cifar-ResNet152: - cmd: bash attacks.sh ++model.init.name=torch_example.ResNet152 stage=attack - model_name=ResNet152 attack.attack_size=100 data=torch_cifar model=torch_cifar - +direction="[maximize,maximize,minimize]" +optimizers="[accuracy,adv_accuracy,adv_success]" - --config-name cifar.yaml - deps: - - path: cifar/reports/attack/default/score_dict.json - hash: md5 - md5: 6aae48892e49e75742b1d67a5e1ec276 - size: 838 - - path: cifar/reports/train/default/score_dict.json - hash: md5 - md5: 73c3315c8f9def00a519bdf6bb9998e2 - size: 508 - outs: - - path: cifar/logs/attack/ResNet152/ - hash: md5 - md5: 08cbdcd3ef354342a69c7bbd361ab6f4.dir - size: 101987846 - nfiles: 7681 - attacks@cifar100-ResNet18: - cmd: bash attacks.sh ++model.init.name=torch_example.ResNet18 stage=attack model_name=ResNet18 - attack.attack_size=100 data=torch_cifar100 model=torch_cifar100 +direction="[maximize,maximize,minimize]" - +optimizers="[accuracy,adv_accuracy,adv_success]" --config-name cifar100.yaml - deps: - - path: cifar100/reports/attack/default/score_dict.json - hash: md5 - md5: 7396ec38b31f7ff9ae1fcab467a98c6c - size: 854 - - path: cifar100/reports/train/default/score_dict.json - hash: md5 - md5: 41191be9e2760b49f21bfe27ccef654c - size: 514 - outs: - - path: cifar100/logs/attack/ResNet18/ - hash: md5 - md5: f6fc033b064588576514c93b8901f040.dir - size: 84069191 - nfiles: 10469 - attacks@cifar100-ResNet34: - cmd: bash attacks.sh ++model.init.name=torch_example.ResNet34 stage=attack model_name=ResNet34 - attack.attack_size=100 data=torch_cifar100 model=torch_cifar100 +direction="[maximize,maximize,minimize]" - +optimizers="[accuracy,adv_accuracy,adv_success]" --config-name cifar100.yaml - deps: - - path: cifar100/reports/attack/default/score_dict.json - hash: md5 - md5: 7396ec38b31f7ff9ae1fcab467a98c6c - size: 854 - - path: cifar100/reports/train/default/score_dict.json - hash: md5 - md5: 41191be9e2760b49f21bfe27ccef654c - size: 514 - outs: - - path: cifar100/logs/attack/ResNet34/ - hash: md5 - md5: 40ab74741f8f446b09778c55522e0d23.dir - size: 45326629 - nfiles: 7681 - attacks@cifar100-ResNet50: - cmd: bash attacks.sh ++model.init.name=torch_example.ResNet50 stage=attack model_name=ResNet50 - attack.attack_size=100 data=torch_cifar100 model=torch_cifar100 +direction="[maximize,maximize,minimize]" - +optimizers="[accuracy,adv_accuracy,adv_success]" --config-name cifar100.yaml - deps: - - path: cifar100/reports/attack/default/score_dict.json - hash: md5 - md5: 7396ec38b31f7ff9ae1fcab467a98c6c - size: 854 - - path: cifar100/reports/train/default/score_dict.json - hash: md5 - md5: 41191be9e2760b49f21bfe27ccef654c - size: 514 - outs: - - path: cifar100/logs/attack/ResNet50/ - hash: md5 - md5: d74f031721b03ab0b6d0eeab26fdfba3.dir - size: 46511652 - nfiles: 7681 - attacks@cifar100-ResNet101: - cmd: bash attacks.sh ++model.init.name=torch_example.ResNet101 stage=attack - model_name=ResNet101 attack.attack_size=100 data=torch_cifar100 model=torch_cifar100 - +direction="[maximize,maximize,minimize]" +optimizers="[accuracy,adv_accuracy,adv_success]" - --config-name cifar100.yaml - deps: - - path: cifar100/reports/attack/default/score_dict.json - hash: md5 - md5: 7396ec38b31f7ff9ae1fcab467a98c6c - size: 854 - - path: cifar100/reports/train/default/score_dict.json - hash: md5 - md5: 41191be9e2760b49f21bfe27ccef654c - size: 514 - outs: - - path: cifar100/logs/attack/ResNet101/ - hash: md5 - md5: 57911c3eaeeb3888a220a1fcf12e8442.dir - size: 63892972 - nfiles: 7681 - attacks@cifar100-ResNet152: - cmd: bash attacks.sh ++model.init.name=torch_example.ResNet152 stage=attack - model_name=ResNet152 attack.attack_size=100 data=torch_cifar100 model=torch_cifar100 - +direction="[maximize,maximize,minimize]" +optimizers="[accuracy,adv_accuracy,adv_success]" - --config-name cifar100.yaml - deps: - - path: cifar100/reports/attack/default/score_dict.json - hash: md5 - md5: 7396ec38b31f7ff9ae1fcab467a98c6c - size: 854 - - path: cifar100/reports/train/default/score_dict.json - hash: md5 - md5: 41191be9e2760b49f21bfe27ccef654c - size: 514 - outs: - - path: cifar100/logs/attack/ResNet152/ - hash: md5 - md5: bef4f0ac68bc29e2b41bd35ee86e5def.dir - size: 80996700 - nfiles: 7681 - compile@cifar-attack: - cmd: python -m deckard.layers.compile --report_folder cifar/reports/attack --results_file - cifar/reports/attack.csv - deps: - - path: cifar/logs/attack/ - hash: md5 - md5: cd234596a07f78c2d3bfc852a38af7ad.dir - size: 265902793 - nfiles: 38405 - - path: cifar/reports/attack/ - hash: md5 - md5: d655cde61b88f9e0d9e014f634ccf84b.dir - size: 76673481 - nfiles: 21300 - outs: - - path: cifar/reports/attack.csv - hash: md5 - md5: 70bcdb22a2abc085664a59873902a508 - size: 31567491 - compile@cifar100-attack: - cmd: python -m deckard.layers.compile --report_folder cifar100/reports/attack - --results_file cifar100/reports/attack.csv - deps: - - path: cifar100/logs/attack/ - hash: md5 - md5: 1f01af02a4a7d87769f4eddcedda6fb0.dir - size: 250371493 - nfiles: 38405 - - path: cifar100/reports/attack/ - hash: md5 - md5: ed1b20a41054766cb993c99a7d2cbf84.dir - size: 68252477 - nfiles: 19208 - outs: - - path: cifar100/reports/attack.csv - hash: md5 - md5: 6416228973971f0719f394c31e0e639d - size: 29836704 - prepare_plot_folder@cifar100-attack: - cmd: cp cifar100/reports/attack.csv plots/data/attack_cifar100.csv - deps: - - path: cifar100/reports/attack.csv - hash: md5 - md5: 6416228973971f0719f394c31e0e639d - size: 29836704 - outs: - - path: plots/data/attack_cifar100.csv - hash: md5 - md5: 6416228973971f0719f394c31e0e639d - size: 29836704 - prepare_plot_folder@cifar-attack: - cmd: cp cifar/reports/attack.csv plots/data/attack_cifar.csv - deps: - - path: cifar/reports/attack.csv - hash: md5 - md5: 70bcdb22a2abc085664a59873902a508 - size: 31567491 - outs: - - path: plots/data/attack_cifar.csv - hash: md5 - md5: 70bcdb22a2abc085664a59873902a508 - size: 31567491 - attacks@mnist-ResNet18-1: - cmd: bash attacks.sh ++model.init.name=torch_example.ResNet18 stage=attack model_name=ResNet18 - attack.attack_size=100 model.trainer.nb_epochs=1 data=torch_mnist model=torch_mnist - +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" - --config-name mnist.yaml - deps: - - path: mnist/reports/attack/default/score_dict.json - hash: md5 - md5: d38a998a2b08eff46f3bab06e7c77d64 - size: 851 - - path: mnist/reports/train/default/score_dict.json - hash: md5 - md5: 1f33102e4833438d6a2151c8eb49d4e7 - size: 525 - outs: - - path: mnist/logs/attack/ResNet18/1/ - hash: md5 - md5: c4e3a8b5197420f5f54553617bc20873.dir - size: 36422730 - nfiles: 7681 - attacks@mnist-ResNet18-10: - cmd: bash attacks.sh ++model.init.name=torch_example.ResNet18 stage=attack model_name=ResNet18 - attack.attack_size=100 model.trainer.nb_epochs=10 data=torch_mnist model=torch_mnist - +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" - --config-name mnist.yaml - deps: - - path: mnist/reports/attack/default/score_dict.json - hash: md5 - md5: d38a998a2b08eff46f3bab06e7c77d64 - size: 851 - - path: mnist/reports/train/default/score_dict.json - hash: md5 - md5: 1f33102e4833438d6a2151c8eb49d4e7 - size: 525 - outs: - - path: mnist/logs/attack/ResNet18/10/ - hash: md5 - md5: d624fd4cefda6b891493b29e876bdc52.dir - size: 32664620 - nfiles: 6037 - attacks@mnist-ResNet34-1: - cmd: bash attacks.sh ++model.init.name=torch_example.ResNet34 stage=attack model_name=ResNet34 - attack.attack_size=100 model.trainer.nb_epochs=1 data=torch_mnist model=torch_mnist - +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" - --config-name mnist.yaml - deps: - - path: mnist/reports/attack/default/score_dict.json - hash: md5 - md5: d38a998a2b08eff46f3bab06e7c77d64 - size: 851 - - path: mnist/reports/train/default/score_dict.json - hash: md5 - md5: 1f33102e4833438d6a2151c8eb49d4e7 - size: 525 - outs: - - path: mnist/logs/attack/ResNet34/1/ - hash: md5 - md5: e497d20650c616bd9a458e3ef5781f10.dir - size: 82396544 - nfiles: 9345 - attacks@mnist-ResNet50-1: - cmd: bash attacks.sh ++model.init.name=torch_example.ResNet50 stage=attack model_name=ResNet50 - attack.attack_size=100 model.trainer.nb_epochs=1 data=torch_mnist model=torch_mnist - +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" - --config-name mnist.yaml - deps: - - path: mnist/reports/attack/default/score_dict.json - hash: md5 - md5: d38a998a2b08eff46f3bab06e7c77d64 - size: 851 - - path: mnist/reports/train/default/score_dict.json - hash: md5 - md5: 1f33102e4833438d6a2151c8eb49d4e7 - size: 525 - outs: - - path: mnist/logs/attack/ResNet50/1/ - hash: md5 - md5: 4cff9becc018ddf89b6a912553fe237e.dir - size: 47146814 - nfiles: 7681 - attacks@mnist-ResNet101-1: - cmd: bash attacks.sh ++model.init.name=torch_example.ResNet101 stage=attack - model_name=ResNet101 attack.attack_size=100 model.trainer.nb_epochs=1 data=torch_mnist - model=torch_mnist +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" - --config-name mnist.yaml - deps: - - path: mnist/reports/attack/default/score_dict.json - hash: md5 - md5: d38a998a2b08eff46f3bab06e7c77d64 - size: 851 - - path: mnist/reports/train/default/score_dict.json - hash: md5 - md5: 1f33102e4833438d6a2151c8eb49d4e7 - size: 525 - outs: - - path: mnist/logs/attack/ResNet101/1/ - hash: md5 - md5: 8459e065681999a3e82830141acddd36.dir - size: 58634505 - nfiles: 7681 - attacks@mnist-ResNet152-1: - cmd: bash attacks.sh ++model.init.name=torch_example.ResNet152 stage=attack - model_name=ResNet152 attack.attack_size=100 model.trainer.nb_epochs=1 data=torch_mnist - model=torch_mnist +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" - --config-name mnist.yaml - deps: - - path: mnist/reports/attack/default/score_dict.json - hash: md5 - md5: d38a998a2b08eff46f3bab06e7c77d64 - size: 851 - - path: mnist/reports/train/default/score_dict.json - hash: md5 - md5: 1f33102e4833438d6a2151c8eb49d4e7 - size: 525 - outs: - - path: mnist/logs/attack/ResNet152/1/ - hash: md5 - md5: 3169ac92a9461de72740ec83d01dd0e1.dir - size: 76579893 - nfiles: 7681 - attacks@cifar-ResNet18-1: - cmd: bash attacks.sh ++model.init.name=torch_example.ResNet18 stage=attack model_name=ResNet18 - attack.attack_size=100 model.trainer.nb_epochs=1 data=torch_cifar model=torch_cifar - +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" - --config-name cifar.yaml - deps: - - path: cifar/reports/attack/default/score_dict.json - hash: md5 - md5: 46802563046219bac6f958866dda96f5 - size: 852 - - path: cifar/reports/train/default/score_dict.json - hash: md5 - md5: 5f47af0c6e48142318d161e0dae6eb06 - size: 526 - outs: - - path: cifar/logs/attack/ResNet18/1/ - hash: md5 - md5: 3b9bae7e98835391bd8683b8e138596c.dir - size: 36146151 - nfiles: 7681 - attacks@cifar-ResNet34-1: - cmd: bash attacks.sh ++model.init.name=torch_example.ResNet34 stage=attack model_name=ResNet34 - attack.attack_size=100 model.trainer.nb_epochs=1 data=torch_cifar model=torch_cifar - +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" - --config-name cifar.yaml - deps: - - path: cifar/reports/attack/default/score_dict.json - hash: md5 - md5: 46802563046219bac6f958866dda96f5 - size: 852 - - path: cifar/reports/train/default/score_dict.json - hash: md5 - md5: 5f47af0c6e48142318d161e0dae6eb06 - size: 526 - outs: - - path: cifar/logs/attack/ResNet34/1/ - hash: md5 - md5: 52ac24dec18dffe1eb8f019548d81a7c.dir - size: 42447682 - nfiles: 7681 - attacks@cifar-ResNet50-1: - cmd: bash attacks.sh ++model.init.name=torch_example.ResNet50 stage=attack model_name=ResNet50 - attack.attack_size=100 model.trainer.nb_epochs=1 data=torch_cifar model=torch_cifar - +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" - --config-name cifar.yaml - deps: - - path: cifar/reports/attack/default/score_dict.json - hash: md5 - md5: 46802563046219bac6f958866dda96f5 - size: 852 - - path: cifar/reports/train/default/score_dict.json - hash: md5 - md5: 5f47af0c6e48142318d161e0dae6eb06 - size: 526 - outs: - - path: cifar/logs/attack/ResNet50/1/ - hash: md5 - md5: 41bdebb342f790e4174c95841b1e514b.dir - size: 61996057 - nfiles: 7681 - attacks@cifar-ResNet101-1: - cmd: bash attacks.sh ++model.init.name=torch_example.ResNet101 stage=attack - model_name=ResNet101 attack.attack_size=100 model.trainer.nb_epochs=1 data=torch_cifar - model=torch_cifar +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" - --config-name cifar.yaml - deps: - - path: cifar/reports/attack/default/score_dict.json - hash: md5 - md5: 46802563046219bac6f958866dda96f5 - size: 852 - - path: cifar/reports/train/default/score_dict.json - hash: md5 - md5: 5f47af0c6e48142318d161e0dae6eb06 - size: 526 - outs: - - path: cifar/logs/attack/ResNet101/1/ - hash: md5 - md5: b7bfc568067aca4ce163b1163e5b5404.dir - size: 56855189 - nfiles: 7681 - attacks@cifar-ResNet152-1: - cmd: bash attacks.sh ++model.init.name=torch_example.ResNet152 stage=attack - model_name=ResNet152 attack.attack_size=100 model.trainer.nb_epochs=1 data=torch_cifar - model=torch_cifar +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" - --config-name cifar.yaml - deps: - - path: cifar/reports/attack/default/score_dict.json - hash: md5 - md5: 46802563046219bac6f958866dda96f5 - size: 852 - - path: cifar/reports/train/default/score_dict.json - hash: md5 - md5: 5f47af0c6e48142318d161e0dae6eb06 - size: 526 - outs: - - path: cifar/logs/attack/ResNet152/1/ - hash: md5 - md5: d68c082a5525f00982df9b769be8e2f8.dir - size: 68457714 - nfiles: 7681 - attacks@cifar100-ResNet18-1: - cmd: bash attacks.sh ++model.init.name=torch_example.ResNet18 stage=attack model_name=ResNet18 - attack.attack_size=100 model.trainer.nb_epochs=1 data=torch_cifar100 model=torch_cifar100 - +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" - --config-name cifar100.yaml - deps: - - path: cifar100/reports/attack/default/score_dict.json - hash: md5 - md5: c2f9e1e3db07148a510a2dd237d22218 - size: 828 - - path: cifar100/reports/train/default/score_dict.json - hash: md5 - md5: 8871d4ed73592450b4ceaa09fdfb4c0f - size: 513 - outs: - - path: cifar100/logs/attack/ResNet18/1/ - hash: md5 - md5: ac985ee1b0c7d97477205ac1b0be64db.dir - size: 35832419 - nfiles: 7681 - attacks@cifar100-ResNet34-1: - cmd: bash attacks.sh ++model.init.name=torch_example.ResNet34 stage=attack model_name=ResNet34 - attack.attack_size=100 model.trainer.nb_epochs=1 data=torch_cifar100 model=torch_cifar100 - +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" - --config-name cifar100.yaml - deps: - - path: cifar100/reports/attack/default/score_dict.json - hash: md5 - md5: c2f9e1e3db07148a510a2dd237d22218 - size: 828 - - path: cifar100/reports/train/default/score_dict.json - hash: md5 - md5: 8871d4ed73592450b4ceaa09fdfb4c0f - size: 513 - outs: - - path: cifar100/logs/attack/ResNet34/1/ - hash: md5 - md5: c3ce673126d443de9434667307930a6c.dir - size: 41411277 - nfiles: 7681 - attacks@cifar100-ResNet50-1: - cmd: bash attacks.sh ++model.init.name=torch_example.ResNet50 stage=attack model_name=ResNet50 - attack.attack_size=100 model.trainer.nb_epochs=1 data=torch_cifar100 model=torch_cifar100 - +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" - --config-name cifar100.yaml - deps: - - path: cifar100/reports/attack/default/score_dict.json - hash: md5 - md5: c2f9e1e3db07148a510a2dd237d22218 - size: 828 - - path: cifar100/reports/train/default/score_dict.json - hash: md5 - md5: 8871d4ed73592450b4ceaa09fdfb4c0f - size: 513 - outs: - - path: cifar100/logs/attack/ResNet50/1/ - hash: md5 - md5: 5206d77e526ee19a768ef63731e4656e.dir - size: 44976271 - nfiles: 7681 - attacks@cifar100-ResNet101-1: - cmd: bash attacks.sh ++model.init.name=torch_example.ResNet101 stage=attack - model_name=ResNet101 attack.attack_size=100 model.trainer.nb_epochs=1 data=torch_cifar100 - model=torch_cifar100 +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" - --config-name cifar100.yaml - deps: - - path: cifar100/reports/attack/default/score_dict.json - hash: md5 - md5: c2f9e1e3db07148a510a2dd237d22218 - size: 828 - - path: cifar100/reports/train/default/score_dict.json - hash: md5 - md5: 8871d4ed73592450b4ceaa09fdfb4c0f - size: 513 - outs: - - path: cifar100/logs/attack/ResNet101/1/ - hash: md5 - md5: 63708391dbb0828d6270a5d78e6ad76c.dir - size: 57235553 - nfiles: 7681 - attacks@cifar100-ResNet152-1: - cmd: bash attacks.sh ++model.init.name=torch_example.ResNet152 stage=attack - model_name=ResNet152 attack.attack_size=100 model.trainer.nb_epochs=1 data=torch_cifar100 - model=torch_cifar100 +direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" - --config-name cifar100.yaml - deps: - - path: cifar100/reports/attack/default/score_dict.json - hash: md5 - md5: c2f9e1e3db07148a510a2dd237d22218 - size: 828 - - path: cifar100/reports/train/default/score_dict.json - hash: md5 - md5: 8871d4ed73592450b4ceaa09fdfb4c0f - size: 513 - outs: - - path: cifar100/logs/attack/ResNet152/1/ - hash: md5 - md5: 9d1a7833ce57f0b5825c1114bc88ce36.dir - size: 70915973 - nfiles: 7681 - compile@mnist-attack: - cmd: python -m deckard.layers.compile --report_folder mnist/reports/attack --results_file - mnist/reports/attack.csv - deps: - - path: mnist/logs/attack/ - hash: md5 - md5: 4d6bdfb3d27cfd532c69a05b2e7cf2d0.dir - size: 343311550 - nfiles: 48119 - - path: mnist/reports/attack/ - hash: md5 - md5: 90555fff902b52c29e799cfac76c2d5b.dir - size: 108999231 - nfiles: 28416 - outs: - - path: mnist/reports/attack.csv - hash: md5 - md5: e50857f937b52c64d262302b4aa9740e - size: 53174519 - prepare_plot_folder@mnist-attack: - cmd: cp mnist/reports/attack.csv plots/data/attack_mnist.csv - deps: - - path: mnist/reports/attack.csv - hash: md5 - md5: e50857f937b52c64d262302b4aa9740e - size: 53174519 - outs: - - path: plots/data/attack_mnist.csv - hash: md5 - md5: e50857f937b52c64d262302b4aa9740e - size: 53174519 diff --git a/examples/pytorch/dvc.yaml b/examples/pytorch/dvc.yaml index f249f80b..ab188018 100644 --- a/examples/pytorch/dvc.yaml +++ b/examples/pytorch/dvc.yaml @@ -86,11 +86,11 @@ stages: # - ${item.dataset}/${files.reports}/train/${item.model}/ attacks: matrix: - dataset : [mnist, cifar, cifar100] - model : [ResNet18, ResNet34, ResNet50, ResNet101, ResNet152] - epochs : [1] #10, 20, 30, 50, 100 + atk_name : [FGM, PGD, Deep, Pixel, Thresh, HSJ] + dataset : [mnist,] # cifar, cifar100 + model : [ResNet18,] # ResNet34, ResNet50, ResNet101, ResNet152 cmd: >- - bash attacks.sh + bash scripts/${item.atk_name}.sh ++model.init.name=torch_example.${item.model} stage=attack model_name=${item.model} @@ -98,15 +98,16 @@ stages: model.trainer.nb_epochs=${item.epochs} data=torch_${item.dataset} model=torch_${item.dataset} - +direction="[maximize,maximize,minimize]" + ++direction="[maximize,maximize,minimize]" ++optimizers="[accuracy,adv_accuracy,adv_success]" --config-name ${item.dataset}.yaml deps: - ${item.dataset}/${files.reports}/attack/${files.name}/${files.score_dict_file} # This is here just to ensure it runs after the attack stage - ${item.dataset}/${files.reports}/train/${files.name}/${files.score_dict_file} - - attacks.sh + - scripts/${item.atk_name}.sh + - scripts/models.sh outs: - - ${item.dataset}/logs/attack/${item.model}/${item.epochs}/: + - ${item.dataset}/logs/attack/${item.model}/${item.atk_name}/: cache: True persist: True compile: diff --git a/examples/pytorch/main.sh b/examples/pytorch/main.sh deleted file mode 100644 index 34e724ac..00000000 --- a/examples/pytorch/main.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -paper_dir=~/ml_afr/ -# set downstream to 2 or nothing -for d in */ ; do - cd $d - # run command and write to log file - dvc repro --downstream clean -f >| dvc_repro.log - # dvc push - cd - -done -# change to paper directory -cd $paper_dir -# run dvc repro and dvc push -dvc repro -dvc push -# change back to original directory -cd - diff --git a/examples/pytorch/old_plots/dvc.lock b/examples/pytorch/old_plots/dvc.lock index a15ff82e..6eea5493 100644 --- a/examples/pytorch/old_plots/dvc.lock +++ b/examples/pytorch/old_plots/dvc.lock @@ -81,8 +81,8 @@ stages: deps: - path: data/merged.csv hash: md5 - md5: 9cc7eb6799f013a2d4259a18d36328d2 - size: 123215063 + md5: 3a9b42ec41f4b281681899c27c91c05e + size: 124150010 params: afr.yaml: covariates: @@ -261,71 +261,71 @@ stages: outs: - path: plots/aft_comparison.csv hash: md5 - md5: becda69f88bca7b3aea88f9d03f8d6b2 - size: 372 + md5: 24b6b2aca259503fb026faebb0004c60 + size: 365 - path: plots/aft_comparison.tex hash: md5 - md5: 573048966254e5c14b42eeceb8bcdcef - size: 640 + md5: fb65f3984f6c55aba8fa9c598d8c5229 + size: 637 - path: plots/cox_aft.pdf hash: md5 - md5: bf2e1bfe07b049890b2305e73124249d - size: 31094 + md5: f883ef1536e1576fa26ca59304dce9ba + size: 31093 - path: plots/cox_aft_dummies.pdf hash: md5 - md5: 1c788a6e5bd4d70a9eb587dc86ae9361 + md5: 14ca27bb36efd12f81e6a6f08c1a1945 size: 29292 - path: plots/cox_qq.pdf hash: md5 - md5: 9c211ae31847d09b7017cb4ba12daa2f - size: 19012 + md5: 584b1554726e1358ec3b6ab57ebc1fde + size: 19053 - path: plots/cox_summary.csv hash: md5 - md5: d03ee02cb0d19a22d3817d71b35a980a - size: 4689 + md5: 91f22fc977e955fa27d99c7dc0450bcf + size: 4698 - path: plots/exponential_aft.pdf hash: md5 - md5: 49765ceb038d8030bff7b40ec4470a01 - size: 33405 + md5: cccbe2acccbd2c65b09bb940a106a8c9 + size: 33411 - path: plots/exponential_aft_dummies.pdf hash: md5 - md5: 4451784f013d2410fc27568cf9fdbebe + md5: 918b7eec35edee4df27af112f1a4516e size: 31645 - path: plots/exponential_qq.pdf hash: md5 - md5: 1917e0e492bdab785f9ab3104d626725 - size: 21568 + md5: 26be3272b5e3c8d45cffcbf8ffcead2c + size: 21558 - path: plots/exponential_summary.csv hash: md5 - md5: 1d080ad56ab9db2ba023ace48f1e755b - size: 5886 + md5: d6aaea7ea17abf46f0ebc748bb45ea38 + size: 6123 - path: plots/gamma_aft.pdf hash: md5 - md5: 80ebccdc533086b3184ac75a5dc32762 - size: 29660 + md5: cbcce75b8aca512d624d6f02d878dd6e + size: 29664 - path: plots/gamma_aft_dummies.pdf hash: md5 - md5: f04b41b92436317e3d45c74470702825 - size: 35881 + md5: de7b0eede33eea795fca64b72a941005 + size: 35927 - path: plots/gamma_qq.pdf hash: md5 - md5: 060ea3a2e5a3153d0977ccd2b47e3056 - size: 20856 + md5: 34baa88e5b787670ac9d2b601c0e002c + size: 20841 - path: plots/gamma_summary.csv hash: md5 - md5: 5794ccf0fc040f8f1765b98b34c6a74d - size: 14378 + md5: 2c4b1cdd32a89b7b98c59449a8a13538 + size: 14496 - path: plots/log_logistic_aft.pdf hash: md5 - md5: 1eba1aea7ea126e6721ea4178d829cd8 + md5: 86ef2ddcc55338b0e148b56c0b8d2953 size: 32396 - path: plots/log_logistic_aft_dummies.pdf hash: md5 - md5: 5503a9d6e9b6e377f1ea8fca3d50f34b + md5: 6a329304b4e2f507f7cb847c3a735633 size: 32187 - path: plots/log_logistic_qq.pdf hash: md5 - md5: 3580ab59b1da38fe38cbe4ccfa70071d + md5: bb1708b30869ac6892c7adf7ee54d77c size: 19062 - path: plots/log_logistic_summary.csv hash: md5 @@ -333,15 +333,15 @@ stages: size: 5183 - path: plots/log_normal_aft.pdf hash: md5 - md5: 656cea7713fe17375dd78fa67db6cb3e + md5: ce80e2621ea9433e5f4cec9f26b79484 size: 32568 - path: plots/log_normal_aft_dummies.pdf hash: md5 - md5: 2812690f988e2cc8a8451f9d0199d563 + md5: 1c39735954e3a54917747e8e6c35a0be size: 32559 - path: plots/log_normal_qq.pdf hash: md5 - md5: 437b6932517a099a203eb9c7487255ee + md5: f105790833fede9c803e93b9f6ddffb4 size: 21726 - path: plots/log_normal_summary.csv hash: md5 @@ -349,15 +349,15 @@ stages: size: 5149 - path: plots/weibull_aft.pdf hash: md5 - md5: 3c7233e6138fea0d7ce234ea9c1ebace + md5: 390da69870105d1c7ca5c1c228a10075 size: 32594 - path: plots/weibull_aft_dummies.pdf hash: md5 - md5: 25f9c4aef4cba8ad79f6a90a52cefdef + md5: 1c698f2630a7f781e3ec6bc009c03c38 size: 31752 - path: plots/weibull_qq.pdf hash: md5 - md5: 1c239b3ae7e7f9e3a6acac95d692b913 + md5: a86245cd4f86a8a42ccf496f78773183 size: 18990 - path: plots/weibull_summary.csv hash: md5 @@ -369,37 +369,37 @@ stages: deps: - path: afr.yaml hash: md5 - md5: ae87070c3ab701c6be86d1098f7b5b23 - size: 6090 + md5: de63055dabfb50ee425757578eb84fdf + size: 6091 - path: data/merged.csv hash: md5 - md5: 9cc7eb6799f013a2d4259a18d36328d2 - size: 123215063 + md5: 3a9b42ec41f4b281681899c27c91c05e + size: 124150010 - path: plots/aft_comparison.tex hash: md5 - md5: 573048966254e5c14b42eeceb8bcdcef - size: 640 + md5: fb65f3984f6c55aba8fa9c598d8c5229 + size: 637 - path: predict_with_best.py hash: md5 - md5: e60a437ab37d2ee22256268a207f2431 - size: 2571 + md5: 33d068d06d1c29f6838fc6667353b9ec + size: 2608 outs: - path: data/merged_afr.csv hash: md5 - md5: 3a9b42ec41f4b281681899c27c91c05e - size: 124150010 + md5: 4cb9ffe968589f98d145c0b959457d88 + size: 124148815 plot: cmd: python -m deckard.layers.plots --path plots/ --file data/merged_afr.csv -c plots.yaml deps: - path: data/merged_afr.csv hash: md5 - md5: 3a9b42ec41f4b281681899c27c91c05e - size: 124150010 + md5: 4cb9ffe968589f98d145c0b959457d88 + size: 124148815 - path: plots.yaml hash: md5 - md5: a0c5c100248543bb5f0de8949b459bc5 - size: 3815 + md5: 07075fb24a3646dcc31e7dde7c34d14f + size: 3806 params: afr.yaml: covariates: @@ -461,7 +461,7 @@ stages: - ResNet50 - ResNet101 - ResNet152 - legend_title: Model + legend_title: - file: ben_accuracy_vs_defence_type.pdf hue: model_name kind: boxen @@ -476,7 +476,7 @@ stages: - ResNet50 - ResNet101 - ResNet152 - legend_title: Model + legend_title: - file: trash_score_vs_defence_type.pdf hue: model_name kind: boxen @@ -493,7 +493,7 @@ stages: - ResNet50 - ResNet101 - ResNet152 - legend_title: Model + legend_title: - file: trash_score_vs_attack_type.pdf hue: model_name kind: boxen @@ -638,49 +638,49 @@ stages: outs: - path: plots/adv_accuracy_vs_attack_type.pdf hash: md5 - md5: 9bd169743b91d14f5d30212cbda4287e + md5: 8ad3717dc51bde1de045887235bae9ca size: 36004 - path: plots/adv_accuracy_vs_defence_type.pdf hash: md5 - md5: 38b1f98d9c54b5d33804f5ebb4728fd9 - size: 33276 + md5: 39e066a1202ec6ace9ea980cb825af06 + size: 30891 - path: plots/adv_failure_rate_vs_train_time.pdf hash: md5 - md5: 885d2b52acf4be40f8bafc83b375ab02 - size: 274080 + md5: c2eb6eb9d09566e7a1b0bbc0dcff74db + size: 272655 - path: plots/atk_param_vs_accuracy.pdf hash: md5 - md5: 16268b2860b047140de9266b99cc0472 - size: 21458 + md5: 55394453ad7edb3295e8225cfeeddf3a + size: 20460 - path: plots/ben_accuracy_vs_defence_type.pdf hash: md5 - md5: 6d67c55146f0c45c5b760e84c984a491 - size: 34713 + md5: ac00105e3e12560422b65ef4a2ba43e9 + size: 31571 - path: plots/def_param_vs_accuracy.pdf hash: md5 - md5: 1f4467da61460335aabd287ecdc466dc - size: 19239 + md5: 09a168c3cf7b082e7c3c516f1ccdf27d + size: 18830 - path: plots/def_param_vs_adv_accuracy.pdf hash: md5 - md5: b5cf5be5f27f02f5c835b5c8529fd694 - size: 18808 + md5: 1652d167b647d3d15c3763e99e1679a7 + size: 18406 - path: plots/def_param_vs_adv_failure_rate.pdf hash: md5 - md5: 1113bf660baf47ed897ae16364315843 - size: 22070 + md5: a4ae57526d5004af876bff06666429fd + size: 21120 - path: plots/trash_score_vs_attack_type.pdf hash: md5 - md5: 8bce1a0e763d7636cbbe71dca655ff7c + md5: 14547e008e629855ee78848a3b770bb2 size: 45329 - path: plots/trash_score_vs_defence_type.pdf hash: md5 - md5: 53678b64ea8db5cf0e727cd3cfd67382 - size: 41417 + md5: aa75be062e0a254fe78be06a4f27a3e9 + size: 38003 copy_results: cmd: mkdir -p ~/ml_afr/plots && cp -r plots/* ~/ml_afr/plots/ deps: - path: plots/ hash: md5 - md5: b95f4289dd66fc205b9813aa1aa479fd.dir - size: 1094175 + md5: e3b452d1517f7f532c38825ac46ea16d.dir + size: 1081475 nfiles: 37 diff --git a/examples/pytorch/old_plots/dvc.yaml b/examples/pytorch/old_plots/dvc.yaml index 958ea2e5..cb2abffb 100644 --- a/examples/pytorch/old_plots/dvc.yaml +++ b/examples/pytorch/old_plots/dvc.yaml @@ -11,36 +11,6 @@ vars: - afr.yaml:cox - afr.yaml:dummies stages: - clean: - foreach: - - cifar - - cifar100 - - mnist - do: - cmd: >- - python -m deckard.layers.clean_data - -i data/attack_${item}.csv - -o data/${item}.csv - -c ../conf/clean_${item}.yaml - --drop_if_empty adv_fit_time accuracy train_time predict_time adv_accuracy Epochs - deps: - - data/attack_${item}.csv - - ../conf/clean_${item}.yaml - outs: - - data/${item}.csv - merge: - deps: - - data/mnist.csv - - data/cifar.csv - - data/cifar100.csv - cmd: >- - python -m deckard.layers.merge - --smaller_file data/cifar.csv data/cifar100.csv data/mnist.csv - --output_folder data - --output_file merged.csv - outs: - - data/merged.csv - afr: cmd: python -m deckard.layers.afr --data_file data/merged.csv --target adv_failures --duration_col adv_fit_time --config_file afr.yaml --plots_folder plots/ deps: diff --git a/examples/pytorch/old_plots/plots.yaml b/examples/pytorch/old_plots/plots.yaml index 7bc30e78..2345ed44 100644 --- a/examples/pytorch/old_plots/plots.yaml +++ b/examples/pytorch/old_plots/plots.yaml @@ -16,7 +16,7 @@ cat_plot: - ResNet101 - ResNet152 # titles: "{col_name} dataset" - legend_title: "Model" + legend_title: null # col: 'id' - file: ben_accuracy_vs_defence_type.pdf hue: model_name @@ -33,7 +33,7 @@ cat_plot: - ResNet101 - ResNet152 # titles: "{col_name} dataset" - legend_title: "Model" + legend_title: null # col: 'id' # col_order: # - mnist @@ -56,7 +56,7 @@ cat_plot: - ResNet101 - ResNet152 # titles: "{col_name} dataset" - legend_title: "Model" + legend_title: null # col: 'id' # col_order: # - mnist diff --git a/examples/pytorch/plots/dvc.lock b/examples/pytorch/plots/dvc.lock deleted file mode 100644 index 792bf29b..00000000 --- a/examples/pytorch/plots/dvc.lock +++ /dev/null @@ -1,707 +0,0 @@ -schema: '2.0' -stages: - merge: - cmd: python -m deckard.layers.merge --smaller_file data/cifar.csv data/cifar100.csv - data/mnist.csv --output_folder data --output_file merged.csv - deps: - - path: data/cifar.csv - hash: md5 - md5: 4c6e158a421999f4eae6c238395f63e4 - size: 20578565 - - path: data/cifar100.csv - hash: md5 - md5: c7bbce2ed692f4697b9ab4cbbe36f9c9 - size: 30226661 - - path: data/mnist.csv - hash: md5 - md5: 083e65989c7b8a5f103d407eb675aaf8 - size: 70077338 - outs: - - path: data/merged.csv - hash: md5 - md5: f98b901fa3ee817a55a36f78d0c95233 - size: 124219079 - plot: - cmd: python -m deckard.layers.plots --path plots/ --file data/merged_afr.csv -c - plots.yaml - deps: - - path: data/merged_afr.csv - hash: md5 - md5: e8c06cf834297c5ee44950ea4d640cc8 - size: 125347398 - - path: plots.yaml - hash: md5 - md5: 0f349596bdb4c2cee3c462325dffe4ba - size: 5070 - params: - plots.yaml: - cat_plot: - - file: adv_accuracy_vs_defence_type.pdf - hue: model_name - kind: boxen - set: - yscale: linear - x: def_gen - xlabels: Defence Type - y: adv_accuracy - ylabels: Adv. Accuracy - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - legend_title: Model - - file: ben_accuracy_vs_defence_type.pdf - hue: model_name - kind: boxen - x: def_gen - xlabels: Defence Type - y: accuracy - ylabels: Ben. Accuracy - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - legend_title: Model - - file: ben_failures_per_train_time_vs_defence_type.pdf - hue: model_name - kind: boxen - set: - yscale: log - x: def_gen - xlabels: Defence Type - y: c_ben - ylabels: $\bar{C}_{ben.}$ - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - legend_title: Model - - file: adv_failures_per_train_time_vs_defence_type.pdf - hue: model_name - kind: boxen - set: - yscale: log - x: def_gen - xlabels: Defence Type - y: c_adv - ylabels: $\bar{C}_{adv.}$ - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - legend_title: Model - - file: adv_failures_per_train_time_vs_attack_type.pdf - hue: model_name - kind: boxen - set: - yscale: log - x: atk_gen - xlabels: Attack Type - y: c_adv - ylabels: $\bar{C}_{adv.}$ - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - legend_title: Model - - file: adv_failures_per_test_time_vs_defence_type.pdf - hue: model_name - kind: boxen - x: def_gen - xlabels: Defence Type - y: adv_failure_rate - ylabels: $f_{adv.}$ - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - legend_title: Model - - file: adv_accuracy_vs_attack_type.pdf - hue: model_name - kind: boxen - x: atk_gen - xlabels: Attack Type - y: adv_accuracy - ylabels: Adv. Accuracy - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - legend_title: Model - - file: ben_failure_rate_vs_defence_type.pdf - hue: model_name - kind: boxen - set: - yscale: log - x: def_gen - xlabels: Defence Type - y: failure_rate - ylabels: $f_{ben}(t; \theta)$ - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - legend_title: Model - line_plot: - - file: def_param_vs_accuracy.pdf - hue: def_gen - legend: - bbox_to_anchor: - - 1.05 - - 1 - title: Defence - title: Ben. Accuracy vs Defence Strength - x: def_value - x_scale: linear - xlabel: Defence Control Parameter - y: accuracy - y_scale: - ylabel: Ben. Accuracy - hue_order: - - Control - - Conf - - Gauss-in - - Gauss-out - - FSQ - errorbar: se - err_style: bars - - file: def_param_vs_adv_accuracy.pdf - hue: def_gen - legend: - bbox_to_anchor: - - 1.05 - - 1 - title: Defence - title: Adv. Accuracy vs Defence Strength - x: def_value - x_scale: linear - xlabel: Defence Control Parameter - y: adv_accuracy - y_scale: - ylabel: Adv. Accuracy - hue_order: - - Control - - Conf - - Gauss-in - - Gauss-out - - FSQ - errorbar: se - err_style: bars - - file: def_param_vs_adv_failure_rate.pdf - hue: def_gen - legend: - bbox_to_anchor: - - 1.05 - - 1 - title: Defence - title: $f_{adv}$ vs Defence Strength - x: def_value - x_scale: linear - xlabel: Defence Control Parameter - y: adv_failure_rate - y_scale: linear - ylabel: $f_{adv.}$ - hue_order: - - Control - - Conf - - Gauss-in - - Gauss-out - - FSQ - errorbar: se - err_style: bars - - file: atk_param_vs_accuracy.pdf - hue: atk_gen - legend: - bbox_to_anchor: - - 1.05 - - 1 - title: Adv. Accuracy vs Attack Strength - x: atk_value - x_scale: linear - xlabel: Attack Control Parameter - y: adv_accuracy - y_scale: - ylabel: Adv. Accuracy - hue_order: - - FGM - - PGD - - Deep - - HSJ - - Pixel - - Thresh - errorbar: se - err_style: bars - scatter_plot: - - x: train_time_per_sample - y: adv_failure_rate - hue: model_name - xlabel: $t_{train}$ - ylabel: $f_{adv}$ - title: $f_{adv}$ vs $t_{train}$ - file: adv_failure_rate_vs_train_time.pdf - x_scale: log - legend: - title: Model Name - bbox_to_anchor: - - 1.05 - - 1 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - outs: - - path: plots/adv_accuracy_vs_attack_type.pdf - hash: md5 - md5: 589df119d033ae26afd171bb532b70b0 - size: 36004 - - path: plots/adv_accuracy_vs_defence_type.pdf - hash: md5 - md5: 234b264b494a6e396dbaff15ba1c06b7 - size: 33276 - - path: plots/adv_failure_rate_vs_train_time.pdf - hash: md5 - md5: 409d1d5afc7f53a0bbd6235ea40f3670 - size: 276434 - - path: plots/adv_failures_per_test_time_vs_defence_type.pdf - hash: md5 - md5: bc209f40e4dc19e9b64aa120907f6029 - size: 41070 - - path: plots/adv_failures_per_train_time_vs_attack_type.pdf - hash: md5 - md5: 18eb7fb28635190e81bb9bd6e670a43f - size: 45248 - - path: plots/adv_failures_per_train_time_vs_defence_type.pdf - hash: md5 - md5: 164dd701541215cd35a31a5506a851b9 - size: 41334 - - path: plots/atk_param_vs_accuracy.pdf - hash: md5 - md5: 2f65cc39685356bb1173b3c84a930d65 - size: 21456 - - path: plots/ben_accuracy_vs_defence_type.pdf - hash: md5 - md5: b582dadd56d56ff587411b17a1435fa2 - size: 34713 - - path: plots/ben_failure_rate_vs_defence_type.pdf - hash: md5 - md5: 1c800cde38bdaa41d2758d75e977d35c - size: 43727 - - path: plots/ben_failures_per_train_time_vs_defence_type.pdf - hash: md5 - md5: 77c8a489ec159b88afeac48ee0aeeb18 - size: 41312 - - path: plots/def_param_vs_accuracy.pdf - hash: md5 - md5: 8247cc225c005509e2fe9ece9d4c88af - size: 19260 - - path: plots/def_param_vs_adv_accuracy.pdf - hash: md5 - md5: 3146258daf3dc7d7cea85f957a77557d - size: 18821 - - path: plots/def_param_vs_adv_failure_rate.pdf - hash: md5 - md5: 600022c5a83acf5532e84f7f0daa635c - size: 23053 - afr: - cmd: python -m deckard.layers.afr --data_file data/merged.csv --target adv_failures - --duration_col adv_fit_time --config_file afr.yaml --plots_folder plots/ - deps: - - path: data/merged.csv - hash: md5 - md5: f98b901fa3ee817a55a36f78d0c95233 - size: 124219079 - params: - afr.yaml: - covariates: - - adv_fit_time - - accuracy - - train_time - - atk_value - - def_value - - data.sample.random_state - - Epochs - - model_layers - - id - - atk_gen - - def_gen - - adv_failures - - adv_accuracy - - predict_time - cox: - plot: - file: cox_aft.pdf - title: Cox Model - title: Cox Model - qq_title: Cox QQ Plot - t0: 0.3 - model: - penalizer: 0.2 - labels: - data.sample.random_state: Random State - atk_value: Attack Strength - train_time: $t_{train}$ - predict_proba_time: $t_{predict}$ - adv_accuracy: Adv. Accuracy - accuracy: Ben. Accuracy - adv_fit_time: $t_{attack}$ - adv_failure_rate: $f_{adv.}(t;\theta)$ - failure_rate: $f_{ben.}(t;\theta)$ - Epochs: No. of Epochs - model.trainer.batch_size: Batch Size - def_gen: Defence - exponential: - plot: - file: exponential_aft.pdf - title: Exponential Model - qq_title: Exponential QQ Plot - t0: 0.1 - model: - breakpoints: - - 0.1 - labels: - 'Intercept: rho_': $\rho$ - 'Intercept: lambda_': $\lambda$ - 'data.sample.random_state: lambda_': Random State - 'atk_value: lambda_': Attack Strength - 'train_time: lambda_': $t_{train}$ - 'predict_proba_time: lambda_': $t_{predict}$ - 'adv_accuracy: lambda_': Adv. Accuracy - 'accuracy: lambda_': Ben. Accuracy - 'adv_fit_time: lambda_': $t_{attack}$ - 'adv_failure_rate: lambda_': $f_{adv.}(t;\theta)$ - 'failure_rate: lambda_': $f_{ben.}(t;\theta)$ - 'Epochs: lambda_': No. of Epochs - 'model.trainer.batch_size: lambda_': Batch Size - def_gen: Defence - ': lambda_': '' - gamma: - plot: - file: gamma_aft.pdf - title: Generalized Gamma Model - qq_title: Gamma QQ Plot - t0: 0.3 - model: - penalizer: 0.3 - labels: - 'Intercept: alpha_': $\alpha$ - 'Intercept: beta_': $\beta$ - 'data.sample.random_state: beta_': Random State - 'atk_value: beta_': Attack Strength - 'train_time: beta_': $t_{train}$ - 'predict_proba_time: beta_': $t_{predict}$ - 'adv_accuracy: beta_': Adv. Accuracy - 'accuracy: beta_': Ben. Accuracy - 'adv_fit_time: beta_': $t_{attack}$ - 'adv_failure_rate: beta_': $h_{adv.}(t;\theta)$ - 'failure_rate: beta_': $h_{ben.}(t;\theta)$ - 'Epochs: beta_': No. of Epochs - 'model.trainer.batch_size: beta_': Batch Size - def_gen: Defence - 'attack.init.eps: beta_': $\varepsilon$ - log_logistic: - plot: - file: log_logistic_aft.pdf - title: Log logistic AFR Model - qq_title: Log Logistic QQ Plot - t0: 0.1 - model: - penalizer: 0.1 - labels: - 'Intercept: beta_': $\beta$ - 'Intercept: alpha_': $\alpha$ - 'data.sample.random_state: alpha_': Random State - 'atk_value: alpha_': Attack Strength - 'train_time: alpha_': $t_{train}$ - 'predict_proba_time: alpha_': $t_{predict}$ - 'adv_accuracy: alpha_': Adv. Accuracy - 'accuracy: alpha_': Ben. Accuracy - 'adv_fit_time: alpha_': $t_{attack}$ - 'adv_failure_rate: alpha_': $h_{adv.}(t;\theta)$ - 'failure_rate: alpha_': $h_{ben.}(t;\theta)$ - 'Epochs: alpha_': No. of Epochs - 'model.trainer.batch_size: alpha_': Batch Size - def_gen: Defence - 'attack.init.eps: alpha_': $\varepsilon$ - log_normal: - plot: - file: log_normal_aft.pdf - title: Log Normal AFR Model - qq_title: Log Normal QQ Plot - t0: 2 - t0: 2 - model: - penalizer: 0.5 - labels: - 'Intercept: sigma_': $\sigma$ - 'Intercept: mu_': $\mu$ - 'atk_value: mu_': Attack Strength - 'train_time: mu_': $t_{train}$ - 'predict_proba_time: mu_': $t_{predict}$ - 'adv_accuracy: mu_': Adv. Accuracy - 'accuracy: mu_': Ben. Accuracy - 'adv_fit_time: mu_': $t_{attack}$ - 'adv_failure_rate: mu_': $h_{adv.}(t;\theta)$ - 'failure_rate: mu_': $h_{ben.}(t;\theta)$ - 'Epochs: mu_': No. of Epochs - 'model.trainer.batch_size: mu_': Batch Size - def_gen: Defence - 'attack.init.eps: mu_': $\varepsilon$ - 'data.sample.random_state: mu_': Random State - weibull: - plot: - file: weibull_aft.pdf - title: Weibull AFR Model - qq_title: Weibull QQ Plot - t0: 1 - model: - penalizer: 0.1 - labels: - 'Intercept: rho_': $\rho$ - 'Intercept: lambda_': $\lambda$ - 'data.sample.random_state: lambda_': Random State - 'atk_value: lambda_': Attack Strength - 'train_time: lambda_': $t_{train}$ - 'predict_proba_time: lambda_': $t_{predict}$ - 'adv_accuracy: lambda_': Adv. Accuracy - 'accuracy: lambda_': Ben. Accuracy - 'adv_fit_time: lambda_': $t_{attack}$ - 'adv_failure_rate: lambda_': $f_{adv.}(t;\theta)$ - 'failure_rate: lambda_': $f_{ben.}(t;\theta)$ - 'Epochs: lambda_': No. of Epochs - 'model.trainer.batch_size: lambda_': Batch Size - def_gen: Defence - 'model_layers: lambda_': Layers - 'def_value: lambda_': Defence Strength - 'predict_time: lambda_': $t_{predict}$ - ': lambda_': '' - outs: - - path: plots/aft_comparison.csv - hash: md5 - md5: c2baa8f717cdc1da5c66c45b6fbe15ed - size: 484 - - path: plots/aft_comparison.tex - hash: md5 - md5: 36326edf164e3e50eb1fa9cdfe779883 - size: 771 - - path: plots/cox_aft.pdf - hash: md5 - md5: 49cc31dfa8deb7a973efa79aa5661023 - size: 29060 - - path: plots/cox_aft_dummies.pdf - hash: md5 - md5: 2886a507102de701086572d45da2dfa8 - size: 29294 - - path: plots/cox_qq.pdf - hash: md5 - md5: 1871952097e3a7ced7597800a08864a1 - size: 19783 - - path: plots/cox_summary.csv - hash: md5 - md5: 5c2afaed99bd7a6459ffd1417b085386 - size: 4704 - - path: plots/exponential_aft.pdf - hash: md5 - md5: bc07de53e57a4b8ac93621e31afce321 - size: 31719 - - path: plots/exponential_aft_dummies.pdf - hash: md5 - md5: 1590788252530487ef7c12a2412533c5 - size: 31689 - - path: plots/exponential_qq.pdf - hash: md5 - md5: 5eb9206940c0556752308da1e30db16e - size: 20360 - - path: plots/exponential_summary.csv - hash: md5 - md5: 14692228f8862cdcf958ca6519f3eeb8 - size: 6115 - - path: plots/gamma_aft.pdf - hash: md5 - md5: c14f76b8a748a81eb652df11c4168430 - size: 28179 - - path: plots/gamma_aft_dummies.pdf - hash: md5 - md5: 0a64f85cf0c0dbf042c2af6ac3f1e514 - size: 34487 - - path: plots/gamma_qq.pdf - hash: md5 - md5: 86b107aed5da777f5e11c7148cece4f6 - size: 19114 - - path: plots/gamma_summary.csv - hash: md5 - md5: eae68c47d2098fd5dcb6a4456e50909d - size: 14596 - - path: plots/log_logistic_aft.pdf - hash: md5 - md5: c7927f6d138223b3d0814e08ae70cf50 - size: 29805 - - path: plots/log_logistic_aft_dummies.pdf - hash: md5 - md5: 3451396aab89eda953b67b7013043df5 - size: 30815 - - path: plots/log_logistic_qq.pdf - hash: md5 - md5: 6db594338475a53d925e72d9a5be69e6 - size: 21187 - - path: plots/log_logistic_summary.csv - hash: md5 - md5: bc339874fa9cb8aa52471140047bab73 - size: 5167 - - path: plots/log_normal_aft.pdf - hash: md5 - md5: 60147688508ead164e45910baace6fd9 - size: 30291 - - path: plots/log_normal_aft_dummies.pdf - hash: md5 - md5: 267b2a890ffd2a034d9825cefec12a45 - size: 31496 - - path: plots/log_normal_qq.pdf - hash: md5 - md5: ff928a6fbd9d7277df3f88c99d22c096 - size: 22085 - - path: plots/log_normal_summary.csv - hash: md5 - md5: d48ff310a60a8ad40313cf3b69ab5e58 - size: 5149 - - path: plots/weibull_aft.pdf - hash: md5 - md5: 799bddf708e9c4a48784783c629e722a - size: 31698 - - path: plots/weibull_aft_dummies.pdf - hash: md5 - md5: 99fef4eb8ec5dbf7844ba6e76c904b73 - size: 31043 - - path: plots/weibull_qq.pdf - hash: md5 - md5: 258bebd08669196cf738183ecd05dc14 - size: 19374 - - path: plots/weibull_summary.csv - hash: md5 - md5: 8ee7e7d66bcf24eb8424460826ab7421 - size: 5217 - copy_results: - cmd: mkdir -p ~/ml_afr/plots && cp -r plots/* ~/ml_afr/plots/ - deps: - - path: plots/ - hash: md5 - md5: 5858e5f1ee079e8c99ac53c7834601da.dir - size: 5938101 - nfiles: 43 - clean: - cmd: python -m deckard.layers.clean_data -i data/merged.csv -o data/clean.csv - -c clean.yaml - deps: - - path: data/merged.csv - hash: md5 - md5: 1b4802747f91a7001c43401ad21d997a - size: 50761113 - params: - clean.yaml: - fillna: - Epochs: 10 - outs: - - path: data/clean.csv - hash: md5 - md5: 2913117ef7997065dafc860c9d3080f5 - size: 54972304 - clean@mnist: - cmd: python -m deckard.layers.clean_data -i data/attack_mnist.csv -o data/mnist.csv -c - ../conf/clean_mnist.yaml --drop_if_empty adv_fit_time accuracy train_time predict_time - adv_accuracy Epochs - deps: - - path: ../conf/clean_mnist.yaml - hash: md5 - md5: bb112947b87ca42a244135a52cc5e7d5 - size: 1003 - - path: data/attack_mnist.csv - hash: md5 - md5: c4db49ae4fc1a0e6fa4c3d52b03d650a - size: 93006845 - outs: - - path: data/mnist.csv - hash: md5 - md5: 083e65989c7b8a5f103d407eb675aaf8 - size: 70077338 - clean@cifar: - cmd: python -m deckard.layers.clean_data -i data/attack_cifar.csv -o data/cifar.csv -c - ../conf/clean_cifar.yaml --drop_if_empty adv_fit_time accuracy train_time predict_time - adv_accuracy Epochs - deps: - - path: ../conf/clean_cifar.yaml - hash: md5 - md5: 4e2abc093db66b77b424854549b80497 - size: 961 - - path: data/attack_cifar.csv - hash: md5 - md5: 5a430aa13b88dff6a8fdf0277c9cd53d - size: 23267442 - outs: - - path: data/cifar.csv - hash: md5 - md5: 4c6e158a421999f4eae6c238395f63e4 - size: 20578565 - clean@cifar100: - cmd: python -m deckard.layers.clean_data -i data/attack_cifar100.csv -o data/cifar100.csv -c - ../conf/clean_cifar100.yaml --drop_if_empty adv_fit_time accuracy train_time - predict_time adv_accuracy Epochs - deps: - - path: ../conf/clean_cifar100.yaml - hash: md5 - md5: 5a8cc7e71f7036e2f590bb882acd34fb - size: 897 - - path: data/attack_cifar100.csv - hash: md5 - md5: ea55355b5b530f5751aea19d13067099 - size: 36035453 - outs: - - path: data/cifar100.csv - hash: md5 - md5: c7bbce2ed692f4697b9ab4cbbe36f9c9 - size: 30226661 - predict_survival_time: - cmd: python predict_with_best.py --data data/merged.csv --config_file afr.yaml --model - weibull --target adv_failures --duration_col adv_fit_time --output data/merged_afr.csv - deps: - - path: afr.yaml - hash: md5 - md5: 82c7a6430d38320263db502d57cbd5c6 - size: 5728 - - path: data/merged.csv - hash: md5 - md5: f98b901fa3ee817a55a36f78d0c95233 - size: 124219079 - - path: plots/aft_comparison.tex - hash: md5 - md5: 36326edf164e3e50eb1fa9cdfe779883 - size: 771 - outs: - - path: data/merged_afr.csv - hash: md5 - md5: e8c06cf834297c5ee44950ea4d640cc8 - size: 125347398 diff --git a/examples/pytorch/scripts/Deep.sh b/examples/pytorch/scripts/Deep.sh new file mode 100644 index 00000000..445f8d15 --- /dev/null +++ b/examples/pytorch/scripts/Deep.sh @@ -0,0 +1,8 @@ +bash scripts/models.sh \ + stage=attack \ + attack=default \ + ++attack.init.name=art.attacks.evasion.DeepFool \ + ++attack.init.max_iter=10 \ + ++attack.init.batch_size=4096 \ + ++attack.init.nb_grads=1,3,5,8,10 \ + atk_name=Deep $@ diff --git a/examples/pytorch/scripts/FGM.sh b/examples/pytorch/scripts/FGM.sh new file mode 100644 index 00000000..835a64be --- /dev/null +++ b/examples/pytorch/scripts/FGM.sh @@ -0,0 +1,7 @@ +bash scripts/models.sh \ + stage=attack \ + attack=default \ + ++attack.init.name=art.attacks.evasion.FastGradientMethod \ + ++attack.init.eps=.001,.01,.1,.5,1 \ + ++attack.init.norm=2 \ + atk_name=FGM $@ diff --git a/examples/pytorch/scripts/HSJ.sh b/examples/pytorch/scripts/HSJ.sh new file mode 100644 index 00000000..be5d2023 --- /dev/null +++ b/examples/pytorch/scripts/HSJ.sh @@ -0,0 +1,9 @@ +bash scripts/models.sh \ + stage=attack \ + attack=default \ + ++attack.init.name=art.attacks.evasion.HopSkipJump \ + ++attack.init.max_iter=1,3,5,10,15 \ + ++attack.init.init_eval=3 \ + ++attack.init.max_eval=10 \ + ++attack.init.norm=2 \ + atk_name=HSJ $@ diff --git a/examples/pytorch/scripts/PGD.sh b/examples/pytorch/scripts/PGD.sh new file mode 100644 index 00000000..34e655a5 --- /dev/null +++ b/examples/pytorch/scripts/PGD.sh @@ -0,0 +1,9 @@ +bash scripts/models.sh \ + stage=attack \ + attack=default \ + ++attack.init.name=art.attacks.evasion.ProjectedGradientDescent \ + ++attack.init.eps=.001,.01,.1,.5,1 \ + ++attack.init.norm=2 \ + ++attack.init.eps_step=.001,.003,.01 \ + atk_name=PGD \ + ++attack.init.max_iter=1,5,10,50,100 $@ diff --git a/examples/pytorch/scripts/Pixel.sh b/examples/pytorch/scripts/Pixel.sh new file mode 100644 index 00000000..568a01d6 --- /dev/null +++ b/examples/pytorch/scripts/Pixel.sh @@ -0,0 +1,7 @@ +bash scripts/models.sh \ + stage=attack \ + attack=default \ + ++attack.init.name=art.attacks.evasion.PixelAttack \ + ~attack.init.batch_size \ + ++attack.init.th=1,4,16,64,256 \ + atk_name=Pixel $@ diff --git a/examples/pytorch/scripts/Thresh.sh b/examples/pytorch/scripts/Thresh.sh new file mode 100644 index 00000000..3cd733ed --- /dev/null +++ b/examples/pytorch/scripts/Thresh.sh @@ -0,0 +1,7 @@ +bash scripts/models.sh \ + stage=attack \ + attack=default \ + ++attack.init.name=art.attacks.evasion.ThresholdAttack \ + ~attack.init.batch_size \ + ++attack.init.th=1,4,16,64,256 \ + atk_name=Thresh $@ diff --git a/examples/pytorch/attacks.sh b/examples/pytorch/scripts/attacks.sh similarity index 81% rename from examples/pytorch/attacks.sh rename to examples/pytorch/scripts/attacks.sh index bba66e7c..eb7acc9a 100644 --- a/examples/pytorch/attacks.sh +++ b/examples/pytorch/scripts/attacks.sh @@ -10,7 +10,7 @@ # ++attack.init.eps=.001,.01,.1,.5,1 \ # ++attack.init.norm=2 \ # atk_name=FGM $@ -# # ##################################################### +# ##################################################### # # Projected Gradient Descent # bash models.sh \ # stage=attack \ @@ -32,6 +32,27 @@ # ++attack.init.nb_grads=1,3,5,8,10 \ # atk_name=Deep $@ # # ##################################################### +# # Projected Gradient Descent +# bash models.sh \ +# stage=attack \ +# attack=default \ +# ++attack.init.name=art.attacks.evasion.HopSkipJump \ +# ++attack.init.max_iter=1,3,5,10,15 \ +# ++attack.init.init_eval=3 \ +# ++attack.init.max_eval=10 \ +# ++attack.init.norm=2 \ +# atk_name=HSJ $@ +# ##################################################### +# # # PixelAttack +# bash models.sh \ +# stage=attack \ +# attack=default \ +# ++attack.init.name=art.attacks.evasion.DeepFool \ +# ++attack.init.max_iter=10 \ +# ++attack.init.batch_size=4096 \ +# ++attack.init.nb_grads=1,3,5,8,10 \ +# atk_name=Deep $@ +# # ##################################################### # # HopSkipJump # bash models.sh \ # stage=attack \ diff --git a/examples/pytorch/models.sh b/examples/pytorch/scripts/models.sh similarity index 84% rename from examples/pytorch/models.sh rename to examples/pytorch/scripts/models.sh index 2a54e307..d34a790b 100644 --- a/examples/pytorch/models.sh +++ b/examples/pytorch/scripts/models.sh @@ -14,9 +14,6 @@ python -m deckard.layers.optimise \ def_name=FSQ \ $@ --multirun -rm -rf mnist/models -rm -rf cifar/models -rm -rf cifar100/models # Gaussian Augmentation (Input) python -m deckard.layers.optimise \ @@ -27,9 +24,6 @@ python -m deckard.layers.optimise \ def_name=Gauss-In \ $@ --multirun -rm -rf mnist/models -rm -rf cifar/models -rm -rf cifar100/model # Gaussian Noise (Output) python -m deckard.layers.optimise \ @@ -39,9 +33,6 @@ python -m deckard.layers.optimise \ def_name=Gauss-Out \ $@ --multirun -rm -rf mnist/models -rm -rf cifar/models -rm -rf cifar100/models # High Confidence python -m deckard.layers.optimise \ @@ -49,7 +40,3 @@ python -m deckard.layers.optimise \ +model.art.postprocessor.cutoff=.1,.3,.5,.9,.99 \ def_name=Conf \ $@ --multirun - -rm -rf mnist/models -rm -rf cifar/models -rm -rf cifar100/models diff --git a/examples/pytorch/wait.sh b/examples/pytorch/scripts/wait.sh similarity index 100% rename from examples/pytorch/wait.sh rename to examples/pytorch/scripts/wait.sh