diff --git a/.gitignore b/.gitignore index 680aabf0..32b61e06 100644 --- a/.gitignore +++ b/.gitignore @@ -140,5 +140,10 @@ env/ # screenlog screenlog.* -# tmp.py -tmp.py +# env310 +env310/* +# dvc read-write protect files in the examples folder +examples/*/.dvc/tmp/lock +examples/*/.dvc/tmp/lock +examples/*/.dvc/tmp/rwlock +examples/*/.dvc/tmp/rwlock.lock diff --git a/deckard/base/attack/attack.py b/deckard/base/attack/attack.py index 976fab63..5df0748c 100644 --- a/deckard/base/attack/attack.py +++ b/deckard/base/attack/attack.py @@ -202,7 +202,7 @@ def __call__( ) except TypeError as e: logger.error(f"Failed to compute success rate. Error: {e}") - if attack_file is not None: + if attack_file is not None and not Path(attack_file).exists(): self.data.save(samples, attack_file) if adv_predictions_file is not None and Path(adv_predictions_file).exists(): adv_predictions = self.data.load(adv_predictions_file) @@ -210,7 +210,7 @@ def __call__( else: adv_predictions = model.predict(samples) results["adv_predictions"] = np.array(adv_predictions) - if adv_predictions_file is not None: + if adv_predictions_file is not None and not Path(adv_predictions_file).exists(): self.data.save(adv_predictions, adv_predictions_file) if adv_probabilities_file is not None and Path(adv_probabilities_file).exists(): adv_probabilities = self.data.load(adv_probabilities_file) diff --git a/deckard/base/data/generator.py b/deckard/base/data/generator.py index b9494756..5a6f69bb 100644 --- a/deckard/base/data/generator.py +++ b/deckard/base/data/generator.py @@ -11,7 +11,11 @@ make_moons, make_circles, ) -from torchvision.io import read_image, read_file + +try: + from torchvision.io import read_image, read_file +except ImportError: + pass from art.utils import load_mnist, load_cifar10, load_diabetes, to_categorical from ..utils import my_hash @@ -225,8 +229,6 @@ def __call__(self): return TorchDataGenerator(self.name, **self.kwargs)() elif self.name in KERAS_DATASETS: return KerasDataGenerator(self.name, **self.kwargs)() - elif isinstance(self.name, str) and Path(self.name).exists(): - return SklearnDataGenerator(self.name, **self.kwargs)() else: # pragma: no cover raise ValueError( f"Invalid name {self.name}. Please choose from {ALL_DATASETS}", diff --git a/deckard/base/data/sklearn_pipeline.py b/deckard/base/data/sklearn_pipeline.py index ccb73745..7880a082 100644 --- a/deckard/base/data/sklearn_pipeline.py +++ b/deckard/base/data/sklearn_pipeline.py @@ -26,6 +26,8 @@ def __call__(self, X_train, X_test, y_train, y_test): name = self.kwargs.pop("_target_", self.name) dict_ = {"_target_": name} dict_.update(**self.kwargs) + while "kwargs" in dict_: + dict_.update(**dict_.pop("kwargs")) obj = instantiate(dict_) X_train = obj.fit(X_train).transform(X_train) X_test = obj.transform(X_test) diff --git a/deckard/base/experiment/experiment.py b/deckard/base/experiment/experiment.py index 0f1828ed..8384eeb0 100644 --- a/deckard/base/experiment/experiment.py +++ b/deckard/base/experiment/experiment.py @@ -112,7 +112,7 @@ def __hash__(self): name = str(self.name).encode("utf-8") return int.from_bytes(name, "little") - def __call__(self): + def __call__(self, **kwargs): """Runs the experiment. If the experiment has already been run, it will load the results from disk. If scorer is not None, it will return the score for the specified scorer. If scorer is None, it will return the score for the first scorer in the ScorerDict. :param scorer: The scorer to return the score for. If None, the score for the first scorer in the ScorerDict will be returned. :type scorer: str @@ -134,7 +134,8 @@ def __call__(self): else: score_dict = {} results = {} - results["score_dict_file"] = score_dict + results["score_dict"] = score_dict + files.update(**results) ######################################################################### # Load or generate data ######################################################################### @@ -143,10 +144,10 @@ def __call__(self): # Load or train model ######################################################################### if self.model is not None: - model_results = self.model(data, **files) + model_results = self.model(**files) score_dict.update(**model_results.pop("time_dict", {})) score_dict.update(**model_results.pop("score_dict", {})) - model = model_results["model"] + files.update(**model_results) # Prefer probabilities, then loss_files, then predictions if ( "probabilities" in model_results @@ -174,8 +175,6 @@ def __call__(self): ########################################################################## if self.attack is not None: adv_results = self.attack( - data, - model, **files, ) if "adv_predictions" in adv_results: @@ -195,6 +194,7 @@ def __call__(self): if "adv_success" in adv_results: adv_success = adv_results["adv_success"] score_dict.update({"adv_success": adv_success}) + files.update(**adv_results) ########################################################################## # Score results ######################################################################### @@ -216,7 +216,7 @@ def __call__(self): logger.debug(f" len(preds) : {len(preds)}") new_score_dict = self.scorers(ground_truth, preds) score_dict.update(**new_score_dict) - results["score_dict_file"] = score_dict + results["score_dict"] = score_dict if "adv_preds" in locals(): ground_truth = data[3][: len(adv_preds)] adv_preds = adv_preds[: len(ground_truth)] @@ -225,7 +225,7 @@ def __call__(self): f"adv_{key}": value for key, value in adv_score_dict.items() } score_dict.update(**adv_score_dict) - results["score_dict_file"] = score_dict + results["score_dict"] = score_dict # # Save results if "score_dict_file" in files and files["score_dict_file"] is not None: if Path(files["score_dict_file"]).exists(): diff --git a/deckard/base/files/files.py b/deckard/base/files/files.py index da55a7c5..e4cb1851 100644 --- a/deckard/base/files/files.py +++ b/deckard/base/files/files.py @@ -83,17 +83,17 @@ def __call__(self): files = dict(self.get_filenames()) return files - def get_filenames(self): + def get_filenames(self, **kwargs): files = deepcopy(self.files) + files.update(**kwargs) files = self._set_filenames(**files) return files def _set_filenames(self, **kwargs): name = self.name stage = self.stage - if hasattr(self, "files"): - kwargs.update(self.files) - files = dict(kwargs) + files = self.files + files.update(**kwargs) new_files = {} directory = self.directory reports = self.reports diff --git a/deckard/base/model/art_pipeline.py b/deckard/base/model/art_pipeline.py index e21c0204..8f5a75d4 100644 --- a/deckard/base/model/art_pipeline.py +++ b/deckard/base/model/art_pipeline.py @@ -34,7 +34,6 @@ class ArtPipelineStage: kwargs: dict = field(default_factory=dict) def __init__(self, name=None, **kwargs): - logger.info(f"Creating pipeline stage: {name} kwargs: {kwargs}") self.name = name kwargs.update(**kwargs.pop("kwargs", {})) self.kwargs = kwargs @@ -69,7 +68,6 @@ def __call__(self): device_type = "gpu" if torch.cuda.is_available() else "cpu" if device_type == "gpu": - logger.info("Using GPU") number_of_devices = torch.cuda.device_count() num = randint(0, number_of_devices - 1) device = torch.device(f"cuda:{num}") @@ -205,7 +203,7 @@ def __call__(self, model: object, data: list) -> BaseEstimator: name = params.pop("name", None) kwargs = params.pop("kwargs", {}) else: - name = None + name = self.library kwargs = {} pre_def = [] post_def = [] diff --git a/deckard/base/model/model.py b/deckard/base/model/model.py index bb435143..06082735 100644 --- a/deckard/base/model/model.py +++ b/deckard/base/model/model.py @@ -76,7 +76,10 @@ def __call__(self, data: list, model: object, library=None): if library in sklearn_dict.keys(): pass elif library in torch_dict.keys(): - pass + trainer["nb_epochs"] = trainer.pop( + "nb_epochs", + trainer.pop("epochs", trainer.pop("nb_epoch", 10)), + ) elif library in keras_dict.keys(): pass elif library in tensorflow_dict.keys(): @@ -142,7 +145,7 @@ def __call__(self, data: list, model: object, library=None): model.fit(data[0], data[2], **trainer) end = process_time_ns() end_timestamp = time() - except Exception as e: + except AttributeError as e: raise e except RuntimeError as e: # pragma: no cover if "eager mode" in str(e) and library in tensorflow_dict.keys(): @@ -176,6 +179,45 @@ def __call__(self, data: list, model: object, library=None): model.fit(data[0], data[2], **trainer) end = process_time_ns() end_timestamp = time() + elif "disable eager execution" in str(e): + logger.warning("Disabling eager execution for Tensorflow.") + import tensorflow as tf + + tf.compat.v1.disable_eager_execution() + start = process_time_ns() + start_timestamp = time() + model.fit(data[0], data[2], **trainer) + end = process_time_ns() + end_timestamp = time() + elif "out of memory" in set(e).lower() and library in torch_dict.keys(): + import torch + + torch.cuda.empty_cache() + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + if device == "cuda": + # Pick the GPU with the most free memory + free_memory = [ + torch.cuda.memory_reserved(i) - torch.cuda.memory_allocated(i) + for i in range(torch.cuda.device_count()) + ] + device = f"cuda:{free_memory.index(max(free_memory))}" + data[0] = torch.from_numpy(data[0]) + data[1] = torch.from_numpy(data[1]) + data[0] = torch.Tensor.float(data[0]) + data[1] = torch.Tensor.float(data[1]) + data[0].to(device) + data[2] = torch.from_numpy(data[2]) + data[3] = torch.from_numpy(data[3]) + data[2] = torch.Tensor.float(data[2]) + data[3] = torch.Tensor.float(data[3]) + data[2].to(device) + model.model.to(device) if hasattr(model, "model") else model.to(device) + start = process_time_ns() + start_timestamp = time() + model.fit(data[0], data[2], **trainer) + end = process_time_ns() + end_timestamp = time() + else: raise e time_dict = { diff --git a/deckard/layers/afr.py b/deckard/layers/afr.py index 13787632..41c7c4dc 100644 --- a/deckard/layers/afr.py +++ b/deckard/layers/afr.py @@ -16,10 +16,15 @@ LogLogisticAFTFitter, CoxPHFitter, CRCSplineFitter, + AalenAdditiveFitter, + GeneralizedGammaRegressionFitter, + PiecewiseExponentialRegressionFitter, ) from lifelines.utils import CensoringType from lifelines.fitters import RegressionFitter -from .clean_data import drop_frames_without_results +from lifelines.exceptions import ConvergenceError +from .clean_data import drop_rows_without_results +from .compile import load_results, save_results logger = logging.getLogger(__name__) @@ -30,6 +35,7 @@ def survival_probability_calibration( df: pd.DataFrame, t0: float, ax=None, + color="red", ): r""" Smoothed calibration curves for time-to-event models. This is analogous to @@ -67,7 +73,6 @@ def ccl(p): if ax is None: ax = plt.gca() - T = model.duration_col E = model.event_col @@ -96,12 +101,55 @@ def ccl(p): crc = CRCSplineFitter(n_baseline_knots=n_knots, penalizer=0.000001) with warnings.catch_warnings(): warnings.filterwarnings("ignore") - if CensoringType.is_right_censoring(model): - crc.fit_right_censoring(prediction_df, T, E, regressors=regressors) - elif CensoringType.is_left_censoring(model): - crc.fit_left_censoring(prediction_df, T, E, regressors=regressors) - elif CensoringType.is_interval_censoring(model): - crc.fit_interval_censoring(prediction_df, T, E, regressors=regressors) + try: + if CensoringType.is_right_censoring(model): + crc.fit_right_censoring(prediction_df, T, E, regressors=regressors) + elif CensoringType.is_left_censoring(model): + crc.fit_left_censoring(prediction_df, T, E, regressors=regressors) + elif CensoringType.is_interval_censoring(model): + crc.fit_interval_censoring(prediction_df, T, E, regressors=regressors) + else: + crc.fit(prediction_df, T, E, regressors=regressors) + except ConvergenceError as e: + if "delta contains nan value(s)" in str(e): + fit_options = { + "step_size": 0.1, + "max_steps": 1000, + "precision": 1e-3, + } + else: + crc._scipy_fit_method = "SLSQP" + fit_options = {} + try: + if CensoringType.is_right_censoring(model): + crc.fit_right_censoring( + prediction_df, + T, + E, + regressors=regressors, + fit_options=fit_options, + ) + elif CensoringType.is_left_censoring(model): + crc.fit_left_censoring( + prediction_df, + T, + E, + regressors=regressors, + fit_options=fit_options, + ) + elif CensoringType.is_interval_censoring(model): + crc.fit_interval_censoring( + prediction_df, + T, + E, + regressors=regressors, + fit_options=fit_options, + ) + else: + crc.fit(prediction_df, T, E, regressors=regressors) + except ConvergenceError as e: + logger.error(f"Could not fit CRC model. due to {e}") + return ax, np.nan, np.nan # predict new model at values 0 to 1, but remember to ccl it! x = np.linspace( @@ -119,11 +167,10 @@ def ccl(p): # plot our results - color = "tab:red" ax.plot(x, y, label="Calibration Curve", color=color) - ax.set_xlabel("Predicted probability of \nt ≤ %d mortality" % t0) - ax.set_ylabel("Observed probability of \nt ≤ %d mortality" % t0, color=color) - ax.tick_params(axis="y", labelcolor=color) + ax.set_xlabel("Predicted P(t ≤ %.2f )" % round(t0, 3)) + ax.set_ylabel("Observed P(t ≤ %.2f )" % round(t0, 3)) + ax.tick_params(axis="y") # plot x=y line ax.plot(x, x, c="k", ls="--") @@ -149,21 +196,32 @@ def fit_aft( duration_col, mtype, summary_file=None, - summary_plot=None, - summary_title=None, folder=None, - replacement_dict={}, **kwargs, ): - + logger.info(f"Trying to initialize model {mtype} with {kwargs}") if mtype == "weibull": - aft = WeibullAFTFitter(**kwargs, penalizer=0.1) + aft = WeibullAFTFitter(penalizer=kwargs.pop("penalizer", 0.1), **kwargs) elif mtype == "log_normal": - aft = LogNormalAFTFitter(**kwargs, penalizer=0.1) + aft = LogNormalAFTFitter(penalizer=kwargs.pop("penalizer", 0.1), **kwargs) elif mtype == "log_logistic": - aft = LogLogisticAFTFitter(**kwargs, penalizer=0.1) + aft = LogLogisticAFTFitter(penalizer=kwargs.pop("penalizer", 0.1), **kwargs) elif mtype == "cox": - aft = CoxPHFitter(**kwargs) + aft = CoxPHFitter(penalizer=kwargs.pop("penalizer", 0.1), **kwargs) + elif mtype == "aalen": + aft = AalenAdditiveFitter(alpha=kwargs.pop("alpha", 0.1), **kwargs) + elif mtype == "gamma": + aft = GeneralizedGammaRegressionFitter( + penalizer=kwargs.pop("penalizer", 0.1), + **kwargs, + ) + elif mtype == "exponential": + aft = PiecewiseExponentialRegressionFitter( + penalizer=kwargs.pop("penalizer", 0.1), + **kwargs, + ) + else: + raise ValueError(f"Model type {mtype} not recognized") assert ( duration_col in df.columns ), f"Column {duration_col} not in dataframe with columns {df.columns}" @@ -171,47 +229,81 @@ def fit_aft( assert ( event_col in df.columns ), f"Column {event_col} not in dataframe with columns {df.columns}" - aft.fit(df, event_col=event_col, duration_col=duration_col) - if summary_file is not None: - summary = pd.DataFrame(aft.summary) - suffix = Path(summary_file).suffix - if folder is not None: - summary_file = Path(folder, summary_file) - if suffix == "": - Path(summary_file).parent.mkdir(exist_ok=True, parents=True) - summary.to_csv(summary_file) - logger.info(f"Saved summary to {summary_file}") - elif suffix == ".csv": - summary.to_csv(summary_file) - logger.info(f"Saved summary to {summary_file}") - elif suffix == ".tex": - summary.to_latex(summary_file, float_format="%.2f") - logger.info(f"Saved summary to {summary_file}") - elif suffix == ".json": - summary.to_json(summary_file) - logger.info(f"Saved summary to {summary_file}") - elif suffix == ".html": - summary.to_html(summary_file) - logger.info(f"Saved summary to {summary_file}") - else: - logger.warning(f"suffix {suffix} not recognized. Saving to csv") - summary.to_csv(summary_file) - logger.info(f"Saved summary to {summary_file}") - if summary_plot is not None: - if summary_title is None: - summary_title = ( - f"{mtype} AFR Summary".replace("_", " ").replace("-", "").title() + start = df[duration_col].min() + end = df[duration_col].max() + start = start - 0.01 * (end - start) + timeline = np.linspace(start, end, 1000) + start = df[duration_col].min() + end = df[duration_col].max() + start = start - 0.01 * (end - start) + timeline = np.linspace(start, end, 1000) + # if the event column isnt a binary column, we need to convert it to binary. + # Assume that floats represent the proportion of failure events + # raise an error on any other type + kwarg_dict = {} + # if df[event_col].dtype == "boolean": + # pass + # elif df[event_col].dtype == "float" : + # # Assume this column represents accuracy + # print(f"Length of data: {len(df)}") + # pos_weights = df[event_col].copy() * 100 + # neg_weights = (1 - df[event_col].copy()) * 100 + # # create a new dataframe for positive/negative + # pos_df = df.copy() + # neg_df = df.copy() + # # Set the event column to 1 for positive, 0 for negative + # pos_df[event_col] = 1 + # neg_df[event_col] = 0 + # # add the weights + # pos_df['weights_col'] = pos_weights + # neg_df['weights_col'] = neg_weights + # # concatenate the dataframes + # df = pd.concat([pos_df, neg_df], axis=0) + # # drop rows with non-positive weights + # df = df[df['weights_col'] > 0] + # # Recast the event column as boolean + # df[event_col] = df[event_col].astype(bool) + # # Recast the weights column as int + # df['weights_col'] = df['weights_col'].astype(int) + # kwarg_dict['weights_col'] = 'weights_col' + # else: + # raise ValueError(f"Event column {event_col} is not boolean or float") + kwarg_dict["duration_col"] = duration_col + kwarg_dict["event_col"] = event_col + if mtype != "aalen": + kwarg_dict["timeline"] = timeline + try: + aft.fit(df, **kwarg_dict) + except AttributeError as e: + raise ConvergenceError(f"Could not fit {mtype} model due to {e}") + except ConvergenceError as e: + if "delta contains nan value(s)" in str(e): + fit_options = { + "step_size": 0.1, + "max_steps": 1000, + "precision": 1e-3, + } + kwarg_dict["fit_options"] = fit_options + logger.info( + "Reducing the step size to 0.1 and increasing the max steps to 1000", ) - plot_summary( - aft=aft, - title=summary_title, - file=summary_plot, - xlabel=kwargs.get("xlabel", "Covariate"), - ylabel=kwargs.get("ylabel", "p-value"), - replacement_dict=replacement_dict, - folder=folder, - filetype=".pdf", - ) + input("Inside the fit function") + else: + logger.info("Trying to fit with SLSQP") + aft._scipy_fit_method = "SLSQP" + try: + aft.fit(df, **kwarg_dict) + except ConvergenceError as e: + logger.error(f"Could not fit {mtype} model due to {e}") + raise ConvergenceError(f"Could not fit {mtype} model due to {e}") + + else: + logger.info(f"Fitted {mtype} model") + if summary_file is not None: + summary = pd.DataFrame(aft.summary).copy() + if folder is None: + folder = "." + save_results(summary, results_file=summary_file, results_folder=folder) return aft @@ -222,6 +314,7 @@ def plot_aft( xlabel, ylabel, replacement_dict={}, + dummy_dict={}, folder=None, filetype=".pdf", ): @@ -236,16 +329,21 @@ def plot_aft( # Only plot the covariates, skipping the intercept and dummy variables # Dummy variables can be examined using the plot_partial_effects function try: - columns = list(aft.summary.index.get_level_values(1)) + columns = list(aft.summary.index.get_level_values(1)).copy() except IndexError: - columns = list(aft.summary.index) + columns = list(aft.summary.index).copy() clean_cols = [] + dummy_cols = [] + dummy_prefixes = tuple(dummy_dict.values()) for col in columns: - if col.startswith("dummy_") or col.startswith("Intercept"): + if str(col).startswith(dummy_prefixes) or str(col).startswith("dummy_"): + dummy_cols.append(col) + elif col.startswith("Intercept"): continue else: clean_cols.append(col) columns = clean_cols + ax = aft.plot(columns=columns) labels = ax.get_yticklabels() labels = [label.get_text() for label in labels] @@ -255,10 +353,42 @@ def plot_aft( ax.set_xlabel(xlabel) ax.set_ylabel(ylabel) ax.set_title(title) + # symlog-scale the x-axis + # ax.set_xscale("linear") ax.get_figure().tight_layout() ax.get_figure().savefig(file) plt.gcf().clear() logger.info(f"Saved graph to {file}") + if len(dummy_cols) > 0: + plt.gcf().clear() + logger.info(f"Dummy variables: {dummy_cols}") + ax2 = aft.plot(columns=dummy_cols) + labels = ax2.get_yticklabels() + labels = [label.get_text() for label in labels] + i = 0 + labels = [x.replace("dummy_", "") for x in labels] + for k, v in dummy_dict.items(): + labels = [label.replace(k, v) for label in labels] + for k, v in replacement_dict.items(): + labels = [label.replace(k, v) for label in labels] + for label in labels: + if label.startswith("Data:"): + # if dataset is in all lowercase, make it uppercase + dataset = label.split(":")[1].strip() + if str(dataset).islower(): + dataset = dataset.upper() + new_label = f"Data: {dataset}" + else: + new_label = label + labels[i] = new_label + i += 1 + ax2.set_yticklabels(labels) + ax2.set_xlabel(xlabel) + ax2.set_ylabel(ylabel) + ax2.set_title(title) + ax2.get_figure().tight_layout() + ax2.get_figure().savefig(file.with_name(file.stem + "_dummies" + file.suffix)) + plt.gcf().clear() return ax @@ -313,15 +443,17 @@ def plot_summary( def plot_qq( - X_test, + X_train, aft, title, file, + X_test=None, xlabel=None, ylabel=None, folder=None, ax=None, filetype=".pdf", + t0=0.35, ): suffix = Path(file).suffix if suffix == "": @@ -331,16 +463,43 @@ def plot_qq( if folder is not None: file = Path(folder, file) plt.gcf().clear() - ax, ici, e50 = survival_probability_calibration(aft, X_test, t0=0.35, ax=ax) - ax.set_xlabel(xlabel) - ax.set_ylabel(ylabel) + + if X_test is not None: + ax, _, _ = survival_probability_calibration( + aft, + X_train, + t0=t0, + ax=ax, + color="red", + ) + ax, _, _ = survival_probability_calibration( + aft, + X_test, + t0=t0, + ax=ax, + color="blue", + ) + else: + ax, _, _ = survival_probability_calibration( + aft, + X_train, + t0=t0, + ax=ax, + color="red", + ) + ax.set_xlim(0, 1) + ax.set_ylim(0, 1) + if xlabel is not None: + ax.set_xlabel(xlabel) + if ylabel is not None: + ax.set_ylabel(ylabel) ax.set_title(title) ax.legend().remove() ax.get_figure().tight_layout() ax.get_figure().savefig(file) logger.info(f"Saved graph to {file}") plt.gcf().clear() - return ax, ici, e50 + return ax def plot_partial_effects( @@ -382,16 +541,19 @@ def plot_partial_effects( return pareto -def score_model(aft, train, test): - train_score = aft.score(train, scoring_method="concordance_index") - test_score = aft.score(test, scoring_method="concordance_index") - train_ll = aft.score(train, scoring_method="log_likelihood") - test_ll = aft.score(test, scoring_method="log_likelihood") +def score_model(aft, train, test, t0=0.35, method="concordance_index"): + train_score = aft.score(train, scoring_method=method) + test_score = aft.score(test, scoring_method=method) + ax, train_ici, train_e50 = survival_probability_calibration(aft, train, t0=t0) + ax, test_ici, test_e50 = survival_probability_calibration(aft, test, t0=t0, ax=ax) scores = { "train_score": train_score, "test_score": test_score, - "train_ll": train_ll, - "test_ll": test_ll, + "train_ici": train_ici, + "test_ici": test_ici, + "train_e50": train_e50, + "test_e50": test_e50, + "qq_plot": ax, } return scores @@ -401,8 +563,7 @@ def make_afr_table( dataset, X_train, X_test, - icis, - e50s, + t0s, folder=".", span_columns=True, ): @@ -414,10 +575,20 @@ def make_afr_table( ] aft_data.index = model_names aft_data["AIC"] = [ - x.AIC_ if not isinstance(x, CoxPHFitter) else np.nan for x in aft_dict.values() + ( + x.AIC_ + if not isinstance(x, (CoxPHFitter, GeneralizedGammaRegressionFitter)) + else np.nan + ) + for x in aft_dict.values() ] aft_data["BIC"] = [ - x.AIC_ if not isinstance(x, CoxPHFitter) else np.nan for x in aft_dict.values() + ( + x.AIC_ + if not isinstance(x, (CoxPHFitter, GeneralizedGammaRegressionFitter)) + else np.nan + ) + for x in aft_dict.values() ] aft_data["Concordance"] = [ x.score(X_train, scoring_method="concordance_index") for x in aft_dict.values() @@ -425,13 +596,38 @@ def make_afr_table( aft_data["Test Concordance"] = [ x.score(X_test, scoring_method="concordance_index") for x in aft_dict.values() ] + icis = [] + e50s = [] + for key in aft_dict.keys(): + model = aft_dict[key] + t0 = t0s[key] + _, ici, e50 = survival_probability_calibration(model, X_train, t0=t0) + icis.append(ici) + e50s.append(e50) + + t_icis = [] + t_e50s = [] + for model in aft_dict.values(): + try: + _, ici, e50 = survival_probability_calibration(model, X_test, t0=t0) + except ConvergenceError as e: + logger.error( + f"Could not calculate ICI and E50 for model {model} due to {e}", + ) + ici = np.nan + e50 = np.nan + t_icis.append(ici) + t_e50s.append(e50) aft_data["ICI"] = icis + aft_data["Test ICI"] = t_icis aft_data["E50"] = e50s - pretty_dataset = ( - dataset.upper() - if dataset in ["combined", "Combined", "COMBINED"] - else dataset.upper() - ) + aft_data["Test E50"] = t_e50s + if dataset in ["combined", "Combined", "COMBINED"]: + pretty_dataset = "combined" + elif dataset is None: + pretty_dataset = None + else: + pretty_dataset = dataset.upper() aft_data = aft_data.round(2) aft_data.to_csv(folder / "aft_comparison.csv") logger.info(f"Saved AFR comparison to {folder / 'aft_comparison.csv'}") @@ -440,8 +636,16 @@ def make_afr_table( aft_data.to_latex( folder / "aft_comparison.tex", float_format="%.3g", # Two decimal places, since we have 100 adversarial examples - label=f"tab:{dataset.lower()}", # Label for cross-referencing - caption=f"Comparison of AFR Models on the {pretty_dataset} dataset.", + label=( + f"tab:{dataset.lower()}" if dataset is not None else "tab:afr_models" + ), # Label for cross-referencing + caption=( + f"Comparison of AFT Models on the {pretty_dataset} dataset." + if pretty_dataset is not None + else None + ), + index=True, + header=True, ) # Change to table* if span_columns is True if span_columns is True: @@ -461,28 +665,38 @@ def clean_data_for_aft( data, covariate_list, target="adv_failure_rate", + dummy_dict={}, ): - subset = data.copy() + # Find categorical vars + assert ( - target in subset - ), f"Target {target} not in dataframe with columns {subset.columns}" - logger.info(f"Shape of dirty data: {subset.shape}") - cleaned = pd.DataFrame() + target in data + ), f"Target {target} not in dataframe with columns {data.columns}" + logger.info(f"Shape of dirty data: {data.shape}") covariate_list.append(target) + covariate_list = list(set(covariate_list)) logger.info(f"Covariates : {covariate_list}") - for kwarg in covariate_list: - assert kwarg in subset.columns, f"{kwarg} not in data.columns" - cleaned[kwarg] = subset[kwarg] - cols = cleaned.columns - cleaned = pd.DataFrame(subset, columns=cols) - for col in cols: - cleaned = cleaned[cleaned[col] != -1e10] - cleaned = cleaned[cleaned[col] != 1e10] - cleaned.dropna(inplace=True, how="any", axis=0) + subset = data[covariate_list] + for col in subset.columns: + subset = subset[subset[col] != -1e10] + subset = subset[subset[col] != 1e10] + if len(dummy_dict) > 0: + dummy_subset = subset[list(dummy_dict.keys())] + dummies = pd.get_dummies( + dummy_subset, + prefix=dummy_dict, + prefix_sep=" ", + columns=list(dummy_dict.keys()), + ) + subset = subset.drop(columns=list(dummy_dict.keys())) + cleaned = pd.concat([subset, dummies], axis=1) + else: + cleaned = subset.astype(float) + cleaned = subset.dropna(axis=0, how="any") assert ( target in cleaned.columns ), f"Target {target} not in dataframe with columns {cleaned.columns}" - logger.info(f"Shape of cleaned data: {cleaned.shape}") + logger.info(f"Shape of data data: {cleaned.shape}") return cleaned @@ -490,22 +704,23 @@ def split_data_for_aft( data, target, duration_col, - test_size=0.2, + test_size=0.25, random_state=42, ): - cleaned = pd.get_dummies(data, prefix="dummy", prefix_sep="_") + assert isinstance(test_size, float), "Test size must be a float" + assert 0 < test_size < 1, "Test size must be between 0 and 1" X_train, X_test = train_test_split( - cleaned, + data, train_size=(1 - test_size), test_size=test_size, random_state=random_state, ) assert ( - target in cleaned.columns - ), f"Target {target} not in dataframe with columns {cleaned.columns}" + target in data.columns + ), f"Target {target} not in dataframe with columns {data.columns}" assert ( - duration_col in cleaned.columns - ), f"Duration {duration_col} not in dataframe with columns {cleaned.columns}" + duration_col in data.columns + ), f"Duration {duration_col} not in dataframe with columns {data.columns}" X_train = X_train.dropna(axis=0, how="any") X_test = X_test.dropna(axis=0, how="any") return X_train, X_test @@ -517,50 +732,53 @@ def run_afr_experiment( X_train, target, duration_col, + t0, X_test=None, + dummy_dict={}, folder=".", ): if len(config.keys()) > 0: plots = [] plot_dict = config.pop("plot", {}) - label_dict = plot_dict.pop("labels", plot_dict.get("labels", {})) + label_dict = config.pop("labels", {}) partial_effect_list = config.pop("partial_effect", []) model_config = config.pop("model", {}) + model_config.update(**config) aft = fit_aft( summary_file=plot_dict.get("summary_file", f"{mtype}_summary.csv"), - summary_plot=plot_dict.get("summary_plot", f"{mtype}_summary.pdf"), folder=folder, df=X_train, event_col=target, duration_col=duration_col, - replacement_dict=label_dict, mtype=mtype, - summary_title=plot_dict.get("summary_title", f"{mtype} AFR Summary"), **model_config, ) afr_plot = plot_aft( aft=aft, title=plot_dict.get( - "qq_title", - f"{mtype}".replace("_", " ").replace("-", " ").title() + " AFR", + "title", + f"{mtype}".replace("_", " ").replace("-", " ").title(), ), file=plot_dict.get("plot", f"{mtype}_aft.pdf"), - xlabel=label_dict.get("xlabel", "Acceleration Factor"), - ylabel=label_dict.get("ylabel", ""), # noqa W605 + xlabel=label_dict.pop("xlabel", r"log$(\phi)$"), + ylabel=label_dict.pop("ylabel", ""), # noqa W605 replacement_dict=label_dict, + dummy_dict=dummy_dict, folder=folder, ) plots.append(afr_plot) - qq_plot, ici, e50 = plot_qq( + qq_plot = plot_qq( + X_train=X_train, X_test=X_test, aft=aft, title=plot_dict.get( - "title", - f"{mtype}".replace("_", " ").replace("-", " ").title() + " AFR QQ Plot", + "qq_title", + f"{mtype}".replace("_", " ").replace("-", " ").title() + " QQ Plot", ), + t0=t0, file=plot_dict.get("qq_file", f"{mtype}_qq.pdf"), - xlabel=label_dict.get("xlabel", "Observed Quantiles"), - ylabel=label_dict.get("ylabel", "Predicted Quantiles"), + xlabel=label_dict.pop("xlabel", None), + ylabel=label_dict.pop("ylabel", None), folder=folder, ) plots.append(qq_plot) @@ -573,7 +791,7 @@ def run_afr_experiment( folder=folder, ) plots.append(partial_effect_plot) - return aft, plots, ici, e50 + return aft, plots def render_all_afr_plots( @@ -582,15 +800,15 @@ def render_all_afr_plots( target, data, dataset, - test_size=0.75, + test_size=0.25, folder=".", + dummy_dict={}, ): - covariate_list = config.pop("covariates", []) - cleaned = clean_data_for_aft(data, covariate_list, target=target) - assert target in cleaned.columns, f"{target} not in data.columns" - assert duration_col in cleaned.columns, f"{duration_col} not in data.columns" + + assert target in data.columns, f"{target} not in data.columns" + assert duration_col in data.columns, f"{duration_col} not in data.columns" X_train, X_test = split_data_for_aft( - cleaned, + data, target, duration_col, test_size=test_size, @@ -598,31 +816,30 @@ def render_all_afr_plots( ) plots = {} models = {} - icis = [] - e50s = [] mtypes = list(config.keys()) + t0s = {} for mtype in mtypes: sub_config = config.get(mtype, {}) - models[mtype], plots[mtype], ici, e50 = run_afr_experiment( + t0 = sub_config.pop("t0", 0.35) + models[mtype], plots[mtype] = run_afr_experiment( + t0=t0, mtype=mtype, config=sub_config, X_train=X_train, X_test=X_test, target=target, + dummy_dict=dummy_dict, duration_col=duration_col, folder=folder, ) - icis.append(ici) - e50s.append(e50) - + t0s[mtype] = t0 aft_data = make_afr_table( models, dataset, X_train, X_test, folder=folder, - icis=icis, - e50s=e50s, + t0s=t0s, ) print("*" * 80) print("*" * 34 + " RESULTS " + "*" * 34) @@ -631,38 +848,27 @@ def render_all_afr_plots( print("*" * 80) -def prepare_aft_data(args, data, config): +def calculate_raw_failures(args, data, config): data = data.applymap(lambda x: x.strip() if isinstance(x, str) else x) assert Path(args.config_file).exists(), f"{args.config_file} does not exist." covariates = config.get("covariates", []) assert len(covariates) > 0, "No covariates specified in config file" - logger.info(f"Shape of data before data before dropping na: {data.shape}") - logger.info(f"Shape of data before data before dropping na: {data.shape}") if "adv_failures" in covariates and "adv_failures" not in data.columns: data.loc[:, "adv_failures"] = (1 - data.loc[:, "adv_accuracy"]) * data.loc[ :, "attack.attack_size", ] + + if "adv_accuracy" in covariates: + covariates.remove("adv_accuracy") if "ben_failures" in covariates and "ben_failures" not in data.columns: - if "predict_time" in data.columns: - data["n_samples"] = data["predict_time"] / data[ - "predict_time_per_sample" - ].astype(int) - elif "predict_proba_time" in data.columns: - data["n_samples"] = data["predict_proba_time"] / data[ - "predict_time_per_sample" - ].astype(int) - elif "predict_loss_time" in data.columns: - data["n_samples"] = data["predict_loss_time"] / data[ - "predict_time_per_sample" - ].astype(int) - else: - assert "n_samples" in data.columns, "n_samples not in data" data.loc[:, "ben_failures"] = (1 - data.loc[:, "accuracy"]) * data.loc[ :, - "n_samples", + "data.sample.test_size", ] - data = drop_frames_without_results(data, covariates) + if "accuracy" in covariates: + covariates.remove("accuracy") + data = drop_rows_without_results(data, covariates) return data @@ -682,7 +888,11 @@ def main(args): csv_file = args.data_file FOLDER = args.plots_folder Path(FOLDER).mkdir(exist_ok=True, parents=True) - data = pd.read_csv(csv_file, index_col=0) + data = load_results( + results_file=Path(csv_file).name, + results_folder=Path(csv_file).parent, + ) + logger.info(f"Shape of data: {data.shape}") data.columns = data.columns.str.strip() with Path(args.config_file).open("r") as f: @@ -691,7 +901,11 @@ def main(args): for k, v in fillna.items(): assert k in data.columns, f"{k} not in data" data[k] = data[k].fillna(v) - data = prepare_aft_data(args, data, config) + dummies = config.pop("dummies", {}) + # Default formerly: {"atk_gen" : "Atk:", "def_gen" : "Def:", "id" : "Data:"} + data = calculate_raw_failures(args, data, config) + covariate_list = config.pop("covariates", []) + data = clean_data_for_aft(data, covariate_list, target=target, dummy_dict=dummies) assert target in data.columns, f"{target} not in data.columns" assert duration_col in data.columns, f"{duration_col} not in data.columns" render_all_afr_plots( @@ -700,8 +914,9 @@ def main(args): target, data, dataset, - test_size=0.8, + test_size=0.25, folder=FOLDER, + dummy_dict=dummies, ) @@ -709,7 +924,7 @@ def main(args): afr_parser = argparse.ArgumentParser() afr_parser.add_argument("--target", type=str, default="adv_failures") afr_parser.add_argument("--duration_col", type=str, default="adv_fit_time") - afr_parser.add_argument("--dataset", type=str, default="mnist") + afr_parser.add_argument("--dataset", type=str, default=None) afr_parser.add_argument("--data_file", type=str, default="data.csv") afr_parser.add_argument("--config_file", type=str, default="afr.yaml") afr_parser.add_argument("--plots_folder", type=str, default="plots") diff --git a/deckard/layers/clean_data.py b/deckard/layers/clean_data.py index 92e2646f..9fdd30d9 100644 --- a/deckard/layers/clean_data.py +++ b/deckard/layers/clean_data.py @@ -7,15 +7,47 @@ import yaml import numpy as np from tqdm import tqdm - from .utils import deckard_nones as nones -from .compile import save_results +from .compile import save_results, load_results logger = logging.getLogger(__name__) sns.set_theme(style="whitegrid", font_scale=1.8, font="times new roman") -def drop_frames_without_results( +def fill_train_time( + data, + match=[ + "model.init.name", + "model.trainer.nb_epoch", + "model.art.preprocessor", + "model.art.postprocessor", + "def_name", + "def_gen", + "defence_name", + ], +): + sort_by = [] + for col in match: + # find out which columns have the string in match + if col in data.columns: + sort_by.append(col) + else: + pass + # Convert "train_time" to numeric + # Fill missing values in "train_time" with the max of the group + data["train_time"] = pd.to_numeric(data["train_time"], errors="coerce").astype( + float, + ) + # Group by everything in the "match" list + assert isinstance(data, pd.DataFrame), "data is not a pandas DataFrame" + # Sort by the entries in "match" + data = data.sort_values(by=sort_by) + data["train_time"] = data["train_time"].fillna(method="ffill") + data["train_time"] = data["train_time"].fillna(method="bfill") + return data + + +def drop_rows_without_results( data, subset=[ "accuracy", @@ -27,7 +59,7 @@ def drop_frames_without_results( ], ): """ - The function `drop_frames_without_results` drops rows from a DataFrame that have missing values in + The function `drop_rows_without_results` drops rows from a DataFrame that have missing values in specified columns. Args: @@ -42,7 +74,18 @@ def drop_frames_without_results( """ logger.info(f"Dropping frames without results for {subset}") - data.dropna(axis=0, subset=subset, inplace=True) + + for col in subset: + logger.info(f"Dropping frames without results for {col}") + before = data.shape[0] + logger.info(f"Shape of data before data before dropping na: {data.shape}") + data.dropna(axis=0, subset=[col], inplace=True) + after = data.shape[0] + logger.info(f"Shape of data before data after dropping na: {data.shape}") + percent_change = (before - after) / before * 100 + if percent_change > 5: + # input(f"{percent_change:.2f}% of data dropped for {col}. Press any key to continue.") + logger.warning(f"{percent_change:.2f}% of data dropped for {col}") return data @@ -59,8 +102,6 @@ def calculate_failure_rate(data): "failure_rate", "training_time_per_failure", and "training_time_per_adv_failure". """ logger.info("Calculating failure rate") - data = data[data.columns.drop(list(data.filter(regex=r"\.1$")))] - data.columns.str.replace(" ", "") assert "accuracy" in data.columns, "accuracy not in data.columns" data.loc[:, "accuracy"] = pd.to_numeric(data.loc[:, "accuracy"]) assert ( @@ -243,6 +284,7 @@ def merge_defences( for _, entry in tqdm(results.iterrows(), desc="Merging defences"): defence = [] i = 0 + # Explicit defences from ART for col in defence_columns: if col in entry and entry[col] not in nones: defence.append(entry[col]) @@ -251,8 +293,8 @@ def merge_defences( i += 1 ############################################################################################################ if len(defence) > 1: - def_gen = [str(x).split(".")[-1] for x in defence] - defence = "_".join(defence) + def_gen = [str(str(x).split(".")[-1]) for x in defence] + defence = "_".join(str(defence)) def_gen = "_".join(def_gen) elif len(defence) == 1 and defence[0] not in nones and defence[0] != np.nan: def_gen = str(defence[0]).split(".")[-1] @@ -447,14 +489,35 @@ def replace_strings_in_data(data, replace_dict): return data +def replace_strings_in_columns(data, replace_dict): + cols = list(data.columns) + for k, v in replace_dict.items(): + logger.debug(f"Replacing {k} with {v} in column names...") + for col in cols: + if k == col: + logger.debug(f"Replacing {k} with {v} in column names...") + data.rename(columns={k: v}, inplace=True) + else: + pass + after = list(data.columns) + if len(replace_dict) > 0: + logger.info(f"Columns after replacement: {after}") + assert cols != after, "Columns not replaced." + assert len(cols) == len( + after, + ), f"Length of columns before and after replacement not equal: {len(cols)} != {len(after)}." + return data + + def clean_data_for_plotting( data, - def_gen_dict, - atk_gen_dict, - control_dict, - fillna, - replace_dict, - pareto_dict, + def_gen_dict={}, + atk_gen_dict={}, + control_dict={}, + fillna={}, + replace_dict={}, + col_replace_dict={}, + pareto_dict={}, ): """ The function `clean_data_for_plotting` cleans and formats data for plotting by dropping empty rows, @@ -494,31 +557,50 @@ def clean_data_for_plotting( data = data.assign(model_layers=model_layers) logger.info(f"Model Names: {data.model_name.unique()}") logger.info(f"Model Layers: {data.model_layers.unique()}") - logger.info("Replacing data.sample.random_state with random_state...") data["data.sample.random_state"].rename("random_state", inplace=True) if len(def_gen_dict) > 0: data = merge_defences(data) logger.info("Replacing attack and defence names with short names...") if hasattr(data, "def_gen"): - def_gen = data.def_gen.map(def_gen_dict) - data.def_gen = def_gen - data.dropna(axis=0, subset=["def_gen"], inplace=True) + shorten_defence_names(data, def_gen_dict) if "attack.init.name" in data: data = merge_attacks(data) if hasattr(data, "atk_gen"): - atk_gen = data.atk_gen.map(atk_gen_dict) - data.atk_gen = atk_gen - data.dropna(axis=0, subset=["atk_gen"], inplace=True) + shorten_attack_names(data, atk_gen_dict) data, fillna = format_control_parameter(data, control_dict, fillna) + data = fill_na(data, fillna) + data = replace_strings_in_data(data, replace_dict) + data = replace_strings_in_columns(data, col_replace_dict) + + if len(pareto_dict) > 0: + data = find_pareto_set(data, pareto_dict) + return data + + +def shorten_defence_names(data, def_gen_dict): + def_gen = data.def_gen.map(def_gen_dict) + data.def_gen = def_gen + data.dropna(axis=0, subset=["def_gen"], inplace=True) + + +def shorten_attack_names(data, atk_gen_dict): + atk_gen = data.atk_gen.map(atk_gen_dict) + data.atk_gen = atk_gen + data.dropna(axis=0, subset=["atk_gen"], inplace=True) + + +def fill_na(data, fillna): for k, v in fillna.items(): if k in data.columns: data[k] = data[k].fillna(v) else: data[k] = str(v) - data = replace_strings_in_data(data, replace_dict) - if len(pareto_dict) > 0: - data = pareto_set(data, pareto_dict) + return data + + +def find_pareto_set(data, pareto_dict): + data = pareto_set(data, pareto_dict) return data @@ -586,22 +668,21 @@ def main(args): assert Path( args.input_file, ).exists(), f"File {args.input_file} does not exist. Please specify a valid file using the -i flag." - data = pd.read_csv(args.input_file) + data = load_results( + results_file=Path(args.input_file).name, + results_folder=Path(args.input_file).parent, + ) # Strip whitespace from column names trim_strings = lambda x: x.strip() if isinstance(x, str) else x # noqa E731 data.rename(columns=trim_strings, inplace=True) # Strip whitespace from column values data = data.applymap(lambda x: x.strip() if isinstance(x, str) else x) - assert "model.init.name" in data.columns, "model.init.name not in data.columns" - if isinstance(args.drop_if_empty, str): args.drop_if_empty = args.drop_if_empty.split(",") else: assert isinstance(args.drop_if_empty, list) - for col in args.drop_if_empty: - assert col in data.columns, f"Column {col} not in data.columns" - data = drop_frames_without_results(data, subset=args.drop_if_empty) + # Reads Config file with open(Path(args.config), "r") as f: big_dict = yaml.load(f, Loader=yaml.FullLoader) @@ -611,6 +692,7 @@ def main(args): fillna = big_dict.get("fillna", {}) min_max = big_dict.get("min_max", []) replace_dict = big_dict.get("replace", {}) + replace_cols = big_dict.get("replace_cols", {}) pareto_dict = big_dict.get("pareto", {}) drop_dict = big_dict.pop("drop_values", {}) data = drop_values(data, drop_dict) @@ -621,11 +703,16 @@ def main(args): control_dict, fillna=fillna, replace_dict=replace_dict, + col_replace_dict=replace_cols, pareto_dict=pareto_dict, ) - + for col in results.columns: + if "def" in col: + print(col) if "adv_accuracy" in results.columns: results = calculate_failure_rate(results) + results = fill_train_time(results) + results = drop_rows_without_results(results, subset=args.drop_if_empty) results = min_max_scaling(results, *min_max) output_file = save_results( results, diff --git a/deckard/layers/compile.py b/deckard/layers/compile.py index 765ee848..4a33e818 100644 --- a/deckard/layers/compile.py +++ b/deckard/layers/compile.py @@ -33,7 +33,11 @@ def flatten_results(df: pd.DataFrame) -> pd.DataFrame: return df -def parse_folder(folder, files=["params.yaml", "score_dict.json"]) -> pd.DataFrame: +def parse_folder( + folder, + files=["params.yaml", "score_dict.json"], + other_files=False, +) -> pd.DataFrame: """ Parse a folder containing files and return a dataframe with the results, excluding the files in the exclude list. :param folder: Path to folder containing files @@ -56,13 +60,12 @@ def parse_folder(folder, files=["params.yaml", "score_dict.json"]) -> pd.DataFra results = {} for file in tqdm(path_gen, desc="Parsing Specified files"): results = read_file(file, results) - for folder in tqdm(folder_gen, desc="Adding other files to results"): - results = add_file(folder, path_gen, results) + if other_files is True: + for folder in tqdm(folder_gen, desc="Adding other files to results"): + results = add_file(folder, path_gen, results) df = pd.DataFrame(results).T df = df.applymap(lambda x: x.strip() if isinstance(x, str) else x) df.columns = df.columns.str.strip() - df = df.applymap(lambda x: x.strip() if isinstance(x, str) else x) - df.columns = df.columns.str.strip() return df @@ -125,6 +128,10 @@ def save_results(results, results_file, results_folder) -> str: """ Compile results from a folder of reports and save to a csv file; return the path to the csv file. It will optionally delete columns from the results. """ + assert isinstance( + results, + pd.DataFrame, + ), f"Results must be a pandas DataFrame, not {type(results)}." results_file = Path(results_folder, results_file) logger.info(f"Saving data to {results_file}") Path(results_file).parent.mkdir(exist_ok=True, parents=True) @@ -137,6 +144,17 @@ def save_results(results, results_file, results_folder) -> str: results.to_html(results_file, index=True) elif suffix == ".json": results.to_json(results_file, index=True, orient="records") + elif suffix == ".tex": + pretty_model = results_file.stem.replace("_", " ").title() + results.to_latex( + results_file, + index=True, + escape=True, + label=f"tab:{results_file.stem}", + caption=f"{pretty_model} Results", + header=True, + position="htbp", + ) else: raise ValueError(f"File type {suffix} not supported.") assert Path( @@ -161,6 +179,15 @@ def load_results(results_file, results_folder) -> pd.DataFrame: results = pd.read_html(results_file) elif suffix == ".json": results = pd.read_json(results_file) + elif suffix == ".tex": + pd.read_csv( + results_file, + sep="&", + header=None, + skiprows=4, + skipfooter=3, + engine="python", + ) else: raise ValueError(f"File type {suffix} not supported.") assert Path( diff --git a/deckard/layers/find_best.py b/deckard/layers/find_best.py index eb582d51..9cb34315 100644 --- a/deckard/layers/find_best.py +++ b/deckard/layers/find_best.py @@ -1,5 +1,6 @@ import optuna import logging +import pandas as pd from pathlib import Path from hydra import initialize_config_dir, compose from omegaconf import OmegaConf @@ -24,13 +25,70 @@ def find_optuna_best( ): logger.info(f"Study name: {study_name}") logger.info(f"Storage name: {storage_name}") - study = optuna.create_study( - study_name=study_name, - storage=storage_name, - load_if_exists=True, - direction=direction, - ) - df = study.trials_dataframe(attrs=("number", "value", "params", "state")) + if isinstance(direction, str): + study = optuna.create_study( + study_name=study_name, + storage=storage_name, + load_if_exists=True, + direction=direction, + ) + directions = [direction] + else: + study = optuna.create_study( + study_name=study_name, + storage=storage_name, + load_if_exists=True, + directions=direction, + ) + directions = direction + assert isinstance(directions, list), f"Directions is not a list: {type(directions)}" + df = study.trials_dataframe(attrs=("number", "value", "params")) + # Find the average of each value over the columns in average_over + not_these = ["number", "value"] + val_cols = [ + col + for col in df.columns + if col.startswith("values_") and col.split("values_")[-1] not in not_these + ] + not_these.extend(val_cols) + print(f"Not these: {not_these}") + groupby_cols = [ + col for col in df.columns if col.split("params_")[-1] not in not_these + ] + print(f"Groupby cols: {groupby_cols}") + dfs = df.groupby(groupby_cols) + new_df = pd.DataFrame(columns=groupby_cols + ["mean", "std", "ntrials", "nuniques"]) + means = [] + stds = [] + ntrials = [] + nuniques = [] + for _, df in dfs: + # find mean of value + mean_ = df.value.mean() + # find the std + std_ = df.value.std() + # find the number of trials + n = df.value.count() + # find the number of unique trials + nunique = df.value.nunique() + means.append(mean_) + stds.append(std_) + ntrials.append(n) + nuniques.append(nunique) + # add first row of df to new_df + new_df = pd.concat([new_df, df.head(1)]) + new_df.drop(columns=["value"], inplace=True) + new_df["mean"] = means + new_df["std"] = stds + new_df["ntrials"] = ntrials + new_df["nuniques"] = nuniques + for direction in directions: + assert direction in [ + "minimize", + "maximize", + ], f"Direction {direction} not recognized." + directions = [False if x == "maximize" else True for x in directions] + assert isinstance(new_df, pd.DataFrame), f"df is not a dataframe: {type(df)}" if study_csv is not None: Path(study_csv).parent.mkdir(parents=True, exist_ok=True) df.to_csv(study_csv) @@ -52,7 +110,7 @@ def find_optuna_best( # Changing the keys to hydra override format for key, value in best_params.items(): if ( - key.startswith("++") or key.startswith("~~") or key.startswith("--") + key.startswith("++") or key.startswith("~") or key.startswith("--") ): # reserved meaning pass elif key.startswith("+"): # appends to config diff --git a/deckard/layers/merge.py b/deckard/layers/merge.py new file mode 100644 index 00000000..13d62ea1 --- /dev/null +++ b/deckard/layers/merge.py @@ -0,0 +1,252 @@ +import pandas as pd +from pathlib import Path +import logging +import yaml +import argparse +from numpy import nan as np_nan +from deckard.layers.compile import save_results + +logger = logging.getLogger(__name__) + + +__all__ = ["merge_csv", "main", "parser"] + + +def merge_csv( + output_file: str, + smaller_file: list = None, + fillna: dict = {}, + metadata: list = [], + subset_metadata: list = [], + how: str = "outer", + **kwargs, +): + """ + The function `merge_csv` merges two CSV files, one from a big directory and one from a little + directory, and saves the merged file. + + Args: + big_dir: The `big_dir` parameter is the directory path where the dataset to be merged into is located. This + dataset is assumed to have a file named "raw.csv" which will be read. + little_dir: The `little_dir` parameter is the directory path where the smaller dataset is located. + data_file: The `data_file` parameter is the name of the CSV file that will be used for both the + `big` and `small` dataframes. If `smaller_file` is not provided, then the `data_file` from + the `big` directory will be used for both dataframes. Defaults to raw.csv + smaller_file: The parameter `smaller_file` is an optional argument that specifies + the name of the data file in the `little_dir` directory. If this argument is provided, the function + will read the data from the specified file in the `little_dir` directory. If this argument is not + provided, the + + Returns: + None. + """ + if Path(output_file).exists(): + big = pd.read_csv(Path(output_file), index_col=0, dtype=str) + else: + big = pd.DataFrame() + # Cast all columns to strings: + big = big.replace(np_nan, "").astype(str) + # Strip whitepsace around entries: + big = big.apply(lambda x: x.str.strip() if x.dtype == "object" else x) + # strip whitespace around column names + big.columns = big.columns.astype(str).str.strip() + if "id" not in big: + big["id"] = None + if smaller_file is not None: + small = pd.read_csv(Path(smaller_file), index_col=0, dtype=str) + if "id" not in small: + small["id"] = Path(smaller_file).stem + # Cast all columns to strings: + small = small.replace(np_nan, "").astype(str) + # Strip whitepsace around entries: + small = small.apply(lambda x: x.str.strip() if x.dtype == "object" else x) + # strip whitespace around column names + small.columns = small.columns.str.strip() + logger.info(f"Shape of big: {big.shape}") + logger.info(f"Shape of small: {small.shape}") + try: + merged = pd.merge(big, small, how=how, **kwargs) + except ( + pd.errors.MergeError + ) as e: # will raise if no common columns to merge on, so just concat + if "No common columns to perform merge on" in str(e): + merged = pd.concat([big, small], axis=0) + else: + raise e + for k, v in fillna.items(): + if k in merged.columns: + merged[k] = merged[k].fillna(v) + else: + merged[k] = v + logger.info(f"Shape of merged: {merged.shape}") + logger.info(f"Saving merged to {output_file}.") + results_folder = Path(output_file).parent + results_file = Path(output_file).name + results_folder.mkdir(parents=True, exist_ok=True) + # Add metadata if it exists + if len(metadata) > 0: + merged = add_metadata(merged, metadata) + # Add subset metadata if it exists + if len(subset_metadata) > 0: + merged = add_subset_metadata(merged, subset_metadata) + saved_path = save_results( + merged, + results_file=results_file, + results_folder=results_folder, + ) + + assert Path(saved_path).exists(), f"Saved path {saved_path} does not exist." + return None + + +def parse_cleaning_config(config_file, metadata_file=None, subset_metadata_file=None): + dict_ = {} + if config_file is not None: + with open(config_file, "r") as stream: + dict_ = yaml.safe_load(stream) + fillna = dict_.get("fillna", dict_) + else: + fillna = {} + dict_["fillna"] = fillna + if metadata_file is not None: + with open(metadata_file, "r") as stream: + metadata = yaml.safe_load(stream) + metadata = metadata.get("metadata", metadata) + elif "metadata" in dict_: + metadata = dict_["metadata"] + else: + metadata = {} + dict_["metadata"] = metadata + if subset_metadata_file is not None: + with open(subset_metadata_file, "r") as stream: + subset_metadata = yaml.safe_load(stream) + subset_metadata = subset_metadata.get("subset_metadata", subset_metadata) + elif "subset_metadata" in dict_: + subset_metadata = dict_["subset_metadata"] + else: + subset_metadata = {} + dict_["subset_metadata"] = subset_metadata + return dict_ + + +def main(args): + config = parse_cleaning_config(args.config, args.metadata, args.subset_metadata) + if args.output_folder is None: + args.output_folder = Path().cwd() + output_file = Path(args.output_folder) / args.output_file + if isinstance(args.smaller_file, list): + for little in args.smaller_file: + merge_csv( + smaller_file=little, + fillna=config.get("fillna", {}), + metadata=config.get("metadata", {}), + subset_metadata=config.get("subset_metadata", {}), + output_file=output_file, + how=args.how, + ) + else: + merge_csv( + smaller_file=args.smaller_file, + fillna=config.get("fillna", {}), + metadata=config.get("metadata", {}), + subset_metadata=config.get("subset_metadata", {}), + output_file=output_file, + how=args.how, + ) + return None + + +def add_metadata(df, metadata_dict={}): + """ + The function `add_metadata` adds metadata to the dataframe. + + Args: + df: The `df` parameter is the dataframe to which metadata will be added. + metadata_dict: The `metadata_dict` parameter is a dictionary containing the metadata to be added to the dataframe. + + Returns: + df: The dataframe with the added metadata. + """ + for thing in metadata_dict: + k = thing.split(":")[0] + v = thing.split(":")[1] + df[k] = v + return df + + +def add_subset_metadata(df, metadata_list=[]): + """ + The function `add_subset_metadata` adds metadata to the dataframe. + + Args: + df: The `df` parameter is the dataframe to which metadata will be added. + metadata_dict: The `metadata_dict` parameter is a dictionary containing the metadata to be added to the dataframe. + + Returns: + df: The dataframe with the added metadata. + """ + for thing in metadata_list: + subset = thing.get("subset", ValueError("subset not found in entry.")) + subset_col = subset.split(" ")[0] + subset_op = subset.split(" ")[1] + subset_val = subset.split(" ")[2] + filter = f"df[{subset_col}] {subset_op} {subset_val}" + key = thing.get("key", ValueError("key not found in entry.")) + value = thing.get("value", ValueError("value not found in entry.")) + subset_df = df[subset_col].where(filter) + subset_df[key] = value + df.update(subset_df) + return df + + +parser = argparse.ArgumentParser() +parser.add_argument( + "--output_file", + type=str, + help="Name of the output file", + default="merged.csv", +) +parser.add_argument( + "--output_folder", + type=str, + help="Name of the output folder", + required=False, +) +parser.add_argument( + "--smaller_file", + type=str, + help="Name(s) of the files to merge into the big file.", + required=False, + nargs="*", +) +parser.add_argument( + "--config", + type=str, + help="Name of file containing a 'fillna' config dictionary.", + required=False, +) +parser.add_argument( + "--metadata", + type=str, + help="Name of file containing a 'metadata' dictionary.", + required=False, + # set default to --config + default=None, +) +parser.add_argument( + "--subset_metadata", + type=str, + help="Name of file containing a 'subset_metadata' dictionary.", + required=False, + default=None, +) +parser.add_argument( + "--how", + type=str, + help="Type of merge to perform. Default is 'outer'.", + default="outer", +) + +if __name__ == "__main__": + args = parser.parse_args() + main(args) diff --git a/deckard/layers/optimise.py b/deckard/layers/optimise.py index 71bf6893..9f96bd9c 100644 --- a/deckard/layers/optimise.py +++ b/deckard/layers/optimise.py @@ -1,7 +1,6 @@ import logging import os import traceback -from copy import deepcopy from pathlib import Path import yaml from hydra.utils import instantiate @@ -172,32 +171,50 @@ def parse_stage(stage: str = None, params: dict = None, path=None) -> dict: if isinstance(params, dict): key_list = [] for stage in stages: + stage = stage.split("@")[0] assert Path( path, "dvc.yaml", ).exists(), f"{Path(path, 'dvc.yaml')} does not exist." with open(Path(path, "dvc.yaml"), "r") as f: - new_keys = yaml.load(f, Loader=yaml.FullLoader)["stages"][stage][ - "params" - ] + print() + keys = yaml.load(f, Loader=yaml.FullLoader)["stages"] + if stage in keys: + new_keys = keys[stage] + if "foreach" in new_keys: + new_keys = new_keys["do"]["params"] + else: + new_keys = new_keys["params"] + key_list.extend(new_keys) else: raise TypeError(f"Expected str or dict, got {type(params)}") params = read_subset_of_params(key_list, params) # Load files from dvc with open(Path(path, "dvc.yaml"), "r") as f: - file_list = [] - for stage in stages: - pipe = yaml.load(f, Loader=yaml.FullLoader)["stages"][stage] - if "deps" in pipe: - dep_list = [str(x).split(":")[0] for x in pipe["deps"]] - file_list.extend(dep_list) - if "outs" in pipe: - out_list = [str(x).split(":")[0] for x in pipe["outs"]] - file_list.extend(out_list) - if "metrics" in pipe: - metric_list = [str(x).split(":")[0] for x in pipe["metrics"]] - file_list.extend(metric_list) + pipe = yaml.load(f, Loader=yaml.FullLoader) + file_list = [] + for stage in stages: + if len(stage.split("@")) > 1: + sub_stage = stage.split("@")[1] + directory = stage.splits("@")[0] + file_list.append(directory) + file_list.append(sub_stage) + else: + sub_stage = None + stage = stage.split("@")[0] + pipe = pipe["stages"][stage] + if "do" in pipe: + pipe = pipe["do"] + if "deps" in pipe: + dep_list = [str(x).split(":")[0] for x in pipe["deps"]] + file_list.extend(dep_list) + if "outs" in pipe: + out_list = [str(x).split(":")[0] for x in pipe["outs"]] + file_list.extend(out_list) + if "metrics" in pipe: + metric_list = [str(x).split(":")[0] for x in pipe["metrics"]] + file_list.extend(metric_list) file_string = str(file_list) files = params["files"] file_list = list(files.keys()) @@ -252,7 +269,7 @@ def optimise(cfg: DictConfig) -> None: stage = cfg.pop("stage", None) cfg = parse_stage(params=cfg, stage=stage, path=working_dir) exp = instantiate(cfg) - files = deepcopy(exp.files)() + files = exp.files.get_filenames() folder = Path(files["score_dict_file"]).parent Path(folder).mkdir(exist_ok=True, parents=True) write_stage(cfg, stage, path=folder, working_dir=working_dir) diff --git a/deckard/layers/plots.py b/deckard/layers/plots.py index 86b44918..af653714 100644 --- a/deckard/layers/plots.py +++ b/deckard/layers/plots.py @@ -31,11 +31,11 @@ def cat_plot( y, hue, kind, - titles, - xlabels, - ylabels, file, folder, + xlabels=None, + ylabels=None, + titles=None, legend_title=None, x_lim=None, y_lim=None, @@ -112,9 +112,12 @@ def cat_plot( data = data.sort_values(by=[x, y]) logger.debug(f"Data sorted by x:{x}, y:{y}, kind:{kind}, and kwargs:{kwargs}.") graph = sns.catplot(data=data, x=x, y=y, kind=kind, **kwargs) - graph.set_xlabels(xlabels) - graph.set_ylabels(ylabels) - graph.set_titles(titles) + if xlabels is not None: + graph.set_xlabels(xlabels) + if ylabels is not None: + graph.set_ylabels(ylabels) + if titles is not None: + graph.set_titles(titles) if legend_title is not None: graph.legend.set_title(title=legend_title) else: diff --git a/deckard/layers/utils.py b/deckard/layers/utils.py index 2217f785..7ce37a15 100644 --- a/deckard/layers/utils.py +++ b/deckard/layers/utils.py @@ -81,8 +81,8 @@ def get_overrides(overrides=None): overrides[k] = v elif k.startswith("+"): overrides[f"++{k[1:]}"] = v - elif k.startswith("~~"): - overrides[f"~~{k[2:]}"] = v + elif k.startswith("~"): + overrides[f"~{k[2:]}"] = v else: overrides[f"++{k}"] = v @@ -168,30 +168,45 @@ def get_dvc_stage_params( directory=".", name=None, ): + tmp_stage = stage.split("@")[0] + sub_stage = stage.split("@")[1] if tmp_stage != stage else None logger.info( - f"Getting params for stage {stage} from {params_file} and {pipeline_file} in {directory}.", + f"Getting params for stage {tmp_stage} from {params_file} and {pipeline_file} in {directory}.", ) - params = dvc.api.params_show(stages=stage, repo=directory) + params = dvc.api.params_show(stages=tmp_stage, repo=directory) params.update({"_target_": "deckard.base.experiment.Experiment"}) params = OmegaConf.to_container(OmegaConf.create(params), resolve=True) flat_params = flatten_dict(params) - pipe_params = dvc.api.params_show(pipeline_file, stages=stage, repo=directory)[ - "stages" - ][stage] - file_list = [] - for key in ["metrics", "deps", "outs", "plots"]: - param_string = str(pipe_params.get(key, {})) - # find all values within ${} and add them to file_list - file_list.extend(re.findall(r"\${(.*?)}", param_string)) - file_dict = {} - for k in file_list: - if k in flat_params: - file_dict[k] = flat_params[k] + keys = dvc.api.params_show(pipeline_file, stages=stage, repo=directory).keys() + if "stages" in keys: + + pipe_params = dvc.api.params_show(pipeline_file, stages=stage, repo=directory)[ + "stages" + ] + if sub_stage is None: + pipe_params = pipe_params[stage] else: - raise ValueError(f"File {k} not found in {pipe_params.keys()}") - file_dict = unflatten_dict(file_dict) + pipe_params = pipe_params[tmp_stage]["do"] + file_list = [] + for key in ["metrics", "deps", "outs", "plots"]: + param_string = str(pipe_params.get(key, {})) + # find all values within ${} and add them to file_list + file_list.extend(re.findall(r"\${(.*?)}", param_string)) + file_dict = {} + for k in file_list: + if k in flat_params: + file_dict[k] = flat_params[k] + elif k == "item": + file_dict["directory"] = sub_stage + else: + raise ValueError(f"File {k} not found in {pipe_params.keys()}") + file_dict = unflatten_dict(file_dict) + else: + pipe_params = dvc.api.params_show(pipeline_file, stages=stage, repo=directory) + file_dict = unflatten_dict(pipe_params) params["files"] = file_dict.pop("files", {}) - params["files"]["stage"] = stage + params["files"].update(file_dict) + params["files"]["stage"] = tmp_stage # Merge remaining params params = OmegaConf.merge(params, file_dict) params = OmegaConf.to_container(OmegaConf.create(params), resolve=True) @@ -200,31 +215,6 @@ def get_dvc_stage_params( return params -# def get_dvc_stage_params( -# stage, -# params_file="params.yaml", -# pipeline_file="dvc.yaml", -# directory=".", -# name=None, -# ): -# logger.info( -# f"Getting params for stage {stage} from {params_file} and {pipeline_file} in {directory}.", -# ) -# params = dvc.api.params_show(stages=stage) -# params.update({"_target_": "deckard.base.experiment.Experiment"}) -# pipe_params = dvc.api.params_show(pipeline_file, stages=stage, repo=directory) -# pipe_params = unflatten_dict(pipe_params) -# params["files"] = dict(pipe_params.pop("files", pipe_params)) -# params["files"]["_target_"] = "deckard.base.files.FileConfig" -# params["files"]["stage"] = stage -# params["stage"] = stage -# if name is not None: -# params["files"]["name"] = name -# # Merge remaining params -# params = OmegaConf.merge(params, pipe_params) -# return params - - def prepare_files(params_file, stage, params, id_): # Turns the dictionary into a FileConfig object. # This creates a new directory at files.directory @@ -232,7 +222,11 @@ def prepare_files(params_file, stage, params, id_): # It also creates a new directory at files.directory/files.reports_dir # If a stage is specified, it also creates a new directory at files.directory/files.reports/stage params["files"]["_target_"] = "deckard.base.files.FileConfig" - params["files"]["stage"] = stage + tmp_stage = stage.split("@")[0] + sub_stage = stage.split("@")[1] if tmp_stage != stage else None + if sub_stage is not None: + params["files"]["directory"] = sub_stage + params["files"]["stage"] = tmp_stage params["files"]["name"] = ( id_ if params["files"].get("name", None) is None else params["files"]["name"] ) @@ -261,9 +255,8 @@ def prepare_files(params_file, stage, params, id_): def get_stages(pipeline_file="dvc.yaml", stages=None, repo=None): try: - def_stages = list( - dvc.api.params_show(pipeline_file, repo=repo)["stages"].keys(), - ) + with open(pipeline_file, "r") as f: + def_stages = yaml.safe_load(f)["stages"].keys() except NotGitRepository: raise ValueError( f"Directory {repo} is not a dvc repository. Please run `dvc init` in {repo} and try again.", @@ -276,8 +269,9 @@ def get_stages(pipeline_file="dvc.yaml", stages=None, repo=None): else: assert isinstance(stages, list), f"args.stage is of type {type(stages)}" for stage in stages: + tmp_stage = stage.split("@")[0] assert ( - stage in def_stages + tmp_stage in def_stages ), f"Stage {stage} not found in {pipeline_file}. Available stages: {def_stages}" return stages @@ -340,8 +334,8 @@ def run_stage( params["_target_"] = "deckard.experiment.Experiment" exp = instantiate(params) id_ = exp.name - _ = prepare_files(params_file, stage, params, id_) - score = exp() + files = prepare_files(params_file, stage, params, id_) + score = exp(**files) else: possible_subdicts = ["data", "model", "attack", "scorers", "plots", "files"] assert ( diff --git a/examples/.gitignore b/examples/.gitignore new file mode 100644 index 00000000..76d646aa --- /dev/null +++ b/examples/.gitignore @@ -0,0 +1,2 @@ +old/ +*_old/ diff --git a/examples/pytorch/cifar10/conf/model/art/preprocessor/default.yaml b/examples/gzip/.dvc/config similarity index 100% rename from examples/pytorch/cifar10/conf/model/art/preprocessor/default.yaml rename to examples/gzip/.dvc/config diff --git a/examples/gzip/classifier.ipynb b/examples/gzip/classifier.ipynb new file mode 100644 index 00000000..d68075b8 --- /dev/null +++ b/examples/gzip/classifier.ipynb @@ -0,0 +1,634 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "# ! python -m pip install numpy scikit-learn tqdm pandas matplotlib seaborn" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"\n", + "This is a module toa be used as a reference for building other modules\n", + "\"\"\"\n", + "\n", + "import numpy as np\n", + "from sklearn.base import BaseEstimator, ClassifierMixin\n", + "from sklearn.utils.validation import check_is_fitted\n", + "from sklearn.utils.multiclass import unique_labels\n", + "from sklearn.metrics import accuracy_score\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.datasets import make_classification, fetch_20newsgroups\n", + "from sklearn.preprocessing import LabelEncoder\n", + "import gzip\n", + "from tqdm import tqdm\n", + "from sklearn.preprocessing import LabelEncoder\n", + "import pandas as pd\n", + "\n", + "\n", + "class GzipClassifier(ClassifierMixin, BaseEstimator):\n", + " \"\"\"An example classifier which implements a 1-NN algorithm.\n", + "\n", + " For more information regarding how to build your own classifier, read more\n", + " in the :ref:`User Guide `.\n", + "\n", + " Parameters\n", + " ----------\n", + " demo_param : str, default='demo'\n", + " A parameter used for demonstation of how to pass and store paramters.\n", + "\n", + " Attributes\n", + " ----------\n", + " X_ : ndarray, shape (n_samples, n_features)\n", + " The input passed during :meth:`fit`.\n", + " y_ : ndarray, shape (n_samples,)\n", + " The labels passed during :meth:`fit`.\n", + " classes_ : ndarray, shape (n_classes,)\n", + " The classes seen at :meth:`fit`.\n", + " \"\"\"\n", + "\n", + " def __init__(self, k=3):\n", + " self.k = k\n", + " self.compressor = \"gzip\"\n", + " self._set_compressor()\n", + "\n", + " def fit(self, X, y):\n", + " \"\"\"A reference implementation of a fitting function for a classifier.\n", + "\n", + " Parameters\n", + " ----------\n", + " X : array-like, shape (n_samples, n_features)\n", + " The training input samples.\n", + " y : array-like, shape (n_samples,)\n", + " The target values. An array of int.\n", + "\n", + " Returns\n", + " -------\n", + " self : object\n", + " Returns self.\n", + " \"\"\"\n", + " # Check that X and y have correct shape\n", + " # X, y = check_X_y(X, y)\n", + " # Store the classes seen during fit\n", + " self.classes_ = unique_labels(y)\n", + "\n", + " self.X_ = X\n", + " self.y_ = y\n", + " Cxs = []\n", + "\n", + " for x in self.X_:\n", + " Cx = self._compress(x)\n", + " Cxs.append(Cx)\n", + " self.Cx_ = Cxs\n", + " # Return the classifier\n", + " return self\n", + "\n", + " def _set_compressor(self):\n", + " if self.compressor == \"gzip\":\n", + " self._compress = self._gzip_compressor\n", + " else:\n", + " raise NotImplementedError(\n", + " f\"Compressing with {self.compressor} not supported.\"\n", + " )\n", + "\n", + " def _gzip_compressor(self, x):\n", + " return len(gzip.compress(str(x).encode()))\n", + "\n", + " def _ncd(self, Cx1, x1):\n", + " distance_from_x1 = []\n", + " for x2, Cx2 in zip(self.X_, self.Cx_):\n", + " x2 = str(x2)\n", + " x1x2 = \" \".join([x1, x2])\n", + " Cx1x2 = self._compress(x1x2)\n", + " ncd = (Cx1x2 - min(Cx1, Cx2)) / max(Cx1, Cx2)\n", + " distance_from_x1.append(ncd)\n", + "\n", + " def predict(self, X):\n", + " \"\"\"A reference implementation of a prediction for a classifier.\n", + "\n", + " Parameters\n", + " ----------\n", + " X : array-like, shape (n_samples, n_features)\n", + " The input samples.\n", + "\n", + " Returns\n", + " -------\n", + " y : ndarray, shape (n_samples,)\n", + " The label for each sample is the label of the closest sample\n", + " seen during fit.\n", + " \"\"\"\n", + " # Check is fit had been called\n", + " check_is_fitted(self, [\"X_\", \"y_\", \"Cx_\", \"_compress\"])\n", + "\n", + " # Input validation\n", + " # X = check_array(X)\n", + " results = []\n", + " for x1 in tqdm(X, desc=\"Predicting...\", leave=False):\n", + " x1 = str(x1)\n", + " Cx1 = self._compress(x1)\n", + " distance_from_x1 = []\n", + " for x2, Cx2 in zip(self.X_, self.Cx_):\n", + " x2 = str(x2)\n", + " x1x2 = \" \".join([x1, x2])\n", + " Cx1x2 = self._compress(x1x2)\n", + " ncd = (Cx1x2 - min(Cx1, Cx2)) / max(Cx1, Cx2)\n", + " distance_from_x1.append(ncd)\n", + " # distance_from_x1 = self._ncd(Cx1, x1)\n", + " sorted_idx = np.argsort(np.array(distance_from_x1))\n", + " top_k_class = list(self.y_[sorted_idx[: self.k]])\n", + " predict_class = max(set(top_k_class), key=top_k_class.count)\n", + " results.append(predict_class)\n", + " return results" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Accuracy score is: 0.912\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r" + ] + } + ], + "source": [ + "# Newsgroup Data\n", + "\n", + "X, y = fetch_20newsgroups(\n", + " subset=\"train\",\n", + " categories=[\"alt.atheism\", \"talk.religion.misc\"],\n", + " shuffle=True,\n", + " random_state=42,\n", + " return_X_y=True,\n", + ")\n", + "y = LabelEncoder().fit(y).transform(y)\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y)\n", + "newsgroup_model = GzipClassifier(k=3)\n", + "newsgroup_model.fit(X_train, y_train)\n", + "preds = newsgroup_model.predict(X_test)\n", + "print()\n", + "print(f\"Accuracy score is: {round(accuracy_score(y_test, preds), 3)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Shape of df: (125973, 43)\n", + "Shape of y: (125973,)\n", + "Shape of X: (125973, 42)\n", + "Set of labels: {0, 1}\n", + "Shape of X_train: (1000, 42)\n", + "Shape of X_test: (100, 42)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "KDD-NSL\n", + "Accuracy score is: 0.97\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r" + ] + } + ], + "source": [ + "# KDD\n", + "\n", + "df = pd.read_csv(\n", + " \"https://gist.githubusercontent.com/simplymathematics/8c6c04bd151950d5ea9e62825db97fdd/raw/34e546e4813f154d11d4f13869b9e3481fc3e829/kdd-nsl.csv\",\n", + " header=None,\n", + ")\n", + "print(\"Shape of df:\", df.shape)\n", + "width = df.shape[1]\n", + "y = df[width - 2]\n", + "print(\"Shape of y:\", y.shape)\n", + "del df[width - 2]\n", + "X = np.array(df)\n", + "print(\"Shape of X:\", X.shape)\n", + "del df\n", + "new_y = []\n", + "for entry in y:\n", + " if entry == \"normal\":\n", + " new_y.append(0)\n", + " else:\n", + " new_y.append(1)\n", + "y = LabelEncoder().fit(new_y).transform(new_y)\n", + "print(f\"Set of labels: {set(y)}\")\n", + "X_train, X_test, y_train, y_test = train_test_split(\n", + " X, y, train_size=1000, test_size=100\n", + ")\n", + "print(\"Shape of X_train:\", X_train.shape)\n", + "print(\"Shape of X_test:\", X_test.shape)\n", + "kdd_model = GzipClassifier(k=3)\n", + "kdd_model.fit(X_train, y_train)\n", + "predictions = kdd_model.predict(X_test)\n", + "print(\"KDD-NSL\")\n", + "print(f\"Accuracy score is: {round(accuracy_score(y_test, predictions), 3)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Shape of df: (134198, 64)\n", + "Shape of y: (134198,)\n", + "Shape of X: (134198, 62)\n", + "Set of labels: {0, 1}\n", + "Shape of X_train: (1000, 62)\n", + "Shape of X_test: (100, 62)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Truthseeker\n", + "Accuracy score is: 0.95\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r" + ] + } + ], + "source": [ + "# Truthseeker\n", + "\n", + "df = pd.read_csv(\n", + " \"https://gist.githubusercontent.com/simplymathematics/8c6c04bd151950d5ea9e62825db97fdd/raw/34e546e4813f154d11d4f13869b9e3481fc3e829/truthseeker.csv\"\n", + ")\n", + "print(\"Shape of df:\", df.shape)\n", + "y = np.array(df[\"BotScoreBinary\"].astype(\"int\"))\n", + "print(\"Shape of y:\", y.shape)\n", + "del df[\"BotScoreBinary\"]\n", + "del df[\"BotScore\"]\n", + "X = np.array(df)\n", + "print(\"Shape of X:\", X.shape)\n", + "print(f\"Set of labels: {set(y)}\")\n", + "X_train, X_test, y_train, y_test = train_test_split(\n", + " X, y, train_size=1000, test_size=100\n", + ")\n", + "print(\"Shape of X_train:\", X_train.shape)\n", + "print(\"Shape of X_test:\", X_test.shape)\n", + "truthseeker_model = GzipClassifier(k=7)\n", + "truthseeker_model.fit(X_train, y_train)\n", + "predictions = truthseeker_model.predict(X_test)\n", + "print(\"Truthseeker\")\n", + "print(f\"Accuracy score is: {round(accuracy_score(y_test, predictions), 3)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Accuracy score is: 0.552\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r" + ] + } + ], + "source": [ + "# Make Classification Data\n", + "\n", + "\n", + "X, y = make_classification(\n", + " n_classes=3,\n", + " n_features=10,\n", + " n_informative=10,\n", + " n_redundant=0,\n", + " n_samples=1000,\n", + " n_clusters_per_class=1,\n", + " class_sep=10,\n", + ")\n", + "y = LabelEncoder().fit(y).transform(y)\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y)\n", + "model = GzipClassifier(k=1)\n", + "model.fit(X_train, y_train)\n", + "predictions = model.predict(X_test)\n", + "print()\n", + "print(f\"Accuracy score is: {round(accuracy_score(y_test, predictions), 3)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model is BaseEstimator: True\n" + ] + } + ], + "source": [ + "# ART setup\n", + "from art.attacks.evasion import HopSkipJump, ZooAttack\n", + "from art.estimators.classification import SklearnClassifier\n", + "\n", + "print(\"Model is BaseEstimator:\", isinstance(model, BaseEstimator))\n", + "classifier = SklearnClassifier(model=model)\n", + "classifier._input_shape = X_train[0].shape\n", + "classifier._nb_classes = len(np.unique(y_train))" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(10,)\n" + ] + } + ], + "source": [ + "print(classifier._input_shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Attack initialized.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "HopSkipJump: 100%|██████████| 100/100 [04:57<00:00, 2.98s/it] \n", + " \r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Adversarial examples generated.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "HopSkipJump\n", + "Accuracy score is: 0.37\n", + "Attack Success score is: 0.35\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r" + ] + } + ], + "source": [ + "# Evasion Attack: HopSkipJump\n", + "n = 100\n", + "attack = HopSkipJump(\n", + " classifier=classifier,\n", + " targeted=False,\n", + " norm=2,\n", + " max_iter=1,\n", + " max_eval=1,\n", + " init_eval=1,\n", + " init_size=1,\n", + ")\n", + "print(\"Attack initialized.\")\n", + "advs = attack.generate(x=X_test[:n])\n", + "print(\"Adversarial examples generated.\")\n", + "adv_preds = model.predict(advs)\n", + "print(\"HopSkipJump\")\n", + "print(f\"Accuracy score is: {round(accuracy_score(y_test[:n], adv_preds), 3)}\")\n", + "print(\n", + " f\"Attack Success score is: {round(accuracy_score(predictions[:n], adv_preds), 3)}\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "ZOO: 100%|██████████| 100/100 [01:00<00:00, 1.66it/s] \n", + " " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ZooAttack\n", + "Accuracy score is: 0.54\n", + "Attack Success score is: 0.96\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r" + ] + } + ], + "source": [ + "# Evasion Attack: ZooAttack\n", + "attack = ZooAttack(\n", + " classifier=classifier,\n", + " confidence=0.9,\n", + " targeted=False,\n", + " learning_rate=1e-1,\n", + " max_iter=1,\n", + " binary_search_steps=1,\n", + " initial_const=1e-3,\n", + " abort_early=True,\n", + " use_resize=False,\n", + " use_importance=False,\n", + " nb_parallel=1,\n", + " batch_size=1,\n", + " variable_h=0.01,\n", + ")\n", + "advs = attack.generate(x=X_test[:n])\n", + "adv_preds = model.predict(advs)\n", + "print(\"ZooAttack\")\n", + "print(f\"Accuracy score is: {round(accuracy_score(y_test[:n], adv_preds), 3)}\")\n", + "print(\n", + " f\"Attack Success score is: {round(accuracy_score(predictions[:n], adv_preds), 3)}\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [], + "source": [ + "# from art.attacks.inference.attribute_inference import AttributeInferenceBlackBox\n", + "# n = 10\n", + "# attack_feature = 1\n", + "# attack = AttributeInferenceBlackBox(classifier)\n", + "# attack.fit(X_train[:n])\n", + "# predictions = np.array(predictions)" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "dict_keys(['_targeted', '_estimator', '_summary_writer_arg', '_summary_writer', 'norm', 'max_iter', 'max_eval', 'init_eval', 'init_size', 'curr_iter', 'batch_size', 'verbose', 'theta'])" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# vars(attack).keys()" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [], + "source": [ + "# attack_result = attack.infer(X_test[:n], y_test[:n], pred=predictions[:n])" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [], + "source": [ + "# from art.attacks.inference.membership_inference import MembershipInferenceBlackBox" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.7" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/gzip/conf/data/large.yaml b/examples/gzip/conf/data/large.yaml new file mode 100644 index 00000000..46ffcc2c --- /dev/null +++ b/examples/gzip/conf/data/large.yaml @@ -0,0 +1,25 @@ +_target_: deckard.base.data.Data +generate: + # _target_: deckard.base.data.generator.DataGenerator + name: classification + random_state : 0 + n_samples : 101000 + n_features : 20 + n_classes: 3 + n_clusters_per_class: 1 + n_informative: 20 + n_redundant: 0 + n_repeated: 0 +sample: + # _target_: deckard.base.data.sampler.SklearnDataSampler + random_state : 0 + stratify: True + train_size : 100000 + test_size : 1000 +sklearn_pipeline: + # _target_: deckard.base.data.sklearn_pipeline.SklearnDataPipeline + preprocessor: + # + name: sklearn.preprocessing.StandardScaler + with_mean: True + with_std: True diff --git a/examples/gzip/conf/data/medium.yaml b/examples/gzip/conf/data/medium.yaml new file mode 100644 index 00000000..1b5f6a22 --- /dev/null +++ b/examples/gzip/conf/data/medium.yaml @@ -0,0 +1,25 @@ +_target_: deckard.base.data.Data +generate: + # _target_: deckard.base.data.generator.DataGenerator + name: classification + random_state : 0 + n_samples : 11000 + n_features : 20 + n_classes: 3 + n_clusters_per_class: 1 + n_informative: 20 + n_redundant: 0 + n_repeated: 0 +sample: + # _target_: deckard.base.data.sampler.SklearnDataSampler + random_state : 0 + stratify: True + train_size : 10000 + test_size : 1000 +sklearn_pipeline: + # _target_: deckard.base.data.sklearn_pipeline.SklearnDataPipeline + preprocessor: + # + name: sklearn.preprocessing.StandardScaler + with_mean: True + with_std: True diff --git a/examples/gzip/conf/data/small.yaml b/examples/gzip/conf/data/small.yaml new file mode 100644 index 00000000..2bc54e4b --- /dev/null +++ b/examples/gzip/conf/data/small.yaml @@ -0,0 +1,22 @@ +_target_: deckard.base.data.Data +generate: + # _target_: deckard.base.data.generator.DataGenerator + name: classification + random_state : 0 + n_samples : 2100 + n_features : 20 + n_classes: 3 + n_clusters_per_class: 1 +sample: + # _target_: deckard.base.data.sampler.SklearnDataSampler + random_state : 0 + stratify: True + train_size : 1000 + test_size : 1000 +sklearn_pipeline: + # _target_: deckard.base.data.sklearn_pipeline.SklearnDataPipeline + preprocessor: + # + name: sklearn.preprocessing.StandardScaler + with_mean: True + with_std: True diff --git a/examples/gzip/conf/data/very_large.yaml b/examples/gzip/conf/data/very_large.yaml new file mode 100644 index 00000000..a699dabf --- /dev/null +++ b/examples/gzip/conf/data/very_large.yaml @@ -0,0 +1,25 @@ +_target_: deckard.base.data.Data +generate: + # _target_: deckard.base.data.generator.DataGenerator + name: classification + random_state : 0 + n_samples : 1001000 + n_features : 20 + n_classes: 3 + n_clusters_per_class: 1 + n_informative: 20 + n_redundant: 0 + n_repeated: 0 +sample: + # _target_: deckard.base.data.sampler.SklearnDataSampler + random_state : 0 + stratify: True + train_size : 100000 + test_size : 1000 +sklearn_pipeline: + # _target_: deckard.base.data.sklearn_pipeline.SklearnDataPipeline + preprocessor: + # + name: sklearn.preprocessing.StandardScaler + with_mean: True + with_std: True diff --git a/examples/gzip/conf/model/art/defense_grid.yaml b/examples/gzip/conf/model/art/defense_grid.yaml new file mode 100644 index 00000000..a3fe3719 --- /dev/null +++ b/examples/gzip/conf/model/art/defense_grid.yaml @@ -0,0 +1,44 @@ +- model.art.preprocessor.name: art.defences.preprocessor.FeatureSqueezing + model.art.preprocessor.params: + clip_values : + - [0,255] + bit_depth : [4, 8, 16, 32, 64] + +- model.art.preprocessor.name: art.defences.preprocessor.GaussianAugmentation + model.art.preprocessor.params: + clip_values : + - [0,255] + sigma : [.1, .3, 1] + ratio : [.1, .5, 1] + +- model.art.preprocessor.name: art.defences.preprocessor.SpatialSmoothing + model.art.preprocessor.params: + clip_values : + - [0,255] + window_size : [2,3,4] + +- model.art.preprocessor.name: art.defences.preprocessor.TotalVarMin + model.art.preprocessor.params: + clip_values : + - [0,255] + prob : [.001, .01, .1] + norm : [1, 2, 3] + lamb : [.05, .5, .95] + max_iter : [100] + +- model.art.postprocessor.name : art.defences.postprocessor.GaussianNoise + model.art.postprocessor.params: + clip_values : + - [0,255] + scale: [.1, .9, .999] + +- model.art.postprocessor.name : art.defences.postprocessor.HighConfidence + model.art.postprocessor.params: + cutoff : [.1, .5, .9, .99] + + +- model.art.postprocessor.name : art.defences.postprocessor.Rounded + model.art.preprocessor.params: + clip_values : + - [0,255] + decimals : [1, 2, 4, 8] diff --git a/examples/gzip/conf/model/art/postprocessor.yaml b/examples/gzip/conf/model/art/postprocessor.yaml new file mode 100644 index 00000000..91f3c00a --- /dev/null +++ b/examples/gzip/conf/model/art/postprocessor.yaml @@ -0,0 +1,6 @@ +_target_ : deckard.base.model.art_pipeline.ArtPipeline +library : sklearn-svc +postprocessor: + name: art.defences.postprocessor.HighConfidence + threshold: 0.0 +initialize: diff --git a/examples/gzip/conf/model/art/postprocessor/high_confidence.yaml b/examples/gzip/conf/model/art/postprocessor/high_confidence.yaml new file mode 100644 index 00000000..16ae0e50 --- /dev/null +++ b/examples/gzip/conf/model/art/postprocessor/high_confidence.yaml @@ -0,0 +1,6 @@ +_target_ : deckard.base.model.art_pipeline.ArtPipeline +library : ${model.library} +postprocessor: + name: art.defences.postprocessor.HighConfidence + threshold: 0.0 +initialize: diff --git a/examples/gzip/conf/model/art/postprocessor/labels.yaml b/examples/gzip/conf/model/art/postprocessor/labels.yaml new file mode 100644 index 00000000..16ae0e50 --- /dev/null +++ b/examples/gzip/conf/model/art/postprocessor/labels.yaml @@ -0,0 +1,6 @@ +_target_ : deckard.base.model.art_pipeline.ArtPipeline +library : ${model.library} +postprocessor: + name: art.defences.postprocessor.HighConfidence + threshold: 0.0 +initialize: diff --git a/examples/gzip/conf/model/art/preprocessor.yaml b/examples/gzip/conf/model/art/preprocessor.yaml new file mode 100644 index 00000000..ebf46909 --- /dev/null +++ b/examples/gzip/conf/model/art/preprocessor.yaml @@ -0,0 +1,6 @@ +_target_ : deckard.base.model.art_pipeline.ArtPipeline +library : sklearn-svc +preprocessor: + name: art.defences.preprocessor.FeatureSqueezing + bit_depth: 32 +initialize: diff --git a/examples/gzip/conf/model/art/preprocessor/feature_squeezing.yaml b/examples/gzip/conf/model/art/preprocessor/feature_squeezing.yaml new file mode 100644 index 00000000..1c48b3e7 --- /dev/null +++ b/examples/gzip/conf/model/art/preprocessor/feature_squeezing.yaml @@ -0,0 +1,6 @@ +_target_ : deckard.base.model.art_pipeline.ArtPipeline +library : ${model.library} +preprocessor: + name: art.defences.preprocessor.FeatureSqueezing + bit_depth: 32 +initialize: diff --git a/examples/gzip/conf/model/gzip_classifier.yaml b/examples/gzip/conf/model/gzip_classifier.yaml new file mode 100644 index 00000000..fcebad74 --- /dev/null +++ b/examples/gzip/conf/model/gzip_classifier.yaml @@ -0,0 +1,16 @@ +data: ${data} +library : sklearn +init: + _target_: deckard.base.model.ModelInitializer + name : gzip_classifier.GzipClassifier + k : 1 + m: -1 + method : random + distance_matrix : ${files.directory}/${files.model_dir}/${dataset}/${model_name}/${model.init.compressor}/${data.sample.random_state}-${data.sample.train_size}.npz + compressor : gzip +_target_: deckard.base.model.Model +art: + _target_ : deckard.base.model.art_pipeline.ArtPipeline + library : sklearn + initialize: + nb_classes: 3 diff --git a/examples/gzip/conf/model/linear_svc.yaml b/examples/gzip/conf/model/linear_svc.yaml new file mode 100644 index 00000000..8068c0b3 --- /dev/null +++ b/examples/gzip/conf/model/linear_svc.yaml @@ -0,0 +1,15 @@ +data: ${data} +library : sklearn-svc +init: + _target_: deckard.base.model.ModelInitializer + name : sklearn.svm.SVC + C : float(.001, .01, .1, 1, 10, 100, 1000) + kernel : linear + probability : true + random_state : 0 + max_iter : 100 +_target_: deckard.base.model.Model +art: + _target_ : deckard.base.model.art_pipeline.ArtPipeline + library : sklearn-svc + initialize: diff --git a/examples/gzip/conf/model/model.yaml b/examples/gzip/conf/model/model.yaml new file mode 100644 index 00000000..eee5736f --- /dev/null +++ b/examples/gzip/conf/model/model.yaml @@ -0,0 +1,13 @@ +_target_: deckard.base.model.Model +art: + _target_: deckard.base.model.art_pipeline.ArtPipeline + initialize: + nb_classes: 3 + library: sklearn +data: ${data} +init: + _target_: deckard.base.model.ModelInitializer + k: 3 + n: 50 + name: gzip_classifier.GzipClassifier +library: sklearn diff --git a/examples/gzip/conf/model/poly_svc.yaml b/examples/gzip/conf/model/poly_svc.yaml new file mode 100644 index 00000000..7a5be797 --- /dev/null +++ b/examples/gzip/conf/model/poly_svc.yaml @@ -0,0 +1,16 @@ +data: ${data} +library : sklearn-svc +init: + _target_: deckard.base.model.ModelInitializer + name : sklearn.svm.SVC + C : 1.0 + coef0 : 0.0 + kernel : poly + probability : true + random_state : 0 + max_iter : 100 +_target_: deckard.base.model.Model +art: + _target_ : deckard.base.model.art_pipeline.ArtPipeline + library : sklearn-svc + initialize: diff --git a/examples/gzip/conf/model/rbf_svc.yaml b/examples/gzip/conf/model/rbf_svc.yaml new file mode 100644 index 00000000..42ca4c07 --- /dev/null +++ b/examples/gzip/conf/model/rbf_svc.yaml @@ -0,0 +1,15 @@ +data: ${data} +library : sklearn-svc +init: + _target_: deckard.base.model.ModelInitializer + name : sklearn.svm.SVC + C : 1.0 + kernel : rbf + probability : true + random_state : 0 + max_iter : 100 +_target_: deckard.base.model.Model +art: + _target_ : deckard.base.model.art_pipeline.ArtPipeline + library : sklearn-svc + initialize: diff --git a/examples/gzip/params.yaml b/examples/gzip/params.yaml new file mode 100644 index 00000000..43dbcb17 --- /dev/null +++ b/examples/gzip/params.yaml @@ -0,0 +1,88 @@ +data: + _target_: deckard.base.data.Data + name: https://gist.githubusercontent.com/simplymathematics/8c6c04bd151950d5ea9e62825db97fdd/raw/d6a22cdb42a1db624c89f0298cb4f654d3812703/kdd_nsl.csv + sample: + _target_: deckard.base.data.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 100 + train_size: 100 + sklearn_pipeline: + encoder: + handle_unknown: use_encoded_value + name: sklearn.preprocessing.OrdinalEncoder + unknown_value: -1 + preprocessor: + name: sklearn.preprocessing.StandardScaler + with_mean: true + with_std: true + target: label +dataset: kdd_nsl +direction: +- maximize +files: + _target_: deckard.base.files.FileConfig + attack_dir: attacks + attack_file: attack + attack_type: .pkl + data_dir: data + data_file: data + data_type: .pkl + directory: output + model_dir: model + model_file: model + model_type: .pkl + name: default + params_file: params.yaml + predictions_file: predictions.json + reports: reports + score_dict_file: score_dict.json +model: + _target_: deckard.base.model.Model + art: + _target_: deckard.base.model.art_pipeline.ArtPipeline + initialize: + nb_classes: 3 + library: sklearn + data: + _target_: deckard.base.data.Data + name: https://gist.githubusercontent.com/simplymathematics/8c6c04bd151950d5ea9e62825db97fdd/raw/d6a22cdb42a1db624c89f0298cb4f654d3812703/kdd_nsl.csv + sample: + _target_: deckard.base.data.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 100 + train_size: 100 + sklearn_pipeline: + encoder: + handle_unknown: use_encoded_value + name: sklearn.preprocessing.OrdinalEncoder + unknown_value: -1 + preprocessor: + name: sklearn.preprocessing.StandardScaler + with_mean: true + with_std: true + target: label + init: + _target_: deckard.base.model.ModelInitializer + compressor: gzip + distance_matrix: output/model/kdd_nsl/gzip_classifier/gzip/0-100.npz + k: 1 + m: -1 + method: random + name: gzip_classifier.GzipClassifier + library: sklearn +model_name: gzip_classifier +optimizers: +- accuracy +scorers: + _target_: deckard.base.scorer.ScorerDict + accuracy: + _target_: deckard.base.scorer.ScorerConfig + direction: maximize + name: sklearn.metrics.accuracy_score + log_loss: + _target_: deckard.base.scorer.ScorerConfig + direction: minimize + name: sklearn.metrics.log_loss +stage: train diff --git a/examples/power/conf/combined_afr.yaml b/examples/power/conf/combined_afr.yaml index 3209d7e4..f65984b9 100644 --- a/examples/power/conf/combined_afr.yaml +++ b/examples/power/conf/combined_afr.yaml @@ -18,62 +18,24 @@ weibull: plot: file : weibull_aft.pdf title : Weibull AFR Model - labels: - "Intercept: rho_": "$\\rho$" - "Intercept: lambda_": "$\\lambda$" - "data.sample.random_state: lambda_": "Random State" - "atk_value: lambda_": "Attack Strength" - "train_time: lambda_": "$T_{t}$" - "predict_proba_time: lambda_": "$T_{i}$" - "adv_accuracy: lambda_": "Adv. Accuracy" - "accuracy: lambda_": "Ben. Accuracy" - "adv_fit_time: lambda_": "$T_{a}$" - "adv_failure_rate: lambda_": "$h_{adv.}(t;\\theta)$" - "failure_rate: lambda_": "$h_{ben.}(t;\\theta)$" - "model.trainer.nb_epoch: lambda_": "No. of Epochs" - "model.trainer.batch_size: lambda_": "Batch Size" - "def_gen": "Defence" - "attack.init.eps: lambda_": "$\\varepsilon$" - partial_effect: - - "file": "weibull_epochs_partial_effect.pdf" - "covariate_array": "model.trainer.nb_epoch" - "values_array": [1,10,25,50, 100] - "title": "$S(t)$ for Weibull AFR" - "ylabel": "$\\mathbb{P}~(T>t)$" - "xlabel": "Time $t$ (seconds)" - "legend_kwargs": { - "title": "Epochs", - "labels": ["1", "10", "25", "50"] - } - - "file": "weibull_batch_size_partial_effect.pdf" - "covariate_array": "model.trainer.batch_size" - "values_array": [1,10,100, 1000] - "title": "$S(t)$ for Weibull AFR" - "ylabel": "$\\mathbb{P}~(T>t)$" - "xlabel": "Time $t$ (seconds)" - "legend_kwargs": { - "title": "Batch Size", - "labels": ["10", "100", "1000", "10000"] - } - - "file" : weibull_attack_eps_partial_effect.pdf - "covariate_array" : attack.init.eps - "values_array" : [1e-4, 1e-3, 1e-2, 1e-1, 1] - "title": "$S(t)$ for Weibull AFR" - "ylabel": "$\\mathbb{P}~(T>t)$" - "xlabel": "Time $t$ (seconds)" - "legend_kwargs": { - "title": "$\\varepsilon$", - "labels": ["1e-4", "1e-3", "1e-2", '1e-1', '1'] - } - - file : weibull_accuracy_partial_effect.pdf - covariate_array : accuracy - values_array : [0.5, .9, .99, .999] - title: "$S(t)$ for Weibull AFR" - ylabel: "$\\mathbb{P}~(T>t)$" - xlabel: "Time $t$ (seconds)" - legend_kwargs: - title: "Benign Accuracy" - labels: [".5", ".9", ".99", ".999"] + labels: + "Intercept: rho_": "$\\rho$" + "Intercept: lambda_": "$\\lambda$" + "data.sample.random_state: lambda_": "Random State" + "atk_value: lambda_": "Attack Strength" + "train_time: lambda_": "$T_{t}$" + "predict_proba_time: lambda_": "$T_{i}$" + "adv_accuracy: lambda_": "Adv. Accuracy" + "accuracy: lambda_": "Ben. Accuracy" + "adv_fit_time: lambda_": "$T_{a}$" + "adv_failure_rate: lambda_": "$h_{adv.}(t;\\theta)$" + "failure_rate: lambda_": "$h_{ben.}(t;\\theta)$" + "model.trainer.nb_epoch: lambda_": "No. of Epochs" + "model.trainer.batch_size: lambda_": "Batch Size" + "def_gen": "Defence" + "attack.init.eps: lambda_": "$\\varepsilon$" + ": lambda_" : "" + "nvidia-" : "" # cox: # plot: # file : cox_aft.pdf @@ -127,119 +89,46 @@ log_logistic: plot: file : log_logistic_aft.pdf title : Log logistic AFR Model - labels: - "Intercept: beta_": "$\\beta$" - "Intercept: alpha_": "$\\alpha$" - "data.sample.random_state: alpha_": "Random State" - "atk_value: alpha_": "Attack Strength" - "train_time: alpha_": "$T_{t}$" - "predict_proba_time: alpha_": "$T_{i}$" - "adv_accuracy: alpha_": "Adv. Accuracy" - "accuracy: alpha_": "Ben. Accuracy" - "adv_fit_time: alpha_": "$T_{a}$" - "adv_failure_rate: alpha_": "$h_{adv.}(t;\\theta)$" - "failure_rate: alpha_": "$h_{ben.}(t;\\theta)$" - "model.trainer.nb_epoch: alpha_": "No. of Epochs" - "model.trainer.batch_size: alpha_": "Batch Size" - "def_gen": "Defence" - "attack.init.eps: alpha_": "$\\varepsilon$" - partial_effect: - - "file": "log_logistic_epochs_partial_effect.pdf" - "covariate_array": "model.trainer.nb_epoch" - "values_array": [1,10,25,50] - "title": "$S(t)$ for Log-Logistic AFR" - "ylabel": "$\\mathbb{P}~(T>t)$" - "xlabel": "Time $t$ (seconds)" - "legend_kwargs": { - "title": "Epochs", - "labels": ["1", "10", "25", "50"] - } - - "file": "log_logistic_batch_size_partial_effect.pdf" - "covariate_array": "model.trainer.batch_size" - "values_array": [1,10,25,50] - "title": "$S(t)$ for Log-Logistic AFR" - "ylabel": "$\\mathbb{P}~(T>t)$" - "xlabel": "Time $t$ (seconds)" - "legend_kwargs": { - "title": "Batch Size", - "labels": ["10", "100", "1000", "10000"] - } - - "file" : log_logistic_attack_eps_partial_effect.pdf - "covariate_array" : attack.init.eps - "values_array" : [1e-4, 1e-3, 1e-2, 1e-1, 1] - "title": "$S(t)$ for Log-Logistic AFR" - "ylabel": "$\\mathbb{P}~(T>t)$" - "xlabel": "Time $t$ (seconds)" - "legend_kwargs": { - "title": "$\\varepsilon$", - "labels": ["1e-4", "1e-3", "1e-2", '1e-1', '1'] - } - - file : log_logistic_accuracy_partial_effect.pdf - covariate_array : accuracy - values_array : [0.5, .9, .99, .999] - title: "$S(t)$ for Log-Logistic AFR" - ylabel: "$\\mathbb{P}~(T>t)$" - xlabel: "Time $t$ (seconds)" - legend_kwargs: - title: "Benign Accuracy" - labels: [".5", ".9", ".99", ".999"] + labels: + "Intercept: beta_": "$\\beta$" + "Intercept: alpha_": "$\\alpha$" + "data.sample.random_state: alpha_": "Random State" + "atk_value: alpha_": "Attack Strength" + "train_time: alpha_": "$T_{t}$" + "predict_proba_time: alpha_": "$T_{i}$" + "adv_accuracy: alpha_": "Adv. Accuracy" + "accuracy: alpha_": "Ben. Accuracy" + "adv_fit_time: alpha_": "$T_{a}$" + "adv_failure_rate: alpha_": "$h_{adv.}(t;\\theta)$" + "failure_rate: alpha_": "$h_{ben.}(t;\\theta)$" + "model.trainer.nb_epoch: alpha_": "No. of Epochs" + "model.trainer.batch_size: alpha_": "Batch Size" + "def_gen": "Defence" + "attack.init.eps: alpha_": "$\\varepsilon$" + ": alpha_" : "" + "nvidia-" : "" log_normal: plot: file : log_normal_aft.pdf title : Log-Normal AFR Model - labels: - "Intercept: sigma_": "$\\sigma$" - "Intercept: mu_": "$\\mu$" - "atk_value: mu_": "Attack Strength" - "train_time: mu_": "$T_{t}$" - "predict_proba_time: mu_": "$T_{i}$" - "adv_accuracy: mu_": "Adv. Accuracy" - "accuracy: mu_": "Ben. Accuracy" - "adv_fit_time: mu_": "$T_{a}$" - "adv_failure_rate: mu_": "$h_{adv.}(t;\\theta)$" - "failure_rate: mu_": "$h_{ben.}(t;\\theta)$" - "model.trainer.nb_epoch: mu_": "No. of Epochs" - "model.trainer.batch_size: mu_": "Batch Size" - "def_gen": "Defence" - "attack.init.eps: mu_": "$\\varepsilon$" - "data.sample.random_state: mu_": "Random State" - partial_effect: - - "file": "log_normal_epochs_partial_effect.pdf" - "covariate_array": "model.trainer.nb_epoch" - "values_array": [1,10,25,50] - "title": "$S(t)$ for Log-Normal AFR" - "ylabel": "$\\mathbb{P}~(T>t)$" - "xlabel": "Time $t$ (seconds)" - "legend_kwargs": { - "title": "Epochs", - "labels": ["1", "10", "25", "50"] - } - - "file": "log_normal_batch_size_partial_effect.pdf" - "covariate_array": "model.trainer.batch_size" - "values_array": [1,10,25,50] - "title": "$S(t)$ for Log-Normal AFR" - "ylabel": "$\\mathbb{P}~(T>t)$" - "xlabel": "Time $t$ (seconds)" - "legend_kwargs": { - "title": "Batch Size", - "labels": ["10", "100", "1000", "10000"] - } - - "file" : log_normal_attack_eps_partial_effect.pdf - "covariate_array" : attack.init.eps - "values_array" : [1e-4, 1e-3, 1e-2, 1e-1, 1] - "title": "$S(t)$ for Log-Normal AFR" - "ylabel": "$\\mathbb{P}~(T>t)$" - "xlabel": "Time $t$ (seconds)" - "legend_kwargs": { - "title": "$\\varepsilon$", - "labels": ["1e-4", "1e-3", "1e-2", '1e-1', '1'] - } - - file : log_normal_accuracy_partial_effect.pdf - covariate_array : accuracy - values_array : [0.5, .9, .99, .999] - title: "$S(t)$ for Log-Normal AFR" - ylabel: "$\\mathbb{P}~(T>t)$" - xlabel: "Time $t$ (seconds)" - legend_kwargs: - title: "Benign Accuracy" - labels: [".5", ".9", ".99", ".999"] + labels: + "Intercept: sigma_": "$\\sigma$" + "Intercept: mu_": "$\\mu$" + "atk_value: mu_": "Attack Strength" + "train_time: mu_": "$T_{t}$" + "predict_proba_time: mu_": "$T_{i}$" + "adv_accuracy: mu_": "Adv. Accuracy" + "accuracy: mu_": "Ben. Accuracy" + "adv_fit_time: mu_": "$T_{a}$" + "adv_failure_rate: mu_": "$h_{adv.}(t;\\theta)$" + "failure_rate: mu_": "$h_{ben.}(t;\\theta)$" + "model.trainer.nb_epoch: mu_": "No. of Epochs" + "model.trainer.batch_size: mu_": "Batch Size" + "def_gen": "Defence" + "attack.init.eps: mu_": "$\\varepsilon$" + "data.sample.random_state: mu_": "Random State" + ": mu_" : "" + "nvidia-" : "" +dummies: + device_id : "GPU:" + dataset : "Data:" diff --git a/examples/power/conf/data/torch_cifar.yaml b/examples/power/conf/data/torch_cifar.yaml index 951cf254..36f29c69 100644 --- a/examples/power/conf/data/torch_cifar.yaml +++ b/examples/power/conf/data/torch_cifar.yaml @@ -6,9 +6,3 @@ sample: _target_: deckard.base.data.sampler.SklearnDataSampler random_state : 0 stratify: True -sklearn_pipeline: - _target_: deckard.base.data.sklearn_pipeline.SklearnDataPipeline - preprocessor: - name: sklearn.preprocessing.StandardScaler - with_mean: True - with_std: True diff --git a/examples/power/conf/data/torch_cifar100.yaml b/examples/power/conf/data/torch_cifar100.yaml index a0e83862..d1603c01 100644 --- a/examples/power/conf/data/torch_cifar100.yaml +++ b/examples/power/conf/data/torch_cifar100.yaml @@ -7,9 +7,3 @@ sample: _target_: deckard.base.data.sampler.SklearnDataSampler random_state : 0 stratify: True -sklearn_pipeline: - _target_: deckard.base.data.sklearn_pipeline.SklearnDataPipeline - preprocessor: - name: sklearn.preprocessing.StandardScaler - with_mean: True - with_std: True diff --git a/examples/power/conf/data/torch_mnist.yaml b/examples/power/conf/data/torch_mnist.yaml index 56e9c38b..3b77487f 100644 --- a/examples/power/conf/data/torch_mnist.yaml +++ b/examples/power/conf/data/torch_mnist.yaml @@ -6,9 +6,3 @@ sample: _target_: deckard.base.data.sampler.SklearnDataSampler random_state : 0 stratify: True -sklearn_pipeline: - _target_: deckard.base.data.sklearn_pipeline.SklearnDataPipeline - preprocessor: - name: sklearn.preprocessing.StandardScaler - with_mean: True - with_std: True diff --git a/examples/power/plots/dvc.lock b/examples/power/plots/dvc.lock index c75ee7ac..97aaf48f 100644 --- a/examples/power/plots/dvc.lock +++ b/examples/power/plots/dvc.lock @@ -1,5 +1,65 @@ schema: '2.0' stages: + merge@mnist: + cmd: python merge.py --big_dir data/bit_depth/mnist --little_dir data/mnist --config + ../conf/afr.yaml --data_file power.csv --output_folder mnist/ --output_file + merged.csv + deps: + - path: ../conf/clean.yaml + hash: md5 + md5: 3b23a3d656ad56e49e69f8624e227852 + size: 513 + - path: data/bit_depth/mnist/power.csv + hash: md5 + md5: ed40bca679961759527ca94ca4cbd984 + size: 5421537 + - path: data/mnist/power.csv + hash: md5 + md5: 6c84e8e91d81504e86db933bab0e9ae2 + size: 9968081 + params: + ../conf/clean.yaml: + fillna: + Epochs: 20 + FGM: 0.0 + FSQ: 32 + Control: 18 + Batch_Size: 1024 + outs: + - path: mnist/merged.csv + hash: md5 + md5: 8f227012f75f50d6f01745b54d2dc5a5 + size: 15408533 + merge@cifar: + cmd: python merge.py --big_dir data/bit_depth/cifar --little_dir data/cifar --config + ../conf/afr.yaml --data_file power.csv --output_folder cifar/ --output_file + merged.csv + deps: + - path: ../conf/clean.yaml + hash: md5 + md5: 3b23a3d656ad56e49e69f8624e227852 + size: 513 + - path: data/bit_depth/cifar/power.csv + hash: md5 + md5: b9ebbe01b2bd93f1c4af02fb8abd910b + size: 5291011 + - path: data/cifar/power.csv + hash: md5 + md5: a8e056a8c4008ce37006237f2e44b2bf + size: 9997291 + params: + ../conf/clean.yaml: + fillna: + Epochs: 20 + FGM: 0.0 + FSQ: 32 + Control: 18 + Batch_Size: 1024 + outs: + - path: cifar/merged.csv + hash: md5 + md5: 2d2cbcb913a61d304950c910bd487b9b + size: 15304603 merge@cifar100: cmd: python merge.py --big_dir data/bit_depth/cifar100 --little_dir data/cifar100 --config ../conf/afr.yaml --data_file power.csv --output_folder cifar100/ --output_file @@ -10,9 +70,11 @@ stages: md5: 3b23a3d656ad56e49e69f8624e227852 size: 513 - path: data/bit_depth/cifar100/power.csv + hash: md5 md5: d52080413e3ba7bd86ef8d92ebd00507 size: 3987447 - path: data/cifar100/power.csv + hash: md5 md5: f51f0d4ebd105f64829af90a0212de00 size: 16081846 params: @@ -28,18 +90,18 @@ stages: hash: md5 md5: f2bf33451a3c6a27e8b37dafc2a30ace size: 20099761 - clean@cifar100: - cmd: python -m deckard.layers.clean_data --i cifar100/merged.csv --o cifar100/clean.csv + clean@mnist: + cmd: python -m deckard.layers.clean_data --i mnist/merged.csv --o mnist/clean.csv --config "../conf/clean.yaml" deps: - path: ../conf/clean.yaml hash: md5 md5: 3b23a3d656ad56e49e69f8624e227852 size: 513 - - path: cifar100/merged.csv + - path: mnist/merged.csv hash: md5 - md5: f2bf33451a3c6a27e8b37dafc2a30ace - size: 20099761 + md5: 8f227012f75f50d6f01745b54d2dc5a5 + size: 15408533 params: ../conf/clean.yaml: attacks: @@ -68,132 +130,66 @@ stages: Epochs: model.trainer.nb_epoch Batch_Size: model.trainer.batch_size outs: - - path: cifar100/clean.csv - hash: md5 - md5: 57d972137d581f726c30f609f6e3dc16 - size: 9316351 - afr@cifar100: - cmd: python -m deckard.layers.afr --dataset cifar100 --data_file cifar100/clean.csv - --config_file "../conf/afr.yaml" --plots_folder cifar100/ --target adv_failures - --duration_col adv_fit_time - deps: - - path: ../conf/afr.yaml - hash: md5 - md5: 2d5b03f6b006b2fe0f875e47b4634e10 - size: 4793 - - path: cifar100/clean.csv - hash: md5 - md5: 57d972137d581f726c30f609f6e3dc16 - size: 9316351 - outs: - - path: cifar100/aft_comparison.csv - hash: md5 - md5: b7c15ae615a33408a0b727601cab890a - size: 212 - - path: cifar100/aft_comparison.tex - hash: md5 - md5: 3466f7e3a4024d83bbaa2ccc7b13da80 - size: 472 - - path: cifar100/cox_aft.pdf - hash: md5 - md5: 8d4a1219efea2157e2dbd6c34f69c6bb - size: 28926 - - path: cifar100/cox_epochs_partial_effect.pdf - hash: md5 - md5: 6e8c834ceef716c83fb2636b2e5e2fb5 - size: 33262 - - path: cifar100/log_logistic_aft.pdf - hash: md5 - md5: c2831702b2c06647e434910fd396c72c - size: 29307 - - path: cifar100/log_logistic_epochs_partial_effect.pdf - hash: md5 - md5: e0210ef32a6a57ed272b59a810a7c240 - size: 27155 - - path: cifar100/log_normal_aft.pdf - hash: md5 - md5: 2313759f670761c972d56b1ab44447de - size: 29841 - - path: cifar100/log_normal_epochs_partial_effect.pdf - hash: md5 - md5: fe1c01462a182aec383408ff0082e7a7 - size: 27993 - - path: cifar100/weibull_aft.pdf - hash: md5 - md5: 6feac2da8227c48d81b4f50619f03518 - size: 29018 - - path: cifar100/weibull_epochs_partial_effect.pdf + - path: mnist/clean.csv hash: md5 - md5: 8447c8226062b6498c1df56a2f94f41e - size: 28027 - merge@cifar: - cmd: python merge.py --big_dir data/bit_depth/cifar --little_dir data/cifar --config - ../conf/afr.yaml --data_file power.csv --output_folder cifar/ --output_file - merged.csv + md5: 4dfbcdb8c5aa27976ec2811ea9c1fda6 + size: 9294911 + clean@cifar: + cmd: python -m deckard.layers.clean_data --i cifar/merged.csv --o cifar/clean.csv + --config "../conf/clean.yaml" deps: - path: ../conf/clean.yaml hash: md5 md5: 3b23a3d656ad56e49e69f8624e227852 size: 513 - - path: data/bit_depth/cifar/power.csv - md5: b9ebbe01b2bd93f1c4af02fb8abd910b - size: 5291011 - - path: data/cifar/power.csv - md5: a8e056a8c4008ce37006237f2e44b2bf - size: 9997291 - params: - ../conf/clean.yaml: - fillna: - Epochs: 20 - FGM: 0.0 - FSQ: 32 - Control: 18 - Batch_Size: 1024 - outs: - path: cifar/merged.csv hash: md5 md5: 2d2cbcb913a61d304950c910bd487b9b size: 15304603 - merge@mnist: - cmd: python merge.py --big_dir data/bit_depth/mnist --little_dir data/mnist --config - ../conf/afr.yaml --data_file power.csv --output_folder mnist/ --output_file - merged.csv - deps: - - path: ../conf/clean.yaml - hash: md5 - md5: 3b23a3d656ad56e49e69f8624e227852 - size: 513 - - path: data/bit_depth/mnist/power.csv - md5: ed40bca679961759527ca94ca4cbd984 - size: 5421537 - - path: data/mnist/power.csv - md5: 6c84e8e91d81504e86db933bab0e9ae2 - size: 9968081 params: ../conf/clean.yaml: + attacks: + FastGradientMethod: FGM + ProjectedGradientDescent: PGD + HopSkipJump: HSJ + DeepFool: Deep + defences: + Control: Control + FeatureSqueezing: FSQ + Epochs: Epochs + model_layers: Control fillna: Epochs: 20 FGM: 0.0 FSQ: 32 Control: 18 Batch_Size: 1024 + params: + FGM: attack.init.eps + PGD: attack.init.eps + HSJ: attack.init.eps + DeepFool: attack.init.eps + FSQ: model.art.preprocessor.bit_depth + Control: model_layers + Epochs: model.trainer.nb_epoch + Batch_Size: model.trainer.batch_size outs: - - path: mnist/merged.csv + - path: cifar/clean.csv hash: md5 - md5: 8f227012f75f50d6f01745b54d2dc5a5 - size: 15408533 - clean@mnist: - cmd: python -m deckard.layers.clean_data --i mnist/merged.csv --o mnist/clean.csv + md5: c546fa08ebcc159fd81fcaa5dda11411 + size: 9305692 + clean@cifar100: + cmd: python -m deckard.layers.clean_data --i cifar100/merged.csv --o cifar100/clean.csv --config "../conf/clean.yaml" deps: - path: ../conf/clean.yaml hash: md5 md5: 3b23a3d656ad56e49e69f8624e227852 size: 513 - - path: mnist/merged.csv + - path: cifar100/merged.csv hash: md5 - md5: 8f227012f75f50d6f01745b54d2dc5a5 - size: 15408533 + md5: f2bf33451a3c6a27e8b37dafc2a30ace + size: 20099761 params: ../conf/clean.yaml: attacks: @@ -222,10 +218,10 @@ stages: Epochs: model.trainer.nb_epoch Batch_Size: model.trainer.batch_size outs: - - path: mnist/clean.csv + - path: cifar100/clean.csv hash: md5 - md5: 6d39faeafde4c9f3c7c1cca0f3dac5a8 - size: 9292855 + md5: d020a3f4406f0f6091ea8f09ea67b824 + size: 9318360 afr@mnist: cmd: python -m deckard.layers.afr --dataset mnist --data_file mnist/clean.csv --config_file "../conf/afr.yaml" --plots_folder mnist/ --target adv_failures @@ -237,93 +233,49 @@ stages: size: 4793 - path: mnist/clean.csv hash: md5 - md5: 6d39faeafde4c9f3c7c1cca0f3dac5a8 - size: 9292855 + md5: 4dfbcdb8c5aa27976ec2811ea9c1fda6 + size: 9294911 outs: - path: mnist/aft_comparison.csv hash: md5 - md5: 83bac6dcedac880fa889c4eb997ef11b - size: 207 + md5: 7cf6100c4a63fafce26ec0dabe60b7dc + size: 258 - path: mnist/aft_comparison.tex hash: md5 - md5: 030489512539372044c8f958ce6bab22 - size: 467 + md5: aca171194f34f77b26ce81432f0be23a + size: 534 - path: mnist/cox_aft.pdf hash: md5 - md5: d7785b4b4e405064c88213aec92d1644 - size: 28581 + md5: 6cf343a6c73d2e39f953071e761011c5 + size: 22648 - path: mnist/cox_epochs_partial_effect.pdf hash: md5 - md5: a5eb4c53909f58fd79fa72cd76de3407 - size: 32663 + md5: ca0df1707b223c591e51c6092d677d9b + size: 29247 - path: mnist/log_logistic_aft.pdf hash: md5 - md5: 915eb55ca3d44b0c97fc491d9f797425 - size: 29342 + md5: dfa718e8ee09a14efe19dbf16c468d56 + size: 23673 - path: mnist/log_logistic_epochs_partial_effect.pdf hash: md5 - md5: 6fed782e1b3acc31c925ad6747b50068 - size: 27325 + md5: 8540d0206d466bcafca2581ce87c40ee + size: 27441 - path: mnist/log_normal_aft.pdf hash: md5 - md5: 3e78554ecd7648c6abd66d5c073359a1 - size: 29874 + md5: 7d7fbda03eafc9b1e881c0469be93f40 + size: 24069 - path: mnist/log_normal_epochs_partial_effect.pdf hash: md5 - md5: c3e612dafecb517a3211549982730ab3 - size: 28077 + md5: 6f0fa989d40cd3d12453594e325c9a1f + size: 28250 - path: mnist/weibull_aft.pdf hash: md5 - md5: f3011ee5be6f1854bbe3e75502f89ad0 - size: 29062 + md5: 27b4ba3c41bc2f01227f80cb5251101a + size: 23671 - path: mnist/weibull_epochs_partial_effect.pdf hash: md5 - md5: 8aadd28ac46599dee591eaaa162f364d - size: 26910 - clean@cifar: - cmd: python -m deckard.layers.clean_data --i cifar/merged.csv --o cifar/clean.csv - --config "../conf/clean.yaml" - deps: - - path: ../conf/clean.yaml - hash: md5 - md5: 3b23a3d656ad56e49e69f8624e227852 - size: 513 - - path: cifar/merged.csv - hash: md5 - md5: 2d2cbcb913a61d304950c910bd487b9b - size: 15304603 - params: - ../conf/clean.yaml: - attacks: - FastGradientMethod: FGM - ProjectedGradientDescent: PGD - HopSkipJump: HSJ - DeepFool: Deep - defences: - Control: Control - FeatureSqueezing: FSQ - Epochs: Epochs - model_layers: Control - fillna: - Epochs: 20 - FGM: 0.0 - FSQ: 32 - Control: 18 - Batch_Size: 1024 - params: - FGM: attack.init.eps - PGD: attack.init.eps - HSJ: attack.init.eps - DeepFool: attack.init.eps - FSQ: model.art.preprocessor.bit_depth - Control: model_layers - Epochs: model.trainer.nb_epoch - Batch_Size: model.trainer.batch_size - outs: - - path: cifar/clean.csv - hash: md5 - md5: f11065790fb385b9ca6ab6a21bcb8ec2 - size: 9303640 + md5: 2ec56ba18ab0446bda375509fa74f5dc + size: 27137 afr@cifar: cmd: python -m deckard.layers.afr --dataset cifar --data_file cifar/clean.csv --config_file "../conf/afr.yaml" --plots_folder cifar/ --target adv_failures @@ -335,85 +287,139 @@ stages: size: 4793 - path: cifar/clean.csv hash: md5 - md5: f11065790fb385b9ca6ab6a21bcb8ec2 - size: 9303640 + md5: c546fa08ebcc159fd81fcaa5dda11411 + size: 9305692 outs: - path: cifar/aft_comparison.csv hash: md5 - md5: 42dad9a57c0b74eee3caffbd85e177d0 - size: 211 + md5: 8e5d3667cfffeb11a7ab923429ca606a + size: 263 - path: cifar/aft_comparison.tex hash: md5 - md5: 112fb42d06cc47aee03cf203865133bf - size: 471 + md5: ae1a73e2065bb6707bbf12281c077291 + size: 535 - path: cifar/cox_aft.pdf hash: md5 - md5: 8a3e6c71760bc98fe64687f518ba0654 - size: 28935 + md5: 60ebd0500a1fe6a30e02084bafaee691 + size: 22303 - path: cifar/cox_epochs_partial_effect.pdf hash: md5 - md5: c9ad760d1e59f90218ca4a7592bcd309 - size: 33196 + md5: 2c71d7194f9c54a7e3fb972b90b7670a + size: 29601 - path: cifar/log_logistic_aft.pdf hash: md5 - md5: af4360e69eb314e5717407a543c9cd37 - size: 29703 + md5: accd69a47f040740316a10fbccd15105 + size: 23671 - path: cifar/log_logistic_epochs_partial_effect.pdf hash: md5 - md5: 17fa89d624dd206070dd6669254e287f - size: 27433 + md5: d4719975431aed2a2bb184038e88c621 + size: 28014 - path: cifar/log_normal_aft.pdf hash: md5 - md5: f7aebd3a3cc48ef9f4c04279af54b2bc - size: 30232 + md5: 6c8be4e785288cbdd92f405e85b7c7ac + size: 24075 - path: cifar/log_normal_epochs_partial_effect.pdf hash: md5 - md5: f2d95584d261226817a894b895d276e7 - size: 28162 + md5: 2d847fcb77e0242e7640557f85413402 + size: 28659 - path: cifar/weibull_aft.pdf hash: md5 - md5: 9c45ad78f653dcdfff593545c5c8921b - size: 28724 + md5: a16749a3658eb60911fafa742b5d9395 + size: 23676 - path: cifar/weibull_epochs_partial_effect.pdf hash: md5 - md5: 8aa46a3fec32679b6a2e29fea439bb43 - size: 28030 + md5: 731486b0d77ffe3c1a10e9266889badc + size: 27620 + afr@cifar100: + cmd: python -m deckard.layers.afr --dataset cifar100 --data_file cifar100/clean.csv + --config_file "../conf/afr.yaml" --plots_folder cifar100/ --target adv_failures + --duration_col adv_fit_time + deps: + - path: ../conf/afr.yaml + hash: md5 + md5: 2d5b03f6b006b2fe0f875e47b4634e10 + size: 4793 + - path: cifar100/clean.csv + hash: md5 + md5: d020a3f4406f0f6091ea8f09ea67b824 + size: 9318360 + outs: + - path: cifar100/aft_comparison.csv + hash: md5 + md5: af5fb567ef3c24779630f0b674863d46 + size: 264 + - path: cifar100/aft_comparison.tex + hash: md5 + md5: 55433d5ecef94bc9c649b1cfbac52071 + size: 536 + - path: cifar100/cox_aft.pdf + hash: md5 + md5: 35ea7e276dd37b2654f9ca879a03f135 + size: 22290 + - path: cifar100/cox_epochs_partial_effect.pdf + hash: md5 + md5: 0754f4454cac185238a8ca0c0f4ef9f6 + size: 33010 + - path: cifar100/log_logistic_aft.pdf + hash: md5 + md5: 0d01170bd1c2524d8a191217377a337f + size: 23679 + - path: cifar100/log_logistic_epochs_partial_effect.pdf + hash: md5 + md5: 8fc02cf3bb19feb5c374ce42546a158c + size: 27770 + - path: cifar100/log_normal_aft.pdf + hash: md5 + md5: 3d2a77921049b8094cea1ba42f4f3267 + size: 24078 + - path: cifar100/log_normal_epochs_partial_effect.pdf + hash: md5 + md5: 6eed47f6dd64ff0ebb7d452af2d81052 + size: 28693 + - path: cifar100/weibull_aft.pdf + hash: md5 + md5: fe52aa7910c2e97d448499d07b655700 + size: 23666 + - path: cifar100/weibull_epochs_partial_effect.pdf + hash: md5 + md5: a5b7cafd33b9fa31dc90b77db05e0165 + size: 27319 combined_plots: cmd: python combined_plots.py deps: - path: cifar/aft_comparison.csv hash: md5 - md5: 42dad9a57c0b74eee3caffbd85e177d0 - size: 211 + md5: 8e5d3667cfffeb11a7ab923429ca606a + size: 263 - path: cifar100/aft_comparison.csv hash: md5 - md5: b7c15ae615a33408a0b727601cab890a - size: 212 + md5: af5fb567ef3c24779630f0b674863d46 + size: 264 - path: combined_plots.py hash: md5 md5: 7ea9df483b87b41bbe1195b55275bd6c size: 7724 - path: mnist/aft_comparison.csv hash: md5 - md5: 83bac6dcedac880fa889c4eb997ef11b - size: 207 + md5: 7cf6100c4a63fafce26ec0dabe60b7dc + size: 258 outs: - path: combined/acc.pdf hash: md5 - md5: c379bcfd7e96effb55ac149fd16e1efe - size: 30390 + md5: 6887b091b83c375be8c81d5c64973b7e + size: 30384 - path: combined/cost.pdf hash: md5 - md5: cab8446b120746a4a9d8a54fcafa2f58 - size: 39900 + md5: 8450471bfedb2f535a868435dfe68d47 + size: 39894 - path: combined/power.pdf hash: md5 - md5: 5034f0a136246c06c9fcb48e6f58c94b - size: 40090 + md5: 53da512ffda57bd846e756be51430d49 + size: 40084 - path: combined/time.pdf hash: md5 - md5: 71b708cc731f911d999db3c5d3fc977a - size: 40222 + md5: 4c3a1fad66932c3ad2b093534551f2c3 + size: 40216 - path: data/combined/combined.csv hash: md5 md5: 5373a0cba09196ebf00bbfbab9378398 @@ -460,8 +466,8 @@ stages: outs: - path: combined/clean.csv hash: md5 - md5: 24f49787a6d20f9172dacf9edefff85f - size: 37298176 + md5: 0c16036112b442a00cff7f91411fb07d + size: 37306094 combined_afr: cmd: python -m deckard.layers.afr --dataset Combined --data_file combined/clean.csv --config_file "../conf/combined_afr.yaml" --plots_folder combined/ --target @@ -469,146 +475,102 @@ stages: deps: - path: ../conf/combined_afr.yaml hash: md5 - md5: 068c842adac014ff65cbf74d28e57263 - size: 8923 + md5: 1deda0b091aac0cc406453185271c97e + size: 4505 - path: combined/clean.csv hash: md5 - md5: 24f49787a6d20f9172dacf9edefff85f - size: 37298176 + md5: 0c16036112b442a00cff7f91411fb07d + size: 37306094 outs: - path: combined/aft_comparison.csv hash: md5 - md5: 238ff959f5768d165bddb38595db5d37 - size: 188 + md5: f0296039b0f7f1ea537a10a3de5c9b81 + size: 238 - path: combined/aft_comparison.tex hash: md5 - md5: 14bc20405c6aadb2dfc4e421cd38ca10 - size: 435 + md5: f125f5ad936db3c9a1c20432b425a795 + size: 489 - path: combined/log_logistic_aft.pdf hash: md5 - md5: d7b9b3f4d713ff30a3ac12338b548a74 - size: 26842 - - path: combined/log_logistic_attack_eps_partial_effect.pdf - hash: md5 - md5: 785a8d08bde202772f2127ee7d8c33de - size: 26868 - - path: combined/log_logistic_batch_size_partial_effect.pdf + md5: 414ad5b58384b570e73b009776aec55b + size: 26799 + - path: combined/log_logistic_aft_dummies.pdf hash: md5 - md5: 4facc4ccdd837b805a1e01d5bbaeaa97 - size: 28790 - - path: combined/log_logistic_epochs_partial_effect.pdf - hash: md5 - md5: 720b3775b9aacafde7db37012ceb54ff - size: 27938 + md5: 43eaa4999b0dc48296d2ae1261d6621b + size: 24362 - path: combined/log_logistic_qq.pdf hash: md5 - md5: b3961143f38c1f19fe26aa05f2cb3ca4 - size: 18763 + md5: de98afb708b08acd57561fcc7b468a71 + size: 20770 - path: combined/log_logistic_summary.csv hash: md5 - md5: c0cbc323a723516dbd3087bb9440caf1 - size: 3486 - - path: combined/log_logistic_summary.pdf - hash: md5 - md5: 283e08bfbb7f67667dd5126358f3d160 - size: 29010 + md5: e89252783f5b8e44efe165632dfd51e6 + size: 3393 - path: combined/log_normal_aft.pdf hash: md5 - md5: ae6452dd66e6b5c2c4d493d2f2e2de27 - size: 27359 - - path: combined/log_normal_attack_eps_partial_effect.pdf - hash: md5 - md5: 982eb06fd39148ec8a540528b4a7886a - size: 27579 - - path: combined/log_normal_batch_size_partial_effect.pdf - hash: md5 - md5: af939d5bf9a83e5e606c5f09aaa481ba - size: 29598 - - path: combined/log_normal_epochs_partial_effect.pdf + md5: 5299f736bc524ea10dddda7b68feba60 + size: 26912 + - path: combined/log_normal_aft_dummies.pdf hash: md5 - md5: b3dd6798f6377a13ac425e53cf34bea8 - size: 28603 + md5: d225a60cc7c433d9ff62113e42c07c2a + size: 24760 - path: combined/log_normal_qq.pdf hash: md5 - md5: 202f50d80f2be2a0a1dfafa691b99eb2 - size: 19906 + md5: c589021c809e186438ce543e4afef58a + size: 22475 - path: combined/log_normal_summary.csv hash: md5 - md5: 9025cfe2f47a1a1aa0d9fb3575072712 - size: 3445 - - path: combined/log_normal_summary.pdf - hash: md5 - md5: af4ebf73e8d223c5fc6dcd274ff79b49 - size: 27645 + md5: fc44e385df146e969bcad89fa9b5bc44 + size: 3349 - path: combined/weibull_aft.pdf hash: md5 - md5: 099c99556c6c16cd21714f6ea96790d9 - size: 26881 - - path: combined/weibull_attack_eps_partial_effect.pdf - hash: md5 - md5: cd9bdcb36c0012ac355a1ef9bf7260c8 - size: 26258 - - path: combined/weibull_batch_size_partial_effect.pdf + md5: 6bf6572c0e6a793259306b3762eb0bcf + size: 26996 + - path: combined/weibull_aft_dummies.pdf hash: md5 - md5: b98d5c459d73e65d994b2a23db1f71ec - size: 28543 - - path: combined/weibull_epochs_partial_effect.pdf - hash: md5 - md5: 8a34f173079fa7f7f5667d84d77d2768 - size: 27623 + md5: dac28c7790653630b806056435d161b7 + size: 24361 - path: combined/weibull_qq.pdf hash: md5 - md5: 5dae104737074ab926ef947938254eda - size: 18121 + md5: 3f862fe7fac70b1a989b552d1aeb4852 + size: 20533 - path: combined/weibull_summary.csv hash: md5 - md5: 60e0589a5a15d42dff93363a7eb4321d - size: 3496 - - path: combined/weibull_summary.pdf - hash: md5 - md5: c9a8302a6c526187170c7f2c3a0e2db8 - size: 27676 + md5: 021bd0236ee988f89633606ac6152804 + size: 3413 copy@mnist: - cmd: rm -rf ~/kepler-ml/plots/mnist/ && mkdir -p ~/kepler-ml/plots/mnist/ && cp - -r mnist/* ~/kepler-ml/plots/mnist/ cmd: rm -rf ~/kepler-ml/plots/mnist/ && mkdir -p ~/kepler-ml/plots/mnist/ && cp -r mnist/* ~/kepler-ml/plots/mnist/ deps: - path: mnist/ hash: md5 - md5: a31729c195d8a0130c0235f50240f8be.dir - size: 25145164 - nfiles: 24 + md5: 5a2da96ca7e0fcdee2ea31522675f4c7.dir + size: 25107819 + nfiles: 21 copy@cifar: - cmd: rm -rf ~/kepler-ml/plots/cifar/ && mkdir -p ~/kepler-ml/plots/cifar/ && cp - -r cifar/* ~/kepler-ml/plots/cifar/ cmd: rm -rf ~/kepler-ml/plots/cifar/ && mkdir -p ~/kepler-ml/plots/cifar/ && cp -r cifar/* ~/kepler-ml/plots/cifar/ deps: - path: cifar/ hash: md5 - md5: b407c30ea9d76581e3e6c2aba9c72c7d.dir - size: 25054465 - nfiles: 24 + md5: fe364ed42297c19f60b48e9775b913c0.dir + size: 24910643 + nfiles: 20 copy@cifar100: - cmd: rm -rf ~/kepler-ml/plots/cifar100/ && mkdir -p ~/kepler-ml/plots/cifar100/ - && cp -r cifar100/* ~/kepler-ml/plots/cifar100/ cmd: rm -rf ~/kepler-ml/plots/cifar100/ && mkdir -p ~/kepler-ml/plots/cifar100/ && cp -r cifar100/* ~/kepler-ml/plots/cifar100/ deps: - path: cifar100/ hash: md5 - md5: 75185a117f2df768d81bf7005ad94a9f.dir - size: 29860670 - nfiles: 24 + md5: 3e4f3b8aad2d77de6f7fff32aaea49c7.dir + size: 29721403 + nfiles: 20 copy@combined: - cmd: rm -rf ~/kepler-ml/plots/combined/ && mkdir -p ~/kepler-ml/plots/combined/ - && cp -r combined/* ~/kepler-ml/plots/combined/ cmd: rm -rf ~/kepler-ml/plots/combined/ && mkdir -p ~/kepler-ml/plots/combined/ && cp -r combined/* ~/kepler-ml/plots/combined/ deps: - path: combined/ hash: md5 - md5: fadbd9a0be85c67c2d8d384d201be0b8.dir - size: 38115902 - nfiles: 35 + md5: 1b15716951d56f93bb2892ad68959f9e.dir + size: 37685542 + nfiles: 20 diff --git a/examples/power/plots/dvc.yaml b/examples/power/plots/dvc.yaml index eee4377a..6e05b2f4 100644 --- a/examples/power/plots/dvc.yaml +++ b/examples/power/plots/dvc.yaml @@ -127,29 +127,19 @@ stages: - ../conf/combined_afr.yaml plots: - combined/weibull_aft.pdf # You can also define them manually - - combined/weibull_epochs_partial_effect.pdf - - combined/weibull_batch_size_partial_effect.pdf - - combined/weibull_attack_eps_partial_effect.pdf - combined/weibull_qq.pdf - - combined/weibull_summary.pdf + - combined/weibull_aft_dummies.pdf # - combined/cox_aft.pdf # - combined/cox_epochs_partial_effect.pdf # - combined/cox_batch_size_partial_effect.pdf # - combined/cox_attack_eps_partial_effect.pdf # - combined/cox_qq.pdf - # - combined/cox_summary.pdf - combined/log_logistic_aft.pdf - - combined/log_logistic_epochs_partial_effect.pdf - - combined/log_logistic_batch_size_partial_effect.pdf - - combined/log_logistic_attack_eps_partial_effect.pdf - combined/log_logistic_qq.pdf - - combined/log_logistic_summary.pdf + - combined/log_logistic_aft_dummies.pdf - combined/log_normal_aft.pdf - - combined/log_normal_epochs_partial_effect.pdf - - combined/log_normal_batch_size_partial_effect.pdf - - combined/log_normal_attack_eps_partial_effect.pdf - combined/log_normal_qq.pdf - - combined/log_normal_summary.pdf + - combined/log_normal_aft_dummies.pdf metrics: - combined/aft_comparison.csv - combined/log_normal_summary.csv diff --git a/examples/pytorch/.dvc/tmp/lock b/examples/pytorch/.dvc/tmp/lock deleted file mode 100644 index d6eb4e1b..00000000 --- a/examples/pytorch/.dvc/tmp/lock +++ /dev/null @@ -1 +0,0 @@ - 1966819 diff --git a/examples/pytorch/.dvc/tmp/rwlock b/examples/pytorch/.dvc/tmp/rwlock deleted file mode 100644 index 0967ef42..00000000 --- a/examples/pytorch/.dvc/tmp/rwlock +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/examples/pytorch/.dvc/tmp/rwlock.lock b/examples/pytorch/.dvc/tmp/rwlock.lock deleted file mode 100644 index 59efaf32..00000000 --- a/examples/pytorch/.dvc/tmp/rwlock.lock +++ /dev/null @@ -1 +0,0 @@ - 10872 diff --git a/examples/pytorch/.gitignore b/examples/pytorch/.gitignore index 9dcfac9d..e2755d59 100644 --- a/examples/pytorch/.gitignore +++ b/examples/pytorch/.gitignore @@ -1 +1,8 @@ .dvc/cache/ +old/ +mnist/ +cifar/ +cifar100/ +/mnist.yaml +/cifar.yaml +original_data/ diff --git a/examples/pytorch/README.md b/examples/pytorch/README.md index 03263133..9dc1f3d5 100644 --- a/examples/pytorch/README.md +++ b/examples/pytorch/README.md @@ -1,6 +1,9 @@ -Each dataset contains a `dvc.yaml` file and a `conf/` directory that contain the reproducible steps. First, install this repository, which will include dvc as a dependency. Then, you can run: +This folder contains a `dvc.yaml` file and a `conf/` directory that contain the reproducible steps. First, install this repository, which will include dvc as a dependency. Then, you can run: ```dvc repro``` -within one of the subdirectories of this directory. The ```dvc.yaml``` contains a series of "stages" that outline the dependencies, parameters, metrics, plots, and/or outputs of each stage. +within one of the subdirectories of this directory. The `dvc.yaml` contains a series of "stages" that outline the dependencies, parameters, metrics, plots, and/or outputs of each stage. If an output from one stage is a dependency of another, then `dvc` will automatically know the order of operations, which is tremendously useful for development, allowing you to develop downstream changes and know that you can't accidentally delete weeks of calculations, though this requires setting up a dvc storage cache as per their docs. +You can view the dvc diagram with +```dvc dag``` +or by looking at the mermaid chart dag.md using a suitable rendering engine. If you want to use a different pytorch-compatible model, modify the `torch-example.py` file and/or the configurations in the `conf/model/art` folder (for modifying training method or run-time parameters like optimizer) or the `conf/model/initialize' to change the initialization parameters or file name for your custom model. diff --git a/examples/pytorch/attacks.sh b/examples/pytorch/attacks.sh new file mode 100644 index 00000000..f31abe05 --- /dev/null +++ b/examples/pytorch/attacks.sh @@ -0,0 +1,98 @@ +# #!/bin/bash + +# # This script is used to generate the attacks for the example. + +# Fast Gradient Method +bash models.sh \ + stage=attack \ + attack=default \ + ++attack.init.name=art.attacks.evasion.FastGradientMethod \ + ++attack.init.eps=.001,.01,.1,.5,1 \ + ++attack.init.norm=2 \ + atk_name=FGM $@ +# ##################################################### +# Projected Gradient Descent +bash models.sh \ + stage=attack \ + attack=default \ + ++attack.init.name=art.attacks.evasion.ProjectedGradientDescent \ + ++attack.init.eps=.001,.01,.1,.5,1 \ + ++attack.init.norm=2 \ + ++attack.init.eps_step=.001,.003,.01 \ + atk_name=PGD \ + ++attack.init.max_iter=1,5,10,50,100 $@ +# ##################################################### +# DeepFool +bash models.sh \ + stage=attack \ + attack=default \ + ++attack.init.name=art.attacks.evasion.DeepFool \ + ++attack.init.max_iter=10 \ + ++attack.init.batch_size=4096 \ + ++attack.init.nb_grads=1,3,5,8,10 \ + atk_name=Deep $@ +# # ##################################################### +# # HopSkipJump +# bash models.sh \ +# stage=attack \ +# attack=default \ +# ++attack.init.name=art.attacks.evasion.HopSkipJump \ +# ++attack.init.max_iter=1,3,5,10,15 \ +# ++attack.init.init_eval=10 \ +# ++attack.init.batch_size=4096 \ +# ++attack.init.max_eval=100 \ +# ++attack.init.norm=2 \ +# atk_name=HSJ $@ +# # ##################################################### +# # PixelAttack +# bash models.sh \ +# stage=attack \ +# attack=default \ +# ++attack.init.name=art.attacks.evasion.PixelAttack \ +# ~attack.init.batch_size \ +# ++attack.init.th=1,4,16,64,256 \ +# atk_name=Pixel $@ +# # ##################################################### +# # ThresholdAttack +# bash models.sh \ +# stage=attack \ +# attack=default \ +# ++attack.init.name=art.attacks.evasion.ThresholdAttack \ +# ~attack.init.batch_size \ +# ++attack.init.th=1,4,16,64,256 \ +# atk_name=Thresh $@ +# ##################################################### +# # ZooAttack +# bash models.sh \ +# stage=attack \ +# attack=default \ +# ++attack.init.name=art.attacks.evasion.ZooAttack \ +# ++attack.init.binary_search_steps=1,2,3,5,10 \ +# ++attack.init.abort_early=True \ +# atk_name=Zoo $@ +# #################################################### +# # Carlini L0 Method +# bash models.sh \ +# attack=default \ +# stage=attack \ +# ++attack.init.name=art.attacks.evasion.CarliniL0Method \ +# ++attack.init.max_iter=10 \ +# ++attack.init.confidence=.1,.3,.5,.9,.99 \ +# atk_name=Patch $@ +# # Carlini L2 Method +# bash models.sh \ +# attack=default \ +# stage=attack \ +# ++attack.init.name=art.attacks.evasion.CarliniL2Method \ +# ++attack.init.confidence=.1,.3,.5,.9,.99 \ +# atk_name=CW2 $@ +# # # Carlini L2 Method +# bash models.sh \ +# attack=default \ +# stage=attack \ +# ++attack.init.name=art.attacks.evasion.CarliniL2Method \ +# ++attack.init.confidence=.1,.9,.99 \ +# atk_name=Patch $@ +# rm -rf mnist/models/* +# rm -rf cifar/models/* +# rm -rf cifar100/models/* diff --git a/examples/pytorch/cifar10/.dvc/.gitignore b/examples/pytorch/cifar10/.dvc/.gitignore deleted file mode 100644 index 528f30c7..00000000 --- a/examples/pytorch/cifar10/.dvc/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/config.local -/tmp -/cache diff --git a/examples/pytorch/cifar10/.dvc/config b/examples/pytorch/cifar10/.dvc/config deleted file mode 100644 index 4cf322d9..00000000 --- a/examples/pytorch/cifar10/.dvc/config +++ /dev/null @@ -1,2 +0,0 @@ -[core] - autostage = true diff --git a/examples/pytorch/cifar10/.dvcignore b/examples/pytorch/cifar10/.dvcignore deleted file mode 100644 index 51973055..00000000 --- a/examples/pytorch/cifar10/.dvcignore +++ /dev/null @@ -1,3 +0,0 @@ -# Add patterns of files dvc should ignore, which could improve -# the performance. Learn more at -# https://dvc.org/doc/user-guide/dvcignore diff --git a/examples/pytorch/cifar10/.gitignore b/examples/pytorch/cifar10/.gitignore deleted file mode 100644 index e2fb84f9..00000000 --- a/examples/pytorch/cifar10/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -cifar/* -.dvc/* -20_epochs/* -**/optimization_results.yaml diff --git a/examples/pytorch/cifar10/attacks.sh b/examples/pytorch/cifar10/attacks.sh deleted file mode 100644 index 8ec1f079..00000000 --- a/examples/pytorch/cifar10/attacks.sh +++ /dev/null @@ -1,37 +0,0 @@ -# #!/bin/bash - -# # This script is used to generate the attacks for the example. - -# Fast Gradient Method -bash models.sh attack=default ++attack.init.name=art.attacks.evasion.FastGradientMethod ++attack.init.eps=.001,.01,.1,.5,1 ++attack.init.norm=inf,1,2 ++attack.init.eps_step=.001,.003,.01 ++attack.init.batch_size=1024 stage=attack ++hydra.sweeper.study_name=fgm ++direction=maximize $@ - -# Projected Gradient Descent -bash models.sh attack=default ++attack.init.name=art.attacks.evasion.ProjectedGradientDescent ++attack.init.eps=.001,.01,.1,.5,1 ++attack.init.norm=inf,1,2 ++attack.init.eps_step=.001,.003,.01 ++attack.init.batch_size=1024 ++attack.init.max_iter=10 stage=attack ++hydra.sweeper.study_name=pgd ++direction=maximize $@ - -# DeepFool -bash models.sh attack=default ++attack.init.name=art.attacks.evasion.DeepFool ++attack.init.max_iter=10 ++attack.init.batch_size=1024 ++attack.init.nb_grads=1,3,5,10 stage=attack ++hydra.sweeper.study_name=deep ++direction=maximize $@ - -# HopSkipJump -bash models.sh attack=default ++attack.init.name=art.attacks.evasion.HopSkipJump ++attack.init.max_iter=1,3,5,10 ++attack.init.init_eval=10 ++attack.init.norm=inf,2 stage=attack ++hydra.sweeper.study_name=hsj ++direction=maximize $@ - -# ##################################################### -# PixelAttack -bash models.sh attack=default ++attack.init.name=art.attacks.evasion.PixelAttack ++attack.init.max_iter=10 ++attack.init.th=1,4,16,64,256 stage=attack ++hydra.sweeper.study_name=pixel ++direction=maximize $@ - -# ThresholdAttack -bash models.sh attack=default ++attack.init.name=art.attacks.evasion.ThresholdAttack ++attack.init.max_iter=10 ++attack.init.th=1,4,16,64,256 stage=attack ++hydra.sweeper.study_name=thresh ++direction=maximize $@ - -# # AdversarialPatch -# bash models.sh attack=default --attack.init.batch_size ++attack.init.name=art.attacks.evasion.AdversarialPatch ++attack.init.max_iter=10 ++attack.init.learning_rate=.5,5.0,50.0 stage=patch ++hydra.sweeper.study_name=attack ++direction=maximize ++attack.init.patch_shape=[1,28,28] $@ -##################################################### - -# # Carlini L0 Method -# bash models.sh attack=default ++attack.init.name=art.attacks.evasion.CarliniL0Method ++attack.init.batch_size=1024 ++attack.init.max_iter=10 ++attack.init.confidence=.1,.9,.99 stage=cw0 ++hydra.sweeper.study_name=cw0 ++direction=maximize $@ - -# # Carlini L2 Method -# bash models.sh attack=default ++attack.init.name=art.attacks.evasion.CarliniL2Method ++attack.init.batch_size=1024 ++attack.init.max_iter=10 ++attack.init.confidence=.1,.9,.99 stage=cw2 ++hydra.sweeper.study_name=cw2 ++direction=maximize $@ - -# # Carlini LInf Method -# bash models.sh attack=default ++attack.init.name=art.attacks.evasion.CarliniLInfMethod ++attack.init.max_iter=10 ++attack.init.confidence=.1,.9,.99 stage=attack ++hydra.sweeper.study_name=cwinf ++direction=maximize $@ - -rm -rf output/models/* diff --git a/examples/pytorch/cifar10/conf/afr.yaml b/examples/pytorch/cifar10/conf/afr.yaml deleted file mode 100644 index 7413432c..00000000 --- a/examples/pytorch/cifar10/conf/afr.yaml +++ /dev/null @@ -1,183 +0,0 @@ -covariates: - - "adv_fit_time" - - "accuracy" - - "train_time" - - "atk_value" - - "def_value" - - "data.sample.random_state" - - "model_layers" - - model.trainer.nb_epoch -# - atk_gen -# - def_gen - - predict_time -fillna: - model.trainer.nb_epoch: 20 -weibull: - plot: - file : weibull_aft.eps - title : Weibull AFR Model - labels: - "Intercept: rho_": "$\\rho$" - "Intercept: lambda_": "$\\lambda$" - "data.sample.random_state: lambda_": "Random State" - "atk_value: lambda_": "Attack Strength" - "train_time: lambda_": "$t_{train}$" - "predict_proba_time: lambda_": "$t_{predict}$" - "adv_accuracy: lambda_": "Adv. Accuracy" - "accuracy: lambda_": "Ben. Accuracy" - "adv_fit_time: lambda_": "$t_{attack}$" - "adv_failure_rate: lambda_": "$f_{adv.}(t;\\theta)$" - "failure_rate: lambda_": "$f_{ben.}(t;\\theta)$" - "model.trainer.nb_epoch: lambda_": "No. of Epochs" - "model.trainer.batch_size: lambda_": "Batch Size" - "def_gen": "Defence" - "model_layers: lambda_" : "Layers" - "def_value: lambda_" : "Defence Strength" - "predict_time: lambda_" : "$t_{predict}$" - partial_effect: - - "file": "weibull_epochs_partial_effect.eps" - "covariate_array": "model.trainer.nb_epoch" - "values_array": [1,10,25,50] - "title": "$S(t)$ for Weibull AFR" - "ylabel": "Expectation of $S(t)$" - "xlabel": "Time $T$ (seconds)" - "legend_kwargs": { - "title": "Epochs", - "labels": ["1", "10", "25", "50"] - } - - "file": "weibull_layers_partial_effect.eps" - "covariate_array": "model_layers" - "values_array": [18, 34, 50, 101, 152] - "title": "$S(t)$ for Weibull AFR" - "ylabel": "Expectation of $S(t)$" - "xlabel": "Time $T$ (seconds)" - "legend_kwargs": { - "title": "Layers", - "labels": ["18", "34", "50", "101", "152"] - } -# cox: -# plot: -# file : cox_aft.eps -# title : Cox AFR Model -# labels: -# "Intercept: rho_": "$\\rho$" -# "Intercept: lambda_": "$\\lambda$" -# "data.sample.random_state: lambda_": "Random State" -# "atk_value: lambda_": "Attack Strength" -# "train_time: lambda_": "$t_{train}$" -# "predict_proba_time: lambda_": "$t_{predict}$" -# "adv_accuracy: lambda_": "Adv. Accuracy" -# "accuracy: lambda_": "Ben. Accuracy" -# "adv_fit_time: lambda_": "$t_{attack}$" -# "adv_failure_rate: lambda_": "$f_{adv.}(t;\\theta)$" -# "failure_rate: lambda_": "$f_{ben.}(t;\\theta)$" -# "model.trainer.nb_epoch: lambda_": "No. of Epochs" -# "model.trainer.batch_size: lambda_": "Batch Size" -# "def_gen": "Defence" -# partial_effect: -# - "file": "cox_epochs_partial_effect.eps" -# "covariate_array": "model.trainer.nb_epoch" -# "values_array": [1,10,25,50] -# "title": "$S(t)$ for Cox AFR" -# "ylabel": "Expectation of $S(t)$" -# "xlabel": "Time $T$ (seconds)" -# "legend_kwargs": { -# "title": "Epochs", -# "labels": ["1", "10", "25", "50"] -# } -# - "file": "cox_layers_partial_effect.eps" -# "covariate_array": "model_layers" -# "values_array": [18, 34, 50, 101, 152] -# "title": "$S(t)$ for Cox AFR" -# "ylabel": "Expectation of $S(t)$" -# "xlabel": "Time $T$ (seconds)" -# "legend_kwargs": { -# "title": "Layers", -# "labels": ["18", "34", "50", "101", "152"] -# } -log_logistic: - plot: - file : log_logistic_aft.eps - title : Log logistic AFR Model - labels: - "Intercept: beta_": "$\\beta$" - "Intercept: alpha_": "$\\alpha$" - "data.sample.random_state: alpha_": "Random State" - "atk_value: alpha_": "Attack Strength" - "train_time: alpha_": "$t_{train}$" - "predict_proba_time: alpha_": "$t_{predict}$" - "adv_accuracy: alpha_": "Adv. Accuracy" - "accuracy: alpha_": "Ben. Accuracy" - "adv_fit_time: alpha_": "$t_{attack}$" - "adv_failure_rate: alpha_": "$f_{adv.}(t;\\theta)$" - "failure_rate: alpha_": "$f_{ben.}(t;\\theta)$" - "model.trainer.nb_epoch: alpha_": "No. of Epochs" - "model.trainer.batch_size: alpha_": "Batch Size" - "def_gen": "Defence" - "model_layers: alpha_" : "Layers" - "def_value: alpha_" : "Defence Strength" - "predict_time: alpha_" : "$t_{predict}$" - partial_effect: - - "file": "log_logistic_epochs_partial_effect.eps" - "covariate_array": "model.trainer.nb_epoch" - "values_array": [1,10,25,50] - "title": "$S(t)$ for Log-Logistic AFR" - "ylabel": "Expectation of $S(t)$" - "xlabel": "Time $T$ (seconds)" - "legend_kwargs": { - "title": "Epochs", - "labels": ["1", "10", "25", "50"] - } - - "file": "log_logistic_layers_partial_effect.eps" - "covariate_array": "model_layers" - "values_array": [18, 34, 50, 101, 152] - "title": "$S(t)$ for Log Logistic AFR" - "ylabel": "Expectation of $S(t)$" - "xlabel": "Time $T$ (seconds)" - "legend_kwargs": { - "title": "Layers", - "labels": ["18", "34", "50", "101", "152"] - } -log_normal: - plot: - file : log_normal_aft.eps - title : Log Normal AFR Model - labels: - "Intercept: sigma_": "$\\rho$" - "Intercept: mu_": "$\\mu$" - "data.sample.random_state: mu_": "Random State" - "atk_value: mu_": "Attack Strength" - "train_time: mu_": "$t_{train}$" - "predict_proba_time: mu_": "$t_{predict}$" - "adv_accuracy: mu_": "Adv. Accuracy" - "accuracy: mu_": "Ben. Accuracy" - "adv_fit_time: mu_": "$t_{attack}$" - "adv_failure_rate: mu_": "$f_{adv.}(t;\\theta)$" - "failure_rate: mu_": "$f_{ben.}(t;\\theta)$" - "model.trainer.nb_epoch: mu_": "No. of Epochs" - "model.trainer.batch_size: mu_": "Batch Size" - "def_gen": "Defence" - "model_layers: mu_" : "Layers" - "def_value: mu_" : "Defence Strength" - "predict_time: mu_" : "$t_{predict}$" - partial_effect: - - "file": "log_normal_epochs_partial_effect.eps" - "covariate_array": "model.trainer.nb_epoch" - "values_array": [1,10,25,50] - "title": "$S(t)$ for Log-Normal AFR" - "ylabel": "Expectation of $S(t)$" - "xlabel": "Time $T$ (seconds)" - "legend_kwargs": { - "title": "Epochs", - "labels": ["1", "10", "25", "50"] - } - - "file": "log_normal_layers_partial_effect.eps" - "covariate_array": "model_layers" - "values_array": [18, 34, 50, 101, 152] - "title": "$S(t)$ for Log Normal AFR" - "ylabel": "Expectation of $S(t)$" - "xlabel": "Time $T$ (seconds)" - "legend_kwargs": { - "title": "Layers", - "labels": ["18", "34", "50", "101", "152"] - } diff --git a/examples/pytorch/cifar10/conf/data/torch_cifar.yaml b/examples/pytorch/cifar10/conf/data/torch_cifar.yaml deleted file mode 100644 index b7603125..00000000 --- a/examples/pytorch/cifar10/conf/data/torch_cifar.yaml +++ /dev/null @@ -1,11 +0,0 @@ -_target_: deckard.base.data.Data -generate: - name: torch_cifar10 -sample: - random_state : 0 - stratify: True -sklearn_pipeline: - preprocessor: - name: sklearn.preprocessing.StandardScaler - with_mean: True - with_std: True diff --git a/examples/pytorch/cifar10/conf/data/torch_mnist.yaml b/examples/pytorch/cifar10/conf/data/torch_mnist.yaml deleted file mode 100644 index 9d79c036..00000000 --- a/examples/pytorch/cifar10/conf/data/torch_mnist.yaml +++ /dev/null @@ -1,15 +0,0 @@ -_target_: deckard.base.data.Data -generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist -sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state : 0 - stratify: True -sklearn_pipeline: - _target_: deckard.base.data.sklearn_pipeline.SklearnDataPipeline - preprocessor: - - name: sklearn.preprocessing.StandardScaler - with_mean: True - with_std: True diff --git a/examples/pytorch/cifar10/conf/model/art/initialize/default.yaml b/examples/pytorch/cifar10/conf/model/art/initialize/default.yaml deleted file mode 100644 index 40e7983e..00000000 --- a/examples/pytorch/cifar10/conf/model/art/initialize/default.yaml +++ /dev/null @@ -1,7 +0,0 @@ -criterion: - name : torch.nn.CrossEntropyLoss -optimizer: - name : torch.optim.SGD - lr : 0.01 - momentum : 0.9 -clip_values : [0.0, 255.0] diff --git a/examples/pytorch/cifar10/conf/model/torch_cifar.yaml b/examples/pytorch/cifar10/conf/model/torch_cifar.yaml deleted file mode 100644 index 5ca2e24e..00000000 --- a/examples/pytorch/cifar10/conf/model/torch_cifar.yaml +++ /dev/null @@ -1,13 +0,0 @@ -defaults: - - art : default -data: ${data} -init: - _target_: deckard.base.model.ModelInitializer - name : torch_example.ResNet18 - num_channels: 3 - num_classes: 10 -_target_: deckard.base.model.Model -trainer: - nb_epoch: 100 - batch_size: 1024 -library : pytorch diff --git a/examples/pytorch/cifar10/conf/model/torch_mnist.yaml b/examples/pytorch/cifar10/conf/model/torch_mnist.yaml deleted file mode 100644 index 9c18c54b..00000000 --- a/examples/pytorch/cifar10/conf/model/torch_mnist.yaml +++ /dev/null @@ -1,13 +0,0 @@ - -defaults: - - art : default -data: ${data} -init: - _target_: deckard.base.model.ModelInitializer - num_channels : 1 - name : torch_example.ResNet18 -_target_: deckard.base.model.Model -trainer: - nb_epoch: 100 - batch_size: 1024 -library : pytorch diff --git a/examples/pytorch/cifar10/dvc.lock b/examples/pytorch/cifar10/dvc.lock deleted file mode 100644 index e371d08c..00000000 --- a/examples/pytorch/cifar10/dvc.lock +++ /dev/null @@ -1,585 +0,0 @@ -schema: '2.0' -stages: - clean@attack: - cmd: python -m deckard.layers.clean_data -i cifar/reports/attack.csv -o cifar/reports/clean_attack.csv - -c conf/clean.yaml - deps: - - path: cifar/reports/attack.csv - md5: f782978aea22d56dbd68fa9f04e4dfcf - size: 26091607 - params: - params.yaml: - files.directory: cifar - files.reports: reports - conf/clean.yaml: - attacks: - DeepFool: Deep - FastGradientMethod: FGM - HopSkipJump: HSJ - PixelAttack: Pixel - ProjectedGradientDescent: PGD - ThresholdAttack: Thresh - defences: - Control: Control - FeatureSqueezing: FSQ - GaussianAugmentation: Gauss-in - GaussianNoise: Gauss-out - HighConfidence: Conf - nb_epoch: Epochs - model_layers: Control - fillna: - model.trainer.nb_epoch: 20 - params: - Deep: attack.init.kwargs.nb_grads - FGM: attack.init.kwargs.eps - HSJ: attack.init.kwargs.max_iter - Pixel: attack.init.kwargs.th - PGD: attack.init.kwargs.eps - Thresh: attack.init.kwargs.th - Gauss-out: model.art.pipeline.postprocessor.kwargs.scale - Conf: model.art.pipeline.postprocessor.kwargs.cutoff - FSQ: model.art.pipeline.preprocessor.kwargs.bit_depth - Gauss-in: model.art.pipeline.preprocessor.kwargs.sigma - Control: model_layers - Epochs: model.trainer.nb_epoch - control: - model_layers: 18 - defaults: - model.trainer.nb_epoch: 20 - outs: - - path: cifar/reports/clean_attack.csv - md5: 81b63f1d5864d65bdaf71aed4ae4c2b0 - size: 14727230 - afr: - cmd: python -m deckard.layers.afr --dataset cifar --data_file cifar/reports/clean_attack.csv --target - adv_accuracy --duration_col predict_time --dataset cifar --config_file conf/afr.yaml - --plots_folder cifar/plots/ - deps: - - path: cifar/reports/clean_attack.csv - md5: 81b63f1d5864d65bdaf71aed4ae4c2b0 - size: 14727230 - params: - params.yaml: - files.directory: cifar - conf/afr.yaml: - covariates: - - adv_fit_time - - accuracy - - train_time - - atk_value - - def_value - - data.sample.random_state - - model_layers - - model.trainer.nb_epoch - - predict_time - log_logistic: - plot: - file: log_logistic_aft.eps - title: Log logistic AFR Model - labels: - 'Intercept: beta_': $\beta$ - 'Intercept: alpha_': $\alpha$ - 'data.sample.random_state: alpha_': Random State - 'atk_value: alpha_': Attack Strength - 'train_time: alpha_': $t_{train}$ - 'predict_proba_time: alpha_': $t_{predict}$ - 'adv_accuracy: alpha_': Adv. Accuracy - 'accuracy: alpha_': Ben. Accuracy - 'adv_fit_time: alpha_': $t_{attack}$ - 'adv_failure_rate: alpha_': $f_{adv.}(t;\theta)$ - 'failure_rate: alpha_': $f_{ben.}(t;\theta)$ - 'model.trainer.nb_epoch: alpha_': No. of Epochs - 'model.trainer.batch_size: alpha_': Batch Size - def_gen: Defence - 'model_layers: alpha_': Layers - 'def_value: alpha_': Defence Strength - 'predict_time: alpha_': $t_{predict}$ - partial_effect: - - file: log_logistic_epochs_partial_effect.eps - covariate_array: model.trainer.nb_epoch - values_array: - - 1 - - 10 - - 25 - - 50 - title: $S(t)$ for Log-Logistic AFR - ylabel: Expectation of $S(t)$ - xlabel: Time $T$ (seconds) - legend_kwargs: - title: Epochs - labels: - - '1' - - '10' - - '25' - - '50' - - file: log_logistic_layers_partial_effect.eps - covariate_array: model_layers - values_array: - - 18 - - 34 - - 50 - - 101 - - 152 - title: $S(t)$ for Log Logistic AFR - ylabel: Expectation of $S(t)$ - xlabel: Time $T$ (seconds) - legend_kwargs: - title: Layers - labels: - - '18' - - '34' - - '50' - - '101' - - '152' - log_normal: - plot: - file: log_normal_aft.eps - title: Log Normal AFR Model - labels: - 'Intercept: sigma_': $\rho$ - 'Intercept: mu_': $\mu$ - 'data.sample.random_state: mu_': Random State - 'atk_value: mu_': Attack Strength - 'train_time: mu_': $t_{train}$ - 'predict_proba_time: mu_': $t_{predict}$ - 'adv_accuracy: mu_': Adv. Accuracy - 'accuracy: mu_': Ben. Accuracy - 'adv_fit_time: mu_': $t_{attack}$ - 'adv_failure_rate: mu_': $f_{adv.}(t;\theta)$ - 'failure_rate: mu_': $f_{ben.}(t;\theta)$ - 'model.trainer.nb_epoch: mu_': No. of Epochs - 'model.trainer.batch_size: mu_': Batch Size - def_gen: Defence - 'model_layers: mu_': Layers - 'def_value: mu_': Defence Strength - 'predict_time: mu_': $t_{predict}$ - partial_effect: - - file: log_normal_epochs_partial_effect.eps - covariate_array: model.trainer.nb_epoch - values_array: - - 1 - - 10 - - 25 - - 50 - title: $S(t)$ for Log-Normal AFR - ylabel: Expectation of $S(t)$ - xlabel: Time $T$ (seconds) - legend_kwargs: - title: Epochs - labels: - - '1' - - '10' - - '25' - - '50' - - file: log_normal_layers_partial_effect.eps - covariate_array: model_layers - values_array: - - 18 - - 34 - - 50 - - 101 - - 152 - title: $S(t)$ for Log Normal AFR - ylabel: Expectation of $S(t)$ - xlabel: Time $T$ (seconds) - legend_kwargs: - title: Layers - labels: - - '18' - - '34' - - '50' - - '101' - - '152' - weibull: - plot: - file: weibull_aft.eps - title: Weibull AFR Model - labels: - 'Intercept: rho_': $\rho$ - 'Intercept: lambda_': $\lambda$ - 'data.sample.random_state: lambda_': Random State - 'atk_value: lambda_': Attack Strength - 'train_time: lambda_': $t_{train}$ - 'predict_proba_time: lambda_': $t_{predict}$ - 'adv_accuracy: lambda_': Adv. Accuracy - 'accuracy: lambda_': Ben. Accuracy - 'adv_fit_time: lambda_': $t_{attack}$ - 'adv_failure_rate: lambda_': $f_{adv.}(t;\theta)$ - 'failure_rate: lambda_': $f_{ben.}(t;\theta)$ - 'model.trainer.nb_epoch: lambda_': No. of Epochs - 'model.trainer.batch_size: lambda_': Batch Size - def_gen: Defence - 'model_layers: lambda_': Layers - 'def_value: lambda_': Defence Strength - 'predict_time: lambda_': $t_{predict}$ - partial_effect: - - file: weibull_epochs_partial_effect.eps - covariate_array: model.trainer.nb_epoch - values_array: - - 1 - - 10 - - 25 - - 50 - title: $S(t)$ for Weibull AFR - ylabel: Expectation of $S(t)$ - xlabel: Time $T$ (seconds) - legend_kwargs: - title: Epochs - labels: - - '1' - - '10' - - '25' - - '50' - - file: weibull_layers_partial_effect.eps - covariate_array: model_layers - values_array: - - 18 - - 34 - - 50 - - 101 - - 152 - title: $S(t)$ for Weibull AFR - ylabel: Expectation of $S(t)$ - xlabel: Time $T$ (seconds) - legend_kwargs: - title: Layers - labels: - - '18' - - '34' - - '50' - - '101' - - '152' - outs: - - path: cifar/plots/log_logistic_aft.eps - md5: 25c507a344b0589027e50473873485f4 - size: 44483 - - path: cifar/plots/log_logistic_epochs_partial_effect.eps - md5: 58ad2cc4455c7a2e14dd8baf5e0bebbc - size: 43547 - - path: cifar/plots/log_logistic_layers_partial_effect.eps - md5: c1ec42636bdbf6ba5a3279da4ff03226 - size: 46098 - - path: cifar/plots/log_normal_aft.eps - md5: c10aef0b38507ea46d0bcab12edd43d7 - size: 43898 - - path: cifar/plots/log_normal_epochs_partial_effect.eps - md5: c7e71cb7f8673260b6ba330e228f9897 - size: 44094 - - path: cifar/plots/log_normal_layers_partial_effect.eps - md5: c516900f508281933e52f08fc6e4cec3 - size: 46510 - - path: cifar/plots/weibull_aft.eps - md5: 42a59a94ef52a0bd0260cc1fe2113bc3 - size: 41411 - - path: cifar/plots/weibull_epochs_partial_effect.eps - md5: e5397975c6e5b322a41b7c5918e278cf - size: 43063 - - path: cifar/plots/weibull_layers_partial_effect.eps - md5: 00b0191fdd4ac6f407a33374dbb7a06b - size: 45397 - plot: - cmd: python -m deckard.layers.plots --path cifar/plots/ --file cifar/reports/clean_attack.csv - -c conf/plots.yaml - deps: - - path: cifar/reports/clean_attack.csv - md5: 81b63f1d5864d65bdaf71aed4ae4c2b0 - size: 14727230 - params: - params.yaml: - files.directory: cifar - files.reports: reports - conf/plots.yaml: - cat_plot: - - file: adv_accuracy_vs_defence_type.eps - hue: model_name - kind: boxen - set: - yscale: linear - titles: Adv. Accuracy vs Defence Type - x: def_gen - xlabels: Defence Type - y: adv_accuracy - ylabels: Adv. Accuracy - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - - file: ben_accuracy_vs_defence_type.eps - hue: model_name - kind: boxen - titles: Ben. Accuracy vs Defence Type - x: def_gen - xlabels: Defence Type - y: accuracy - ylabels: Ben. Accuracy - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - - file: ben_failures_per_train_time_vs_defence_type.eps - hue: model_name - kind: boxen - set: - yscale: log - titles: $\bar{C}_{ben.}$ vs Defence Type - x: def_gen - xlabels: Defence Type - y: training_time_per_failure - ylabels: $\bar{C}_{ben.}$ - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - - file: adv_failures_per_train_time_vs_defence_type.eps - hue: model_name - kind: boxen - titles: $\bar{C}_{adv.}$ vs Defence Type - x: def_gen - xlabels: Defence Type - y: training_time_per_adv_failure - ylabels: $\bar{C}_{adv.}$ - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - - file: adv_failures_per_train_time_vs_attack_type.eps - hue: model_name - kind: boxen - legend_title: Model Name - titles: $\bar{C}_{adv.}$ vs Attack Type - x: atk_gen - xlabels: Attack Type - y: training_time_per_adv_failure - ylabels: $\bar{C}_{adv.}$ - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - - file: adv_failures_per_test_time_vs_defence_type.eps - hue: model_name - kind: boxen - legend_title: Model Name - titles: $f_{adv}$ vs Defence Type - x: def_gen - xlabels: Defence Type - y: adv_failure_rate - ylabels: $f_{adv.}$ - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - - file: adv_accuracy_vs_attack_type.eps - hue: model_name - kind: boxen - legend_title: Model Name - titles: Adv. Accuracy vs Attack Type - x: atk_gen - xlabels: Attack Type - y: adv_accuracy - ylabels: Adv. Accuracy - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - - file: ben_failure_rate_vs_defence_type.eps - hue: model_name - kind: boxen - legend_title: Model Name - set: - yscale: log - titles: $f_{ben}(t; \theta)$ vs Defence Type - x: def_gen - xlabels: Defence Type - y: failure_rate - ylabels: $f_{ben}(t; \theta)$ - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - line_plot: - - file: def_param_vs_accuracy.eps - hue: def_gen - legend: - bbox_to_anchor: - - 1.05 - - 1 - title: Defence - title: Ben. Accuracy vs Defence Strength - x: def_value - x_scale: linear - xlabel: Defence Control Parameter - y: accuracy - y_scale: - ylabel: Ben. Accuracy - hue_order: - - Control - - Conf - - Epochs - - Gauss-in - - Gauss-out - - Conf - - FSQ - errorbar: se - err_style: bars - - file: def_param_vs_adv_accuracy.eps - hue: def_gen - legend: - bbox_to_anchor: - - 1.05 - - 1 - title: Defence - title: Adv. Accuracy vs Defence Strength - x: def_value - x_scale: linear - xlabel: Defence Control Parameter - y: adv_accuracy - y_scale: - ylabel: Adv. Accuracy - hue_order: - - Control - - Conf - - Epochs - - Gauss-in - - Gauss-out - - Conf - - FSQ - errorbar: se - err_style: bars - - file: def_param_vs_adv_failure_rate.eps - hue: def_gen - legend: - bbox_to_anchor: - - 1.05 - - 1 - title: Defence - title: $f_{adv}$ vs Defence Strength - x: def_value - x_scale: log - xlabel: Defence Control Parameter - y: adv_failure_rate - y_scale: log - ylabel: $f_{adv.}$ - hue_order: - - Control - - Conf - - Epochs - - Gauss-in - - Gauss-out - - Conf - - FSQ - errorbar: se - err_style: bars - - file: atk_param_vs_accuracy.eps - hue: atk_gen - legend: - bbox_to_anchor: - - 1.05 - - 1 - title: Adv. Accuracy vs Attack Strength - x: atk_value - x_scale: linear - xlabel: Attack Control Parameter - y: adv_accuracy - y_scale: - ylabel: Adv. Accuracy - hue_order: - - FGM - - PGD - - Deep - - HSJ - - Pixel - - Thresh - errorbar: se - err_style: bars - scatter_plot: - - x: train_time_per_sample - y: adv_failure_rate - hue: model_name - xlabel: $t_{train}$ - ylabel: $f_{adv}$ - title: $f_{adv}$ vs $t_{train}$ - file: adv_failure_rate_vs_train_time.eps - y_scale: linear - x_scale: log - legend: - title: Model Name - bbox_to_anchor: - - 1.05 - - 1 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - outs: - - path: cifar/plots/adv_accuracy_vs_attack_type.eps - md5: 5138387d43aa6b80861beda375a668a4 - size: 108266 - - path: cifar/plots/adv_accuracy_vs_defence_type.eps - md5: 413761c7484aed43f3d3f6f75da81e82 - size: 101582 - - path: cifar/plots/adv_failure_rate_vs_train_time.eps - md5: 2cdddda5ce788d5813ca55880fdb7a91 - size: 631508 - - path: cifar/plots/adv_failures_per_test_time_vs_defence_type.eps - md5: 23aff41fad24c912f0301f2e986ae96d - size: 121711 - - path: cifar/plots/adv_failures_per_train_time_vs_attack_type.eps - md5: 85efb2307112f220946f0c4b623df366 - size: 120765 - - path: cifar/plots/adv_failures_per_train_time_vs_defence_type.eps - md5: f914efca4568b2f5163b7e8723f0df5c - size: 113874 - - path: cifar/plots/atk_param_vs_accuracy.eps - md5: 35ee04493de38424453b9514c45d66f1 - size: 39212 - - path: cifar/plots/ben_accuracy_vs_defence_type.eps - md5: 61569c543a2891c9fcdc95314924ceb4 - size: 96824 - - path: cifar/plots/ben_failure_rate_vs_defence_type.eps - md5: 792603cf59c01c922408c18d1dfeda3e - size: 122740 - - path: cifar/plots/ben_failures_per_train_time_vs_defence_type.eps - md5: 25f8992268b8eefa2eaa60b73e1f6370 - size: 112749 - - path: cifar/plots/def_param_vs_accuracy.eps - md5: 21b2ace7923bdeb9c5bb950caf2307f1 - size: 38940 - - path: cifar/plots/def_param_vs_adv_accuracy.eps - md5: 43d5d57465019a4067af163be4178fa8 - size: 38631 - - path: cifar/plots/def_param_vs_adv_failure_rate.eps - md5: 2a6cfab767a224847deb07eb1d451b4a - size: 39304 - copy_results: - cmd: mkdir -p ~/ml_afr/cifar/ && cp -r cifar/plots/* ~/ml_afr/cifar/ - deps: - - path: cifar/plots/ - md5: c22ad5afc78d35f682d247d6bd61643f.dir - size: 16455580 - nfiles: 25 diff --git a/examples/pytorch/cifar10/dvc.yaml b/examples/pytorch/cifar10/dvc.yaml deleted file mode 100644 index 6b445d4c..00000000 --- a/examples/pytorch/cifar10/dvc.yaml +++ /dev/null @@ -1,168 +0,0 @@ -vars: - - conf/plots.yaml:line_plot - - conf/plots.yaml:scatter_plot - - conf/plots.yaml:cat_plot - - conf/afr.yaml:covariates - - conf/afr.yaml:weibull - - conf/afr.yaml:log_logistic - - conf/afr.yaml:log_normal - - conf/clean.yaml:attacks - - conf/clean.yaml:defences - - conf/clean.yaml:params - - conf/clean.yaml:fillna -stages: - train: - cmd: python -m deckard.layers.experiment train --config_file cifar.yaml - params: - - data - - model - - scorers - - files - outs: - - ${files.directory}/${files.data_dir}/${files.data_file}${files.data_type} - - ${files.directory}/${files.model_dir}/${files.model_file}${files.model_type} - - ${files.directory}/${files.model_dir}/${files.model_file}.optimizer${files.model_type} - # - ${files.directory}/${files.reports}/train/${files.name}/${files.params_file} - # - ${files.directory}/${files.reports}/train/${files.name}/${files.test_labels_file} # Omit to save space - - ${files.directory}/${files.reports}/train/${files.name}/${files.predictions_file} # logit outputs for our model - # - ${files.directory}/${files.reports}/train/${files.name}/${files.probabilities_file} # Omit to save space - metrics: - - ${files.directory}/${files.reports}/train/${files.name}/${files.score_dict_file} - attack: - cmd: python -m deckard.layers.experiment attack --config_file cifar.yaml - params: - - data - - model - - attack - - scorers - - files - outs: - - ${files.directory}/${files.attack_dir}/${files.attack_file}${files.attack_type} - - ${files.directory}/${files.reports}/attack/${files.name}/${files.adv_predictions_file} - # - ${files.directory}/${files.reports}/attack/${files.name}/${files.params_file} - deps: - - ${files.directory}/${files.data_dir}/${files.data_file}${files.data_type} - - ${files.directory}/${files.model_dir}/${files.model_file}${files.model_type} - metrics: - - ${files.directory}/${files.reports}/attack/${files.name}/${files.score_dict_file} - - ############################################################################## - # models: # This is a loop over the ResNet models - # foreach: - # - ResNet18 - # # - ResNet34 - # # - ResNet50 - # # - ResNet101 - # # - ResNet152 - # do: # This script configures eazch defence - # cmd: bash models.sh ++model.init.name=torch_example.${item} stage=train ++hydra.sweeper.storage=sqlite:///${files.directory}/${files.reports}/train/${item}.db --config-name cifar.yaml - # deps: - # - models.sh - # - ${files.directory}/${files.model_dir}/${files.model_file}${files.model_type} - # - ${files.directory}/${files.model_dir}/${files.model_file}.optimizer${files.model_type} - # outs: - # - ${files.directory}/${files.reports}/train/${item}.db: # This outputs a database file for each model - # cache: True - # persist: True - attacks: - foreach: # This is a loop over the ResNet models - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - do: - cmd: bash attacks.sh ++attack.attack_size=100 ++model.init.name=torch_example.${item} stage=attack ++hydra.sweeper.storage=sqlite:///${files.directory}/${files.reports}/attack/${item}.db --config-name cifar.yaml - deps: - - models.sh # This script configures each defence - - attacks.sh # This script configures each attack - - ${files.directory}/${files.reports}/attack/${files.name}/${files.score_dict_file} # This is here just to ensure it runs after the attack stage - # - ${files.directory}/${files.reports}/train/${item}.db - outs: - - ${files.directory}/${files.reports}/attack/${item}.db: # This outputs a database file for each model - cache: True - persist: True - compile: - foreach: # iterates through each stage - # - train - - attack - do: - cmd: python -m deckard.layers.compile --report_folder ${files.directory}/${files.reports}/${item} --results_file ${files.directory}/${files.reports}/${item}.csv - deps: - - ${files.directory}/${files.reports}/${item}/ - - ${files.directory}/${files.reports}/${item}/ResNet18.db - - ${files.directory}/${files.reports}/${item}/ResNet34.db - - ${files.directory}/${files.reports}/${item}/ResNet50.db - - ${files.directory}/${files.reports}/${item}/ResNet101.db - - ${files.directory}/${files.reports}/${item}/ResNet152.db - outs: - - ${files.directory}/${files.reports}/${item}.csv - clean: - foreach: # iterates through each stage - # - train - - attack - do: - cmd: python -m deckard.layers.clean_data -i ${files.directory}/${files.reports}/${item}.csv -o ${files.directory}/${files.reports}/clean_${item}.csv -c conf/clean.yaml - deps: - - ${files.directory}/${files.reports}/${item}.csv - outs: - - ${files.directory}/${files.reports}/clean_${item}.csv - params: - - files.directory - - files.reports - - conf/clean.yaml: - - attacks - - defences - - params - - fillna - plot: - cmd : python -m deckard.layers.plots --path ${files.directory}/plots/ --file ${files.directory}/${files.reports}/clean_attack.csv -c conf/plots.yaml - deps: - - ${files.directory}/${files.reports}/clean_attack.csv - plots: - - ${files.directory}/plots/${cat_plot[0].file} - - ${files.directory}/plots/${cat_plot[1].file} - - ${files.directory}/plots/${cat_plot[2].file} - - ${files.directory}/plots/${cat_plot[3].file} - - ${files.directory}/plots/${cat_plot[4].file} - - ${files.directory}/plots/${cat_plot[5].file} - - ${files.directory}/plots/${cat_plot[6].file} - - ${files.directory}/plots/${cat_plot[7].file} - - ${files.directory}/plots/${line_plot[0].file} - - ${files.directory}/plots/${line_plot[1].file} - - ${files.directory}/plots/${line_plot[2].file} - - ${files.directory}/plots/${line_plot[3].file} - - ${files.directory}/plots/${scatter_plot[0].file} - params: - - files.directory - - files.reports - - conf/plots.yaml: - - line_plot - - scatter_plot - - cat_plot - afr: - cmd: python -m deckard.layers.afr --dataset ${files.directory} --data_file ${files.directory}/${files.reports}/clean_attack.csv --target adv_accuracy --duration_col predict_time --dataset cifar --config_file conf/afr.yaml --plots_folder ${files.directory}/plots/ - deps: - - ${files.directory}/${files.reports}/clean_attack.csv - plots: - - ${files.directory}/plots/weibull_aft.eps - - ${files.directory}/plots/weibull_epochs_partial_effect.eps - - ${files.directory}/plots/weibull_layers_partial_effect.eps - - ${files.directory}/plots/log_logistic_aft.eps - - ${files.directory}/plots/log_logistic_epochs_partial_effect.eps - - ${files.directory}/plots/log_logistic_layers_partial_effect.eps - - ${files.directory}/plots/log_normal_aft.eps - - ${files.directory}/plots/log_normal_epochs_partial_effect.eps - - ${files.directory}/plots/log_normal_layers_partial_effect.eps - params: - - files.directory - - conf/afr.yaml: - - covariates - - weibull - - log_logistic - - log_normal - copy_results: - cmd: mkdir -p ~/ml_afr/cifar/ && cp -r ${files.directory}/plots/* ~/ml_afr/cifar/ - deps: - - ${files.directory}/plots/ - diff --git a/examples/pytorch/cifar10/models.sh b/examples/pytorch/cifar10/models.sh deleted file mode 100644 index e8e0944f..00000000 --- a/examples/pytorch/cifar10/models.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -# This script is used to generate the models for the sklearn example. - -# # Default model -echo "python -m deckard.layers.optimise ++model.trainer.nb_epoch=1,10,30,50,100" $@ "--multirun" -python -m deckard.layers.optimise ++model.trainer.nb_epoch=1,10,30,50,100 $@ --multirun - -# # This line generates the model and adds the FeatureSqueezing preprocessing defence. -# python -m deckard.layers.optimise \ -# ++model.art.preprocessor.name=art.defences.preprocessor.FeatureSqueezing \ -# +model.art.preprocessor.params.bit_depth=4,8,16,32,64 \ -# +model.art.preprocessor.params.clip_values=[0,255] \ -# ++hydra.sweeper.study_name=FSQ $@ --multirun - -# # # # Gaussian Augmentation (Input) -# python -m deckard.layers.optimise \ -# ++model.art.preprocessor.name=art.defences.preprocessor.GaussianAugmentation \ -# +model.art.preprocessor.params.sigma=.01,.1,.3,.5,1 \ -# +model.art.preprocessor.params.ratio=.5 \ -# +model.art.preprocessor.params.augmentation=False \ -# ++hydra.sweeper.study_name=gauss-in $@ --multirun - -# # # # # Gaussian Noise (Output) -# python -m deckard.layers.optimise \ -# ++model.art.postprocessor.name=art.defences.postprocessor.GaussianNoise \ -# ++model.art.postprocessor.params.scale=.01,.1,.3,.5,1 \ -# ++hydra.sweeper.study_name=gauss-out $@ --multirun - -# # # # # High Confidence -# python -m deckard.layers.optimise \ -# +model.art.postprocessor.name=art.defences.postprocessor.HighConfidence \ -# +model.art.postprocessor.params.cutoff=.1,.3,.5,.9,.99 \ -# ++hydra.sweeper.study_name=conf $@ --multirun diff --git a/examples/pytorch/cifar10/params.yaml b/examples/pytorch/cifar10/params.yaml deleted file mode 100644 index d8ad2b2e..00000000 --- a/examples/pytorch/cifar10/params.yaml +++ /dev/null @@ -1,207 +0,0 @@ -_target_: deckard.base.experiment.Experiment -attack: - _target_: deckard.base.attack.Attack - attack_size: 10 - data: - _target_: deckard.base.data.Data - generate: - name: torch_cifar10 - sample: - random_state: 0 - stratify: true - sklearn_pipeline: - preprocessor: - name: sklearn.preprocessing.StandardScaler - with_mean: true - with_std: true - init: - _target_: deckard.base.attack.AttackInitializer - model: - _target_: deckard.base.model.Model - art: - _target_: deckard.base.model.art_pipeline.ArtPipeline - data: - _target_: deckard.base.data.Data - generate: - name: torch_cifar10 - sample: - random_state: 0 - stratify: true - sklearn_pipeline: - preprocessor: - name: sklearn.preprocessing.StandardScaler - with_mean: true - with_std: true - initialize: - clip_values: - - 0.0 - - 255.0 - criterion: - name: torch.nn.CrossEntropyLoss - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - library: pytorch - data: - _target_: deckard.base.data.Data - generate: - name: torch_cifar10 - sample: - random_state: 0 - stratify: true - sklearn_pipeline: - preprocessor: - name: sklearn.preprocessing.StandardScaler - with_mean: true - with_std: true - init: - _target_: deckard.base.model.ModelInitializer - name: torch_example.ResNet18 - num_channels: 3 - num_classes: 10 - library: pytorch - trainer: - batch_size: 1024 - nb_epoch: 100 - name: art.attacks.evasion.HopSkipJump - method: evasion - model: - _target_: deckard.base.model.Model - art: - _target_: deckard.base.model.art_pipeline.ArtPipeline - data: - _target_: deckard.base.data.Data - generate: - name: torch_cifar10 - sample: - random_state: 0 - stratify: true - sklearn_pipeline: - preprocessor: - name: sklearn.preprocessing.StandardScaler - with_mean: true - with_std: true - initialize: - clip_values: - - 0.0 - - 255.0 - criterion: - name: torch.nn.CrossEntropyLoss - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - library: pytorch - data: - _target_: deckard.base.data.Data - generate: - name: torch_cifar10 - sample: - random_state: 0 - stratify: true - sklearn_pipeline: - preprocessor: - name: sklearn.preprocessing.StandardScaler - with_mean: true - with_std: true - init: - _target_: deckard.base.model.ModelInitializer - name: torch_example.ResNet18 - num_channels: 3 - num_classes: 10 - library: pytorch - trainer: - batch_size: 1024 - nb_epoch: 100 -data: - _target_: deckard.base.data.Data - generate: - name: torch_cifar10 - sample: - random_state: 0 - stratify: true - sklearn_pipeline: - preprocessor: - name: sklearn.preprocessing.StandardScaler - with_mean: true - with_std: true -direction: maximize -files: - _target_: deckard.base.files.FileConfig - adv_predictions_file: adv_predictions.json - attack_dir: attacks - attack_file: attack - attack_type: .pkl - data_dir: data - data_file: data - data_type: .pkl - directory: cifar - model_dir: models - model_file: model - model_type: .pt - name: default - params_file: params.yaml - predictions_file: predictions.json - reports: reports - score_dict_file: score_dict.json -model: - _target_: deckard.base.model.Model - art: - _target_: deckard.base.model.art_pipeline.ArtPipeline - data: - _target_: deckard.base.data.Data - generate: - name: torch_cifar10 - sample: - random_state: 0 - stratify: true - sklearn_pipeline: - preprocessor: - name: sklearn.preprocessing.StandardScaler - with_mean: true - with_std: true - initialize: - clip_values: - - 0.0 - - 255.0 - criterion: - name: torch.nn.CrossEntropyLoss - optimizer: - lr: 0.01 - momentum: 0.9 - name: torch.optim.SGD - library: pytorch - data: - _target_: deckard.base.data.Data - generate: - name: torch_cifar10 - sample: - random_state: 0 - stratify: true - sklearn_pipeline: - preprocessor: - name: sklearn.preprocessing.StandardScaler - with_mean: true - with_std: true - init: - _target_: deckard.base.model.ModelInitializer - name: torch_example.ResNet18 - num_channels: 3 - num_classes: 10 - library: pytorch - trainer: - batch_size: 1024 - nb_epoch: 100 -optimizers: accuracy -scorers: - _target_: deckard.base.scorer.ScorerDict - accuracy: - _target_: deckard.base.scorer.ScorerConfig - direction: maximize - name: sklearn.metrics.accuracy_score - log_loss: - _target_: deckard.base.scorer.ScorerConfig - direction: minimize - name: sklearn.metrics.log_loss -stage: ??? diff --git a/examples/pytorch/cifar10/torch_example.py b/examples/pytorch/cifar10/torch_example.py deleted file mode 100644 index 32d333bd..00000000 --- a/examples/pytorch/cifar10/torch_example.py +++ /dev/null @@ -1,79 +0,0 @@ -import torch.nn as nn -from torchvision import models - -__all__ = [ - "ResNet18", - "ResNet50", - "ResNet101", - "ResNet152", -] - - -def ResNet18(num_channels=1, num_classes=10): - model = models.resnet18(pretrained=True) - model.conv1 = nn.Conv2d( - num_channels, - 64, - kernel_size=7, - stride=2, - padding=3, - bias=False, - ) - model.fc = nn.Linear(512, num_classes) - return model - - -def ResNet34(num_channels=1, num_classes=10): - model = models.resnet34(pretrained=True) - model.conv1 = nn.Conv2d( - num_channels, - 64, - kernel_size=7, - stride=2, - padding=3, - bias=False, - ) - model.fc = nn.Linear(512, num_classes) - return model - - -def ResNet50(num_channels=1, num_classes=10): - model = models.resnet50(pretrained=True) - model.conv1 = nn.Conv2d( - num_channels, - 64, - kernel_size=7, - stride=2, - padding=3, - bias=False, - ) - model.fc = nn.Linear(2048, num_classes) - return model - - -def ResNet101(num_channels=1, num_classes=10): - model = models.resnet101(pretrained=True) - model.conv1 = nn.Conv2d( - num_channels, - 64, - kernel_size=7, - stride=2, - padding=3, - bias=False, - ) - model.fc = nn.Linear(2048, num_classes) - return model - - -def ResNet152(num_channels=1, num_classes=10): - model = models.resnet152(pretrained=True) - model.conv1 = nn.Conv2d( - num_channels, - 64, - kernel_size=7, - stride=2, - padding=3, - bias=False, - ) - model.fc = nn.Linear(2048, num_classes) - return model diff --git a/examples/pytorch/cifar100/params.yaml b/examples/pytorch/cifar100.yaml similarity index 69% rename from examples/pytorch/cifar100/params.yaml rename to examples/pytorch/cifar100.yaml index c67df2e1..471c78a7 100644 --- a/examples/pytorch/cifar100/params.yaml +++ b/examples/pytorch/cifar100.yaml @@ -1,39 +1,40 @@ _target_: deckard.base.experiment.Experiment +atk_name: hsj attack: _target_: deckard.base.attack.Attack - attack_size: 10 + attack_size: 100 data: _target_: deckard.base.data.Data generate: name: torch_cifar100 - path: original_data + path: original_data/ sample: random_state: 0 stratify: true - sklearn_pipeline: - preprocessor: - name: sklearn.preprocessing.StandardScaler - with_mean: true - with_std: true + test_size: 12000 + train_size: 48000 init: _target_: deckard.base.attack.AttackInitializer + batch_size: 128 model: _target_: deckard.base.model.Model art: _target_: deckard.base.model.art_pipeline.ArtPipeline + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss data: _target_: deckard.base.data.Data generate: name: torch_cifar100 - path: original_data + path: original_data/ sample: random_state: 0 stratify: true - sklearn_pipeline: - preprocessor: - name: sklearn.preprocessing.StandardScaler - with_mean: true - with_std: true + test_size: 12000 + train_size: 48000 initialize: clip_values: - 0 @@ -45,19 +46,20 @@ attack: momentum: 0.9 name: torch.optim.SGD library: pytorch + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD data: _target_: deckard.base.data.Data generate: name: torch_cifar100 - path: original_data + path: original_data/ sample: random_state: 0 stratify: true - sklearn_pipeline: - preprocessor: - name: sklearn.preprocessing.StandardScaler - with_mean: true - with_std: true + test_size: 12000 + train_size: 48000 init: _target_: deckard.base.model.ModelInitializer name: torch_example.ResNet18 @@ -65,27 +67,30 @@ attack: num_classes: 100 library: pytorch trainer: - batch_size: 1024 - nb_epoch: 10 + batch_size: 128 + nb_epochs: 1 + verbose: true name: art.attacks.evasion.HopSkipJump method: evasion model: _target_: deckard.base.model.Model art: _target_: deckard.base.model.art_pipeline.ArtPipeline + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss data: _target_: deckard.base.data.Data generate: name: torch_cifar100 - path: original_data + path: original_data/ sample: random_state: 0 stratify: true - sklearn_pipeline: - preprocessor: - name: sklearn.preprocessing.StandardScaler - with_mean: true - with_std: true + test_size: 12000 + train_size: 48000 initialize: clip_values: - 0 @@ -97,19 +102,20 @@ attack: momentum: 0.9 name: torch.optim.SGD library: pytorch + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD data: _target_: deckard.base.data.Data generate: name: torch_cifar100 - path: original_data + path: original_data/ sample: random_state: 0 stratify: true - sklearn_pipeline: - preprocessor: - name: sklearn.preprocessing.StandardScaler - with_mean: true - with_std: true + test_size: 12000 + train_size: 48000 init: _target_: deckard.base.model.ModelInitializer name: torch_example.ResNet18 @@ -117,31 +123,30 @@ attack: num_classes: 100 library: pytorch trainer: - batch_size: 1024 - nb_epoch: 10 + batch_size: 128 + nb_epochs: 1 + verbose: true data: _target_: deckard.base.data.Data generate: name: torch_cifar100 - path: original_data + path: original_data/ sample: random_state: 0 stratify: true - sklearn_pipeline: - preprocessor: - name: sklearn.preprocessing.StandardScaler - with_mean: true - with_std: true -direction: maximize + test_size: 12000 + train_size: 48000 +dataset: cifar100 +def_name: control +device_id: gpu +direction: +- maximize files: _target_: deckard.base.files.FileConfig adv_predictions_file: adv_predictions.json attack_dir: attacks attack_file: attack attack_type: .pkl - data_dir: data - data_file: data - data_type: .pkl directory: cifar100 model_dir: models model_file: model @@ -155,19 +160,21 @@ model: _target_: deckard.base.model.Model art: _target_: deckard.base.model.art_pipeline.ArtPipeline + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss data: _target_: deckard.base.data.Data generate: name: torch_cifar100 - path: original_data + path: original_data/ sample: random_state: 0 stratify: true - sklearn_pipeline: - preprocessor: - name: sklearn.preprocessing.StandardScaler - with_mean: true - with_std: true + test_size: 12000 + train_size: 48000 initialize: clip_values: - 0 @@ -179,19 +186,20 @@ model: momentum: 0.9 name: torch.optim.SGD library: pytorch + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD data: _target_: deckard.base.data.Data generate: name: torch_cifar100 - path: original_data + path: original_data/ sample: random_state: 0 stratify: true - sklearn_pipeline: - preprocessor: - name: sklearn.preprocessing.StandardScaler - with_mean: true - with_std: true + test_size: 12000 + train_size: 48000 init: _target_: deckard.base.model.ModelInitializer name: torch_example.ResNet18 @@ -199,9 +207,12 @@ model: num_classes: 100 library: pytorch trainer: - batch_size: 1024 - nb_epoch: 10 -optimizers: accuracy + batch_size: 128 + nb_epochs: 1 + verbose: true +model_name: ResNet18 +optimizers: +- accuracy scorers: _target_: deckard.base.scorer.ScorerDict accuracy: diff --git a/examples/pytorch/cifar100/.dvc/.gitignore b/examples/pytorch/cifar100/.dvc/.gitignore deleted file mode 100644 index 528f30c7..00000000 --- a/examples/pytorch/cifar100/.dvc/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/config.local -/tmp -/cache diff --git a/examples/pytorch/cifar100/.dvc/config b/examples/pytorch/cifar100/.dvc/config deleted file mode 100644 index 4cf322d9..00000000 --- a/examples/pytorch/cifar100/.dvc/config +++ /dev/null @@ -1,2 +0,0 @@ -[core] - autostage = true diff --git a/examples/pytorch/cifar100/.dvcignore b/examples/pytorch/cifar100/.dvcignore deleted file mode 100644 index 51973055..00000000 --- a/examples/pytorch/cifar100/.dvcignore +++ /dev/null @@ -1,3 +0,0 @@ -# Add patterns of files dvc should ignore, which could improve -# the performance. Learn more at -# https://dvc.org/doc/user-guide/dvcignore diff --git a/examples/pytorch/cifar100/.gitignore b/examples/pytorch/cifar100/.gitignore deleted file mode 100644 index 69ff62b5..00000000 --- a/examples/pytorch/cifar100/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -cifar100/* -.dvc/* -20_epochs/* -**/optimization_results.yaml -*.rdb -original_data/* diff --git a/examples/pytorch/cifar100/attacks.sh b/examples/pytorch/cifar100/attacks.sh deleted file mode 100644 index b491f97d..00000000 --- a/examples/pytorch/cifar100/attacks.sh +++ /dev/null @@ -1,37 +0,0 @@ -# #!/bin/bash - -# # This script is used to generate the attacks for the example. - -# Fast Gradient Method -bash models.sh attack=default ++attack.init.name=art.attacks.evasion.FastGradientMethod ++attack.init.eps=.001,.01,.1,.5,1 ++attack.init.norm=inf ++attack.init.eps_step=.001,.003,.01 ++attack.init.batch_size=1024 stage=attack ++hydra.sweeper.study_name=fgm ++direction=maximize $@ - -# Projected Gradient Descent -bash models.sh attack=default ++attack.init.name=art.attacks.evasion.ProjectedGradientDescent ++attack.init.eps=.001,.01,.1,.5,1 ++attack.init.norm=inf ++attack.init.eps_step=.001,.003,.01 ++attack.init.batch_size=1024 ++attack.init.max_iter=10 stage=attack ++hydra.sweeper.study_name=pgd ++direction=maximize $@ - -# DeepFool -bash models.sh attack=default ++attack.init.name=art.attacks.evasion.DeepFool ++attack.init.max_iter=10 ++attack.init.batch_size=1024 ++attack.init.nb_grads=1,3,5,10 stage=attack ++hydra.sweeper.study_name=deep ++direction=maximize $@ - -# HopSkipJump -bash models.sh attack=default ++attack.init.name=art.attacks.evasion.HopSkipJump ++attack.init.max_iter=1,3,5,10 ++attack.init.init_eval=10 ++attack.init.norm=inf stage=attack ++hydra.sweeper.study_name=hsj ++direction=maximize $@ - -# ##################################################### -# PixelAttack -bash models.sh attack=default ++attack.init.name=art.attacks.evasion.PixelAttack ++attack.init.max_iter=10 ++attack.init.th=1,4,16,64,256 stage=attack ++hydra.sweeper.study_name=pixel ++direction=maximize $@ - -# ThresholdAttack -bash models.sh attack=default ++attack.init.name=art.attacks.evasion.ThresholdAttack ++attack.init.max_iter=10 ++attack.init.th=1,4,16,64,256 stage=attack ++hydra.sweeper.study_name=thresh ++direction=maximize $@ - -# # AdversarialPatch -# bash models.sh attack=default --attack.init.batch_size ++attack.init.name=art.attacks.evasion.AdversarialPatch ++attack.init.max_iter=10 ++attack.init.learning_rate=.5,5.0,50.0 stage=patch ++hydra.sweeper.study_name=attack ++direction=maximize ++attack.init.patch_shape=[1,28,28] $@ -##################################################### - -# # Carlini L0 Method -# bash models.sh attack=default ++attack.init.name=art.attacks.evasion.CarliniL0Method ++attack.init.batch_size=1024 ++attack.init.max_iter=10 ++attack.init.confidence=.1,.9,.99 stage=cw0 ++hydra.sweeper.study_name=cw0 ++direction=maximize $@ - -# # Carlini L2 Method -# bash models.sh attack=default ++attack.init.name=art.attacks.evasion.CarliniL2Method ++attack.init.batch_size=1024 ++attack.init.max_iter=10 ++attack.init.confidence=.1,.9,.99 stage=cw2 ++hydra.sweeper.study_name=cw2 ++direction=maximize $@ - -# # Carlini LInf Method -# bash models.sh attack=default ++attack.init.name=art.attacks.evasion.CarliniLInfMethod ++attack.init.max_iter=10 ++attack.init.confidence=.1,.9,.99 stage=attack ++hydra.sweeper.study_name=cwinf ++direction=maximize $@ - -rm -rf output/models/* diff --git a/examples/pytorch/cifar100/conf/afr.yaml b/examples/pytorch/cifar100/conf/afr.yaml deleted file mode 100644 index a2f65dc9..00000000 --- a/examples/pytorch/cifar100/conf/afr.yaml +++ /dev/null @@ -1,184 +0,0 @@ -covariates: - - "adv_fit_time" - - "accuracy" - - "train_time" - - "atk_value" - - "def_value" - - "data.sample.random_state" - - "model_layers" - - model.trainer.nb_epoch - - predict_time -# - atk_gen -# - def_gen -fillna: - model.trainer.nb_epoch: 20 -weibull: - plot: - file : weibull_aft.eps - title : Weibull AFR Model - labels: - "Intercept: rho_": "$\\rho$" - "Intercept: lambda_": "$\\lambda$" - "data.sample.random_state: lambda_": "Random State" - "atk_value: lambda_": "Attack Strength" - "train_time: lambda_": "$t_{train}$" - "predict_proba_time: lambda_": "$t_{predict}$" - "adv_accuracy: lambda_": "Adv. Accuracy" - "accuracy: lambda_": "Ben. Accuracy" - "adv_fit_time: lambda_": "$t_{attack}$" - "adv_failure_rate: lambda_": "$h_{adv.}(t;\\theta)$" - "failure_rate: lambda_": "$h_{ben.}(t;\\theta)$" - "model.trainer.nb_epoch: lambda_": "No. of Epochs" - "model.trainer.batch_size: lambda_": "Batch Size" - "def_gen": "Defence" - "model_layers: lambda_" : "Layers" - "def_value: lambda_" : "Defence Strength" - "predict_time: lambda_" : "$t_{predict}$" - partial_effect: - - "file": "weibull_epochs_partial_effect.eps" - "covariate_array": "model.trainer.nb_epoch" - "values_array": [1,10,25,50] - "title": "$S(t)$ for Weibull AFR" - "ylabel": "Expectation of $S(t)$" - "xlabel": "Time $T$ (seconds)" - "legend_kwargs": { - "title": "Epochs", - "labels": ["1", "10", "25", "50"] - } - - "file": "weibull_layers_partial_effect.eps" - "covariate_array": "model_layers" - "values_array": [18, 34, 50, 101, 152] - "title": "$S(t)$ for Weibull AFR" - "ylabel": "Expectation of $S(t)$" - "xlabel": "Time $T$ (seconds)" - "legend_kwargs": { - "title": "Layers", - "labels": ["18", "34", "50", "101", "152"] - } -# cox: -# plot: -# file : cox_aft.eps -# title : Cox AFR Model -# labels: -# "Intercept: rho_": "$\\rho$" -# "Intercept: lambda_": "$\\lambda$" -# "data.sample.random_state: lambda_": "Random State" -# "atk_value: lambda_": "Attack Strength" -# "train_time: lambda_": "$t_{train}$" -# "predict_proba_time: lambda_": "$t_{predict}$" -# "adv_accuracy: lambda_": "Adv. Accuracy" -# "accuracy: lambda_": "Ben. Accuracy" -# "adv_fit_time: lambda_": "$t_{attack}$" -# "adv_failure_rate: lambda_": "$h_{adv.}(t;\\theta)$" -# "failure_rate: lambda_": "$h_{ben.}(t;\\theta)$" -# "model.trainer.nb_epoch: lambda_": "No. of Epochs" -# "model.trainer.batch_size: lambda_": "Batch Size" -# "def_gen": "Defence" -# partial_effect: -# - "file": "cox_epochs_partial_effect.eps" -# "covariate_array": "model.trainer.nb_epoch" -# "values_array": [1,10,25,50] -# "title": "$S(t)$ for Cox AFR" -# "ylabel": "Expectation of $S(t)$" -# "xlabel": "Time $T$ (seconds)" -# "legend_kwargs": { -# "title": "Epochs", -# "labels": ["1", "10", "25", "50"] -# } -# - "file": "cox_layers_partial_effect.eps" -# "covariate_array": "model_layers" -# "values_array": [18, 34, 50, 101, 152] -# "title": "$S(t)$ for Cox AFR" -# "ylabel": "Expectation of $S(t)$" -# "xlabel": "Time $T$ (seconds)" -# "legend_kwargs": { -# "title": "Layers", -# "labels": ["18", "34", "50", "101", "152"] -# } -log_logistic: - plot: - file : log_logistic_aft.eps - title : Log logistic AFR Model - labels: - "Intercept: beta_": "$\\beta$" - "Intercept: alpha_": "$\\alpha$" - "data.sample.random_state: alpha_": "Random State" - "atk_value: alpha_": "Attack Strength" - "train_time: alpha_": "$t_{train}$" - "predict_proba_time: alpha_": "$t_{predict}$" - "adv_accuracy: alpha_": "Adv. Accuracy" - "accuracy: alpha_": "Ben. Accuracy" - "adv_fit_time: alpha_": "$t_{attack}$" - "adv_failure_rate: alpha_": "$h_{adv.}(t;\\theta)$" - "failure_rate: alpha_": "$h_{ben.}(t;\\theta)$" - "model.trainer.nb_epoch: alpha_": "No. of Epochs" - "model.trainer.batch_size: alpha_": "Batch Size" - "def_gen": "Defence" - "model_layers: alpha_" : "Layers" - "def_value: alpha_" : "Defence Strength" - "predict_time: alpha_" : "$t_{predict}$" - partial_effect: - - "file": "log_logistic_epochs_partial_effect.eps" - "covariate_array": "model.trainer.nb_epoch" - "values_array": [1,10,25,50] - "title": "$S(t)$ for Log-Logistic AFR" - "ylabel": "Expectation of $S(t)$" - "xlabel": "Time $T$ (seconds)" - "legend_kwargs": { - "title": "Epochs", - "labels": ["1", "10", "25", "50"] - } - - "file": "log_logistic_layers_partial_effect.eps" - "covariate_array": "model_layers" - "values_array": [18, 34, 50, 101, 152] - "title": "$S(t)$ for Log Logistic AFR" - "ylabel": "Expectation of $S(t)$" - "xlabel": "Time $T$ (seconds)" - "legend_kwargs": { - "title": "Layers", - "labels": ["18", "34", "50", "101", "152"] - } -log_normal: - plot: - file : log_normal_aft.eps - title : Log Normal AFR Model - labels: - "Intercept: sigma_": "$\\rho$" - "Intercept: mu_": "$\\mu$" - "data.sample.random_state: mu_": "Random State" - "atk_value: mu_": "Attack Strength" - "train_time: mu_": "$t_{train}$" - "predict_proba_time: mu_": "$t_{predict}$" - "adv_accuracy: mu_": "Adv. Accuracy" - "accuracy: mu_": "Ben. Accuracy" - "adv_fit_time: mu_": "$t_{attack}$" - "adv_failure_rate: mu_": "$h_{adv.}(t;\\theta)$" - "failure_rate: mu_": "$h_{ben.}(t;\\theta)$" - "model.trainer.nb_epoch: mu_": "No. of Epochs" - "model.trainer.batch_size: mu_": "Batch Size" - "def_gen": "Defence" - "model_layers: mu_" : "Layers" - "def_value: mu_" : "Defence Strength" - "predict_time: mu_" : "$t_{predict}$" - partial_effect: - - "file": "log_normal_epochs_partial_effect.eps" - "covariate_array": "model.trainer.nb_epoch" - "values_array": [1,10,25,50] - "title": "$S(t)$ for Log-Normal AFR" - "ylabel": "Expectation of $S(t)$" - "xlabel": "Time $T$ (seconds)" - "legend_kwargs": { - "title": "Epochs", - "labels": ["1", "10", "25", "50"] - } - - "file": "log_normal_layers_partial_effect.eps" - "covariate_array": "model_layers" - "values_array": [18, 34, 50, 101, 152] - "title": "$S(t)$ for Log Normal AFR" - "ylabel": "Expectation of $S(t)$" - "xlabel": "Time $T$ (seconds)" - "legend_kwargs": { - "title": "Layers", - "labels": ["18", "34", "50", "101", "152"], - "bbox_to_anchor": [1.05, 1], - } diff --git a/examples/pytorch/cifar100/conf/attack/default.yaml b/examples/pytorch/cifar100/conf/attack/default.yaml deleted file mode 100644 index 76b4d62d..00000000 --- a/examples/pytorch/cifar100/conf/attack/default.yaml +++ /dev/null @@ -1,9 +0,0 @@ -data: ${data} -model: ${model} -_target_ : deckard.base.attack.Attack -init: - model: ${model} - _target_: deckard.base.attack.AttackInitializer - name: art.attacks.evasion.HopSkipJump -attack_size : 10 -method : evasion diff --git a/examples/pytorch/cifar100/conf/cifar100.yaml b/examples/pytorch/cifar100/conf/cifar100.yaml deleted file mode 100644 index 4f9da157..00000000 --- a/examples/pytorch/cifar100/conf/cifar100.yaml +++ /dev/null @@ -1,43 +0,0 @@ -defaults: - - _self_ - - data: torch_cifar100 - - model: torch_cifar100 - - attack: default - - files: cifar100 - - scorers: default - - override hydra/sweeper : optuna - - override hydra/sweeper/sampler : grid - - override hydra/launcher : joblib -stage : '???' -direction : "maximize" -_target_ : deckard.base.experiment.Experiment -optimizers : accuracy -hydra: - run: - dir: ${files.directory}/${files.reports}/${stage}/logs - sweep: - dir: ${files.directory}/${stage}/${model.init.name} - subdir : ${hydra.sweeper.study_name}/${hydra.job.num} - sweeper: - sampler: - _target_: optuna.samplers.GridSampler - direction: ${direction} - study_name: control - storage: sqlite:///model.db - n_jobs: 4 - n_trials : 10 - params: - ++data.sample.random_state: choice(0, 1, 2, 3, 4, 5, 6, 7, 8, 9) - ++model.art.initialize.optimizer.lr: choice(10, 1, 0.1, 0.01, 0.001, .0001, .00001, 0.000001) - _target_: hydra_plugins.hydra_optuna_sweeper.optuna_sweeper.OptunaSweeper - launcher: - _target_: hydra_plugins.hydra_joblib_launcher.joblib_launcher.JoblibLauncher - n_jobs: 4 - prefer : processes - verbose: 10 - timeout: null - pre_dispatch: n_jobs - batch_size: auto - temp_folder: /tmp/deckard - max_nbytes: 100000 - mmap_mode: r diff --git a/examples/pytorch/cifar100/conf/data/default.yaml b/examples/pytorch/cifar100/conf/data/default.yaml deleted file mode 100644 index 5eb78278..00000000 --- a/examples/pytorch/cifar100/conf/data/default.yaml +++ /dev/null @@ -1,2 +0,0 @@ -defaults: - - torch_mnist diff --git a/examples/pytorch/cifar100/conf/data/torch_cifar100.yaml b/examples/pytorch/cifar100/conf/data/torch_cifar100.yaml deleted file mode 100644 index 756bedca..00000000 --- a/examples/pytorch/cifar100/conf/data/torch_cifar100.yaml +++ /dev/null @@ -1,12 +0,0 @@ -_target_: deckard.base.data.Data -generate: - name: torch_cifar100 - path: original_data -sample: - random_state : 0 - stratify: True -sklearn_pipeline: - preprocessor: - name: sklearn.preprocessing.StandardScaler - with_mean: True - with_std: True diff --git a/examples/pytorch/cifar100/conf/files/cifar100.yaml b/examples/pytorch/cifar100/conf/files/cifar100.yaml deleted file mode 100644 index 6c1cef70..00000000 --- a/examples/pytorch/cifar100/conf/files/cifar100.yaml +++ /dev/null @@ -1,21 +0,0 @@ -_target_: deckard.base.files.FileConfig -reports: reports -# data_dir: data -model_dir: -attack_dir: attacks -directory: cifar100 -score_dict_file: score_dict.json -# data_file : data -model_file : -attack_file : attack -attack_type : .pkl -# data_type : .pkl -model_type : -params_file : params.yaml -# train_labels_file : train_labels.json -# test_labels_file : test_labels.json -# probabilities_file: probabilities.json -predictions_file : predictions.json -adv_predictions_file : adv_predictions.json -# adv_probabilities_file: adv_probabilities.json -name: default diff --git a/examples/pytorch/cifar100/conf/files/mnist.yaml b/examples/pytorch/cifar100/conf/files/mnist.yaml deleted file mode 100644 index efdb1c42..00000000 --- a/examples/pytorch/cifar100/conf/files/mnist.yaml +++ /dev/null @@ -1,21 +0,0 @@ -_target_: deckard.base.files.FileConfig -reports: reports -data_dir: data -model_dir: models -attack_dir: attacks -directory: mnist -score_dict_file: score_dict.json -data_file : data -model_file : model -attack_file : attack -attack_type : .pkl -data_type : .pkl -model_type : .pt -params_file : params.yaml -# train_labels_file : train_labels.json -# test_labels_file : test_labels.json -# probabilities_file: probabilities.json -predictions_file : predictions.json -adv_predictions_file : adv_predictions.json -# adv_probabilities_file: adv_probabilities.json -name: default diff --git a/examples/pytorch/cifar100/conf/model/art/default.yaml b/examples/pytorch/cifar100/conf/model/art/default.yaml deleted file mode 100644 index 4251627d..00000000 --- a/examples/pytorch/cifar100/conf/model/art/default.yaml +++ /dev/null @@ -1,7 +0,0 @@ -defaults: - - initialize : default - # - preprocessor: fsq - # - postprocessor: confidence -data : ${..data} -library : ${..library} -_target_ : deckard.base.model.art_pipeline.ArtPipeline diff --git a/examples/pytorch/cifar100/conf/model/art/postprocessor/confidence.yaml b/examples/pytorch/cifar100/conf/model/art/postprocessor/confidence.yaml deleted file mode 100644 index 6fcdde95..00000000 --- a/examples/pytorch/cifar100/conf/model/art/postprocessor/confidence.yaml +++ /dev/null @@ -1,3 +0,0 @@ - -name : art.defences.postprocessor.HighConfidence -cutoff : .1 diff --git a/examples/pytorch/cifar100/conf/model/art/postprocessor/gauss-out.yaml b/examples/pytorch/cifar100/conf/model/art/postprocessor/gauss-out.yaml deleted file mode 100644 index c912ebd9..00000000 --- a/examples/pytorch/cifar100/conf/model/art/postprocessor/gauss-out.yaml +++ /dev/null @@ -1,2 +0,0 @@ -name : art.defences.postprocessor.GaussianNoise -scale : 1 diff --git a/examples/pytorch/cifar100/conf/model/art/preprocessor/fsq.yaml b/examples/pytorch/cifar100/conf/model/art/preprocessor/fsq.yaml deleted file mode 100644 index 27ddf58b..00000000 --- a/examples/pytorch/cifar100/conf/model/art/preprocessor/fsq.yaml +++ /dev/null @@ -1,3 +0,0 @@ -name : art.defences.preprocessor.FeatureSqueezing -clip_values : ${model.art.initialize.clip_values} -bit_depth : 64 diff --git a/examples/pytorch/cifar100/conf/model/art/preprocessor/gauss-in.yaml b/examples/pytorch/cifar100/conf/model/art/preprocessor/gauss-in.yaml deleted file mode 100644 index c438e8f5..00000000 --- a/examples/pytorch/cifar100/conf/model/art/preprocessor/gauss-in.yaml +++ /dev/null @@ -1,4 +0,0 @@ -name : art.defences.preprocessor.GaussianAugmentation -clip_values : ${model.art.initialize.clip_values} -sigma : 1 -ratio : .5 diff --git a/examples/pytorch/cifar100/conf/model/torch_mnist.yaml b/examples/pytorch/cifar100/conf/model/torch_mnist.yaml deleted file mode 100644 index 26c462f6..00000000 --- a/examples/pytorch/cifar100/conf/model/torch_mnist.yaml +++ /dev/null @@ -1,13 +0,0 @@ - -defaults: - - art : default -data: ${data} -init: - _target_: deckard.base.model.ModelInitializer - num_channels : 1 - name : torch_example.ResNet18 -_target_: deckard.base.model.Model -trainer: - nb_epoch: 1 - batch_size: 1024 -library : pytorch diff --git a/examples/pytorch/cifar100/conf/scorers/default.yaml b/examples/pytorch/cifar100/conf/scorers/default.yaml deleted file mode 100644 index 4503c5bf..00000000 --- a/examples/pytorch/cifar100/conf/scorers/default.yaml +++ /dev/null @@ -1,9 +0,0 @@ -_target_: deckard.base.scorer.ScorerDict -accuracy: - _target_: deckard.base.scorer.ScorerConfig - name: sklearn.metrics.accuracy_score - direction: maximize -log_loss: - _target_: deckard.base.scorer.ScorerConfig - name: sklearn.metrics.log_loss - direction: minimize diff --git a/examples/pytorch/cifar100/dvc.lock b/examples/pytorch/cifar100/dvc.lock deleted file mode 100644 index 70bd1857..00000000 --- a/examples/pytorch/cifar100/dvc.lock +++ /dev/null @@ -1,598 +0,0 @@ -schema: '2.0' -stages: - clean@attack: - cmd: python -m deckard.layers.clean_data -i cifar100/reports/attack.csv -o cifar100/reports/clean_attack.csv - -c conf/clean.yaml - deps: - - path: cifar100/reports/attack.csv - md5: e1e1b67a591afdb5c9ea8f35003e5394 - size: 36167864 - params: - params.yaml: - files.directory: cifar100 - files.reports: reports - conf/clean.yaml: - attacks: - DeepFool: Deep - FastGradientMethod: FGM - HopSkipJump: HSJ - PixelAttack: Pixel - ProjectedGradientDescent: PGD - ThresholdAttack: Thresh - defences: - Control: Control - FeatureSqueezing: FSQ - GaussianAugmentation: Gauss-in - GaussianNoise: Gauss-out - HighConfidence: Conf - nb_epoch: Epochs - model_layers: Control - fillna: - Epochs: 10 - params: - Deep: attack.init.nb_grads - FGM: attack.init.eps - HSJ: attack.init.max_iter - Pixel: attack.init.th - PGD: attack.init.eps - Thresh: attack.init.th - Gauss-out: model.art.postprocessor.params.scale - Conf: model.art.postprocessor.params.cutoff - FSQ: model.art.preprocessor.params.bit_depth - Gauss-in: model.art.preprocessor.params.sigma - Control: model_layers - Epochs: model.trainer.nb_epoch - control: - model_layers: 18 - defaults: - Epochs: 10 - outs: - - path: cifar100/reports/clean_attack.csv - md5: 1e74314d258faf3aa6cd57f4f0aee4d4 - size: 30258589 - afr: - cmd: python -m deckard.layers.afr --dataset cifar100 --data_file cifar100/reports/clean_attack.csv --target - adv_accuracy --duration_col predict_time --dataset cifar100 --config_file conf/afr.yaml - --plots_folder cifar100/plots/ - deps: - - path: cifar100/reports/clean_attack.csv - md5: 1e74314d258faf3aa6cd57f4f0aee4d4 - size: 30258589 - params: - params.yaml: - files.directory: cifar100 - conf/afr.yaml: - covariates: - - adv_fit_time - - accuracy - - train_time - - atk_value - - def_value - - data.sample.random_state - - model_layers - - model.trainer.nb_epoch - - predict_time - log_logistic: - plot: - file: log_logistic_aft.eps - title: Log logistic AFR Model - labels: - 'Intercept: beta_': $\beta$ - 'Intercept: alpha_': $\alpha$ - 'data.sample.random_state: alpha_': Random State - 'atk_value: alpha_': Attack Strength - 'train_time: alpha_': $t_{train}$ - 'predict_proba_time: alpha_': $t_{predict}$ - 'adv_accuracy: alpha_': Adv. Accuracy - 'accuracy: alpha_': Ben. Accuracy - 'adv_fit_time: alpha_': $t_{attack}$ - 'adv_failure_rate: alpha_': $h_{adv.}(t;\theta)$ - 'failure_rate: alpha_': $h_{ben.}(t;\theta)$ - 'model.trainer.nb_epoch: alpha_': No. of Epochs - 'model.trainer.batch_size: alpha_': Batch Size - def_gen: Defence - 'model_layers: alpha_': Layers - 'def_value: alpha_': Defence Strength - 'predict_time: alpha_': $t_{predict}$ - partial_effect: - - file: log_logistic_epochs_partial_effect.eps - covariate_array: model.trainer.nb_epoch - values_array: - - 1 - - 10 - - 25 - - 50 - title: $S(t)$ for Log-Logistic AFR - ylabel: Expectation of $S(t)$ - xlabel: Time $T$ (seconds) - legend_kwargs: - title: Epochs - labels: - - '1' - - '10' - - '25' - - '50' - - file: log_logistic_layers_partial_effect.eps - covariate_array: model_layers - values_array: - - 18 - - 34 - - 50 - - 101 - - 152 - title: $S(t)$ for Log Logistic AFR - ylabel: Expectation of $S(t)$ - xlabel: Time $T$ (seconds) - legend_kwargs: - title: Layers - labels: - - '18' - - '34' - - '50' - - '101' - - '152' - log_normal: - plot: - file: log_normal_aft.eps - title: Log Normal AFR Model - labels: - 'Intercept: sigma_': $\rho$ - 'Intercept: mu_': $\mu$ - 'data.sample.random_state: mu_': Random State - 'atk_value: mu_': Attack Strength - 'train_time: mu_': $t_{train}$ - 'predict_proba_time: mu_': $t_{predict}$ - 'adv_accuracy: mu_': Adv. Accuracy - 'accuracy: mu_': Ben. Accuracy - 'adv_fit_time: mu_': $t_{attack}$ - 'adv_failure_rate: mu_': $h_{adv.}(t;\theta)$ - 'failure_rate: mu_': $h_{ben.}(t;\theta)$ - 'model.trainer.nb_epoch: mu_': No. of Epochs - 'model.trainer.batch_size: mu_': Batch Size - def_gen: Defence - 'model_layers: mu_': Layers - 'def_value: mu_': Defence Strength - 'predict_time: mu_': $t_{predict}$ - partial_effect: - - file: log_normal_epochs_partial_effect.eps - covariate_array: model.trainer.nb_epoch - values_array: - - 1 - - 10 - - 25 - - 50 - title: $S(t)$ for Log-Normal AFR - ylabel: Expectation of $S(t)$ - xlabel: Time $T$ (seconds) - legend_kwargs: - title: Epochs - labels: - - '1' - - '10' - - '25' - - '50' - - file: log_normal_layers_partial_effect.eps - covariate_array: model_layers - values_array: - - 18 - - 34 - - 50 - - 101 - - 152 - title: $S(t)$ for Log Normal AFR - ylabel: Expectation of $S(t)$ - xlabel: Time $T$ (seconds) - legend_kwargs: - title: Layers - labels: - - '18' - - '34' - - '50' - - '101' - - '152' - bbox_to_anchor: - - 1.05 - - 1 - weibull: - plot: - file: weibull_aft.eps - title: Weibull AFR Model - labels: - 'Intercept: rho_': $\rho$ - 'Intercept: lambda_': $\lambda$ - 'data.sample.random_state: lambda_': Random State - 'atk_value: lambda_': Attack Strength - 'train_time: lambda_': $t_{train}$ - 'predict_proba_time: lambda_': $t_{predict}$ - 'adv_accuracy: lambda_': Adv. Accuracy - 'accuracy: lambda_': Ben. Accuracy - 'adv_fit_time: lambda_': $t_{attack}$ - 'adv_failure_rate: lambda_': $h_{adv.}(t;\theta)$ - 'failure_rate: lambda_': $h_{ben.}(t;\theta)$ - 'model.trainer.nb_epoch: lambda_': No. of Epochs - 'model.trainer.batch_size: lambda_': Batch Size - def_gen: Defence - 'model_layers: lambda_': Layers - 'def_value: lambda_': Defence Strength - 'predict_time: lambda_': $t_{predict}$ - partial_effect: - - file: weibull_epochs_partial_effect.eps - covariate_array: model.trainer.nb_epoch - values_array: - - 1 - - 10 - - 25 - - 50 - title: $S(t)$ for Weibull AFR - ylabel: Expectation of $S(t)$ - xlabel: Time $T$ (seconds) - legend_kwargs: - title: Epochs - labels: - - '1' - - '10' - - '25' - - '50' - - file: weibull_layers_partial_effect.eps - covariate_array: model_layers - values_array: - - 18 - - 34 - - 50 - - 101 - - 152 - title: $S(t)$ for Weibull AFR - ylabel: Expectation of $S(t)$ - xlabel: Time $T$ (seconds) - legend_kwargs: - title: Layers - labels: - - '18' - - '34' - - '50' - - '101' - - '152' - outs: - - path: cifar100/plots/aft_comparison.csv - md5: 9c486aa8ca1b035f2d501971353760b0 - size: 381 - - path: cifar100/plots/aft_comparison.tex - md5: b8d8117defa391250aeaace7a9d0f31d - size: 420 - - path: cifar100/plots/log_logistic_aft.eps - md5: 25570be7c47838516dc2ce6737ea23cf - size: 43150 - - path: cifar100/plots/log_logistic_epochs_partial_effect.eps - md5: dce601a023295267b4219a3d344c6390 - size: 42195 - - path: cifar100/plots/log_logistic_layers_partial_effect.eps - md5: 422b035d3bffc50166597ad26e175400 - size: 43155 - - path: cifar100/plots/log_normal_aft.eps - md5: 2cf3e5852ca6cd9bee9aa66974db1ad7 - size: 44116 - - path: cifar100/plots/log_normal_epochs_partial_effect.eps - md5: 0cb86ef85c693118c8b6abc83a503ac3 - size: 42931 - - path: cifar100/plots/log_normal_layers_partial_effect.eps - md5: 4386a26119906fa28709501875c95660 - size: 43230 - - path: cifar100/plots/weibull_aft.eps - md5: 9509c6ffd09f15d01d411d76074695d3 - size: 41413 - - path: cifar100/plots/weibull_epochs_partial_effect.eps - md5: 65e05f0fee7247d061ce52014821d3dd - size: 41438 - - path: cifar100/plots/weibull_layers_partial_effect.eps - md5: 8903ae7d7c4fd97247d80abde3febaed - size: 42657 - plot: - cmd: python -m deckard.layers.plots --path cifar100/plots/ --file cifar100/reports/clean_attack.csv - -c conf/plots.yaml - deps: - - path: cifar100/reports/clean_attack.csv - md5: 1e74314d258faf3aa6cd57f4f0aee4d4 - size: 30258589 - params: - params.yaml: - files.directory: cifar100 - files.reports: reports - conf/plots.yaml: - cat_plot: - - file: adv_accuracy_vs_defence_type.eps - hue: model_name - kind: boxen - set: - yscale: linear - titles: Adv. Accuracy vs Defence Type - x: def_gen - xlabels: Defence Type - y: adv_accuracy - ylabels: Adv. Accuracy - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - - file: ben_accuracy_vs_defence_type.eps - hue: model_name - kind: boxen - titles: Ben. Accuracy vs Defence Type - x: def_gen - xlabels: Defence Type - y: accuracy - ylabels: Ben. Accuracy - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - - file: ben_failures_per_train_time_vs_defence_type.eps - hue: model_name - kind: boxen - set: - yscale: log - titles: $\bar{C}_{ben.}$ vs Defence Type - x: def_gen - xlabels: Defence Type - y: training_time_per_failure - ylabels: $\bar{C}_{ben.}$ - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - - file: adv_failures_per_train_time_vs_defence_type.eps - hue: model_name - kind: boxen - set: - yscale: log - titles: $\bar{C}_{adv.}$ vs Defence Type - x: def_gen - xlabels: Defence Type - y: training_time_per_adv_failure - ylabels: $\bar{C}_{adv.}$ - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - - file: adv_failures_per_train_time_vs_attack_type.eps - hue: model_name - kind: boxen - legend_title: Model Name - set: - yscale: log - titles: $\bar{C}_{adv.}$ vs Attack Type - x: atk_gen - xlabels: Attack Type - y: training_time_per_adv_failure - ylabels: $\bar{C}_{adv.}$ - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - - file: adv_failures_per_test_time_vs_defence_type.eps - hue: model_name - kind: boxen - legend_title: Model Name - titles: $f_{adv}$ vs Defence Type - x: def_gen - xlabels: Defence Type - y: adv_failure_rate - ylabels: $f_{adv.}$ - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - - file: adv_accuracy_vs_attack_type.eps - hue: model_name - kind: boxen - legend_title: Model Name - titles: Adv. Accuracy vs Attack Type - x: atk_gen - xlabels: Attack Type - y: adv_accuracy - ylabels: Adv. Accuracy - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - - file: ben_failure_rate_vs_defence_type.eps - hue: model_name - kind: boxen - legend_title: Model Name - set: - yscale: log - titles: $f_{ben}(t; \theta)$ vs Defence Type - x: def_gen - xlabels: Defence Type - y: failure_rate - ylabels: $f_{ben}(t; \theta)$ - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - line_plot: - - file: def_param_vs_accuracy.eps - hue: def_gen - legend: - bbox_to_anchor: - - 1.05 - - 1 - title: Defence - title: Ben. Accuracy vs Defence Strength - x: def_value - x_scale: linear - xlabel: Defence Control Parameter - y: accuracy - y_scale: - ylabel: Ben. Accuracy - hue_order: - - Control - - Conf - - Epochs - - Gauss-in - - Gauss-out - - Conf - - FSQ - errorbar: se - err_style: bars - - file: def_param_vs_adv_accuracy.eps - hue: def_gen - legend: - bbox_to_anchor: - - 1.05 - - 1 - title: Defence - title: Adv. Accuracy vs Defence Strength - x: def_value - x_scale: linear - xlabel: Defence Control Parameter - y: adv_accuracy - y_scale: - ylabel: Adv. Accuracy - hue_order: - - Control - - Conf - - Epochs - - Gauss-in - - Gauss-out - - Conf - - FSQ - errorbar: se - err_style: bars - - file: def_param_vs_adv_failure_rate.eps - hue: def_gen - legend: - bbox_to_anchor: - - 1.05 - - 1 - title: Defence - title: $f_{adv}$ vs Defence Strength - x: def_value - x_scale: log - xlabel: Defence Control Parameter - y: adv_failure_rate - y_scale: log - ylabel: $f_{adv.}$ - hue_order: - - Control - - Conf - - Epochs - - Gauss-in - - Gauss-out - - Conf - - FSQ - errorbar: se - err_style: bars - - file: atk_param_vs_accuracy.eps - hue: atk_gen - legend: - bbox_to_anchor: - - 1.05 - - 1 - title: Adv. Accuracy vs Attack Strength - x: atk_value - x_scale: linear - xlabel: Attack Control Parameter - y: adv_accuracy - y_scale: - ylabel: Adv. Accuracy - hue_order: - - FGM - - PGD - - Deep - - HSJ - - Pixel - - Thresh - errorbar: se - err_style: bars - scatter_plot: - - x: train_time_per_sample - y: adv_failure_rate - hue: model_name - xlabel: $t_{train}$ - ylabel: $f_{adv}$ - title: $f_{adv}$ vs $t_{train}$ - file: adv_failure_rate_vs_train_time.eps - y_scale: linear - x_scale: log - legend: - title: Model Name - bbox_to_anchor: - - 1.05 - - 1 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - outs: - - path: cifar100/plots/adv_accuracy_vs_attack_type.eps - md5: c8da4b76e14a314a366460692dc783c5 - size: 110928 - - path: cifar100/plots/adv_accuracy_vs_defence_type.eps - md5: 28fc6c3ec076122c0817ff72bf445724 - size: 89728 - - path: cifar100/plots/adv_failure_rate_vs_train_time.eps - md5: 6b120de323dda22b1d1f7c865a3fee40 - size: 1197829 - - path: cifar100/plots/adv_failures_per_test_time_vs_defence_type.eps - md5: 992a14b470a5297898effaacef21da58 - size: 117343 - - path: cifar100/plots/adv_failures_per_train_time_vs_attack_type.eps - md5: da342b3a5f817de813e3831fb704a137 - size: 135357 - - path: cifar100/plots/adv_failures_per_train_time_vs_defence_type.eps - md5: 02573bd82b9a8e91f222b223265d4c88 - size: 110305 - - path: cifar100/plots/atk_param_vs_accuracy.eps - md5: 3a9ee8b1f8d7632d5e5346ee98fd7302 - size: 38228 - - path: cifar100/plots/ben_accuracy_vs_defence_type.eps - md5: a87ff0bfe4abeb62c8a11b14625094c6 - size: 100131 - - path: cifar100/plots/ben_failure_rate_vs_defence_type.eps - md5: 353409db77a5e01b3e1cace395724747 - size: 118669 - - path: cifar100/plots/ben_failures_per_train_time_vs_defence_type.eps - md5: c5b1247944a237ece405e2cc6d8d5f17 - size: 109170 - - path: cifar100/plots/def_param_vs_accuracy.eps - md5: a1d1bca25b228523ca3518e85407d9f9 - size: 38880 - - path: cifar100/plots/def_param_vs_adv_accuracy.eps - md5: 361afc1a730165a3c05d80ad16fe4c66 - size: 37677 - - path: cifar100/plots/def_param_vs_adv_failure_rate.eps - md5: cd8cc762d8541f690335ea6118eff227 - size: 41408 - copy_results: - cmd: mkdir -p ~/ml_afr/cifar100 && cp -r cifar100/plots/* ~/ml_afr/cifar100 - deps: - - path: cifar100/plots/ - md5: 5862826c5918c815b4814de0b1d5e53b.dir - size: 2876717 - nfiles: 27 diff --git a/examples/pytorch/cifar100/dvc.yaml b/examples/pytorch/cifar100/dvc.yaml deleted file mode 100644 index 60c91cf5..00000000 --- a/examples/pytorch/cifar100/dvc.yaml +++ /dev/null @@ -1,171 +0,0 @@ -vars: - - conf/plots.yaml:line_plot - - conf/plots.yaml:scatter_plot - - conf/plots.yaml:cat_plot - - conf/afr.yaml:covariates - - conf/afr.yaml:weibull - - conf/afr.yaml:log_logistic - - conf/afr.yaml:log_normal - - conf/clean.yaml:attacks - - conf/clean.yaml:defences - - conf/clean.yaml:params - - conf/clean.yaml:fillna -stages: - train: - cmd: python -m deckard.layers.experiment train --config_file cifar100.yaml - params: - - data - - model - - scorers - - files - outs: - - ${files.directory}/${files.data_dir}/${files.data_file}${files.data_type} - - ${files.directory}/${files.model_dir}/${files.model_file}${files.model_type} - - ${files.directory}/${files.model_dir}/${files.model_file}.optimizer${files.model_type} - # - ${files.directory}/${files.reports}/train/${files.name}/${files.params_file} - # - ${files.directory}/${files.reports}/train/${files.name}/${files.test_labels_file} # Omit to save space - - ${files.directory}/${files.reports}/train/${files.name}/${files.predictions_file} # logit outputs for our model - # - ${files.directory}/${files.reports}/train/${files.name}/${files.probabilities_file} # Omit to save space - metrics: - - ${files.directory}/${files.reports}/train/${files.name}/${files.score_dict_file} - attack: - cmd: python -m deckard.layers.experiment attack --config_file cifar100.yaml - params: - - data - - model - - attack - - scorers - - files - outs: - - ${files.directory}/${files.attack_dir}/${files.attack_file}${files.attack_type} - - ${files.directory}/${files.reports}/attack/${files.name}/${files.adv_predictions_file} - # - ${files.directory}/${files.reports}/attack/${files.name}/${files.params_file} - deps: - - ${files.directory}/${files.data_dir}/${files.data_file}${files.data_type} - - ${files.directory}/${files.model_dir}/${files.model_file}${files.model_type} - metrics: - - ${files.directory}/${files.reports}/attack/${files.name}/${files.score_dict_file} - - ############################################################################## - # models: # This is a loop over the ResNet models - # foreach: - # - ResNet18 - # # - ResNet34 - # # - ResNet50 - # # - ResNet101 - # # - ResNet152 - # do: # This script configures eazch defence - # cmd: bash models.sh ++model.init.name=torch_example.${item} stage=train ++hydra.sweeper.storage=sqlite:///${files.directory}/${files.reports}/train/${item}.db --config-name cifar100.yaml - # deps: - # - models.sh - # - ${files.directory}/${files.model_dir}/${files.model_file}${files.model_type} - # - ${files.directory}/${files.model_dir}/${files.model_file}.optimizer${files.model_type} - # outs: - # - ${files.directory}/${files.reports}/train/${item}.db: # This outputs a database file for each model - # cache: True - # persist: True - attacks: - foreach: # This is a loop over the ResNet models - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - do: - cmd: bash attacks.sh ++attack.attack_size=100 ++model.init.name=torch_example.${item} stage=attack ++hydra.sweeper.storage=sqlite:///${files.directory}/${files.reports}/attack/${item}.db --config-name cifar100.yaml - deps: - - models.sh # This script configures each defence - - attacks.sh # This script configures each attack - - ${files.directory}/${files.reports}/attack/${files.name}/${files.score_dict_file} # This is here just to ensure it runs after the attack stage - # - ${files.directory}/${files.reports}/train/${item}.db - outs: - - ${files.directory}/${files.reports}/attack/${item}.db: # This outputs a database file for each model - cache: True - persist: True - compile: - foreach: # iterates through each stage - # - train - - attack - do: - cmd: python -m deckard.layers.compile --report_folder ${files.directory}/${files.reports}/${item} --results_file ${files.directory}/${files.reports}/${item}.csv - deps: - - ${files.directory}/${files.reports}/${item}/ - - ${files.directory}/${files.reports}/${item}/ResNet18.db - - ${files.directory}/${files.reports}/${item}/ResNet34.db - - ${files.directory}/${files.reports}/${item}/ResNet50.db - - ${files.directory}/${files.reports}/${item}/ResNet101.db - # - ${files.directory}/${files.reports}/${item}/ResNet152.db - outs: - - ${files.directory}/${files.reports}/${item}.csv - clean: - foreach: # iterates through each stage - # - train - - attack - do: - cmd: python -m deckard.layers.clean_data -i ${files.directory}/${files.reports}/${item}.csv -o ${files.directory}/${files.reports}/clean_${item}.csv -c conf/clean.yaml - deps: - - ${files.directory}/${files.reports}/${item}.csv - outs: - - ${files.directory}/${files.reports}/clean_${item}.csv - params: - - files.directory - - files.reports - - conf/clean.yaml: - - attacks - - defences - - params - - fillna - plot: - cmd : python -m deckard.layers.plots --path ${files.directory}/plots/ --file ${files.directory}/${files.reports}/clean_attack.csv -c conf/plots.yaml - deps: - - ${files.directory}/${files.reports}/clean_attack.csv - plots: - - ${files.directory}/plots/${cat_plot[0].file} - - ${files.directory}/plots/${cat_plot[1].file} - - ${files.directory}/plots/${cat_plot[2].file} - - ${files.directory}/plots/${cat_plot[3].file} - - ${files.directory}/plots/${cat_plot[4].file} - - ${files.directory}/plots/${cat_plot[5].file} - - ${files.directory}/plots/${cat_plot[6].file} - - ${files.directory}/plots/${cat_plot[7].file} - - ${files.directory}/plots/${line_plot[0].file} - - ${files.directory}/plots/${line_plot[1].file} - - ${files.directory}/plots/${line_plot[2].file} - - ${files.directory}/plots/${line_plot[3].file} - - ${files.directory}/plots/${scatter_plot[0].file} - params: - - files.directory - - files.reports - - conf/plots.yaml: - - line_plot - - scatter_plot - - cat_plot - afr: - cmd: python -m deckard.layers.afr --dataset ${files.directory} --data_file ${files.directory}/${files.reports}/clean_attack.csv --target adv_accuracy --duration_col predict_time --dataset cifar100 --config_file conf/afr.yaml --plots_folder ${files.directory}/plots/ - deps: - - ${files.directory}/${files.reports}/clean_attack.csv - plots: - - ${files.directory}/plots/weibull_aft.eps - - ${files.directory}/plots/weibull_epochs_partial_effect.eps - - ${files.directory}/plots/weibull_layers_partial_effect.eps - - ${files.directory}/plots/log_logistic_aft.eps - - ${files.directory}/plots/log_logistic_epochs_partial_effect.eps - - ${files.directory}/plots/log_logistic_layers_partial_effect.eps - - ${files.directory}/plots/log_normal_aft.eps - - ${files.directory}/plots/log_normal_epochs_partial_effect.eps - - ${files.directory}/plots/log_normal_layers_partial_effect.eps - metrics: - - ${files.directory}/plots/aft_comparison.csv - outs: - - ${files.directory}/plots/aft_comparison.tex - params: - - files.directory - - conf/afr.yaml: - - covariates - - weibull - - log_logistic - - log_normal - copy_results: - cmd: mkdir -p ~/ml_afr/cifar100 && cp -r ${files.directory}/plots/* ~/ml_afr/cifar100 - deps: - - ${files.directory}/plots/ diff --git a/examples/pytorch/cifar100/models.sh b/examples/pytorch/cifar100/models.sh deleted file mode 100644 index da7204d3..00000000 --- a/examples/pytorch/cifar100/models.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -# This script is used to generate the models for the sklearn example. - -# # Default model -echo "python -m deckard.layers.optimise ++model.trainer.nb_epoch=1,10,30,50,100" $@ "--multirun" -python -m deckard.layers.optimise ++model.trainer.nb_epoch=1,10,30,50,100 $@ --multirun - -# This line generates the model and adds the FeatureSqueezing preprocessing defence. -python -m deckard.layers.optimise \ - ++model.art.preprocessor.name=art.defences.preprocessor.FeatureSqueezing \ - +model.art.preprocessor.params.bit_depth=4,8,16,32,64 \ - +model.art.preprocessor.params.clip_values=[0,255] \ - ++hydra.sweeper.study_name=FSQ $@ --multirun - -# # # Gaussian Augmentation (Input) -python -m deckard.layers.optimise \ - ++model.art.preprocessor.name=art.defences.preprocessor.GaussianAugmentation \ - +model.art.preprocessor.params.sigma=.01,.1,.3,.5,1 \ - +model.art.preprocessor.params.ratio=.5 \ - +model.art.preprocessor.params.augmentation=False \ - ++hydra.sweeper.study_name=gauss-in $@ --multirun - -# # # # Gaussian Noise (Output) -python -m deckard.layers.optimise \ - ++model.art.postprocessor.name=art.defences.postprocessor.GaussianNoise \ - ++model.art.postprocessor.params.scale=.01,.1,.3,.5,1 \ - ++hydra.sweeper.study_name=gauss-out $@ --multirun - -# # # # High Confidence -python -m deckard.layers.optimise \ - +model.art.postprocessor.name=art.defences.postprocessor.HighConfidence \ - +model.art.postprocessor.params.cutoff=.1,.3,.5,.9,.99 \ - ++hydra.sweeper.study_name=conf $@ --multirun diff --git a/examples/pytorch/cifar100/torch_example.py b/examples/pytorch/cifar100/torch_example.py deleted file mode 100644 index 2007d9d0..00000000 --- a/examples/pytorch/cifar100/torch_example.py +++ /dev/null @@ -1,79 +0,0 @@ -import torch.nn as nn -from torchvision import models - -__all__ = [ - "ResNet18", - "ResNet50", - "ResNet101", - "ResNet152", -] - - -def ResNet18(num_channels=1, num_classes=10): - model = models.resnet18() - model.conv1 = nn.Conv2d( - num_channels, - 64, - kernel_size=7, - stride=2, - padding=3, - bias=False, - ) - model.fc = nn.Linear(512, num_classes) - return model - - -def ResNet34(num_channels=1, num_classes=10): - model = models.resnet34() - model.conv1 = nn.Conv2d( - num_channels, - 64, - kernel_size=7, - stride=2, - padding=3, - bias=False, - ) - model.fc = nn.Linear(512, num_classes) - return model - - -def ResNet50(num_channels=1, num_classes=10): - model = models.resnet50() - model.conv1 = nn.Conv2d( - num_channels, - 64, - kernel_size=7, - stride=2, - padding=3, - bias=False, - ) - model.fc = nn.Linear(2048, num_classes) - return model - - -def ResNet101(num_channels=1, num_classes=10): - model = models.resnet101() - model.conv1 = nn.Conv2d( - num_channels, - 64, - kernel_size=7, - stride=2, - padding=3, - bias=False, - ) - model.fc = nn.Linear(2048, num_classes) - return model - - -def ResNet152(num_channels=1, num_classes=10): - model = models.resnet152() - model.conv1 = nn.Conv2d( - num_channels, - 64, - kernel_size=7, - stride=2, - padding=3, - bias=False, - ) - model.fc = nn.Linear(2048, num_classes) - return model diff --git a/examples/pytorch/cifar10/conf/attack/default.yaml b/examples/pytorch/conf/attack/default.yaml similarity index 76% rename from examples/pytorch/cifar10/conf/attack/default.yaml rename to examples/pytorch/conf/attack/default.yaml index 76b4d62d..0ed37317 100644 --- a/examples/pytorch/cifar10/conf/attack/default.yaml +++ b/examples/pytorch/conf/attack/default.yaml @@ -5,5 +5,6 @@ init: model: ${model} _target_: deckard.base.attack.AttackInitializer name: art.attacks.evasion.HopSkipJump -attack_size : 10 + batch_size : ${model.trainer.batch_size} +attack_size : 100 method : evasion diff --git a/examples/pytorch/cifar10/conf/cifar10.yaml b/examples/pytorch/conf/cifar.yaml similarity index 51% rename from examples/pytorch/cifar10/conf/cifar10.yaml rename to examples/pytorch/conf/cifar.yaml index 5f701a6a..3a4dff94 100644 --- a/examples/pytorch/cifar10/conf/cifar10.yaml +++ b/examples/pytorch/conf/cifar.yaml @@ -8,32 +8,40 @@ defaults: - override hydra/sweeper : optuna - override hydra/sweeper/sampler : grid - override hydra/launcher : joblib +def_name : control +atk_name : hsj +dataset : cifar +model_name : ResNet18 +device_id : gpu stage : '???' -direction : "maximize" +direction : + - "maximize" _target_ : deckard.base.experiment.Experiment -optimizers : accuracy +optimizers : + - accuracy hydra: run: - dir: ${files.directory}/${files.reports}/${stage}/logs + dir: ${files.directory}/logs/${stage}/ sweep: - dir: ${files.directory}/${stage}/${model.init.name} - subdir : ${hydra.sweeper.study_name}/${hydra.job.num} + dir: ${files.directory}/logs/${stage}/${model_name} + subdir : ${def_name}/${atk_name}/${hydra.job.num} sweeper: sampler: _target_: optuna.samplers.GridSampler direction: ${direction} - study_name: control - storage: sqlite:///model.db - n_jobs: ${hydra.launcher.n_jobs} - n_trials: 32 + study_name: ${model_name}_${def_name}_${atk_name} + storage: sqlite:///${dataset}.db + n_jobs: ${oc.env:HYDRA_SWEEPER_N_JOBS, 8} + n_trials: ${oc.env:HYDRA_SWEEPER_N_TRIALS, 128} + max_failure_rate: 1.0 params: - ++data.sample.random_state: choice(0, 1, 2, 3, 4, 5, 6, 7, 8, 9) - ++model.art.initialize.optimizer.lr: choice(10, 1, 0.1, 0.01, 0.001, .0001, .00001, 0.000001) + ++model.art.initialize.optimizer.lr: choice( 0.1, 0.01, 0.001, .0001, .00001, 0.000001) + ++model.trainer.nb_epoch: choice(1, 10, 30, 50, 100) _target_: hydra_plugins.hydra_optuna_sweeper.optuna_sweeper.OptunaSweeper launcher: _target_: hydra_plugins.hydra_joblib_launcher.joblib_launcher.JoblibLauncher - n_jobs: 8 - prefer : processes + n_jobs: ${oc.env:HYDRA_SWEEPER_N_JOBS, 8} + prefer : threads verbose: 10 timeout: null pre_dispatch: n_jobs diff --git a/examples/pytorch/conf/cifar100.yaml b/examples/pytorch/conf/cifar100.yaml new file mode 100644 index 00000000..1189c84d --- /dev/null +++ b/examples/pytorch/conf/cifar100.yaml @@ -0,0 +1,52 @@ +defaults: + - _self_ + - data: torch_cifar100 + - model: torch_cifar100 + - attack: default + - files: cifar100 + - scorers: default + - override hydra/sweeper : optuna + - override hydra/sweeper/sampler : grid + - override hydra/launcher : joblib +def_name : control +atk_name : hsj +dataset : cifar100 +model_name : ResNet18 +device_id : gpu +stage : '???' +direction : + - "maximize" +_target_ : deckard.base.experiment.Experiment +optimizers : + - accuracy +hydra: + run: + dir: ${files.directory}/logs/${stage}/ + sweep: + dir: ${files.directory}/logs/${stage}/${model_name} + subdir : ${def_name}/${atk_name}/${hydra.job.num} + sweeper: + sampler: + _target_: optuna.samplers.GridSampler + direction: ${direction} + study_name: ${model_name}_${def_name}_${atk_name} + storage: sqlite:///${dataset}.db + n_jobs: ${oc.env:HYDRA_SWEEPER_N_JOBS, 8} + n_trials: ${oc.env:HYDRA_SWEEPER_N_TRIALS, 128} + max_failure_rate: 1.0 + params: + ++model.art.initialize.optimizer.lr: choice( 0.1, 0.01, 0.001, .0001, .00001, 0.000001) + ++model.trainer.nb_epoch: choice(1, 10, 30, 50, 100) + ++model.art.initialize.optimizer.momentum: choice(0.1, 0.9, 0.95, 0.99) + _target_: hydra_plugins.hydra_optuna_sweeper.optuna_sweeper.OptunaSweeper + launcher: + _target_: hydra_plugins.hydra_joblib_launcher.joblib_launcher.JoblibLauncher + n_jobs: ${oc.env:HYDRA_SWEEPER_N_JOBS, 8} + prefer : threads + verbose: 10 + timeout: null + pre_dispatch: n_jobs + batch_size: auto + temp_folder: /tmp/deckard + max_nbytes: 100000 + mmap_mode: r diff --git a/examples/pytorch/cifar10/conf/clean.yaml b/examples/pytorch/conf/clean.yaml similarity index 75% rename from examples/pytorch/cifar10/conf/clean.yaml rename to examples/pytorch/conf/clean.yaml index e9c215ab..78d138dd 100644 --- a/examples/pytorch/cifar10/conf/clean.yaml +++ b/examples/pytorch/conf/clean.yaml @@ -5,20 +5,21 @@ attacks: PixelAttack: Pixel ProjectedGradientDescent: PGD ThresholdAttack: Thresh + ZooAttack: ZOO defences: Control: Control FeatureSqueezing: FSQ GaussianAugmentation: Gauss-in GaussianNoise: Gauss-out HighConfidence: Conf - nb_epoch : Epochs + nb_epochs : Epochs model_layers: Control params: Deep: attack.init.kwargs.nb_grads - FGM: attack.init.kwargs.eps + FGM: attack.init.eps HSJ: attack.init.kwargs.max_iter Pixel: attack.init.kwargs.th - PGD: attack.init.kwargs.eps + PGD: attack.init.eps Thresh: attack.init.kwargs.th Gauss-out: model.art.pipeline.postprocessor.kwargs.scale Conf: model.art.pipeline.postprocessor.kwargs.cutoff @@ -26,9 +27,12 @@ params: Gauss-in: model.art.pipeline.preprocessor.kwargs.sigma Control: model_layers Epochs: model.trainer.nb_epoch + ZOO : attack.init.binary_search_steps control: model_layers: 18 - defaults: - model.trainer.nb_epoch: 20 +replace_cols: + model.trainer.nb_epochs: Epochs fillna: - model.trainer.nb_epoch : 20 + Epochs: 20 + data.sample.train_size : 48000 + data.sample.test_size: 12000 diff --git a/examples/pytorch/conf/clean_cifar.yaml b/examples/pytorch/conf/clean_cifar.yaml new file mode 100644 index 00000000..a671a033 --- /dev/null +++ b/examples/pytorch/conf/clean_cifar.yaml @@ -0,0 +1,36 @@ +attacks: + DeepFool: Deep + FastGradientMethod: FGM + HopSkipJump: HSJ + PixelAttack: Pixel + ProjectedGradientDescent: PGD + ThresholdAttack: Thresh +defences: + Control: Control + FeatureSqueezing: FSQ + GaussianAugmentation: Gauss-in + GaussianNoise: Gauss-out + HighConfidence: Conf + nb_epochs : Epochs + model_layers: Control +params: + Deep: attack.init.nb_grads + FGM: attack.init.eps + HSJ: attack.init.max_iter + Pixel: attack.init.th + PGD: attack.init.eps + Thresh: attack.init.th + Gauss-out: model.art.postprocessor.scale + Conf: model.art.postprocessor.cutoff + FSQ: model.art.preprocessor.bit_depth + Gauss-in: model.art.preprocessor.sigma + Control: model_layers + Epochs: model.trainer.nb_epoch + control: + model_layers: 18 + defaults: + model.trainer.nb_epochs: 20 +fillna: + model.trainer.nb_epochs : 20 +replace_cols: + model.trainer.nb_epochs: Epochs diff --git a/examples/pytorch/cifar100/conf/clean.yaml b/examples/pytorch/conf/clean_cifar100.yaml similarity index 87% rename from examples/pytorch/cifar100/conf/clean.yaml rename to examples/pytorch/conf/clean_cifar100.yaml index 52f6c912..52e82920 100644 --- a/examples/pytorch/cifar100/conf/clean.yaml +++ b/examples/pytorch/conf/clean_cifar100.yaml @@ -29,6 +29,8 @@ params: control: model_layers: 18 defaults: - Epochs: 10 + model.trainer.nb_epoch: 10 fillna: - Epochs: 10 + model.trainer.nb_epoch : 10 +replace_cols: + model.trainer.nb_epoch: Epochs diff --git a/examples/pytorch/mnist/conf/clean.yaml b/examples/pytorch/conf/clean_mnist.yaml similarity index 54% rename from examples/pytorch/mnist/conf/clean.yaml rename to examples/pytorch/conf/clean_mnist.yaml index e9c215ab..39bc97a8 100644 --- a/examples/pytorch/mnist/conf/clean.yaml +++ b/examples/pytorch/conf/clean_mnist.yaml @@ -14,16 +14,16 @@ defences: nb_epoch : Epochs model_layers: Control params: - Deep: attack.init.kwargs.nb_grads - FGM: attack.init.kwargs.eps - HSJ: attack.init.kwargs.max_iter - Pixel: attack.init.kwargs.th - PGD: attack.init.kwargs.eps - Thresh: attack.init.kwargs.th - Gauss-out: model.art.pipeline.postprocessor.kwargs.scale - Conf: model.art.pipeline.postprocessor.kwargs.cutoff - FSQ: model.art.pipeline.preprocessor.kwargs.bit_depth - Gauss-in: model.art.pipeline.preprocessor.kwargs.sigma + Deep: attack.init.nb_grads + FGM: attack.init.eps + HSJ: attack.init.max_iter + Pixel: attack.init.th + PGD: attack.init.eps + Thresh: attack.init.th + Gauss-out: model.art.postprocessor.scale + Conf: model.art.postprocessor.cutoff + FSQ: model.art.preprocessor.bit_depth + Gauss-in: model.art.preprocessor.sigma Control: model_layers Epochs: model.trainer.nb_epoch control: @@ -32,3 +32,5 @@ params: model.trainer.nb_epoch: 20 fillna: model.trainer.nb_epoch : 20 +replace_cols: + model.trainer.nb_epoch: Epochs diff --git a/examples/pytorch/cifar10/conf/data/default.yaml b/examples/pytorch/conf/data/default.yaml similarity index 100% rename from examples/pytorch/cifar10/conf/data/default.yaml rename to examples/pytorch/conf/data/default.yaml diff --git a/examples/pytorch/conf/data/torch_cifar.yaml b/examples/pytorch/conf/data/torch_cifar.yaml new file mode 100644 index 00000000..c967a2aa --- /dev/null +++ b/examples/pytorch/conf/data/torch_cifar.yaml @@ -0,0 +1,8 @@ +_target_: deckard.base.data.Data +generate: + name: torch_cifar10 +sample: + random_state : 0 + stratify: True + train_size : 48000 + test_size : 12000 diff --git a/examples/pytorch/conf/data/torch_cifar100.yaml b/examples/pytorch/conf/data/torch_cifar100.yaml new file mode 100644 index 00000000..feb8a3bc --- /dev/null +++ b/examples/pytorch/conf/data/torch_cifar100.yaml @@ -0,0 +1,9 @@ +_target_: deckard.base.data.Data +generate: + name: torch_cifar100 + path: original_data/ +sample: + random_state : 0 + stratify: True + train_size : 48000 + test_size : 12000 diff --git a/examples/pytorch/cifar100/conf/data/torch_mnist.yaml b/examples/pytorch/conf/data/torch_mnist.yaml similarity index 52% rename from examples/pytorch/cifar100/conf/data/torch_mnist.yaml rename to examples/pytorch/conf/data/torch_mnist.yaml index 56e9c38b..1dbeed67 100644 --- a/examples/pytorch/cifar100/conf/data/torch_mnist.yaml +++ b/examples/pytorch/conf/data/torch_mnist.yaml @@ -6,9 +6,5 @@ sample: _target_: deckard.base.data.sampler.SklearnDataSampler random_state : 0 stratify: True -sklearn_pipeline: - _target_: deckard.base.data.sklearn_pipeline.SklearnDataPipeline - preprocessor: - name: sklearn.preprocessing.StandardScaler - with_mean: True - with_std: True + train_size : 48000 + test_size : 12000 diff --git a/examples/pytorch/cifar100/conf/deploy/pod.yaml b/examples/pytorch/conf/deploy/pod.yaml similarity index 100% rename from examples/pytorch/cifar100/conf/deploy/pod.yaml rename to examples/pytorch/conf/deploy/pod.yaml diff --git a/examples/pytorch/cifar100/conf/deploy/pvc.yaml b/examples/pytorch/conf/deploy/pvc.yaml similarity index 100% rename from examples/pytorch/cifar100/conf/deploy/pvc.yaml rename to examples/pytorch/conf/deploy/pvc.yaml diff --git a/examples/pytorch/cifar100/conf/deploy/sclass.yaml b/examples/pytorch/conf/deploy/sclass.yaml similarity index 100% rename from examples/pytorch/cifar100/conf/deploy/sclass.yaml rename to examples/pytorch/conf/deploy/sclass.yaml diff --git a/examples/pytorch/cifar10/conf/files/cifar.yaml b/examples/pytorch/conf/files/cifar.yaml similarity index 91% rename from examples/pytorch/cifar10/conf/files/cifar.yaml rename to examples/pytorch/conf/files/cifar.yaml index e43cab72..782d0b73 100644 --- a/examples/pytorch/cifar10/conf/files/cifar.yaml +++ b/examples/pytorch/conf/files/cifar.yaml @@ -1,15 +1,12 @@ _target_: deckard.base.files.FileConfig reports: reports -data_dir: data model_dir: models attack_dir: attacks directory: cifar score_dict_file: score_dict.json -data_file : data model_file : model attack_file : attack attack_type : .pkl -data_type : .pkl model_type : .pt params_file : params.yaml # train_labels_file : train_labels.json diff --git a/examples/pytorch/cifar10/conf/files/mnist.yaml b/examples/pytorch/conf/files/cifar100.yaml similarity index 88% rename from examples/pytorch/cifar10/conf/files/mnist.yaml rename to examples/pytorch/conf/files/cifar100.yaml index efdb1c42..e1dd7414 100644 --- a/examples/pytorch/cifar10/conf/files/mnist.yaml +++ b/examples/pytorch/conf/files/cifar100.yaml @@ -1,15 +1,12 @@ _target_: deckard.base.files.FileConfig reports: reports -data_dir: data model_dir: models attack_dir: attacks -directory: mnist +directory: cifar100 score_dict_file: score_dict.json -data_file : data model_file : model attack_file : attack attack_type : .pkl -data_type : .pkl model_type : .pt params_file : params.yaml # train_labels_file : train_labels.json diff --git a/examples/pytorch/mnist/conf/files/mnist.yaml b/examples/pytorch/conf/files/mnist.yaml similarity index 91% rename from examples/pytorch/mnist/conf/files/mnist.yaml rename to examples/pytorch/conf/files/mnist.yaml index efdb1c42..0b111070 100644 --- a/examples/pytorch/mnist/conf/files/mnist.yaml +++ b/examples/pytorch/conf/files/mnist.yaml @@ -1,15 +1,12 @@ _target_: deckard.base.files.FileConfig reports: reports -data_dir: data model_dir: models attack_dir: attacks directory: mnist score_dict_file: score_dict.json -data_file : data model_file : model attack_file : attack attack_type : .pkl -data_type : .pkl model_type : .pt params_file : params.yaml # train_labels_file : train_labels.json diff --git a/examples/pytorch/mnist/conf/grid/attack/cw_0.yaml b/examples/pytorch/conf/grid/attack/cw_0.yaml similarity index 100% rename from examples/pytorch/mnist/conf/grid/attack/cw_0.yaml rename to examples/pytorch/conf/grid/attack/cw_0.yaml diff --git a/examples/pytorch/mnist/conf/grid/attack/cw_2.yaml b/examples/pytorch/conf/grid/attack/cw_2.yaml similarity index 100% rename from examples/pytorch/mnist/conf/grid/attack/cw_2.yaml rename to examples/pytorch/conf/grid/attack/cw_2.yaml diff --git a/examples/pytorch/mnist/conf/grid/attack/cw_inf.yaml b/examples/pytorch/conf/grid/attack/cw_inf.yaml similarity index 100% rename from examples/pytorch/mnist/conf/grid/attack/cw_inf.yaml rename to examples/pytorch/conf/grid/attack/cw_inf.yaml diff --git a/examples/pytorch/mnist/conf/grid/attack/deepfool.yaml b/examples/pytorch/conf/grid/attack/deepfool.yaml similarity index 100% rename from examples/pytorch/mnist/conf/grid/attack/deepfool.yaml rename to examples/pytorch/conf/grid/attack/deepfool.yaml diff --git a/examples/pytorch/mnist/conf/grid/attack/fgm.yaml b/examples/pytorch/conf/grid/attack/fgm.yaml similarity index 100% rename from examples/pytorch/mnist/conf/grid/attack/fgm.yaml rename to examples/pytorch/conf/grid/attack/fgm.yaml diff --git a/examples/pytorch/mnist/conf/grid/attack/hsj.yaml b/examples/pytorch/conf/grid/attack/hsj.yaml similarity index 100% rename from examples/pytorch/mnist/conf/grid/attack/hsj.yaml rename to examples/pytorch/conf/grid/attack/hsj.yaml diff --git a/examples/pytorch/mnist/conf/grid/attack/patch.yaml b/examples/pytorch/conf/grid/attack/patch.yaml similarity index 100% rename from examples/pytorch/mnist/conf/grid/attack/patch.yaml rename to examples/pytorch/conf/grid/attack/patch.yaml diff --git a/examples/pytorch/mnist/conf/grid/attack/pgd.yaml b/examples/pytorch/conf/grid/attack/pgd.yaml similarity index 100% rename from examples/pytorch/mnist/conf/grid/attack/pgd.yaml rename to examples/pytorch/conf/grid/attack/pgd.yaml diff --git a/examples/pytorch/mnist/conf/grid/attack/pixel.yaml b/examples/pytorch/conf/grid/attack/pixel.yaml similarity index 100% rename from examples/pytorch/mnist/conf/grid/attack/pixel.yaml rename to examples/pytorch/conf/grid/attack/pixel.yaml diff --git a/examples/pytorch/mnist/conf/grid/attack/threshold.yaml b/examples/pytorch/conf/grid/attack/threshold.yaml similarity index 100% rename from examples/pytorch/mnist/conf/grid/attack/threshold.yaml rename to examples/pytorch/conf/grid/attack/threshold.yaml diff --git a/examples/pytorch/mnist/conf/grid/model/confidence.yaml b/examples/pytorch/conf/grid/model/confidence.yaml similarity index 76% rename from examples/pytorch/mnist/conf/grid/model/confidence.yaml rename to examples/pytorch/conf/grid/model/confidence.yaml index d07598f8..4f710246 100644 --- a/examples/pytorch/mnist/conf/grid/model/confidence.yaml +++ b/examples/pytorch/conf/grid/model/confidence.yaml @@ -1,4 +1,4 @@ art.postprocessor.name : [art.defences.postprocessor.HighConfidence] art.postprocessor.cutoff : [.1, .2, .3, .4, .5, .6, .7, .8, .9, .95, .99] -art.postprocessor.apply_fit : [True, False] +art.postprocessor.apply_fit : [True] diff --git a/examples/pytorch/cifar100/conf/model/art/preprocessor/default.yaml b/examples/pytorch/conf/grid/model/default.yaml similarity index 100% rename from examples/pytorch/cifar100/conf/model/art/preprocessor/default.yaml rename to examples/pytorch/conf/grid/model/default.yaml diff --git a/examples/pytorch/mnist/conf/grid/model/fsq.yaml b/examples/pytorch/conf/grid/model/fsq.yaml similarity index 100% rename from examples/pytorch/mnist/conf/grid/model/fsq.yaml rename to examples/pytorch/conf/grid/model/fsq.yaml diff --git a/examples/pytorch/mnist/conf/grid/model/gauss-in.yaml b/examples/pytorch/conf/grid/model/gauss-in.yaml similarity index 100% rename from examples/pytorch/mnist/conf/grid/model/gauss-in.yaml rename to examples/pytorch/conf/grid/model/gauss-in.yaml diff --git a/examples/pytorch/mnist/conf/grid/model/gauss-out.yaml b/examples/pytorch/conf/grid/model/gauss-out.yaml similarity index 100% rename from examples/pytorch/mnist/conf/grid/model/gauss-out.yaml rename to examples/pytorch/conf/grid/model/gauss-out.yaml diff --git a/examples/pytorch/mnist/conf/mnist.yaml b/examples/pytorch/conf/mnist.yaml similarity index 55% rename from examples/pytorch/mnist/conf/mnist.yaml rename to examples/pytorch/conf/mnist.yaml index aa05956f..1b144ff7 100644 --- a/examples/pytorch/mnist/conf/mnist.yaml +++ b/examples/pytorch/conf/mnist.yaml @@ -8,33 +8,40 @@ defaults: - override hydra/sweeper : optuna - override hydra/sweeper/sampler : grid - override hydra/launcher : joblib +def_name : control +atk_name : hsj +dataset : mnist +model_name : ResNet18 +device_id : gpu stage : '???' -direction : "maximize" +direction : + - "maximize" _target_ : deckard.base.experiment.Experiment -optimizers : accuracy +optimizers : + - accuracy hydra: run: - dir: ${files.directory}/${files.reports}/${stage}/logs + dir: ${files.directory}/logs/${stage}/ sweep: - dir: ${files.directory}/${stage}/${model.init.name} - subdir : ${hydra.sweeper.study_name}/${hydra.job.num} + dir: ${files.directory}/logs/${stage}/${model_name} + subdir : ${def_name}/${atk_name}/${hydra.job.num} sweeper: sampler: _target_: optuna.samplers.GridSampler direction: ${direction} - study_name: control - storage: sqlite:///model.db - n_jobs: ${hydra.launcher.n_jobs} - n_trials : 32 + study_name: ${model_name}_${def_name}_${atk_name} + storage: sqlite:///${dataset}.db + n_jobs: ${oc.env:HYDRA_SWEEPER_N_JOBS, 8} + n_trials: ${oc.env:HYDRA_SWEEPER_N_TRIALS, 128} + max_failure_rate: 1.0 params: - ++data.sample.random_state: choice(0, 1, 2, 3, 4, 5, 6, 7, 8, 9) - ++model.art.initialize.optimizer.lr: choice(10, 1, 0.1, 0.01, 0.001, .0001, .00001, 0.000001) + ++model.art.initialize.optimizer.lr: choice( 0.1, 0.01, 0.001, .0001, .00001, 0.000001) ++model.trainer.nb_epoch: choice(1, 10, 30, 50, 100) _target_: hydra_plugins.hydra_optuna_sweeper.optuna_sweeper.OptunaSweeper launcher: _target_: hydra_plugins.hydra_joblib_launcher.joblib_launcher.JoblibLauncher - n_jobs: 8 - prefer : processes + n_jobs: ${oc.env:HYDRA_SWEEPER_N_JOBS, 8} + prefer : threads verbose: 10 timeout: null pre_dispatch: n_jobs diff --git a/examples/pytorch/cifar10/conf/model/art/default.yaml b/examples/pytorch/conf/model/art/default.yaml similarity index 100% rename from examples/pytorch/cifar10/conf/model/art/default.yaml rename to examples/pytorch/conf/model/art/default.yaml diff --git a/examples/pytorch/cifar100/conf/model/art/initialize/default.yaml b/examples/pytorch/conf/model/art/initialize/default.yaml similarity index 100% rename from examples/pytorch/cifar100/conf/model/art/initialize/default.yaml rename to examples/pytorch/conf/model/art/initialize/default.yaml diff --git a/examples/pytorch/cifar10/conf/model/art/postprocessor/confidence.yaml b/examples/pytorch/conf/model/art/postprocessor/confidence.yaml similarity index 100% rename from examples/pytorch/cifar10/conf/model/art/postprocessor/confidence.yaml rename to examples/pytorch/conf/model/art/postprocessor/confidence.yaml diff --git a/examples/pytorch/cifar10/conf/model/art/postprocessor/gauss-out.yaml b/examples/pytorch/conf/model/art/postprocessor/gauss-out.yaml similarity index 100% rename from examples/pytorch/cifar10/conf/model/art/postprocessor/gauss-out.yaml rename to examples/pytorch/conf/model/art/postprocessor/gauss-out.yaml diff --git a/examples/pytorch/mnist/conf/grid/model/default.yaml b/examples/pytorch/conf/model/art/preprocessor/default.yaml similarity index 100% rename from examples/pytorch/mnist/conf/grid/model/default.yaml rename to examples/pytorch/conf/model/art/preprocessor/default.yaml diff --git a/examples/pytorch/cifar10/conf/model/art/preprocessor/fsq.yaml b/examples/pytorch/conf/model/art/preprocessor/fsq.yaml similarity index 100% rename from examples/pytorch/cifar10/conf/model/art/preprocessor/fsq.yaml rename to examples/pytorch/conf/model/art/preprocessor/fsq.yaml diff --git a/examples/pytorch/cifar10/conf/model/art/preprocessor/gauss-in.yaml b/examples/pytorch/conf/model/art/preprocessor/gauss-in.yaml similarity index 100% rename from examples/pytorch/cifar10/conf/model/art/preprocessor/gauss-in.yaml rename to examples/pytorch/conf/model/art/preprocessor/gauss-in.yaml diff --git a/examples/pytorch/mnist/conf/model/torch_cifar.yaml b/examples/pytorch/conf/model/torch_cifar.yaml similarity index 55% rename from examples/pytorch/mnist/conf/model/torch_cifar.yaml rename to examples/pytorch/conf/model/torch_cifar.yaml index 5ca2e24e..e97f651c 100644 --- a/examples/pytorch/mnist/conf/model/torch_cifar.yaml +++ b/examples/pytorch/conf/model/torch_cifar.yaml @@ -8,6 +8,15 @@ init: num_classes: 10 _target_: deckard.base.model.Model trainer: - nb_epoch: 100 + nb_epochs: 1 batch_size: 1024 + verbose: True library : pytorch +art: + criterion: + name : torch.nn.CrossEntropyLoss + optimizer: + name : torch.optim.SGD + lr : 0.01 + momentum : 0.9 + clip_values : [0, 255] diff --git a/examples/pytorch/cifar100/conf/model/torch_cifar100.yaml b/examples/pytorch/conf/model/torch_cifar100.yaml similarity index 50% rename from examples/pytorch/cifar100/conf/model/torch_cifar100.yaml rename to examples/pytorch/conf/model/torch_cifar100.yaml index d7d0f91d..5ba3a915 100644 --- a/examples/pytorch/cifar100/conf/model/torch_cifar100.yaml +++ b/examples/pytorch/conf/model/torch_cifar100.yaml @@ -8,6 +8,15 @@ init: num_classes: 100 _target_: deckard.base.model.Model trainer: - nb_epoch: 1 - batch_size: 1024 + nb_epochs: 1 + batch_size: 128 + verbose: True library : pytorch +art: + criterion: + name : torch.nn.CrossEntropyLoss + optimizer: + name : torch.optim.SGD + lr : 0.01 + momentum : 0.9 + clip_values : [0, 255] diff --git a/examples/pytorch/conf/model/torch_mnist.yaml b/examples/pytorch/conf/model/torch_mnist.yaml new file mode 100644 index 00000000..21682316 --- /dev/null +++ b/examples/pytorch/conf/model/torch_mnist.yaml @@ -0,0 +1,23 @@ + +defaults: + - art : default +data: ${data} +init: + _target_: deckard.base.model.ModelInitializer + num_channels : 1 + num_classes : 10 + name : torch_example.ResNet18 +_target_: deckard.base.model.Model +trainer: + nb_epochs: 1 + batch_size: 4096 + verbose: True +library : pytorch +art: + criterion: + name : torch.nn.CrossEntropyLoss + optimizer: + name : torch.optim.SGD + lr : 0.01 + momentum : 0.9 + clip_values : [0, 255] diff --git a/examples/pytorch/cifar10/conf/plots.yaml b/examples/pytorch/conf/plots.yaml similarity index 98% rename from examples/pytorch/cifar10/conf/plots.yaml rename to examples/pytorch/conf/plots.yaml index 57c016f7..b684aec3 100644 --- a/examples/pytorch/cifar10/conf/plots.yaml +++ b/examples/pytorch/conf/plots.yaml @@ -51,8 +51,8 @@ cat_plot: - file: adv_failures_per_train_time_vs_defence_type.eps hue: model_name kind: boxen - # set: - # yscale: log + set: + yscale: log titles: $\bar{C}_{adv.}$ vs Defence Type x: def_gen xlabels: Defence Type @@ -69,8 +69,8 @@ cat_plot: hue: model_name kind: boxen legend_title: Model Name - # set: - # yscale: log + set: + yscale: log titles: $\bar{C}_{adv.}$ vs Attack Type x: atk_gen xlabels: Attack Type @@ -99,6 +99,7 @@ cat_plot: - ResNet50 - ResNet101 - ResNet152 + - file: adv_accuracy_vs_attack_type.eps hue: model_name kind: boxen @@ -179,10 +180,10 @@ line_plot: legend: {"bbox_to_anchor": [1.05, 1], "title": "Defence"} title: $f_{adv}$ vs Defence Strength x: def_value - x_scale: log + x_scale: linear xlabel: Defence Control Parameter y: adv_failure_rate - y_scale: log + y_scale: linear ylabel: $f_{adv.}$ hue_order: - Control @@ -221,7 +222,7 @@ scatter_plot: ylabel: $f_{adv}$ title: $f_{adv}$ vs $t_{train}$ file: adv_failure_rate_vs_train_time.eps - y_scale: linear + # y_scale: log x_scale: log legend: title: Model Name diff --git a/examples/pytorch/cifar10/conf/scorers/default.yaml b/examples/pytorch/conf/scorers/default.yaml similarity index 100% rename from examples/pytorch/cifar10/conf/scorers/default.yaml rename to examples/pytorch/conf/scorers/default.yaml diff --git a/examples/pytorch/dag.md b/examples/pytorch/dag.md new file mode 100644 index 00000000..2df2760a --- /dev/null +++ b/examples/pytorch/dag.md @@ -0,0 +1,111 @@ +```mermaid +flowchart TD + node1["../dvc.yaml:attack@cifar"] + node2["../dvc.yaml:attack@cifar100"] + node3["../dvc.yaml:attack@mnist"] + node4["../dvc.yaml:attacks@ResNet101-cifar"] + node5["../dvc.yaml:attacks@ResNet101-cifar100"] + node6["../dvc.yaml:attacks@ResNet101-mnist"] + node7["../dvc.yaml:attacks@ResNet152-cifar"] + node8["../dvc.yaml:attacks@ResNet152-cifar100"] + node9["../dvc.yaml:attacks@ResNet152-mnist"] + node10["../dvc.yaml:attacks@ResNet18-cifar"] + node11["../dvc.yaml:attacks@ResNet18-cifar100"] + node12["../dvc.yaml:attacks@ResNet18-mnist"] + node13["../dvc.yaml:attacks@ResNet34-cifar"] + node14["../dvc.yaml:attacks@ResNet34-cifar100"] + node15["../dvc.yaml:attacks@ResNet34-mnist"] + node16["../dvc.yaml:attacks@ResNet50-cifar"] + node17["../dvc.yaml:attacks@ResNet50-cifar100"] + node18["../dvc.yaml:attacks@ResNet50-mnist"] + node19["../dvc.yaml:compile@cifar-attack"] + node20["../dvc.yaml:compile@cifar100-attack"] + node21["../dvc.yaml:compile@mnist-attack"] + node22["../dvc.yaml:parse"] + node23["../dvc.yaml:prepare_plot_folder@cifar-attack"] + node24["../dvc.yaml:prepare_plot_folder@cifar100-attack"] + node25["../dvc.yaml:prepare_plot_folder@mnist-attack"] + node26["../dvc.yaml:train@cifar"] + node27["../dvc.yaml:train@cifar100"] + node28["../dvc.yaml:train@mnist"] + node29["afr"] + node30["clean@cifar"] + node31["clean@cifar100"] + node32["clean@mnist"] + node33["copy_results"] + node34["merge"] + node35["plot"] + node36["predict_survival_time"] + node1-->node4 + node1-->node7 + node1-->node10 + node1-->node13 + node1-->node16 + node1-->node19 + node2-->node5 + node2-->node8 + node2-->node11 + node2-->node14 + node2-->node17 + node2-->node20 + node3-->node6 + node3-->node9 + node3-->node12 + node3-->node15 + node3-->node18 + node3-->node21 + node4-->node19 + node5-->node20 + node6-->node21 + node7-->node19 + node8-->node20 + node9-->node21 + node10-->node19 + node11-->node20 + node12-->node21 + node13-->node19 + node14-->node20 + node15-->node21 + node16-->node19 + node17-->node20 + node18-->node21 + node19-->node23 + node20-->node24 + node21-->node25 + node22-->node1 + node22-->node2 + node22-->node3 + node22-->node26 + node22-->node27 + node22-->node28 + node23-->node30 + node24-->node31 + node25-->node32 + node26-->node1 + node26-->node4 + node26-->node7 + node26-->node10 + node26-->node13 + node26-->node16 + node27-->node2 + node27-->node5 + node27-->node8 + node27-->node11 + node27-->node14 + node27-->node17 + node28-->node3 + node28-->node6 + node28-->node9 + node28-->node12 + node28-->node15 + node28-->node18 + node29-->node33 + node29-->node36 + node30-->node34 + node31-->node34 + node32-->node34 + node34-->node29 + node34-->node36 + node35-->node33 + node36-->node35 +``` diff --git a/examples/pytorch/dvc.lock b/examples/pytorch/dvc.lock new file mode 100644 index 00000000..d17b377a --- /dev/null +++ b/examples/pytorch/dvc.lock @@ -0,0 +1,1489 @@ +schema: '2.0' +stages: + train@mnist: + cmd: python -m deckard.layers.experiment train@mnist --config_file mnist.yaml + --params_file mnist.yaml + deps: + - path: mnist.yaml + hash: md5 + md5: 494df904ccf4eb077c0642efde302fee + size: 6005 + params: + params.yaml: + atk_name: hsj + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + def_name: control + device_id: gpu + files: + _target_: deckard.base.files.FileConfig + adv_predictions_file: adv_predictions.json + attack_dir: attacks + attack_file: attack + attack_type: .pkl + directory: mnist + model_dir: models + model_file: model + model_type: .pt + name: default + params_file: params.yaml + predictions_file: predictions.json + reports: reports + score_dict_file: score_dict.json + model: + _target_: deckard.base.model.Model + art: + _target_: deckard.base.model.art_pipeline.ArtPipeline + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + initialize: + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + library: pytorch + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + init: + _target_: deckard.base.model.ModelInitializer + name: torch_example.ResNet18 + num_channels: 1 + num_classes: 10 + library: pytorch + trainer: + batch_size: 1024 + nb_epochs: 1 + verbose: true + scorers: + _target_: deckard.base.scorer.ScorerDict + accuracy: + _target_: deckard.base.scorer.ScorerConfig + direction: maximize + name: sklearn.metrics.accuracy_score + log_loss: + _target_: deckard.base.scorer.ScorerConfig + direction: minimize + name: sklearn.metrics.log_loss + outs: + - path: mnist/models/model.optimizer.pt + hash: md5 + md5: e16de4ab0b1f1a5808aaeb8cdd76553c + size: 44781294 + - path: mnist/models/model.pt + hash: md5 + md5: ae8a80a475c5ccfbf50eaa62adb24e1e + size: 44787138 + - path: mnist/reports/train/default/score_dict.json + hash: md5 + md5: f55673763c20d36336fc1a23bd057a5f + size: 515 + train@cifar: + cmd: python -m deckard.layers.experiment train@cifar --config_file cifar.yaml + --params_file cifar.yaml + deps: + - path: cifar.yaml + hash: md5 + md5: 15fc7cec034cfea8e390c77f0ebd2d26 + size: 5037 + params: + params.yaml: + atk_name: hsj + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + def_name: control + device_id: gpu + files: + _target_: deckard.base.files.FileConfig + adv_predictions_file: adv_predictions.json + attack_dir: attacks + attack_file: attack + attack_type: .pkl + directory: mnist + model_dir: models + model_file: model + model_type: .pt + name: default + params_file: params.yaml + predictions_file: predictions.json + reports: reports + score_dict_file: score_dict.json + model: + _target_: deckard.base.model.Model + art: + _target_: deckard.base.model.art_pipeline.ArtPipeline + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + initialize: + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + library: pytorch + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + init: + _target_: deckard.base.model.ModelInitializer + name: torch_example.ResNet18 + num_channels: 1 + num_classes: 10 + library: pytorch + trainer: + batch_size: 1024 + nb_epochs: 1 + verbose: true + scorers: + _target_: deckard.base.scorer.ScorerDict + accuracy: + _target_: deckard.base.scorer.ScorerConfig + direction: maximize + name: sklearn.metrics.accuracy_score + log_loss: + _target_: deckard.base.scorer.ScorerConfig + direction: minimize + name: sklearn.metrics.log_loss + outs: + - path: cifar/models/model.optimizer.pt + hash: md5 + md5: 325f6119d397c468be7d7fe01b9b04ea + size: 44781294 + - path: cifar/models/model.pt + hash: md5 + md5: 64271d0a795a3bdf5a46951ea1e3ded9 + size: 44787138 + - path: cifar/reports/train/default/score_dict.json + hash: md5 + md5: 73c3315c8f9def00a519bdf6bb9998e2 + size: 508 + train@cifar100: + cmd: python -m deckard.layers.experiment train@cifar100 --config_file cifar100.yaml + --params_file cifar100.yaml + deps: + - path: cifar100.yaml + hash: md5 + md5: deb1b04848c71b5819cf21880b4368f0 + size: 5282 + params: + params.yaml: + atk_name: hsj + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + def_name: control + device_id: gpu + files: + _target_: deckard.base.files.FileConfig + adv_predictions_file: adv_predictions.json + attack_dir: attacks + attack_file: attack + attack_type: .pkl + directory: mnist + model_dir: models + model_file: model + model_type: .pt + name: default + params_file: params.yaml + predictions_file: predictions.json + reports: reports + score_dict_file: score_dict.json + model: + _target_: deckard.base.model.Model + art: + _target_: deckard.base.model.art_pipeline.ArtPipeline + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + initialize: + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + library: pytorch + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + init: + _target_: deckard.base.model.ModelInitializer + name: torch_example.ResNet18 + num_channels: 1 + num_classes: 10 + library: pytorch + trainer: + batch_size: 1024 + nb_epochs: 1 + verbose: true + scorers: + _target_: deckard.base.scorer.ScorerDict + accuracy: + _target_: deckard.base.scorer.ScorerConfig + direction: maximize + name: sklearn.metrics.accuracy_score + log_loss: + _target_: deckard.base.scorer.ScorerConfig + direction: minimize + name: sklearn.metrics.log_loss + outs: + - path: cifar100/models/model.optimizer.pt + hash: md5 + md5: b469db4312d99c3ccf8d198c9936f7ab + size: 44781294 + - path: cifar100/models/model.pt + hash: md5 + md5: 76593e090e9815dfac73c6f9d1d84d57 + size: 44787138 + - path: cifar100/reports/train/default/score_dict.json + hash: md5 + md5: 41191be9e2760b49f21bfe27ccef654c + size: 514 + attack@mnist: + cmd: python -m deckard.layers.experiment attack@mnist --config_file mnist.yaml + --params_file mnist.yaml + deps: + - path: mnist/models/model.optimizer.pt + hash: md5 + md5: e16de4ab0b1f1a5808aaeb8cdd76553c + size: 44781294 + - path: mnist/models/model.pt + hash: md5 + md5: ae8a80a475c5ccfbf50eaa62adb24e1e + size: 44787138 + params: + params.yaml: + atk_name: hsj + attack: + _target_: deckard.base.attack.Attack + attack_size: 100 + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + init: + _target_: deckard.base.attack.AttackInitializer + batch_size: 1024 + model: + _target_: deckard.base.model.Model + art: + _target_: deckard.base.model.art_pipeline.ArtPipeline + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + initialize: + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + library: pytorch + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + init: + _target_: deckard.base.model.ModelInitializer + name: torch_example.ResNet18 + num_channels: 1 + num_classes: 10 + library: pytorch + trainer: + batch_size: 1024 + nb_epochs: 1 + verbose: true + name: art.attacks.evasion.HopSkipJump + method: evasion + model: + _target_: deckard.base.model.Model + art: + _target_: deckard.base.model.art_pipeline.ArtPipeline + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + initialize: + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + library: pytorch + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + init: + _target_: deckard.base.model.ModelInitializer + name: torch_example.ResNet18 + num_channels: 1 + num_classes: 10 + library: pytorch + trainer: + batch_size: 1024 + nb_epochs: 1 + verbose: true + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + dataset: mnist + def_name: control + device_id: gpu + files: + _target_: deckard.base.files.FileConfig + adv_predictions_file: adv_predictions.json + attack_dir: attacks + attack_file: attack + attack_type: .pkl + directory: mnist + model_dir: models + model_file: model + model_type: .pt + name: default + params_file: params.yaml + predictions_file: predictions.json + reports: reports + score_dict_file: score_dict.json + model: + _target_: deckard.base.model.Model + art: + _target_: deckard.base.model.art_pipeline.ArtPipeline + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + initialize: + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + library: pytorch + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + init: + _target_: deckard.base.model.ModelInitializer + name: torch_example.ResNet18 + num_channels: 1 + num_classes: 10 + library: pytorch + trainer: + batch_size: 1024 + nb_epochs: 1 + verbose: true + scorers: + _target_: deckard.base.scorer.ScorerDict + accuracy: + _target_: deckard.base.scorer.ScorerConfig + direction: maximize + name: sklearn.metrics.accuracy_score + log_loss: + _target_: deckard.base.scorer.ScorerConfig + direction: minimize + name: sklearn.metrics.log_loss + outs: + - path: mnist/attacks/attack.pkl + hash: md5 + md5: bb64f427b6df9067d9c59d5021937164 + size: 313766 + - path: mnist/reports/attack/default/score_dict.json + hash: md5 + md5: b63b8d925ab6db88e3b707ec025800d3 + size: 834 + parse@mnist: + cmd: python -m deckard.layers.parse --config_file mnist.yaml --params_file mnist.yaml + deps: + - path: conf/attack + hash: md5 + md5: f5637e0106257966dd59bca49db76523.dir + size: 268 + nfiles: 1 + - path: conf/data + hash: md5 + md5: ab878db613853b9479b012b228a00d50.dir + size: 643 + nfiles: 4 + - path: conf/files + hash: md5 + md5: b10c002322e63c5ed04e88dda4f5b9fa.dir + size: 1530 + nfiles: 3 + - path: conf/mnist.yaml + hash: md5 + md5: ed726950d032472aedbb267ae88c0f9a + size: 1501 + - path: conf/model + hash: md5 + md5: 5a1ef81c82a950e745c930067d8c870d.dir + size: 2093 + nfiles: 10 + - path: conf/scorers + hash: md5 + md5: 1316006b6b9fbc05fd8ed56d46e15718.dir + size: 279 + nfiles: 1 + outs: + - path: mnist.yaml + hash: md5 + md5: 494df904ccf4eb077c0642efde302fee + size: 6005 + parse@cifar: + cmd: python -m deckard.layers.parse --config_file cifar.yaml --params_file cifar.yaml + deps: + - path: conf/attack + hash: md5 + md5: f5637e0106257966dd59bca49db76523.dir + size: 268 + nfiles: 1 + - path: conf/cifar.yaml + hash: md5 + md5: 20703e37d826c71d762491158f302bfe + size: 1501 + - path: conf/data + hash: md5 + md5: ab878db613853b9479b012b228a00d50.dir + size: 643 + nfiles: 4 + - path: conf/files + hash: md5 + md5: b10c002322e63c5ed04e88dda4f5b9fa.dir + size: 1530 + nfiles: 3 + - path: conf/model + hash: md5 + md5: 5a1ef81c82a950e745c930067d8c870d.dir + size: 2093 + nfiles: 10 + - path: conf/scorers + hash: md5 + md5: 1316006b6b9fbc05fd8ed56d46e15718.dir + size: 279 + nfiles: 1 + outs: + - path: cifar.yaml + hash: md5 + md5: 15fc7cec034cfea8e390c77f0ebd2d26 + size: 5037 + parse@cifar100: + cmd: python -m deckard.layers.parse --config_file cifar100.yaml --params_file + cifar100.yaml + deps: + - path: conf/attack + hash: md5 + md5: f5637e0106257966dd59bca49db76523.dir + size: 268 + nfiles: 1 + - path: conf/cifar100.yaml + hash: md5 + md5: 008f14b0f28ad1abdb09f56b702ae0ab + size: 1591 + - path: conf/data + hash: md5 + md5: ab878db613853b9479b012b228a00d50.dir + size: 643 + nfiles: 4 + - path: conf/files + hash: md5 + md5: b10c002322e63c5ed04e88dda4f5b9fa.dir + size: 1530 + nfiles: 3 + - path: conf/model + hash: md5 + md5: 5a1ef81c82a950e745c930067d8c870d.dir + size: 2093 + nfiles: 10 + - path: conf/scorers + hash: md5 + md5: 1316006b6b9fbc05fd8ed56d46e15718.dir + size: 279 + nfiles: 1 + outs: + - path: cifar100.yaml + hash: md5 + md5: deb1b04848c71b5819cf21880b4368f0 + size: 5282 + attack@cifar: + cmd: python -m deckard.layers.experiment attack@cifar --config_file cifar.yaml + --params_file cifar.yaml + deps: + - path: cifar/models/model.optimizer.pt + hash: md5 + md5: 325f6119d397c468be7d7fe01b9b04ea + size: 44781294 + - path: cifar/models/model.pt + hash: md5 + md5: 64271d0a795a3bdf5a46951ea1e3ded9 + size: 44787138 + params: + params.yaml: + atk_name: hsj + attack: + _target_: deckard.base.attack.Attack + attack_size: 100 + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + init: + _target_: deckard.base.attack.AttackInitializer + batch_size: 1024 + model: + _target_: deckard.base.model.Model + art: + _target_: deckard.base.model.art_pipeline.ArtPipeline + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + initialize: + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + library: pytorch + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + init: + _target_: deckard.base.model.ModelInitializer + name: torch_example.ResNet18 + num_channels: 1 + num_classes: 10 + library: pytorch + trainer: + batch_size: 1024 + nb_epochs: 1 + verbose: true + name: art.attacks.evasion.HopSkipJump + method: evasion + model: + _target_: deckard.base.model.Model + art: + _target_: deckard.base.model.art_pipeline.ArtPipeline + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + initialize: + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + library: pytorch + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + init: + _target_: deckard.base.model.ModelInitializer + name: torch_example.ResNet18 + num_channels: 1 + num_classes: 10 + library: pytorch + trainer: + batch_size: 1024 + nb_epochs: 1 + verbose: true + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + dataset: mnist + def_name: control + device_id: gpu + files: + _target_: deckard.base.files.FileConfig + adv_predictions_file: adv_predictions.json + attack_dir: attacks + attack_file: attack + attack_type: .pkl + directory: mnist + model_dir: models + model_file: model + model_type: .pt + name: default + params_file: params.yaml + predictions_file: predictions.json + reports: reports + score_dict_file: score_dict.json + model: + _target_: deckard.base.model.Model + art: + _target_: deckard.base.model.art_pipeline.ArtPipeline + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + initialize: + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + library: pytorch + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + init: + _target_: deckard.base.model.ModelInitializer + name: torch_example.ResNet18 + num_channels: 1 + num_classes: 10 + library: pytorch + trainer: + batch_size: 1024 + nb_epochs: 1 + verbose: true + scorers: + _target_: deckard.base.scorer.ScorerDict + accuracy: + _target_: deckard.base.scorer.ScorerConfig + direction: maximize + name: sklearn.metrics.accuracy_score + log_loss: + _target_: deckard.base.scorer.ScorerConfig + direction: minimize + name: sklearn.metrics.log_loss + outs: + - path: cifar/attacks/attack.pkl + hash: md5 + md5: c3e171e067df56e31ee5d4d968aad9d5 + size: 313766 + - path: cifar/reports/attack/default/score_dict.json + hash: md5 + md5: 6aae48892e49e75742b1d67a5e1ec276 + size: 838 + attack@cifar100: + cmd: python -m deckard.layers.experiment attack@cifar100 --config_file cifar100.yaml + --params_file cifar100.yaml + deps: + - path: cifar100/models/model.optimizer.pt + hash: md5 + md5: b469db4312d99c3ccf8d198c9936f7ab + size: 44781294 + - path: cifar100/models/model.pt + hash: md5 + md5: 76593e090e9815dfac73c6f9d1d84d57 + size: 44787138 + params: + params.yaml: + atk_name: hsj + attack: + _target_: deckard.base.attack.Attack + attack_size: 100 + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + init: + _target_: deckard.base.attack.AttackInitializer + batch_size: 1024 + model: + _target_: deckard.base.model.Model + art: + _target_: deckard.base.model.art_pipeline.ArtPipeline + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + initialize: + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + library: pytorch + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + init: + _target_: deckard.base.model.ModelInitializer + name: torch_example.ResNet18 + num_channels: 1 + num_classes: 10 + library: pytorch + trainer: + batch_size: 1024 + nb_epochs: 1 + verbose: true + name: art.attacks.evasion.HopSkipJump + method: evasion + model: + _target_: deckard.base.model.Model + art: + _target_: deckard.base.model.art_pipeline.ArtPipeline + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + initialize: + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + library: pytorch + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + init: + _target_: deckard.base.model.ModelInitializer + name: torch_example.ResNet18 + num_channels: 1 + num_classes: 10 + library: pytorch + trainer: + batch_size: 1024 + nb_epochs: 1 + verbose: true + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + dataset: mnist + def_name: control + device_id: gpu + files: + _target_: deckard.base.files.FileConfig + adv_predictions_file: adv_predictions.json + attack_dir: attacks + attack_file: attack + attack_type: .pkl + directory: mnist + model_dir: models + model_file: model + model_type: .pt + name: default + params_file: params.yaml + predictions_file: predictions.json + reports: reports + score_dict_file: score_dict.json + model: + _target_: deckard.base.model.Model + art: + _target_: deckard.base.model.art_pipeline.ArtPipeline + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + initialize: + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + library: pytorch + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD + data: + _target_: deckard.base.data.Data + generate: + _target_: deckard.base.data.generator.DataGenerator + name: torch_mnist + sample: + _target_: deckard.base.data.sampler.SklearnDataSampler + random_state: 0 + stratify: true + test_size: 12000 + train_size: 48000 + init: + _target_: deckard.base.model.ModelInitializer + name: torch_example.ResNet18 + num_channels: 1 + num_classes: 10 + library: pytorch + trainer: + batch_size: 1024 + nb_epochs: 1 + verbose: true + scorers: + _target_: deckard.base.scorer.ScorerDict + accuracy: + _target_: deckard.base.scorer.ScorerConfig + direction: maximize + name: sklearn.metrics.accuracy_score + log_loss: + _target_: deckard.base.scorer.ScorerConfig + direction: minimize + name: sklearn.metrics.log_loss + outs: + - path: cifar100/attacks/attack.pkl + hash: md5 + md5: fa9665c5d8fc6a1118123c25a30e8978 + size: 313766 + - path: cifar100/reports/attack/default/score_dict.json + hash: md5 + md5: 7396ec38b31f7ff9ae1fcab467a98c6c + size: 854 + attacks@mnist-ResNet18: + cmd: bash attacks.sh ++model.init.name=torch_example.ResNet18 stage=attack model_name=ResNet18 + attack.attack_size=100 data=torch_mnist model=torch_mnist +direction="[maximize,maximize,minimize]" + +optimizers="[accuracy,adv_accuracy,adv_success]" --config-name mnist.yaml + deps: + - path: mnist/reports/attack/default/score_dict.json + hash: md5 + md5: b63b8d925ab6db88e3b707ec025800d3 + size: 834 + - path: mnist/reports/train/default/score_dict.json + hash: md5 + md5: f55673763c20d36336fc1a23bd057a5f + size: 515 + outs: + - path: mnist/logs/attack/ResNet18/ + hash: md5 + md5: 240762a55b7e67e04f4b337e2aa58e50.dir + size: 67964097 + nfiles: 7809 + attacks@mnist-ResNet50: + cmd: bash attacks.sh ++model.init.name=torch_example.ResNet50 stage=attack model_name=ResNet50 + attack.attack_size=100 data=torch_mnist model=torch_mnist +direction="[maximize,maximize,minimize]" + +optimizers="[accuracy,adv_accuracy,adv_success]" --config-name mnist.yaml + deps: + - path: mnist/reports/attack/default/score_dict.json + hash: md5 + md5: b63b8d925ab6db88e3b707ec025800d3 + size: 834 + - path: mnist/reports/train/default/score_dict.json + hash: md5 + md5: f55673763c20d36336fc1a23bd057a5f + size: 515 + outs: + - path: mnist/logs/attack/ResNet50/ + hash: md5 + md5: 481f3de00d4fe02fde5534cdcc1c2b30.dir + size: 78656482 + nfiles: 7681 + attacks@mnist-ResNet152: + cmd: bash attacks.sh ++model.init.name=torch_example.ResNet152 stage=attack + model_name=ResNet152 attack.attack_size=100 data=torch_mnist model=torch_mnist + +direction="[maximize,maximize,minimize]" +optimizers="[accuracy,adv_accuracy,adv_success]" + --config-name mnist.yaml + deps: + - path: mnist/reports/attack/default/score_dict.json + hash: md5 + md5: b63b8d925ab6db88e3b707ec025800d3 + size: 834 + - path: mnist/reports/train/default/score_dict.json + hash: md5 + md5: f55673763c20d36336fc1a23bd057a5f + size: 515 + outs: + - path: mnist/logs/attack/ResNet152/ + hash: md5 + md5: caf83ba317d2b12ecc75f638dc07e7fb.dir + size: 152486967 + nfiles: 7681 + attacks@mnist-ResNet101: + cmd: bash attacks.sh ++model.init.name=torch_example.ResNet101 stage=attack + model_name=ResNet101 attack.attack_size=100 data=torch_mnist model=torch_mnist + +direction="[maximize,maximize,minimize]" +optimizers="[accuracy,adv_accuracy,adv_success]" + --config-name mnist.yaml + deps: + - path: mnist/reports/attack/default/score_dict.json + hash: md5 + md5: b63b8d925ab6db88e3b707ec025800d3 + size: 834 + - path: mnist/reports/train/default/score_dict.json + hash: md5 + md5: f55673763c20d36336fc1a23bd057a5f + size: 515 + outs: + - path: mnist/logs/attack/ResNet101/ + hash: md5 + md5: 92951fdd84c6ad7dfa6e724499196111.dir + size: 90289426 + nfiles: 7681 + attacks@mnist-ResNet34: + cmd: bash attacks.sh ++model.init.name=torch_example.ResNet34 stage=attack model_name=ResNet34 + attack.attack_size=100 data=torch_mnist model=torch_mnist +direction="[maximize,maximize,minimize]" + +optimizers="[accuracy,adv_accuracy,adv_success]" --config-name mnist.yaml + deps: + - path: mnist/reports/attack/default/score_dict.json + hash: md5 + md5: b63b8d925ab6db88e3b707ec025800d3 + size: 834 + - path: mnist/reports/train/default/score_dict.json + hash: md5 + md5: f55673763c20d36336fc1a23bd057a5f + size: 515 + outs: + - path: mnist/logs/attack/ResNet34/ + hash: md5 + md5: 60ec86982d5cf4693ffd7e5420ce2863.dir + size: 110646599 + nfiles: 9985 + attacks@cifar-ResNet18: + cmd: bash attacks.sh ++model.init.name=torch_example.ResNet18 stage=attack model_name=ResNet18 + attack.attack_size=100 data=torch_cifar model=torch_cifar +direction="[maximize,maximize,minimize]" + +optimizers="[accuracy,adv_accuracy,adv_success]" --config-name cifar.yaml + deps: + - path: cifar/reports/attack/default/score_dict.json + hash: md5 + md5: 6aae48892e49e75742b1d67a5e1ec276 + size: 838 + - path: cifar/reports/train/default/score_dict.json + hash: md5 + md5: 73c3315c8f9def00a519bdf6bb9998e2 + size: 508 + outs: + - path: cifar/logs/attack/ResNet18/ + hash: md5 + md5: 1f8b0fd2b4e94d6e67d017d9ae9d4ea0.dir + size: 66807197 + nfiles: 9345 + attacks@cifar-ResNet34: + cmd: bash attacks.sh ++model.init.name=torch_example.ResNet34 stage=attack model_name=ResNet34 + attack.attack_size=100 data=torch_cifar model=torch_cifar +direction="[maximize,maximize,minimize]" + +optimizers="[accuracy,adv_accuracy,adv_success]" --config-name cifar.yaml + deps: + - path: cifar/reports/attack/default/score_dict.json + hash: md5 + md5: 6aae48892e49e75742b1d67a5e1ec276 + size: 838 + - path: cifar/reports/train/default/score_dict.json + hash: md5 + md5: 73c3315c8f9def00a519bdf6bb9998e2 + size: 508 + outs: + - path: cifar/logs/attack/ResNet34/ + hash: md5 + md5: ca652fb37adbb2bc48c785d3db8cb90e.dir + size: 57831382 + nfiles: 7681 + attacks@cifar-ResNet50: + cmd: bash attacks.sh ++model.init.name=torch_example.ResNet50 stage=attack model_name=ResNet50 + attack.attack_size=100 data=torch_cifar model=torch_cifar +direction="[maximize,maximize,minimize]" + +optimizers="[accuracy,adv_accuracy,adv_success]" --config-name cifar.yaml + deps: + - path: cifar/reports/attack/default/score_dict.json + hash: md5 + md5: 6aae48892e49e75742b1d67a5e1ec276 + size: 838 + - path: cifar/reports/train/default/score_dict.json + hash: md5 + md5: 73c3315c8f9def00a519bdf6bb9998e2 + size: 508 + outs: + - path: cifar/logs/attack/ResNet50/ + hash: md5 + md5: 63b0e940b64d9b941c4877ff05ceb4e6.dir + size: 61565958 + nfiles: 7681 + attacks@cifar-ResNet101: + cmd: bash attacks.sh ++model.init.name=torch_example.ResNet101 stage=attack + model_name=ResNet101 attack.attack_size=100 data=torch_cifar model=torch_cifar + +direction="[maximize,maximize,minimize]" +optimizers="[accuracy,adv_accuracy,adv_success]" + --config-name cifar.yaml + deps: + - path: cifar/reports/attack/default/score_dict.json + hash: md5 + md5: 6aae48892e49e75742b1d67a5e1ec276 + size: 838 + - path: cifar/reports/train/default/score_dict.json + hash: md5 + md5: 73c3315c8f9def00a519bdf6bb9998e2 + size: 508 + outs: + - path: cifar/logs/attack/ResNet101/ + hash: md5 + md5: 6899712a5e067be74511fd4b85c5531b.dir + size: 80647437 + nfiles: 7681 + attacks@cifar-ResNet152: + cmd: bash attacks.sh ++model.init.name=torch_example.ResNet152 stage=attack + model_name=ResNet152 attack.attack_size=100 data=torch_cifar model=torch_cifar + +direction="[maximize,maximize,minimize]" +optimizers="[accuracy,adv_accuracy,adv_success]" + --config-name cifar.yaml + deps: + - path: cifar/reports/attack/default/score_dict.json + hash: md5 + md5: 6aae48892e49e75742b1d67a5e1ec276 + size: 838 + - path: cifar/reports/train/default/score_dict.json + hash: md5 + md5: 73c3315c8f9def00a519bdf6bb9998e2 + size: 508 + outs: + - path: cifar/logs/attack/ResNet152/ + hash: md5 + md5: 08cbdcd3ef354342a69c7bbd361ab6f4.dir + size: 101987846 + nfiles: 7681 + attacks@cifar100-ResNet18: + cmd: bash attacks.sh ++model.init.name=torch_example.ResNet18 stage=attack model_name=ResNet18 + attack.attack_size=100 data=torch_cifar100 model=torch_cifar100 +direction="[maximize,maximize,minimize]" + +optimizers="[accuracy,adv_accuracy,adv_success]" --config-name cifar100.yaml + deps: + - path: cifar100/reports/attack/default/score_dict.json + hash: md5 + md5: 7396ec38b31f7ff9ae1fcab467a98c6c + size: 854 + - path: cifar100/reports/train/default/score_dict.json + hash: md5 + md5: 41191be9e2760b49f21bfe27ccef654c + size: 514 + outs: + - path: cifar100/logs/attack/ResNet18/ + hash: md5 + md5: f6fc033b064588576514c93b8901f040.dir + size: 84069191 + nfiles: 10469 + attacks@cifar100-ResNet34: + cmd: bash attacks.sh ++model.init.name=torch_example.ResNet34 stage=attack model_name=ResNet34 + attack.attack_size=100 data=torch_cifar100 model=torch_cifar100 +direction="[maximize,maximize,minimize]" + +optimizers="[accuracy,adv_accuracy,adv_success]" --config-name cifar100.yaml + deps: + - path: cifar100/reports/attack/default/score_dict.json + hash: md5 + md5: 7396ec38b31f7ff9ae1fcab467a98c6c + size: 854 + - path: cifar100/reports/train/default/score_dict.json + hash: md5 + md5: 41191be9e2760b49f21bfe27ccef654c + size: 514 + outs: + - path: cifar100/logs/attack/ResNet34/ + hash: md5 + md5: 40ab74741f8f446b09778c55522e0d23.dir + size: 45326629 + nfiles: 7681 + attacks@cifar100-ResNet50: + cmd: bash attacks.sh ++model.init.name=torch_example.ResNet50 stage=attack model_name=ResNet50 + attack.attack_size=100 data=torch_cifar100 model=torch_cifar100 +direction="[maximize,maximize,minimize]" + +optimizers="[accuracy,adv_accuracy,adv_success]" --config-name cifar100.yaml + deps: + - path: cifar100/reports/attack/default/score_dict.json + hash: md5 + md5: 7396ec38b31f7ff9ae1fcab467a98c6c + size: 854 + - path: cifar100/reports/train/default/score_dict.json + hash: md5 + md5: 41191be9e2760b49f21bfe27ccef654c + size: 514 + outs: + - path: cifar100/logs/attack/ResNet50/ + hash: md5 + md5: d74f031721b03ab0b6d0eeab26fdfba3.dir + size: 46511652 + nfiles: 7681 + attacks@cifar100-ResNet101: + cmd: bash attacks.sh ++model.init.name=torch_example.ResNet101 stage=attack + model_name=ResNet101 attack.attack_size=100 data=torch_cifar100 model=torch_cifar100 + +direction="[maximize,maximize,minimize]" +optimizers="[accuracy,adv_accuracy,adv_success]" + --config-name cifar100.yaml + deps: + - path: cifar100/reports/attack/default/score_dict.json + hash: md5 + md5: 7396ec38b31f7ff9ae1fcab467a98c6c + size: 854 + - path: cifar100/reports/train/default/score_dict.json + hash: md5 + md5: 41191be9e2760b49f21bfe27ccef654c + size: 514 + outs: + - path: cifar100/logs/attack/ResNet101/ + hash: md5 + md5: 57911c3eaeeb3888a220a1fcf12e8442.dir + size: 63892972 + nfiles: 7681 diff --git a/examples/pytorch/dvc.yaml b/examples/pytorch/dvc.yaml new file mode 100644 index 00000000..d1ddd790 --- /dev/null +++ b/examples/pytorch/dvc.yaml @@ -0,0 +1,127 @@ +stages: + parse: + foreach: + - mnist + - cifar + - cifar100 + do: + cmd: python -m deckard.layers.parse --config_file ${item}.yaml --params_file ${item}.yaml + deps: + - conf/data + - conf/model + - conf/attack + - conf/scorers + - conf/files + - conf/${item}.yaml + outs: + - ${item}.yaml : + cache: True + persist: True + train: + foreach: + - mnist + - cifar + - cifar100 + do: + cmd: python -m deckard.layers.experiment train@${item} --config_file ${item}.yaml --params_file ${item}.yaml + params: + - data + - model + - scorers + - files + - def_name + - atk_name + - device_id + outs: + - ${item}/${files.model_dir}/${files.model_file}${files.model_type} + - ${item}/${files.model_dir}/${files.model_file}.optimizer${files.model_type} + metrics: + - ${item}/${files.reports}/train/${files.name}/${files.score_dict_file} + deps: + - ${item}.yaml + attack: + foreach: + - mnist + - cifar + - cifar100 + do: + cmd: python -m deckard.layers.experiment attack@${item} --config_file ${item}.yaml --params_file ${item}.yaml + params: + - data + - model + - attack + - scorers + - files + - def_name + - atk_name + - dataset + - device_id + outs: + - ${item}/${files.attack_dir}/${files.attack_file}${files.attack_type} + deps: + - ${item}/${files.model_dir}/${files.model_file}${files.model_type} + - ${item}/${files.model_dir}/${files.model_file}.optimizer${files.model_type} + metrics: + - ${item}/${files.reports}/attack/${files.name}/${files.score_dict_file} + ############################################################################## + # models: # This is a loop over the ResNet models + # matrix: + # model : [ResNet18, ResNet34, ResNet50, ResNet101, ResNet152] + # dataset : [mnist, cifar, cifar100] + # cmd: >- + # bash models.sh + # hydra.sweeper.n_jobs=8 + # ++model.init.name=torch_example.${item.model} + # ~attack + # atk_name="Benign" + # model_name=${item.model} + # dataset=${item.dataset} + # data=${item.dataset} + # stage=train + # --config-name ${item.dataset}.yaml + # deps: + # - models.sh + # - ${item.dataset}/${files.reports}/train/${files.name}/${files.score_dict_file} + # outs: + # - ${item.dataset}/${files.reports}/train/${item.model}/ + attacks: + matrix: + dataset : [mnist, cifar, cifar100] + model : [ResNet18, ResNet34, ResNet50, ResNet101, ResNet152] + cmd: >- + bash attacks.sh + ++model.init.name=torch_example.${item.model} + stage=attack + model_name=${item.model} + attack.attack_size=100 + data=torch_${item.dataset} + model=torch_${item.dataset} + +direction="[maximize,maximize,minimize]" + +optimizers="[accuracy,adv_accuracy,adv_success]" + --config-name ${item.dataset}.yaml + deps: + - ${item.dataset}/${files.reports}/attack/${files.name}/${files.score_dict_file} # This is here just to ensure it runs after the attack stage + - ${item.dataset}/${files.reports}/train/${files.name}/${files.score_dict_file} + outs: + - ${item.dataset}/logs/attack/${item.model}/: + cache: True + persist: True + compile: + matrix: + dataset : [mnist, cifar, cifar100] + stage : [attack] + cmd: python -m deckard.layers.compile --report_folder ${item.dataset}/${files.reports}/${item.stage} --results_file ${item.dataset}/${files.reports}/${item.stage}.csv + deps: + - ${item.dataset}/reports/${item.stage}/ + - ${item.dataset}/logs/${item.stage}/ + outs: + - ${item.dataset}/${files.reports}/${item.stage}.csv + prepare_plot_folder: + matrix: + dataset: [mnist, cifar, cifar100] + stage: [attack] + cmd: cp ${item.dataset}/${files.reports}/${item.stage}.csv plots/data/${item.stage}_${item.dataset}.csv + deps: + - ${item.dataset}/${files.reports}/${item.stage}.csv + outs: + - plots/data/${item.stage}_${item.dataset}.csv diff --git a/examples/pytorch/mnist/.dvc/.gitignore b/examples/pytorch/mnist/.dvc/.gitignore deleted file mode 100644 index 528f30c7..00000000 --- a/examples/pytorch/mnist/.dvc/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/config.local -/tmp -/cache diff --git a/examples/pytorch/mnist/.dvc/config b/examples/pytorch/mnist/.dvc/config deleted file mode 100644 index 4cf322d9..00000000 --- a/examples/pytorch/mnist/.dvc/config +++ /dev/null @@ -1,2 +0,0 @@ -[core] - autostage = true diff --git a/examples/pytorch/mnist/.dvcignore b/examples/pytorch/mnist/.dvcignore deleted file mode 100644 index 51973055..00000000 --- a/examples/pytorch/mnist/.dvcignore +++ /dev/null @@ -1,3 +0,0 @@ -# Add patterns of files dvc should ignore, which could improve -# the performance. Learn more at -# https://dvc.org/doc/user-guide/dvcignore diff --git a/examples/pytorch/mnist/.gitignore b/examples/pytorch/mnist/.gitignore deleted file mode 100644 index 85837977..00000000 --- a/examples/pytorch/mnist/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -mnist/* -mnist-backup/* -.dvc/* -20_epochs/* -**/optimization_results.yaml -*.rdb diff --git a/examples/pytorch/mnist/attacks.sh b/examples/pytorch/mnist/attacks.sh deleted file mode 100644 index 717f8dc4..00000000 --- a/examples/pytorch/mnist/attacks.sh +++ /dev/null @@ -1,37 +0,0 @@ -# #!/bin/bash - -# # This script is used to generate the attacks for the example. - -# Fast Gradient Method -# bash models.sh attack=default ++attack.init.name=art.attacks.evasion.FastGradientMethod ++attack.init.eps=.001,.01,.1,.5,1 ++attack.init.norm=inf,1,2 ++attack.init.eps_step=.001,.003,.01 ++attack.init.batch_size=1024 stage=attack ++hydra.sweeper.study_name=fgm ++direction=maximize $@ - -# # Projected Gradient Descent -# bash models.sh attack=default ++attack.init.name=art.attacks.evasion.ProjectedGradientDescent ++attack.init.eps=.001,.01,.1,.5,1 ++attack.init.norm=inf,1,2 ++attack.init.eps_step=.001,.003,.01 ++attack.init.batch_size=1024 ++attack.init.max_iter=10 stage=attack ++hydra.sweeper.study_name=pgd ++direction=maximize $@ - -# # DeepFool -# bash models.sh attack=default ++attack.init.name=art.attacks.evasion.DeepFool ++attack.init.max_iter=10 ++attack.init.batch_size=1024 ++attack.init.nb_grads=1,3,5,10 stage=attack ++hydra.sweeper.study_name=deep ++direction=maximize $@ - -# # HopSkipJump -# bash models.sh attack=default ++attack.init.name=art.attacks.evasion.HopSkipJump ++attack.init.max_iter=1,3,5,10 ++attack.init.init_eval=10 ++attack.init.norm=inf,2 stage=attack ++hydra.sweeper.study_name=hsj ++direction=maximize $@ - -# # ##################################################### -# # PixelAttack -# bash models.sh attack=default ++attack.init.name=art.attacks.evasion.PixelAttack ++attack.init.max_iter=10 ++attack.init.th=1,4,16,64,256 stage=attack ++hydra.sweeper.study_name=pixel ++direction=maximize $@ - -# ThresholdAttack -bash models.sh attack=default ++attack.init.name=art.attacks.evasion.ThresholdAttack ++attack.init.max_iter=10 ++attack.init.th=1,4,16,64,256 stage=attack ++hydra.sweeper.study_name=thresh ++direction=maximize $@ - -# # AdversarialPatch -# bash models.sh attack=default --attack.init.batch_size ++attack.init.name=art.attacks.evasion.AdversarialPatch ++attack.init.max_iter=10 ++attack.init.learning_rate=.5,5.0,50.0 stage=patch ++hydra.sweeper.study_name=attack ++direction=maximize ++attack.init.patch_shape=[1,28,28] $@ -##################################################### - -# # Carlini L0 Method -# bash models.sh attack=default ++attack.init.name=art.attacks.evasion.CarliniL0Method ++attack.init.batch_size=1024 ++attack.init.max_iter=10 ++attack.init.confidence=.1,.9,.99 stage=cw0 ++hydra.sweeper.study_name=cw0 ++direction=maximize $@ - -# # Carlini L2 Method -# bash models.sh attack=default ++attack.init.name=art.attacks.evasion.CarliniL2Method ++attack.init.batch_size=1024 ++attack.init.max_iter=10 ++attack.init.confidence=.1,.9,.99 stage=cw2 ++hydra.sweeper.study_name=cw2 ++direction=maximize $@ - -# # Carlini LInf Method -# bash models.sh attack=default ++attack.init.name=art.attacks.evasion.CarliniLInfMethod ++attack.init.max_iter=10 ++attack.init.confidence=.1,.9,.99 stage=attack ++hydra.sweeper.study_name=cwinf ++direction=maximize $@ - -rm -rf output/models/* diff --git a/examples/pytorch/mnist/average_across_random_states.ipynb b/examples/pytorch/mnist/average_across_random_states.ipynb deleted file mode 100644 index eca5c655..00000000 --- a/examples/pytorch/mnist/average_across_random_states.ipynb +++ /dev/null @@ -1,381 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "from pathlib import Path\n", - "import json\n", - "import logging\n", - "from tqdm import tqdm\n", - "import yaml\n", - "import numpy as np\n", - "from math import isnan\n", - "from deckard.layers.utils import deckard_nones as nones\n", - "\n", - "from deckard.layers.compile import parse_results\n", - "\n", - "\n", - "import pandas as pd\n", - "import seaborn as sns\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "from pathlib import Path\n", - "from paretoset import paretoset" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "ename": "FileNotFoundError", - "evalue": "[Errno 2] No such file or directory: 'mnist/reports/attack.csv'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32m/home/cmeyers/deckard/examples/pytorch/mnist/average_across_random_states.ipynb Cell 2\u001b[0m line \u001b[0;36m5\n\u001b[1;32m 2\u001b[0m data_file \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mmnist/reports/attack.csv\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 4\u001b[0m \u001b[39m# Read data\u001b[39;00m\n\u001b[0;32m----> 5\u001b[0m df \u001b[39m=\u001b[39m pd\u001b[39m.\u001b[39;49mread_csv(data_file)\n\u001b[1;32m 6\u001b[0m df\u001b[39m.\u001b[39mhead()\n", - "File \u001b[0;32m~/deckard/env/lib/python3.8/site-packages/pandas/io/parsers/readers.py:912\u001b[0m, in \u001b[0;36mread_csv\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, date_format, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options, dtype_backend)\u001b[0m\n\u001b[1;32m 899\u001b[0m kwds_defaults \u001b[39m=\u001b[39m _refine_defaults_read(\n\u001b[1;32m 900\u001b[0m dialect,\n\u001b[1;32m 901\u001b[0m delimiter,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 908\u001b[0m dtype_backend\u001b[39m=\u001b[39mdtype_backend,\n\u001b[1;32m 909\u001b[0m )\n\u001b[1;32m 910\u001b[0m kwds\u001b[39m.\u001b[39mupdate(kwds_defaults)\n\u001b[0;32m--> 912\u001b[0m \u001b[39mreturn\u001b[39;00m _read(filepath_or_buffer, kwds)\n", - "File \u001b[0;32m~/deckard/env/lib/python3.8/site-packages/pandas/io/parsers/readers.py:577\u001b[0m, in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m 574\u001b[0m _validate_names(kwds\u001b[39m.\u001b[39mget(\u001b[39m\"\u001b[39m\u001b[39mnames\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39mNone\u001b[39;00m))\n\u001b[1;32m 576\u001b[0m \u001b[39m# Create the parser.\u001b[39;00m\n\u001b[0;32m--> 577\u001b[0m parser \u001b[39m=\u001b[39m TextFileReader(filepath_or_buffer, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwds)\n\u001b[1;32m 579\u001b[0m \u001b[39mif\u001b[39;00m chunksize \u001b[39mor\u001b[39;00m iterator:\n\u001b[1;32m 580\u001b[0m \u001b[39mreturn\u001b[39;00m parser\n", - "File \u001b[0;32m~/deckard/env/lib/python3.8/site-packages/pandas/io/parsers/readers.py:1407\u001b[0m, in \u001b[0;36mTextFileReader.__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m 1404\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39moptions[\u001b[39m\"\u001b[39m\u001b[39mhas_index_names\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m kwds[\u001b[39m\"\u001b[39m\u001b[39mhas_index_names\u001b[39m\u001b[39m\"\u001b[39m]\n\u001b[1;32m 1406\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mhandles: IOHandles \u001b[39m|\u001b[39m \u001b[39mNone\u001b[39;00m \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[0;32m-> 1407\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_engine \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_make_engine(f, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mengine)\n", - "File \u001b[0;32m~/deckard/env/lib/python3.8/site-packages/pandas/io/parsers/readers.py:1661\u001b[0m, in \u001b[0;36mTextFileReader._make_engine\u001b[0;34m(self, f, engine)\u001b[0m\n\u001b[1;32m 1659\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39m\"\u001b[39m\u001b[39mb\u001b[39m\u001b[39m\"\u001b[39m \u001b[39mnot\u001b[39;00m \u001b[39min\u001b[39;00m mode:\n\u001b[1;32m 1660\u001b[0m mode \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mb\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m-> 1661\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mhandles \u001b[39m=\u001b[39m get_handle(\n\u001b[1;32m 1662\u001b[0m f,\n\u001b[1;32m 1663\u001b[0m mode,\n\u001b[1;32m 1664\u001b[0m encoding\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49moptions\u001b[39m.\u001b[39;49mget(\u001b[39m\"\u001b[39;49m\u001b[39mencoding\u001b[39;49m\u001b[39m\"\u001b[39;49m, \u001b[39mNone\u001b[39;49;00m),\n\u001b[1;32m 1665\u001b[0m compression\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49moptions\u001b[39m.\u001b[39;49mget(\u001b[39m\"\u001b[39;49m\u001b[39mcompression\u001b[39;49m\u001b[39m\"\u001b[39;49m, \u001b[39mNone\u001b[39;49;00m),\n\u001b[1;32m 1666\u001b[0m memory_map\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49moptions\u001b[39m.\u001b[39;49mget(\u001b[39m\"\u001b[39;49m\u001b[39mmemory_map\u001b[39;49m\u001b[39m\"\u001b[39;49m, \u001b[39mFalse\u001b[39;49;00m),\n\u001b[1;32m 1667\u001b[0m is_text\u001b[39m=\u001b[39;49mis_text,\n\u001b[1;32m 1668\u001b[0m errors\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49moptions\u001b[39m.\u001b[39;49mget(\u001b[39m\"\u001b[39;49m\u001b[39mencoding_errors\u001b[39;49m\u001b[39m\"\u001b[39;49m, \u001b[39m\"\u001b[39;49m\u001b[39mstrict\u001b[39;49m\u001b[39m\"\u001b[39;49m),\n\u001b[1;32m 1669\u001b[0m storage_options\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49moptions\u001b[39m.\u001b[39;49mget(\u001b[39m\"\u001b[39;49m\u001b[39mstorage_options\u001b[39;49m\u001b[39m\"\u001b[39;49m, \u001b[39mNone\u001b[39;49;00m),\n\u001b[1;32m 1670\u001b[0m )\n\u001b[1;32m 1671\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mhandles \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m 1672\u001b[0m f \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mhandles\u001b[39m.\u001b[39mhandle\n", - "File \u001b[0;32m~/deckard/env/lib/python3.8/site-packages/pandas/io/common.py:859\u001b[0m, in \u001b[0;36mget_handle\u001b[0;34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001b[0m\n\u001b[1;32m 854\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39misinstance\u001b[39m(handle, \u001b[39mstr\u001b[39m):\n\u001b[1;32m 855\u001b[0m \u001b[39m# Check whether the filename is to be opened in binary mode.\u001b[39;00m\n\u001b[1;32m 856\u001b[0m \u001b[39m# Binary mode does not support 'encoding' and 'newline'.\u001b[39;00m\n\u001b[1;32m 857\u001b[0m \u001b[39mif\u001b[39;00m ioargs\u001b[39m.\u001b[39mencoding \u001b[39mand\u001b[39;00m \u001b[39m\"\u001b[39m\u001b[39mb\u001b[39m\u001b[39m\"\u001b[39m \u001b[39mnot\u001b[39;00m \u001b[39min\u001b[39;00m ioargs\u001b[39m.\u001b[39mmode:\n\u001b[1;32m 858\u001b[0m \u001b[39m# Encoding\u001b[39;00m\n\u001b[0;32m--> 859\u001b[0m handle \u001b[39m=\u001b[39m \u001b[39mopen\u001b[39;49m(\n\u001b[1;32m 860\u001b[0m handle,\n\u001b[1;32m 861\u001b[0m ioargs\u001b[39m.\u001b[39;49mmode,\n\u001b[1;32m 862\u001b[0m encoding\u001b[39m=\u001b[39;49mioargs\u001b[39m.\u001b[39;49mencoding,\n\u001b[1;32m 863\u001b[0m errors\u001b[39m=\u001b[39;49merrors,\n\u001b[1;32m 864\u001b[0m newline\u001b[39m=\u001b[39;49m\u001b[39m\"\u001b[39;49m\u001b[39m\"\u001b[39;49m,\n\u001b[1;32m 865\u001b[0m )\n\u001b[1;32m 866\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 867\u001b[0m \u001b[39m# Binary mode\u001b[39;00m\n\u001b[1;32m 868\u001b[0m handle \u001b[39m=\u001b[39m \u001b[39mopen\u001b[39m(handle, ioargs\u001b[39m.\u001b[39mmode)\n", - "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'mnist/reports/attack.csv'" - ] - } - ], - "source": [ - "# Compiled data file\n", - "data_file = \"mnist/reports/attack.csv\"\n", - "\n", - "# Read data\n", - "df = pd.read_csv(data_file)\n", - "df.head()" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "ename": "AttributeError", - "evalue": "'DataFrame' object has no attribute 'model_layers'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m/tmp/ipykernel_894716/4216344897.py\u001b[0m in \u001b[0;36m?\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlayers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel_layers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munique\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0mlayers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msort\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0mepochs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"epochs\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munique\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msort\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/deckard/env/lib/python3.8/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m?\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m 5985\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_accessors\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5986\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_info_axis\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_can_hold_identifiers_and_holds_name\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5987\u001b[0m ):\n\u001b[1;32m 5988\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 5989\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mobject\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getattribute__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;31mAttributeError\u001b[0m: 'DataFrame' object has no attribute 'model_layers'" - ] - } - ], - "source": [ - "layers = [int(x) for x in df.model_layers.unique()]\n", - "layers.sort()\n", - "epochs = df[\"epochs\"].unique()\n", - "epochs.sort()\n", - "attacks = df.atk_gen.unique()\n", - "attacks.sort()\n", - "defenses = df.def_gen.unique()\n", - "defenses.sort()\n", - "\n", - "print(\n", - " f\"Number of models: {len(layers)}\\n\"\n", - " f\"Layers: {layers}\\n\"\n", - " f\"Number of epochs: {len(epochs)}\\n\"\n", - " f\"Epochs: {epochs}\\n\"\n", - " f\"Number of attacks: {len(attacks)}\\n\"\n", - " f\"Attacks: {attacks}\\n\"\n", - " f\"Number of defenses: {len(defenses)}\\n\"\n", - " f\"Defenses: {defenses}\\n\"\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "ename": "KeyError", - "evalue": "'def_param'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32m/home/cmeyers/deckard/examples/pytorch/mnist/average_across_random_states.ipynb Cell 5\u001b[0m line \u001b[0;36m6\n\u001b[1;32m 63\u001b[0m \u001b[39m# sub_df = df[[*sense_dict.keys()]]\u001b[39;00m\n\u001b[1;32m 64\u001b[0m \u001b[39m# bools = paretoset(sub_df, list(sense_dict.values()))\u001b[39;00m\n\u001b[1;32m 65\u001b[0m \u001b[39m# df = df[bools]\u001b[39;00m\n\u001b[1;32m 66\u001b[0m \u001b[39mreturn\u001b[39;00m df\n\u001b[0;32m---> 69\u001b[0m df \u001b[39m=\u001b[39m find_pareto_set_for_graph(df, sense_dict)\n\u001b[1;32m 72\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mdrop_col_if_no_variance\u001b[39m(df):\n\u001b[1;32m 73\u001b[0m drop_these \u001b[39m=\u001b[39m []\n", - "\u001b[1;32m/home/cmeyers/deckard/examples/pytorch/mnist/average_across_random_states.ipynb Cell 5\u001b[0m line \u001b[0;36m5\n\u001b[1;32m 53\u001b[0m \u001b[39mcontinue\u001b[39;00m\n\u001b[1;32m 54\u001b[0m \u001b[39mfor\u001b[39;00m scorer \u001b[39min\u001b[39;00m scorers:\n\u001b[1;32m 55\u001b[0m scores \u001b[39m=\u001b[39m df[scorer]\u001b[39m.\u001b[39mfillna(\n\u001b[0;32m---> 56\u001b[0m df\u001b[39m.\u001b[39;49mgroupby(group_list_wo_attack)[scorer]\u001b[39m.\u001b[39mtransform(\u001b[39m\"\u001b[39m\u001b[39mmean\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 57\u001b[0m )\n\u001b[1;32m 58\u001b[0m df[scorer] \u001b[39m=\u001b[39m scores\u001b[39m.\u001b[39mfillna(scores\u001b[39m.\u001b[39mmean())\n\u001b[1;32m 59\u001b[0m df \u001b[39m=\u001b[39m average_across_random_states(df, scorer, sense_dict)\n", - "File \u001b[0;32m~/deckard/env/lib/python3.8/site-packages/pandas/core/frame.py:8252\u001b[0m, in \u001b[0;36mDataFrame.groupby\u001b[0;34m(self, by, axis, level, as_index, sort, group_keys, observed, dropna)\u001b[0m\n\u001b[1;32m 8249\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mTypeError\u001b[39;00m(\u001b[39m\"\u001b[39m\u001b[39mYou have to supply one of \u001b[39m\u001b[39m'\u001b[39m\u001b[39mby\u001b[39m\u001b[39m'\u001b[39m\u001b[39m and \u001b[39m\u001b[39m'\u001b[39m\u001b[39mlevel\u001b[39m\u001b[39m'\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 8250\u001b[0m axis \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_get_axis_number(axis)\n\u001b[0;32m-> 8252\u001b[0m \u001b[39mreturn\u001b[39;00m DataFrameGroupBy(\n\u001b[1;32m 8253\u001b[0m obj\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m,\n\u001b[1;32m 8254\u001b[0m keys\u001b[39m=\u001b[39;49mby,\n\u001b[1;32m 8255\u001b[0m axis\u001b[39m=\u001b[39;49maxis,\n\u001b[1;32m 8256\u001b[0m level\u001b[39m=\u001b[39;49mlevel,\n\u001b[1;32m 8257\u001b[0m as_index\u001b[39m=\u001b[39;49mas_index,\n\u001b[1;32m 8258\u001b[0m sort\u001b[39m=\u001b[39;49msort,\n\u001b[1;32m 8259\u001b[0m group_keys\u001b[39m=\u001b[39;49mgroup_keys,\n\u001b[1;32m 8260\u001b[0m observed\u001b[39m=\u001b[39;49mobserved,\n\u001b[1;32m 8261\u001b[0m dropna\u001b[39m=\u001b[39;49mdropna,\n\u001b[1;32m 8262\u001b[0m )\n", - "File \u001b[0;32m~/deckard/env/lib/python3.8/site-packages/pandas/core/groupby/groupby.py:931\u001b[0m, in \u001b[0;36mGroupBy.__init__\u001b[0;34m(self, obj, keys, axis, level, grouper, exclusions, selection, as_index, sort, group_keys, observed, dropna)\u001b[0m\n\u001b[1;32m 928\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdropna \u001b[39m=\u001b[39m dropna\n\u001b[1;32m 930\u001b[0m \u001b[39mif\u001b[39;00m grouper \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m--> 931\u001b[0m grouper, exclusions, obj \u001b[39m=\u001b[39m get_grouper(\n\u001b[1;32m 932\u001b[0m obj,\n\u001b[1;32m 933\u001b[0m keys,\n\u001b[1;32m 934\u001b[0m axis\u001b[39m=\u001b[39;49maxis,\n\u001b[1;32m 935\u001b[0m level\u001b[39m=\u001b[39;49mlevel,\n\u001b[1;32m 936\u001b[0m sort\u001b[39m=\u001b[39;49msort,\n\u001b[1;32m 937\u001b[0m observed\u001b[39m=\u001b[39;49mobserved,\n\u001b[1;32m 938\u001b[0m dropna\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mdropna,\n\u001b[1;32m 939\u001b[0m )\n\u001b[1;32m 941\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mobj \u001b[39m=\u001b[39m obj\n\u001b[1;32m 942\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39maxis \u001b[39m=\u001b[39m obj\u001b[39m.\u001b[39m_get_axis_number(axis)\n", - "File \u001b[0;32m~/deckard/env/lib/python3.8/site-packages/pandas/core/groupby/grouper.py:985\u001b[0m, in \u001b[0;36mget_grouper\u001b[0;34m(obj, key, axis, level, sort, observed, validate, dropna)\u001b[0m\n\u001b[1;32m 983\u001b[0m in_axis, level, gpr \u001b[39m=\u001b[39m \u001b[39mFalse\u001b[39;00m, gpr, \u001b[39mNone\u001b[39;00m\n\u001b[1;32m 984\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m--> 985\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mKeyError\u001b[39;00m(gpr)\n\u001b[1;32m 986\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39misinstance\u001b[39m(gpr, Grouper) \u001b[39mand\u001b[39;00m gpr\u001b[39m.\u001b[39mkey \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m 987\u001b[0m \u001b[39m# Add key to exclusions\u001b[39;00m\n\u001b[1;32m 988\u001b[0m exclusions\u001b[39m.\u001b[39madd(gpr\u001b[39m.\u001b[39mkey)\n", - "\u001b[0;31mKeyError\u001b[0m: 'def_param'" - ] - } - ], - "source": [ - "sense_dict = {\n", - " \"model_layers\": \"diff\",\n", - " \"accuracy\": \"max\",\n", - " \"data.sample.random_state\": \"diff\",\n", - " \"epochs\": \"diff\",\n", - " \"model_layers\": \"diff\",\n", - " \"atk_gen\": \"diff\",\n", - " \"def_gen\": \"diff\",\n", - " \"def_param\": \"diff\",\n", - " \"atk_param\": \"diff\",\n", - " \"adv_fit_time\": \"min\",\n", - " \"adv_accuracy\": \"min\",\n", - " \"predict_time\": \"min\",\n", - " \"train_time\": \"min\",\n", - " \"attack.attack_size\": \"diff\",\n", - "}\n", - "\n", - "# Average across random states\n", - "scorer = \"accuracy\"\n", - "\n", - "\n", - "def average_across_random_states(df, scorer, sense_dict):\n", - " sense_dict.pop(\"data.sample.random_state\", None)\n", - " group_list = [k for k, v in sense_dict.items() if v == \"diff\"]\n", - " group_list_wo_random_state = group_list.copy()\n", - " print(f\"Grouping by {group_list_wo_random_state} for {scorer}\")\n", - " df[f\"mean_{scorer}\"] = df.groupby(group_list_wo_random_state)[scorer].transform(\n", - " \"mean\"\n", - " )\n", - " return df\n", - "\n", - "\n", - "def drop_poorly_merged_columns(df):\n", - " cols = df.columns\n", - " for col in cols:\n", - " if col.endswith(\".1\") and col[:-2] in cols:\n", - " df = df.drop(col, axis=1)\n", - " return df\n", - "\n", - "\n", - "def find_pareto_set_for_graph(df, sense_dict):\n", - " scorers = [k for k, v in sense_dict.items() if v in [\"max\", \"min\"]]\n", - " group_list = [k for k, v in sense_dict.items() if v == \"diff\"]\n", - " group_list_wo_attack = group_list.copy()\n", - " for group in group_list:\n", - " if group in [\"atk_gen\", \"atk_value\", \"atk_param\"]:\n", - " group_list_wo_attack.remove(group)\n", - " elif group.startswith(\"attack_\") or group.startswith(\"attack_\"):\n", - " group_list_wo_attack.remove(group)\n", - " elif group.startswith(\"adv.\") or group.startswith(\"adv_\"):\n", - " group_list_wo_attack.remove(group)\n", - " else:\n", - " continue\n", - " for scorer in scorers:\n", - " scores = df[scorer].fillna(\n", - " df.groupby(group_list_wo_attack)[scorer].transform(\"mean\")\n", - " )\n", - " df[scorer] = scores.fillna(scores.mean())\n", - " df = average_across_random_states(df, scorer, sense_dict)\n", - " value = sense_dict.get(scorer)\n", - " sense_dict.update({f\"mean_{scorer}\": value})\n", - " del sense_dict[scorer]\n", - " # sub_df = df[[*sense_dict.keys()]]\n", - " # bools = paretoset(sub_df, list(sense_dict.values()))\n", - " # df = df[bools]\n", - " return df\n", - "\n", - "\n", - "df = find_pareto_set_for_graph(df, sense_dict)\n", - "\n", - "\n", - "def drop_col_if_no_variance(df):\n", - " drop_these = []\n", - " for col in df.columns:\n", - " if df[col].nunique() == 1:\n", - " drop_these.append(col)\n", - " tmp = df.drop(drop_these, axis=1)\n", - " return tmp\n", - "\n", - "\n", - "df = drop_poorly_merged_columns(df)\n", - "\n", - "df" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "sns.lineplot(data=df, y=\"adv_log_loss\", x=\"model.trainer.nb_epoch\", hue=\"model_layers\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from lifelines import (\n", - " CoxPHFitter,\n", - " KaplanMeierFitter,\n", - " NelsonAalenFitter,\n", - " AalenAdditiveFitter,\n", - " WeibullAFTFitter,\n", - " LogNormalAFTFitter,\n", - " LogLogisticAFTFitter,\n", - " PiecewiseExponentialRegressionFitter,\n", - ")\n", - "\n", - "\n", - "model_dict = {\n", - " \"cox\": CoxPHFitter,\n", - " # \"kaplan_meier\" : KaplanMeierFitter,\n", - " # \"nelson_aalen\" : NelsonAalenFitter,\n", - " # \"aalen_additive\" : AalenAdditiveFitter,\n", - " \"weibull\": WeibullAFTFitter,\n", - " \"log_normal\": LogNormalAFTFitter,\n", - " \"log_logistic\": LogLogisticAFTFitter,\n", - " # \"piecewise_exponential\" : PiecewiseExponentialRegressionFitter,\n", - "}\n", - "\n", - "\n", - "def fit_aft_model(df, sense_dict, model_name):\n", - " stratify = [\n", - " \"atk_gen\",\n", - " \"def_gen\",\n", - " ]\n", - " subset_df = df.copy()\n", - " subset_df = subset_df.drop(stratify, axis=1)\n", - " model = model_dict[model_name]()\n", - " model.fit(df, duration_col=\"mean_adv_fit_time\", event_col=\"adv_failures\")\n", - " model.print_summary()\n", - " plot = model.plot()\n", - " concordance = model.score(df, scoring_method=\"concordance_index\")\n", - " print(f\"Concordance index: {concordance}\")\n", - " measured_median = np.median(\n", - " df.mean_adv_fit_time / df[\"attack.attack_size\"] * ((1 - df.adv_failures) / 100)\n", - " )\n", - " print(\"Measured median attack time:\", measured_median)\n", - " modelled_median = np.median(model.predict_median(df, ancillary=df))\n", - " print(\"Predicted median attack time:\", modelled_median)\n", - " score = model.score(df, scoring_method=\"log_likelihood\")\n", - " score_dict = {\n", - " \"model\": model_name,\n", - " \"concordance\": concordance,\n", - " \"measured_median\": measured_median,\n", - " \"modelled_median\": modelled_median,\n", - " \"log_likelihood\": score,\n", - " }\n", - " return model, plot, score\n", - "\n", - "\n", - "models = {}\n", - "scores = {}\n", - "plots = {}\n", - "stratify = [\"atk_gen\", \"def_gen\"]\n", - "subset_cols = [k for k in sense_dict if k not in stratify]\n", - "aft_df = df[subset_cols].copy()\n", - "aft_df[\"adv_failures\"] = (1 - df[\"mean_adv_accuracy\"]) * df[\"attack.attack_size\"]\n", - "del aft_df[\"mean_adv_accuracy\"]\n", - "new_sense_dict = sense_dict.copy()\n", - "new_sense_dict.update({\"adv_failures\": sense_dict[\"mean_adv_accuracy\"]})\n", - "new_sense_dict.pop(\"mean_adv_accuracy\", None)\n", - "new_sense_dict\n", - "\n", - "for model_name in model_dict:\n", - " print(f\"Fitting {model_name} model\")\n", - " model, plot, score = fit_aft_model(aft_df, new_sense_dict, model_name)\n", - " models.update({model_name: model})\n", - " scores.update({model_name: score})\n", - " plots.update({model_name: plot})\n", - " plt.xscale(\"linear\")\n", - " plt.show()\n", - " plt.gcf().clear()\n", - "\n", - "# scores = pd.DataFrame.from_dict(scores, orient='index', columns=['score'])\n", - "\n", - "# covariates = [k for k,v in sense_dict.items() if v == 'diff']\n", - "# values = [np.array(df[k].unique()) for k in covariates]\n", - "# print(f\"Number of covariates: {len(covariates)}\")\n", - "# print(f\"Number of values: {len(values)}\")\n", - "# print(f\"Values: \\n{[value.tolist() for value in values]}\")\n", - "# for i in range(len(covariates)):\n", - "# if covariates[i] in stratify:\n", - "# continue\n", - "# else:\n", - "# print(f\"Plotting {covariates[i]} with values {values[i]}\")\n", - "# graph = model.plot_partial_effects_on_outcome(covariates = covariates[i], values =values[i], cmap='coolwarm', figsize=(10, 10))\n", - "# print(type(graph))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "model = models[\"weibull\"]\n", - "expectations = model.predict_expectation(df, ancillary=df)\n", - "survival_function = model.predict_survival_function(df, ancillary=df)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "scores.T" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "env", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.8" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/pytorch/mnist/conf/afr.yaml b/examples/pytorch/mnist/conf/afr.yaml deleted file mode 100644 index 3585a28e..00000000 --- a/examples/pytorch/mnist/conf/afr.yaml +++ /dev/null @@ -1,183 +0,0 @@ -covariates: - - "adv_fit_time" - - "accuracy" - - "train_time" - - "atk_value" - - "def_value" - - "data.sample.random_state" - - model.trainer.nb_epoch - - "model_layers" -# - atk_gen -# - def_gen - - predict_time -fillna: - model.trainer.nb_epoch: 20 -weibull: - plot: - file : weibull_aft.eps - title : Weibull AFR Model - labels: - "Intercept: rho_": "$\\rho$" - "Intercept: lambda_": "$\\lambda$" - "data.sample.random_state: lambda_": "Random State" - "atk_value: lambda_": "Attack Strength" - "train_time: lambda_": "$t_{train}$" - "predict_proba_time: lambda_": "$t_{predict}$" - "adv_accuracy: lambda_": "Adv. Accuracy" - "accuracy: lambda_": "Ben. Accuracy" - "adv_fit_time: lambda_": "$t_{attack}$" - "adv_failure_rate: lambda_": "$f_{adv.}(t;\\theta)$" - "failure_rate: lambda_": "$f_{ben.}(t;\\theta)$" - "model.trainer.nb_epoch: lambda_": "No. of Epochs" - "model.trainer.batch_size: lambda_": "Batch Size" - "def_gen": "Defence" - "model_layers: lambda_" : "Layers" - "def_value: lambda_" : "Defence Strength" - "predict_time: lambda_" : "$t_{predict}$" - partial_effect: - - "file": "weibull_epochs_partial_effect.eps" - "covariate_array": "model.trainer.nb_epoch" - "values_array": [1,10,25,50] - "title": "$S(t)$ for Weibull AFR" - "ylabel": "Expectation of $S(t)$" - "xlabel": "Time $T$ (seconds)" - "legend_kwargs": { - "title": "Epochs", - "labels": ["1", "10", "25", "50"] - } - - "file": "weibull_layers_partial_effect.eps" - "covariate_array": "model_layers" - "values_array": [18, 34, 50, 101, 152] - "title": "$S(t)$ for Weibull AFR" - "ylabel": "Expectation of $S(t)$" - "xlabel": "Time $T$ (seconds)" - "legend_kwargs": { - "title": "Layers", - "labels": ["18", "34", "50", "101", "152"] - } -# cox: -# plot: -# file : cox_aft.eps -# title : Cox AFR Model -# labels: -# "Intercept: rho_": "$\\rho$" -# "Intercept: lambda_": "$\\lambda$" -# "data.sample.random_state: lambda_": "Random State" -# "atk_value: lambda_": "Attack Strength" -# "train_time: lambda_": "$t_{train}$" -# "predict_proba_time: lambda_": "$t_{predict}$" -# "adv_accuracy: lambda_": "Adv. Accuracy" -# "accuracy: lambda_": "Ben. Accuracy" -# "adv_fit_time: lambda_": "$t_{attack}$" -# "adv_failure_rate: lambda_": "$f_{adv.}(t;\\theta)$" -# "failure_rate: lambda_": "$f_{ben.}(t;\\theta)$" -# "model.trainer.nb_epoch: lambda_": "No. of Epochs" -# "model.trainer.batch_size: lambda_": "Batch Size" -# "def_gen": "Defence" -# partial_effect: -# - "file": "cox_epochs_partial_effect.eps" -# "covariate_array": "model.trainer.nb_epoch" -# "values_array": [1,10,25,50] -# "title": "$S(t)$ for Cox AFR" -# "ylabel": "Expectation of $S(t)$" -# "xlabel": "Time $T$ (seconds)" -# "legend_kwargs": { -# "title": "Epochs", -# "labels": ["1", "10", "25", "50"] -# } -# - "file": "cox_layers_partial_effect.eps" -# "covariate_array": "model_layers" -# "values_array": [18, 34, 50, 101, 152] -# "title": "$S(t)$ for Cox AFR" -# "ylabel": "Expectation of $S(t)$" -# "xlabel": "Time $T$ (seconds)" -# "legend_kwargs": { -# "title": "Layers", -# "labels": ["18", "34", "50", "101", "152"] -# } -log_logistic: - plot: - file : log_logistic_aft.eps - title : Log logistic AFR Model - labels: - "Intercept: beta_": "$\\beta$" - "Intercept: alpha_": "$\\alpha$" - "data.sample.random_state: alpha_": "Random State" - "atk_value: alpha_": "Attack Strength" - "train_time: alpha_": "$t_{train}$" - "predict_proba_time: alpha_": "$t_{predict}$" - "adv_accuracy: alpha_": "Adv. Accuracy" - "accuracy: alpha_": "Ben. Accuracy" - "adv_fit_time: alpha_": "$t_{attack}$" - "adv_failure_rate: alpha_": "$f_{adv.}(t;\\theta)$" - "failure_rate: alpha_": "$f_{ben.}(t;\\theta)$" - "model.trainer.nb_epoch: alpha_": "No. of Epochs" - "model.trainer.batch_size: alpha_": "Batch Size" - "def_gen": "Defence" - "model_layers: alpha_" : "Layers" - "def_value: alpha_" : "Defence Strength" - "predict_time: alpha_" : "$t_{predict}$" - partial_effect: - - "file": "log_logistic_epochs_partial_effect.eps" - "covariate_array": "model.trainer.nb_epoch" - "values_array": [1,10,25,50] - "title": "$S(t)$ for Log-Logistic AFR" - "ylabel": "Expectation of $S(t)$" - "xlabel": "Time $T$ (seconds)" - "legend_kwargs": { - "title": "Epochs", - "labels": ["1", "10", "25", "50"] - } - - "file": "log_logistic_layers_partial_effect.eps" - "covariate_array": "model_layers" - "values_array": [18, 34, 50, 101, 152] - "title": "$S(t)$ for Log Logistic AFR" - "ylabel": "Expectation of $S(t)$" - "xlabel": "Time $T$ (seconds)" - "legend_kwargs": { - "title": "Layers", - "labels": ["18", "34", "50", "101", "152"] - } -log_normal: - plot: - file : log_normal_aft.eps - title : Log Normal AFR Model - labels: - "Intercept: sigma_": "$\\rho$" - "Intercept: mu_": "$\\mu$" - "data.sample.random_state: mu_": "Random State" - "atk_value: mu_": "Attack Strength" - "train_time: mu_": "$t_{train}$" - "predict_proba_time: mu_": "$t_{predict}$" - "adv_accuracy: mu_": "Adv. Accuracy" - "accuracy: mu_": "Ben. Accuracy" - "adv_fit_time: mu_": "$t_{attack}$" - "adv_failure_rate: mu_": "$f_{adv.}(t;\\theta)$" - "failure_rate: mu_": "$f_{ben.}(t;\\theta)$" - "model.trainer.nb_epoch: mu_": "No. of Epochs" - "model.trainer.batch_size: mu_": "Batch Size" - "def_gen": "Defence" - "model_layers: mu_" : "Layers" - "def_value: mu_" : "Defence Strength" - "predict_time: mu_" : "$t_{predict}$" - partial_effect: - - "file": "log_normal_epochs_partial_effect.eps" - "covariate_array": "model.trainer.nb_epoch" - "values_array": [1,10,25,50] - "title": "$S(t)$ for Log-Normal AFR" - "ylabel": "Expectation of $S(t)$" - "xlabel": "Time $T$ (seconds)" - "legend_kwargs": { - "title": "Epochs", - "labels": ["1", "10", "25", "50"] - } - - "file": "log_normal_layers_partial_effect.eps" - "covariate_array": "model_layers" - "values_array": [18, 34, 50, 101, 152] - "title": "$S(t)$ for Log Normal AFR" - "ylabel": "Expectation of $S(t)$" - "xlabel": "Time $T$ (seconds)" - "legend_kwargs": { - "title": "Layers", - "labels": ["18", "34", "50", "101", "152"] - } diff --git a/examples/pytorch/mnist/conf/attack/default.yaml b/examples/pytorch/mnist/conf/attack/default.yaml deleted file mode 100644 index 76b4d62d..00000000 --- a/examples/pytorch/mnist/conf/attack/default.yaml +++ /dev/null @@ -1,9 +0,0 @@ -data: ${data} -model: ${model} -_target_ : deckard.base.attack.Attack -init: - model: ${model} - _target_: deckard.base.attack.AttackInitializer - name: art.attacks.evasion.HopSkipJump -attack_size : 10 -method : evasion diff --git a/examples/pytorch/mnist/conf/data/default.yaml b/examples/pytorch/mnist/conf/data/default.yaml deleted file mode 100644 index 5eb78278..00000000 --- a/examples/pytorch/mnist/conf/data/default.yaml +++ /dev/null @@ -1,2 +0,0 @@ -defaults: - - torch_mnist diff --git a/examples/pytorch/mnist/conf/data/torch_cifar.yaml b/examples/pytorch/mnist/conf/data/torch_cifar.yaml deleted file mode 100644 index b7603125..00000000 --- a/examples/pytorch/mnist/conf/data/torch_cifar.yaml +++ /dev/null @@ -1,11 +0,0 @@ -_target_: deckard.base.data.Data -generate: - name: torch_cifar10 -sample: - random_state : 0 - stratify: True -sklearn_pipeline: - preprocessor: - name: sklearn.preprocessing.StandardScaler - with_mean: True - with_std: True diff --git a/examples/pytorch/mnist/conf/data/torch_mnist.yaml b/examples/pytorch/mnist/conf/data/torch_mnist.yaml deleted file mode 100644 index 9d79c036..00000000 --- a/examples/pytorch/mnist/conf/data/torch_mnist.yaml +++ /dev/null @@ -1,15 +0,0 @@ -_target_: deckard.base.data.Data -generate: - _target_: deckard.base.data.generator.DataGenerator - name: torch_mnist -sample: - _target_: deckard.base.data.sampler.SklearnDataSampler - random_state : 0 - stratify: True -sklearn_pipeline: - _target_: deckard.base.data.sklearn_pipeline.SklearnDataPipeline - preprocessor: - - name: sklearn.preprocessing.StandardScaler - with_mean: True - with_std: True diff --git a/examples/pytorch/mnist/conf/deploy/pod.yaml b/examples/pytorch/mnist/conf/deploy/pod.yaml deleted file mode 100644 index fc68306a..00000000 --- a/examples/pytorch/mnist/conf/deploy/pod.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: deckard -spec: - containers: - - name: deckard - image: ghcr.io/simplymathematics/deckard:main - imagePullPolicy: Always - workingDir: /deckard/examples/pytorch - args: ["python", "-m", "dvc", "repro"] - env: - - name: REDIS_HOST - value: "redis" - - name: REDIS_PORT - value: "6379" - - name: REDIS_DB - value: "0" - - name: REDIS_PASSWORD - value: "" - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - mountPath: /data - name: mypvc - volumes: - - name: mypvc - persistentVolumeClaim: - claimName: podpvc diff --git a/examples/pytorch/mnist/conf/deploy/pvc.yaml b/examples/pytorch/mnist/conf/deploy/pvc.yaml deleted file mode 100644 index a7deee4d..00000000 --- a/examples/pytorch/mnist/conf/deploy/pvc.yaml +++ /dev/null @@ -1,11 +0,0 @@ -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: podpvc -spec: - accessModes: - - ReadWriteMany - storageClassName: filestore-sc - resources: - requests: - storage: 256Gi diff --git a/examples/pytorch/mnist/conf/deploy/sclass.yaml b/examples/pytorch/mnist/conf/deploy/sclass.yaml deleted file mode 100644 index d566656c..00000000 --- a/examples/pytorch/mnist/conf/deploy/sclass.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: filestore-sc -provisioner: filestore.csi.storage.gke.io -volumeBindingMode: Immediate -allowVolumeExpansion: true -parameters: - tier: standard - network: default diff --git a/examples/pytorch/mnist/conf/files/cifar.yaml b/examples/pytorch/mnist/conf/files/cifar.yaml deleted file mode 100644 index e43cab72..00000000 --- a/examples/pytorch/mnist/conf/files/cifar.yaml +++ /dev/null @@ -1,21 +0,0 @@ -_target_: deckard.base.files.FileConfig -reports: reports -data_dir: data -model_dir: models -attack_dir: attacks -directory: cifar -score_dict_file: score_dict.json -data_file : data -model_file : model -attack_file : attack -attack_type : .pkl -data_type : .pkl -model_type : .pt -params_file : params.yaml -# train_labels_file : train_labels.json -# test_labels_file : test_labels.json -# probabilities_file: probabilities.json -predictions_file : predictions.json -adv_predictions_file : adv_predictions.json -# adv_probabilities_file: adv_probabilities.json -name: default diff --git a/examples/pytorch/mnist/conf/model/art/default.yaml b/examples/pytorch/mnist/conf/model/art/default.yaml deleted file mode 100644 index 4251627d..00000000 --- a/examples/pytorch/mnist/conf/model/art/default.yaml +++ /dev/null @@ -1,7 +0,0 @@ -defaults: - - initialize : default - # - preprocessor: fsq - # - postprocessor: confidence -data : ${..data} -library : ${..library} -_target_ : deckard.base.model.art_pipeline.ArtPipeline diff --git a/examples/pytorch/mnist/conf/model/art/initialize/default.yaml b/examples/pytorch/mnist/conf/model/art/initialize/default.yaml deleted file mode 100644 index b694473b..00000000 --- a/examples/pytorch/mnist/conf/model/art/initialize/default.yaml +++ /dev/null @@ -1,7 +0,0 @@ -criterion: - name : torch.nn.CrossEntropyLoss -optimizer: - name : torch.optim.SGD - lr : 0.01 - momentum : 0.9 -clip_values : [0, 255] diff --git a/examples/pytorch/mnist/conf/model/art/postprocessor/confidence.yaml b/examples/pytorch/mnist/conf/model/art/postprocessor/confidence.yaml deleted file mode 100644 index 6fcdde95..00000000 --- a/examples/pytorch/mnist/conf/model/art/postprocessor/confidence.yaml +++ /dev/null @@ -1,3 +0,0 @@ - -name : art.defences.postprocessor.HighConfidence -cutoff : .1 diff --git a/examples/pytorch/mnist/conf/model/art/postprocessor/gauss-out.yaml b/examples/pytorch/mnist/conf/model/art/postprocessor/gauss-out.yaml deleted file mode 100644 index c912ebd9..00000000 --- a/examples/pytorch/mnist/conf/model/art/postprocessor/gauss-out.yaml +++ /dev/null @@ -1,2 +0,0 @@ -name : art.defences.postprocessor.GaussianNoise -scale : 1 diff --git a/examples/pytorch/mnist/conf/model/art/preprocessor/default.yaml b/examples/pytorch/mnist/conf/model/art/preprocessor/default.yaml deleted file mode 100644 index e69de29b..00000000 diff --git a/examples/pytorch/mnist/conf/model/art/preprocessor/fsq.yaml b/examples/pytorch/mnist/conf/model/art/preprocessor/fsq.yaml deleted file mode 100644 index 27ddf58b..00000000 --- a/examples/pytorch/mnist/conf/model/art/preprocessor/fsq.yaml +++ /dev/null @@ -1,3 +0,0 @@ -name : art.defences.preprocessor.FeatureSqueezing -clip_values : ${model.art.initialize.clip_values} -bit_depth : 64 diff --git a/examples/pytorch/mnist/conf/model/art/preprocessor/gauss-in.yaml b/examples/pytorch/mnist/conf/model/art/preprocessor/gauss-in.yaml deleted file mode 100644 index c438e8f5..00000000 --- a/examples/pytorch/mnist/conf/model/art/preprocessor/gauss-in.yaml +++ /dev/null @@ -1,4 +0,0 @@ -name : art.defences.preprocessor.GaussianAugmentation -clip_values : ${model.art.initialize.clip_values} -sigma : 1 -ratio : .5 diff --git a/examples/pytorch/mnist/conf/model/torch_mnist.yaml b/examples/pytorch/mnist/conf/model/torch_mnist.yaml deleted file mode 100644 index 9c18c54b..00000000 --- a/examples/pytorch/mnist/conf/model/torch_mnist.yaml +++ /dev/null @@ -1,13 +0,0 @@ - -defaults: - - art : default -data: ${data} -init: - _target_: deckard.base.model.ModelInitializer - num_channels : 1 - name : torch_example.ResNet18 -_target_: deckard.base.model.Model -trainer: - nb_epoch: 100 - batch_size: 1024 -library : pytorch diff --git a/examples/pytorch/mnist/conf/scorers/default.yaml b/examples/pytorch/mnist/conf/scorers/default.yaml deleted file mode 100644 index 4503c5bf..00000000 --- a/examples/pytorch/mnist/conf/scorers/default.yaml +++ /dev/null @@ -1,9 +0,0 @@ -_target_: deckard.base.scorer.ScorerDict -accuracy: - _target_: deckard.base.scorer.ScorerConfig - name: sklearn.metrics.accuracy_score - direction: maximize -log_loss: - _target_: deckard.base.scorer.ScorerConfig - name: sklearn.metrics.log_loss - direction: minimize diff --git a/examples/pytorch/mnist/dvc.lock b/examples/pytorch/mnist/dvc.lock deleted file mode 100644 index db59a303..00000000 --- a/examples/pytorch/mnist/dvc.lock +++ /dev/null @@ -1,828 +0,0 @@ -schema: '2.0' -stages: - clean@attack: - cmd: python -m deckard.layers.clean_data -i mnist/reports/attack.csv -o mnist/reports/clean_attack.csv - -c conf/clean.yaml - deps: - - path: mnist/reports/attack.csv - md5: 51d779c26865540247a82408cd6a46d0 - size: 189645846 - params: - params.yaml: - files.directory: mnist - files.reports: reports - conf/clean.yaml: - attacks: - DeepFool: Deep - FastGradientMethod: FGM - HopSkipJump: HSJ - PixelAttack: Pixel - ProjectedGradientDescent: PGD - ThresholdAttack: Thresh - defences: - Control: Control - FeatureSqueezing: FSQ - GaussianAugmentation: Gauss-in - GaussianNoise: Gauss-out - HighConfidence: Conf - nb_epoch: Epochs - model_layers: Control - fillna: - model.trainer.nb_epoch: 20 - params: - Deep: attack.init.kwargs.nb_grads - FGM: attack.init.kwargs.eps - HSJ: attack.init.kwargs.max_iter - Pixel: attack.init.kwargs.th - PGD: attack.init.kwargs.eps - Thresh: attack.init.kwargs.th - Gauss-out: model.art.pipeline.postprocessor.kwargs.scale - Conf: model.art.pipeline.postprocessor.kwargs.cutoff - FSQ: model.art.pipeline.preprocessor.kwargs.bit_depth - Gauss-in: model.art.pipeline.preprocessor.kwargs.sigma - Control: model_layers - Epochs: model.trainer.nb_epoch - control: - model_layers: 18 - defaults: - model.trainer.nb_epoch: 20 - outs: - - path: mnist/reports/clean_attack.csv - md5: 2c19e993bd04189f0f47a38eda17f3b9 - size: 43039659 - afr: - cmd: python -m deckard.layers.afr --dataset mnist --data_file mnist/reports/clean_attack.csv --target - adv_accuracy --duration_col predict_time --dataset mnist --config_file conf/afr.yaml - --plots_folder mnist/plots/ - deps: - - path: mnist/reports/clean_attack.csv - md5: 2c19e993bd04189f0f47a38eda17f3b9 - size: 43039659 - params: - params.yaml: - files.directory: mnist - conf/afr.yaml: - covariates: - - adv_fit_time - - accuracy - - train_time - - atk_value - - def_value - - data.sample.random_state - - model.trainer.nb_epoch - - model_layers - - predict_time - log_logistic: - plot: - file: log_logistic_aft.eps - title: Log logistic AFR Model - labels: - 'Intercept: beta_': $\beta$ - 'Intercept: alpha_': $\alpha$ - 'data.sample.random_state: alpha_': Random State - 'atk_value: alpha_': Attack Strength - 'train_time: alpha_': $t_{train}$ - 'predict_proba_time: alpha_': $t_{predict}$ - 'adv_accuracy: alpha_': Adv. Accuracy - 'accuracy: alpha_': Ben. Accuracy - 'adv_fit_time: alpha_': $t_{attack}$ - 'adv_failure_rate: alpha_': $f_{adv.}(t;\theta)$ - 'failure_rate: alpha_': $f_{ben.}(t;\theta)$ - 'model.trainer.nb_epoch: alpha_': No. of Epochs - 'model.trainer.batch_size: alpha_': Batch Size - def_gen: Defence - 'model_layers: alpha_': Layers - 'def_value: alpha_': Defence Strength - 'predict_time: alpha_': $t_{predict}$ - partial_effect: - - file: log_logistic_epochs_partial_effect.eps - covariate_array: model.trainer.nb_epoch - values_array: - - 1 - - 10 - - 25 - - 50 - title: $S(t)$ for Log-Logistic AFR - ylabel: Expectation of $S(t)$ - xlabel: Time $T$ (seconds) - legend_kwargs: - title: Epochs - labels: - - '1' - - '10' - - '25' - - '50' - - file: log_logistic_layers_partial_effect.eps - covariate_array: model_layers - values_array: - - 18 - - 34 - - 50 - - 101 - - 152 - title: $S(t)$ for Log Logistic AFR - ylabel: Expectation of $S(t)$ - xlabel: Time $T$ (seconds) - legend_kwargs: - title: Layers - labels: - - '18' - - '34' - - '50' - - '101' - - '152' - log_normal: - plot: - file: log_normal_aft.eps - title: Log Normal AFR Model - labels: - 'Intercept: sigma_': $\rho$ - 'Intercept: mu_': $\mu$ - 'data.sample.random_state: mu_': Random State - 'atk_value: mu_': Attack Strength - 'train_time: mu_': $t_{train}$ - 'predict_proba_time: mu_': $t_{predict}$ - 'adv_accuracy: mu_': Adv. Accuracy - 'accuracy: mu_': Ben. Accuracy - 'adv_fit_time: mu_': $t_{attack}$ - 'adv_failure_rate: mu_': $f_{adv.}(t;\theta)$ - 'failure_rate: mu_': $f_{ben.}(t;\theta)$ - 'model.trainer.nb_epoch: mu_': No. of Epochs - 'model.trainer.batch_size: mu_': Batch Size - def_gen: Defence - 'model_layers: mu_': Layers - 'def_value: mu_': Defence Strength - 'predict_time: mu_': $t_{predict}$ - partial_effect: - - file: log_normal_epochs_partial_effect.eps - covariate_array: model.trainer.nb_epoch - values_array: - - 1 - - 10 - - 25 - - 50 - title: $S(t)$ for Log-Normal AFR - ylabel: Expectation of $S(t)$ - xlabel: Time $T$ (seconds) - legend_kwargs: - title: Epochs - labels: - - '1' - - '10' - - '25' - - '50' - - file: log_normal_layers_partial_effect.eps - covariate_array: model_layers - values_array: - - 18 - - 34 - - 50 - - 101 - - 152 - title: $S(t)$ for Log Normal AFR - ylabel: Expectation of $S(t)$ - xlabel: Time $T$ (seconds) - legend_kwargs: - title: Layers - labels: - - '18' - - '34' - - '50' - - '101' - - '152' - weibull: - plot: - file: weibull_aft.eps - title: Weibull AFR Model - labels: - 'Intercept: rho_': $\rho$ - 'Intercept: lambda_': $\lambda$ - 'data.sample.random_state: lambda_': Random State - 'atk_value: lambda_': Attack Strength - 'train_time: lambda_': $t_{train}$ - 'predict_proba_time: lambda_': $t_{predict}$ - 'adv_accuracy: lambda_': Adv. Accuracy - 'accuracy: lambda_': Ben. Accuracy - 'adv_fit_time: lambda_': $t_{attack}$ - 'adv_failure_rate: lambda_': $f_{adv.}(t;\theta)$ - 'failure_rate: lambda_': $f_{ben.}(t;\theta)$ - 'model.trainer.nb_epoch: lambda_': No. of Epochs - 'model.trainer.batch_size: lambda_': Batch Size - def_gen: Defence - 'model_layers: lambda_': Layers - 'def_value: lambda_': Defence Strength - 'predict_time: lambda_': $t_{predict}$ - partial_effect: - - file: weibull_epochs_partial_effect.eps - covariate_array: model.trainer.nb_epoch - values_array: - - 1 - - 10 - - 25 - - 50 - title: $S(t)$ for Weibull AFR - ylabel: Expectation of $S(t)$ - xlabel: Time $T$ (seconds) - legend_kwargs: - title: Epochs - labels: - - '1' - - '10' - - '25' - - '50' - - file: weibull_layers_partial_effect.eps - covariate_array: model_layers - values_array: - - 18 - - 34 - - 50 - - 101 - - 152 - title: $S(t)$ for Weibull AFR - ylabel: Expectation of $S(t)$ - xlabel: Time $T$ (seconds) - legend_kwargs: - title: Layers - labels: - - '18' - - '34' - - '50' - - '101' - - '152' - outs: - - path: mnist/plots/aft_comparison.csv - md5: f59decd89bd1e9a684ea77c280dd4977 - size: 375 - - path: mnist/plots/aft_comparison.tex - md5: 9ba120db593233d5811417ba894d3551 - size: 566 - - path: mnist/plots/log_logistic_aft.eps - md5: b7b0671479b42062562f0c03d8c0975f - size: 43143 - - path: mnist/plots/log_logistic_epochs_partial_effect.eps - md5: 6d4454d35005dfd34dc25de5691b05b3 - size: 45110 - - path: mnist/plots/log_logistic_layers_partial_effect.eps - md5: 04f63bb1af4c6cc182f031b0a6f2f570 - size: 46247 - - path: mnist/plots/log_normal_aft.eps - md5: 3c5895eed8d86f1bb68e50a9e8d705fb - size: 44109 - - path: mnist/plots/log_normal_epochs_partial_effect.eps - md5: e273ac270f3c7a04257edf27250f0451 - size: 45828 - - path: mnist/plots/log_normal_layers_partial_effect.eps - md5: 0bd9b1734275002d5fba203829d9ace6 - size: 46723 - - path: mnist/plots/weibull_aft.eps - md5: 7671058fb0db407eef4a01464e324a5c - size: 41411 - - path: mnist/plots/weibull_epochs_partial_effect.eps - md5: b77d411ad6499a3a77a38486ef67e600 - size: 44468 - - path: mnist/plots/weibull_layers_partial_effect.eps - md5: 90c8f92165a288641f249f866d8c25d0 - size: 45557 - plot: - cmd: python -m deckard.layers.plots --path mnist/plots/ --file mnist/reports/clean_attack.csv - -c conf/plots.yaml - deps: - - path: mnist/reports/clean_attack.csv - md5: 2c19e993bd04189f0f47a38eda17f3b9 - size: 43039659 - params: - conf/plots.yaml: - cat_plot: - - file: adv_accuracy_vs_defence_type.eps - hue: model_name - kind: boxen - set: - yscale: linear - titles: Adv. Accuracy vs Defence Type - x: def_gen - xlabels: Defence Type - y: adv_accuracy - ylabels: Adv. Accuracy - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - - file: ben_accuracy_vs_defence_type.eps - hue: model_name - kind: boxen - titles: Ben. Accuracy vs Defence Type - x: def_gen - xlabels: Defence Type - y: accuracy - ylabels: Ben. Accuracy - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - - file: ben_failures_per_train_time_vs_defence_type.eps - hue: model_name - kind: boxen - set: - yscale: log - titles: $\bar{C}_{ben.}$ vs Defence Type - x: def_gen - xlabels: Defence Type - y: training_time_per_failure - ylabels: $\bar{C}_{ben.}$ - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - - file: adv_failures_per_train_time_vs_defence_type.eps - hue: model_name - kind: boxen - set: - yscale: log - titles: $\bar{C}_{adv.}$ vs Defence Type - x: def_gen - xlabels: Defence Type - y: training_time_per_adv_failure - ylabels: $\bar{C}_{adv.}$ - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - - file: adv_failures_per_train_time_vs_attack_type.eps - hue: model_name - kind: boxen - legend_title: Model Name - set: - yscale: log - titles: $\bar{C}_{adv.}$ vs Attack Type - x: atk_gen - xlabels: Attack Type - y: training_time_per_adv_failure - ylabels: $\bar{C}_{adv.}$ - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - - file: adv_failures_per_test_time_vs_defence_type.eps - hue: model_name - kind: boxen - legend_title: Model Name - titles: $f_{adv}$ vs Defence Type - x: def_gen - xlabels: Defence Type - y: adv_failure_rate - ylabels: $f_{adv.}$ - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - - file: adv_accuracy_vs_attack_type.eps - hue: model_name - kind: boxen - legend_title: Model Name - titles: Adv. Accuracy vs Attack Type - x: atk_gen - xlabels: Attack Type - y: adv_accuracy - ylabels: Adv. Accuracy - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - - file: ben_failure_rate_vs_defence_type.eps - hue: model_name - kind: boxen - legend_title: Model Name - set: - yscale: log - titles: $f_{ben}(t; \theta)$ vs Defence Type - x: def_gen - xlabels: Defence Type - y: failure_rate - ylabels: $f_{ben}(t; \theta)$ - rotation: 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - line_plot: - - file: def_param_vs_accuracy.eps - hue: def_gen - legend: - bbox_to_anchor: - - 1.05 - - 1 - title: Defence - title: Ben. Accuracy vs Defence Strength - x: def_value - x_scale: linear - xlabel: Defence Control Parameter - y: accuracy - y_scale: - ylabel: Ben. Accuracy - hue_order: - - Control - - Conf - - Epochs - - Gauss-in - - Gauss-out - - Conf - - FSQ - errorbar: se - err_style: bars - - file: def_param_vs_adv_accuracy.eps - hue: def_gen - legend: - bbox_to_anchor: - - 1.05 - - 1 - title: Defence - title: Adv. Accuracy vs Defence Strength - x: def_value - x_scale: linear - xlabel: Defence Control Parameter - y: adv_accuracy - y_scale: - ylabel: Adv. Accuracy - hue_order: - - Control - - Conf - - Epochs - - Gauss-in - - Gauss-out - - Conf - - FSQ - errorbar: se - err_style: bars - - file: def_param_vs_adv_failure_rate.eps - hue: def_gen - legend: - bbox_to_anchor: - - 1.05 - - 1 - title: Defence - title: $f_{adv}$ vs Defence Strength - x: def_value - x_scale: linear - xlabel: Defence Control Parameter - y: adv_failure_rate - y_scale: linear - ylabel: $f_{adv.}$ - hue_order: - - Control - - Conf - - Epochs - - Gauss-in - - Gauss-out - - Conf - - FSQ - errorbar: se - err_style: bars - - file: atk_param_vs_accuracy.eps - hue: atk_gen - legend: - bbox_to_anchor: - - 1.05 - - 1 - title: Adv. Accuracy vs Attack Strength - x: atk_value - x_scale: linear - xlabel: Attack Control Parameter - y: adv_accuracy - y_scale: - ylabel: Adv. Accuracy - hue_order: - - FGM - - PGD - - Deep - - HSJ - - Pixel - - Thresh - errorbar: se - err_style: bars - scatter_plot: - - x: train_time_per_sample - y: adv_failure_rate - hue: model_name - xlabel: $t_{train}$ - ylabel: $f_{adv}$ - title: $f_{adv}$ vs $t_{train}$ - file: adv_failure_rate_vs_train_time.eps - x_scale: log - legend: - title: Model Name - bbox_to_anchor: - - 1.05 - - 1 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - outs: - - path: mnist/plots/adv_accuracy_vs_attack_type.eps - md5: d51200cf3e8cefd7517a97c507c32a92 - size: 123974 - - path: mnist/plots/adv_accuracy_vs_defence_type.eps - md5: 1142a65c254df12224d88187d580c756 - size: 118866 - - path: mnist/plots/adv_failure_rate_vs_train_time.eps - md5: 584a893a5e386ce8af1b49dd4333f27a - size: 987785 - - path: mnist/plots/adv_failures_per_test_time_vs_defence_type.eps - md5: ae31c6898b60fc8c4b2d5ed523e8648b - size: 138229 - - path: mnist/plots/adv_failures_per_train_time_vs_attack_type.eps - md5: 32d8a97ee6fa8bfb385cd10cef9985a0 - size: 128046 - - path: mnist/plots/adv_failures_per_train_time_vs_defence_type.eps - md5: 191c566d64c2f94b5832578951a1ca3a - size: 122522 - - path: mnist/plots/atk_param_vs_accuracy.eps - md5: 2d20b9e7214b8b04c8d6ba60e59be5a0 - size: 39701 - - path: mnist/plots/ben_accuracy_vs_defence_type.eps - md5: c94bb2af2ed7a26bf6d1f232d010671a - size: 112716 - - path: mnist/plots/ben_failure_rate_vs_defence_type.eps - md5: ab115d786b80015aba9efe0b03df9ada - size: 140041 - - path: mnist/plots/ben_failures_per_train_time_vs_defence_type.eps - md5: e2e8499a4b6fc81d50e18d6b4bdf29f7 - size: 123124 - - path: mnist/plots/def_param_vs_accuracy.eps - md5: a37bec164c0ba420ef4c20cab952cd9a - size: 39660 - - path: mnist/plots/def_param_vs_adv_accuracy.eps - md5: 09af779c5fd3c9b132bd5270516a6e53 - size: 39631 - - path: mnist/plots/def_param_vs_adv_failure_rate.eps - md5: 706238ba734e5a65642f12e105365aa9 - size: 39067 - copy_results: - cmd: mkdir -p ~/ml_afr/mnist/ && cp -r mnist/plots/* ~/ml_afr/mnist/ - deps: - - path: mnist/plots/ - md5: 37a3df39d57394928f1d659d522f9544.dir - size: 3021764 - nfiles: 35 - - path: mnist/plots/ - md5: 37a3df39d57394928f1d659d522f9544.dir - size: 3021764 - nfiles: 35 - afr_with_dummies: - cmd: python -m deckard.layers.afr --dataset mnist --data_file mnist/reports/clean_attack.csv --target - adv_accuracy --duration_col predict_time --dataset mnist --config_file conf/afr_dummy.yaml - --plots_folder mnist/plots/ --summary_file aft_comparison_with_dummy - deps: - - path: mnist/reports/clean_attack.csv - md5: 2c19e993bd04189f0f47a38eda17f3b9 - size: 43039659 - params: - params.yaml: - files.directory: mnist - conf/afr.yaml: - covariates: - - adv_fit_time - - accuracy - - train_time - - atk_value - - def_value - - data.sample.random_state - - model.trainer.nb_epoch - - model_layers - - predict_time - log_logistic: - plot: - file: log_logistic_aft.eps - title: Log logistic AFR Model - labels: - 'Intercept: beta_': $\beta$ - 'Intercept: alpha_': $\alpha$ - 'data.sample.random_state: alpha_': Random State - 'atk_value: alpha_': Attack Strength - 'train_time: alpha_': $t_{train}$ - 'predict_proba_time: alpha_': $t_{predict}$ - 'adv_accuracy: alpha_': Adv. Accuracy - 'accuracy: alpha_': Ben. Accuracy - 'adv_fit_time: alpha_': $t_{attack}$ - 'adv_failure_rate: alpha_': $f_{adv.}(t;\theta)$ - 'failure_rate: alpha_': $f_{ben.}(t;\theta)$ - 'model.trainer.nb_epoch: alpha_': No. of Epochs - 'model.trainer.batch_size: alpha_': Batch Size - def_gen: Defence - 'model_layers: alpha_': ResNet Layers - 'def_value: alpha_': Defence Strength - 'predict_time: alpha_': $t_{predict}$ - partial_effect: - - file: log_logistic_epochs_partial_effect.eps - covariate_array: model.trainer.nb_epoch - values_array: - - 1 - - 10 - - 25 - - 50 - title: $S(t)$ for Log-Logistic AFR - ylabel: Expectation of $S(t)$ - xlabel: Time $T$ (seconds) - legend_kwargs: - title: Epochs - labels: - - '1' - - '10' - - '25' - - '50' - - file: log_logistic_layers_partial_effect.eps - covariate_array: model_layers - values_array: - - 18 - - 34 - - 50 - - 101 - - 152 - title: $S(t)$ for Log Logistic AFR - ylabel: Expectation of $S(t)$ - xlabel: Time $T$ (seconds) - legend_kwargs: - title: ResNet Layers - labels: - - '18' - - '34' - - '50' - - '101' - - '152' - log_normal: - plot: - file: log_normal_aft.eps - title: Log Normal AFR Model - labels: - 'Intercept: sigma_': $\rho$ - 'Intercept: mu_': $\mu$ - 'data.sample.random_state: mu_': Random State - 'atk_value: mu_': Attack Strength - 'train_time: mu_': $t_{train}$ - 'predict_proba_time: mu_': $t_{predict}$ - 'adv_accuracy: mu_': Adv. Accuracy - 'accuracy: mu_': Ben. Accuracy - 'adv_fit_time: mu_': $t_{attack}$ - 'adv_failure_rate: mu_': $f_{adv.}(t;\theta)$ - 'failure_rate: mu_': $f_{ben.}(t;\theta)$ - 'model.trainer.nb_epoch: mu_': No. of Epochs - 'model.trainer.batch_size: mu_': Batch Size - def_gen: Defence - 'model_layers: mu_': ResNet Layers - 'def_value: mu_': Defence Strength - 'predict_time: mu_': $t_{predict}$ - partial_effect: - - file: log_normal_epochs_partial_effect.eps - covariate_array: model.trainer.nb_epoch - values_array: - - 1 - - 10 - - 25 - - 50 - title: $S(t)$ for Log-Normal AFR - ylabel: Expectation of $S(t)$ - xlabel: Time $T$ (seconds) - legend_kwargs: - title: Epochs - labels: - - '1' - - '10' - - '25' - - '50' - - file: log_normal_layers_partial_effect.eps - covariate_array: model_layers - values_array: - - 18 - - 34 - - 50 - - 101 - - 152 - title: $S(t)$ for Log Normal AFR - ylabel: Expectation of $S(t)$ - xlabel: Time $T$ (seconds) - legend_kwargs: - title: ResNet Layers - labels: - - '18' - - '34' - - '50' - - '101' - - '152' - weibull: - plot: - file: weibull_aft.eps - title: Weibull AFR Model - labels: - 'Intercept: rho_': $\rho$ - 'Intercept: lambda_': $\lambda$ - 'data.sample.random_state: lambda_': Random State - 'atk_value: lambda_': Attack Strength - 'train_time: lambda_': $t_{train}$ - 'predict_proba_time: lambda_': $t_{predict}$ - 'adv_accuracy: lambda_': Adv. Accuracy - 'accuracy: lambda_': Ben. Accuracy - 'adv_fit_time: lambda_': $t_{attack}$ - 'adv_failure_rate: lambda_': $f_{adv.}(t;\theta)$ - 'failure_rate: lambda_': $f_{ben.}(t;\theta)$ - 'model.trainer.nb_epoch: lambda_': No. of Epochs - 'model.trainer.batch_size: lambda_': Batch Size - def_gen: Defence - 'model_layers: lambda_': ResNet Layers - 'def_value: lambda_': Defence Strength - 'predict_time: lambda_': $t_{predict}$ - partial_effect: - - file: weibull_epochs_partial_effect.eps - covariate_array: model.trainer.nb_epoch - values_array: - - 1 - - 10 - - 25 - - 50 - title: $S(t)$ for Weibull AFR - ylabel: Expectation of $S(t)$ - xlabel: Time $T$ (seconds) - legend_kwargs: - title: Epochs - labels: - - '1' - - '10' - - '25' - - '50' - - file: weibull_layers_partial_effect.eps - covariate_array: model_layers - values_array: - - 18 - - 34 - - 50 - - 101 - - 152 - title: $S(t)$ for Weibull AFR - ylabel: Expectation of $S(t)$ - xlabel: Time $T$ (seconds) - legend_kwargs: - title: ResNet Layers - labels: - - '18' - - '34' - - '50' - - '101' - - '152' - outs: - - path: mnist/plots/aft_comparison_with_dummy.csv - md5: 00cfc38147e5ffed62c71eef81d76604 - size: 382 - - path: mnist/plots/aft_comparison_with_dummy.tex - md5: a7b04f60efcb56609a0374f4a614b322 - size: 566 - - path: mnist/plots/log_logistic_aft_dummy.eps - md5: 6c3c08024c4464c7069cc1ba0b0b2a29 - size: 62987 - - path: mnist/plots/log_logistic_epochs_partial_effect_dummy.eps - md5: 8c83fe60085369058bce8c4860a82e96 - size: 43270 - - path: mnist/plots/log_logistic_layers_partial_effect_dummy.eps - md5: df5b4c525afc9799f66348f89e4c0f11 - size: 48378 - - path: mnist/plots/log_normal_aft_dummy.eps - md5: 9792419c39453877f3a37361b5605ca0 - size: 61512 - - path: mnist/plots/log_normal_epochs_partial_effect_dummy.eps - md5: e55b95768eeaf862ba16d23e3ca17c53 - size: 45738 - - path: mnist/plots/log_normal_layers_partial_effect_dummy.eps - md5: 91d32fe10a4c39014c2b310388a34e1f - size: 48808 - - path: mnist/plots/weibull_aft_dummy.eps - md5: 75ca056699d7c578d7492d6dcacf9e57 - size: 60898 - - path: mnist/plots/weibull_epochs_partial_effect_dummy.eps - md5: d93f5cd06f362504b56215a89783990f - size: 44530 - - path: mnist/plots/weibull_layers_partial_effect_dummy.eps - md5: 0434dbd9406305ab7551e9ad8ce6bdc6 - size: 47796 diff --git a/examples/pytorch/mnist/dvc.yaml b/examples/pytorch/mnist/dvc.yaml deleted file mode 100644 index a7b80490..00000000 --- a/examples/pytorch/mnist/dvc.yaml +++ /dev/null @@ -1,195 +0,0 @@ -vars: - - conf/plots.yaml:line_plot - - conf/plots.yaml:scatter_plot - - conf/plots.yaml:cat_plot - - conf/afr.yaml:covariates - - conf/afr.yaml:weibull - - conf/afr.yaml:log_logistic - - conf/afr.yaml:log_normal - - conf/clean.yaml:attacks - - conf/clean.yaml:defences - - conf/clean.yaml:params - - conf/clean.yaml:fillna -stages: - train: - cmd: python -m deckard.layers.experiment train --config_file mnist.yaml - params: - - data - - model - - scorers - - files - outs: - - ${files.directory}/${files.data_dir}/${files.data_file}${files.data_type} - - ${files.directory}/${files.model_dir}/${files.model_file}${files.model_type} - - ${files.directory}/${files.model_dir}/${files.model_file}.optimizer${files.model_type} - # - ${files.directory}/${files.reports}/train/${files.name}/${files.params_file} - # - ${files.directory}/${files.reports}/train/${files.name}/${files.test_labels_file} # Omit to save space - - ${files.directory}/${files.reports}/train/${files.name}/${files.predictions_file} # logit outputs for our model - # - ${files.directory}/${files.reports}/train/${files.name}/${files.probabilities_file} # Omit to save space - metrics: - - ${files.directory}/${files.reports}/train/${files.name}/${files.score_dict_file} - attack: - cmd: python -m deckard.layers.experiment attack --config_file mnist.yaml - params: - - data - - model - - attack - - scorers - - files - outs: - - ${files.directory}/${files.attack_dir}/${files.attack_file}${files.attack_type} - - ${files.directory}/${files.reports}/attack/${files.name}/${files.adv_predictions_file} - # - ${files.directory}/${files.reports}/attack/${files.name}/${files.params_file} - deps: - - ${files.directory}/${files.data_dir}/${files.data_file}${files.data_type} - - ${files.directory}/${files.model_dir}/${files.model_file}${files.model_type} - metrics: - - ${files.directory}/${files.reports}/attack/${files.name}/${files.score_dict_file} - - ############################################################################## - # models: # This is a loop over the ResNet models - # foreach: - # - ResNet18 - # # - ResNet34 - # # - ResNet50 - # # - ResNet101 - # # - ResNet152 - # do: # This script configures eazch defence - # cmd: bash models.sh ++model.init.name=torch_example.${item} stage=train ++hydra.sweeper.storage=sqlite:///${files.directory}/${files.reports}/train/${item}.db --config-name mnist.yaml - # deps: - # - models.sh - # - ${files.directory}/${files.model_dir}/${files.model_file}${files.model_type} - # - ${files.directory}/${files.model_dir}/${files.model_file}.optimizer${files.model_type} - # outs: - # - ${files.directory}/${files.reports}/train/${item}.db: # This outputs a database file for each model - # cache: True - # persist: True - attacks: - foreach: # This is a loop over the ResNet models - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - do: - cmd: bash attacks.sh ++attack.attack_size=100 ++model.init.name=torch_example.${item} stage=attack ++hydra.sweeper.storage=sqlite:///${files.directory}/${files.reports}/attack/${item}.db --config-name mnist.yaml - deps: - - models.sh # This script configures each defence - - attacks.sh # This script configures each attack - - ${files.directory}/${files.reports}/attack/${files.name}/${files.score_dict_file} # This is here just to ensure it runs after the attack stage - # - ${files.directory}/${files.reports}/train/${item}.db - outs: - - ${files.directory}/${files.reports}/attack/${item}.db: # This outputs a database file for each model - cache: True - persist: True - compile: - foreach: # iterates through each stage - # - train - - attack - do: - cmd: python -m deckard.layers.compile --report_folder ${files.directory}/${files.reports}/${item} --results_file ${files.directory}/${files.reports}/${item}.csv - deps: - - ${files.directory}/${files.reports}/${item}/ - - ${files.directory}/${files.reports}/${item}/ResNet18.db - - ${files.directory}/${files.reports}/${item}/ResNet34.db - - ${files.directory}/${files.reports}/${item}/ResNet50.db - - ${files.directory}/${files.reports}/${item}/ResNet101.db - # - ${files.directory}/${files.reports}/${item}/ResNet152.db - outs: - - ${files.directory}/${files.reports}/${item}.csv - clean: - foreach: # iterates through each stage - # - train - - attack - do: - cmd: python -m deckard.layers.clean_data -i ${files.directory}/${files.reports}/${item}.csv -o ${files.directory}/${files.reports}/clean_${item}.csv -c conf/clean.yaml - deps: - - ${files.directory}/${files.reports}/${item}.csv - outs: - - ${files.directory}/${files.reports}/clean_${item}.csv - params: - - files.directory - - files.reports - - conf/clean.yaml: - - attacks - - defences - - params - - fillna - clean: - foreach: # iterates through each stage - # - train - - attack - do: - cmd: python -m deckard.layers.clean_data -i ${files.directory}/${files.reports}/${item}.csv -o ${files.directory}/${files.reports}/clean_${item}.csv -c conf/clean.yaml - deps: - - ${files.directory}/${files.reports}/${item}.csv - outs: - - ${files.directory}/${files.reports}/clean_${item}.csv - params: - - files.directory - - files.reports - - conf/clean.yaml: - - attacks - - defences - - params - - fillna - plot: - cmd : python -m deckard.layers.plots --path ${files.directory}/plots/ --file ${files.directory}/${files.reports}/clean_attack.csv -c conf/plots.yaml - cmd : python -m deckard.layers.plots --path ${files.directory}/plots/ --file ${files.directory}/${files.reports}/clean_attack.csv -c conf/plots.yaml - deps: - - ${files.directory}/${files.reports}/clean_attack.csv - plots: - - ${files.directory}/plots/${cat_plot[0].file} - - ${files.directory}/plots/${cat_plot[1].file} - - ${files.directory}/plots/${cat_plot[2].file} - - ${files.directory}/plots/${cat_plot[3].file} - - ${files.directory}/plots/${cat_plot[4].file} - - ${files.directory}/plots/${cat_plot[5].file} - - ${files.directory}/plots/${cat_plot[6].file} - - ${files.directory}/plots/${cat_plot[7].file} - - ${files.directory}/plots/${line_plot[0].file} - - ${files.directory}/plots/${line_plot[1].file} - - ${files.directory}/plots/${line_plot[2].file} - - ${files.directory}/plots/${line_plot[3].file} - - ${files.directory}/plots/${scatter_plot[0].file} - params: - - conf/plots.yaml: - - line_plot - - scatter_plot - - cat_plot - afr: - cmd: python -m deckard.layers.afr --dataset ${files.directory} --data_file ${files.directory}/${files.reports}/clean_attack.csv --target adv_accuracy --duration_col predict_time --dataset mnist --config_file conf/afr.yaml --plots_folder ${files.directory}/plots/ - deps: - - ${files.directory}/${files.reports}/clean_attack.csv - plots: - - ${files.directory}/plots/weibull_aft.eps - - ${files.directory}/plots/weibull_epochs_partial_effect.eps - - ${files.directory}/plots/weibull_layers_partial_effect.eps - - ${files.directory}/plots/log_logistic_aft.eps - - ${files.directory}/plots/log_logistic_epochs_partial_effect.eps - - ${files.directory}/plots/log_logistic_layers_partial_effect.eps - - ${files.directory}/plots/log_normal_aft.eps - - ${files.directory}/plots/log_normal_epochs_partial_effect.eps - - ${files.directory}/plots/log_normal_layers_partial_effect.eps - metrics: - - ${files.directory}/plots/aft_comparison.csv - outs: - - ${files.directory}/plots/aft_comparison.tex - params: - - files.directory - - conf/afr.yaml: - - covariates - - weibull - - log_logistic - - log_normal - params: - - files.directory - - conf/afr.yaml: - - covariates - - weibull - - log_logistic - - log_normal - copy_results: - cmd: mkdir -p ~/ml_afr/mnist/ && cp -r ${files.directory}/plots/* ~/ml_afr/mnist/ - deps: - - ${files.directory}/plots/ diff --git a/examples/pytorch/mnist/models.sh b/examples/pytorch/mnist/models.sh deleted file mode 100644 index 2978d445..00000000 --- a/examples/pytorch/mnist/models.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# This script is used to generate the models for the sklearn example. - -# # Default model -echo "python -m deckard.layers.optimise " $@ "--multirun" -python -m deckard.layers.optimise $@ --multirun - -# # This line generates the model and adds the FeatureSqueezing preprocessing defence. -# python -m deckard.layers.optimise ++model.art.preprocessor.name=art.defences.preprocessor.FeatureSqueezing +model.art.preprocessor.params.bit_depth=4,8,16,32,64 +model.art.preprocessor.params.clip_values=[0,255] ++hydra.sweeper.study_name=FSQ $@ --multirun - -# # # Gaussian Augmentation (Input) -# python -m deckard.layers.optimise ++model.art.preprocessor.name=art.defences.preprocessor.GaussianAugmentation +model.art.preprocessor.params.sigma=.01,.1,.3,.5,1 +model.art.preprocessor.params.ratio=.5 +model.art.preprocessor.params.augmentation=False ++hydra.sweeper.study_name=gauss-in $@ --multirun - -# # # # Gaussian Noise (Output) -# python -m deckard.layers.optimise ++model.art.postprocessor.name=art.defences.postprocessor.GaussianNoise ++model.art.postprocessor.params.scale=.01,.1,.3,.5,1 ++hydra.sweeper.study_name=gauss-out $@ --multirun - -# # # # High Confidence -# python -m deckard.layers.optimise +model.art.postprocessor.name=art.defences.postprocessor.HighConfidence +model.art.postprocessor.params.cutoff=.1,.3,.5,.9,.99 ++hydra.sweeper.study_name=conf $@ --multirun diff --git a/examples/pytorch/mnist/optuna_viz.ipynb b/examples/pytorch/mnist/optuna_viz.ipynb deleted file mode 100644 index 7e005824..00000000 --- a/examples/pytorch/mnist/optuna_viz.ipynb +++ /dev/null @@ -1,68 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/cmeyers/deckard/env/lib/python3.8/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "ename": "SyntaxError", - "evalue": "invalid syntax (371257998.py, line 2)", - "output_type": "error", - "traceback": [ - "\u001b[0;36m Cell \u001b[0;32mIn[2], line 2\u001b[0;36m\u001b[0m\n\u001b[0;31m study = optuna.create_study(storage=)\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n" - ] - } - ], - "source": [ - "import optuna\n", - "\n", - "study = optuna.create_study(storage=\"sqlite:///mnist/reports/attack/ResNet18.db\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "env", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.8" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/pytorch/models.sh b/examples/pytorch/models.sh new file mode 100644 index 00000000..2a54e307 --- /dev/null +++ b/examples/pytorch/models.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +# This script is used to generate the models for the sklearn example. + +# # Default model +echo "python -m deckard.layers.optimise" $@ "--multirun" +python -m deckard.layers.optimise $@ --multirun + +# This line generates the model and adds the FeatureSqueezing preprocessing defence. +python -m deckard.layers.optimise \ + ++model.art.preprocessor.name=art.defences.preprocessor.FeatureSqueezing \ + +model.art.preprocessor.bit_depth=4,8,16,32,64 \ + +model.art.preprocessor.clip_values=[0,255] \ + def_name=FSQ \ + $@ --multirun + +rm -rf mnist/models +rm -rf cifar/models +rm -rf cifar100/models + +# Gaussian Augmentation (Input) +python -m deckard.layers.optimise \ + ++model.art.preprocessor.name=art.defences.preprocessor.GaussianAugmentation \ + +model.art.preprocessor.sigma=.01,.1,.3,.5,1 \ + +model.art.preprocessor.ratio=.5 \ + +model.art.preprocessor.augmentation=False \ + def_name=Gauss-In \ + $@ --multirun + +rm -rf mnist/models +rm -rf cifar/models +rm -rf cifar100/model + +# Gaussian Noise (Output) +python -m deckard.layers.optimise \ + ++model.art.postprocessor.name=art.defences.postprocessor.GaussianNoise \ + ++model.art.postprocessor.scale=.01,.1,.3,.5,1 \ + ++model.art.postprocessor.apply_fit=True \ + def_name=Gauss-Out \ + $@ --multirun + +rm -rf mnist/models +rm -rf cifar/models +rm -rf cifar100/models + +# High Confidence +python -m deckard.layers.optimise \ + +model.art.postprocessor.name=art.defences.postprocessor.HighConfidence \ + +model.art.postprocessor.cutoff=.1,.3,.5,.9,.99 \ + def_name=Conf \ + $@ --multirun + +rm -rf mnist/models +rm -rf cifar/models +rm -rf cifar100/models diff --git a/examples/pytorch/old_plots/afr.yaml b/examples/pytorch/old_plots/afr.yaml new file mode 100644 index 00000000..92aa814d --- /dev/null +++ b/examples/pytorch/old_plots/afr.yaml @@ -0,0 +1,201 @@ +covariates: + - adv_fit_time + - accuracy + - train_time + - atk_value + - def_value + - data.sample.random_state + - Epochs + - model_layers + - id + - atk_gen + - def_gen + - adv_failures + - adv_accuracy + - predict_time +fillna: + Epochs: 20 + data.sample.train_size : 48000 + data.sample.test_size: 12000 +cox: + plot: + file : cox_aft.pdf + title : Cox Model + qq_title : Cox QQ Plot + t0: .3 + model: + penalizer: .1 + labels: + "data.sample.random_state": "Random State" + "atk_value": "Attack Strength" + "train_time": "$T_{train}$" + "predict_time": "$T_{predict}$" + "adv_accuracy": "Adv. Accuracy" + "def_value": "Defence Strength" + "accuracy": "Ben. Accuracy" + "model_layers": "Layers" + "adv_fit_time": "$T_{attack}$" + "adv_failure_rate": "$f_{adv.}(t;\\theta)$" + "failure_rate": "$f_{ben.}(t;\\theta)$" + "Epochs": "No. of Epochs" + "model.trainer.batch_size": "Batch Size" + "def_gen": "Defence" +# aalen: +# plot: +# file : aalen_aft.pdf +# title : Aalen Model +# qq_title : Aalen QQ Plot +# t0: 1 +# model: +# alpha: 1 +# coef_penalizer: .1 +# smoothing_penalizer: .1 +# labels: +# "data.sample.random_state": "Random State" +# "atk_value": "Attack Strength" +# "train_time": "$T_{train}$" +# "predict_time": "$T_{predict}$" +# "adv_accuracy": "Adv. Accuracy" +# "accuracy": "Ben. Accuracy" +# "adv_fit_time": "$T_{attack}$" +# "adv_failure_rate": "$f_{adv.}(t;\\theta)$" +# "failure_rate": "$f_{ben.}(t;\\theta)$" +# "Epochs": "No. of Epochs" +# "model.trainer.batch_size": "Batch Size" +# "def_gen": "Defence" +gamma: + plot: + file : gamma_aft.pdf + title : Generalized Gamma Model + qq_title : Gamma QQ Plot + + t0: .3 + model: + penalizer : .4 + labels: + "Intercept: alpha_": "$\\alpha$" + "Intercept: beta_": "$\\beta$" + "data.sample.random_state: beta_": "Random State" + "def_value: beta_": "Defence Strength" + "atk_value: beta_": "Attack Strength" + "train_time: beta_": "$T_{train}$" + "model_layers: beta_": "Layers" + "predict_time: beta_": "$T_{predict}$" + "adv_accuracy: beta_": "Adv. Accuracy" + "accuracy: beta_": "Ben. Accuracy" + "adv_fit_time: beta_": "$T_{attack}$" + "adv_failure_rate: beta_": "$h_{adv.}(t;\\theta)$" + "failure_rate: beta_": "$h_{ben.}(t;\\theta)$" + "Epochs: beta_": "No. of Epochs" + "model.trainer.batch_size: beta_": "Batch Size" + "def_gen": "Defence" + "attack.init.eps: beta_": "$\\varepsilon$" +weibull: + plot: + file : weibull_aft.pdf + title : Weibull AFT Model + qq_title : Weibull QQ Plot + t0: .3 + model: + penalizer: .1 + labels: + "Intercept: rho_": "$\\rho$" + "Intercept: lambda_": "$\\lambda$" + "data.sample.random_state: lambda_": "Random State" + "atk_value: lambda_": "Attack Strength" + "model_layers: lambda_": "Layers" + "train_time: lambda_": "$T_{train}$" + "predict_time: lambda_": "$T_{predict}$" + "adv_accuracy: lambda_": "Adv. Accuracy" + "accuracy: lambda_": "Ben. Accuracy" + "adv_fit_time: lambda_": "$T_{attack}$" + "adv_failure_rate: lambda_": "$f_{adv.}(t;\\theta)$" + "failure_rate: lambda_": "$f_{ben.}(t;\\theta)$" + "Epochs: lambda_": "No. of Epochs" + "model.trainer.batch_size: lambda_": "Batch Size" + "def_gen": "Defence" + "def_value: lambda_" : "Defence Strength" + ": lambda_" : "" +exponential: + plot: + file : exponential_aft.pdf + title : Exponential Model + qq_title : Exponential QQ Plot + t0: .1 + model: + breakpoints: + - .1 + labels: + "Intercept: rho_": "$\\rho$" + "Intercept: lambda_": "$\\lambda$" + "data.sample.random_state: lambda_": "Random State" + "atk_value: lambda_": "Attack Strength" + "def_value: lambda_": "Defence Strength" + "model_layers: lambda_": "Layers" + "train_time: lambda_": "$T_{train}$" + "predict_time: lambda_": "$T_{predict}$" + "adv_accuracy: lambda_": "Adv. Accuracy" + "accuracy: lambda_": "Ben. Accuracy" + "adv_fit_time: lambda_": "$T_{attack}$" + "adv_failure_rate: lambda_": "$f_{adv.}(t;\\theta)$" + "failure_rate: lambda_": "$f_{ben.}(t;\\theta)$" + "Epochs: lambda_": "No. of Epochs" + "model.trainer.batch_size: lambda_": "Batch Size" + "def_gen": "Defence" + ": lambda_" : "" +log_logistic: + plot: + file : log_logistic_aft.pdf + title : Log logistic AFT Model + qq_title : Log Logistic QQ Plot + t0: 1 + model: + penalizer: .2 + labels: + "Intercept: beta_": "$\\beta$" + "Intercept: alpha_": "$\\alpha$" + "data.sample.random_state: alpha_": "Random State" + "atk_value: alpha_": "Attack Strength" + "def_value: alpha_": "Defence Strength" + "model_layers: alpha_": "Layers" + "train_time: alpha_": "$T_{train}$" + "predict_time: alpha_": "$T_{predict}$" + "adv_accuracy: alpha_": "Adv. Accuracy" + "accuracy: alpha_": "Ben. Accuracy" + "adv_fit_time: alpha_": "$T_{attack}$" + "adv_failure_rate: alpha_": "$h_{adv.}(t;\\theta)$" + "failure_rate: alpha_": "$h_{ben.}(t;\\theta)$" + "Epochs: alpha_": "No. of Epochs" + "model.trainer.batch_size: alpha_": "Batch Size" + "def_gen": "Defence" + "attack.init.eps: alpha_": "$\\varepsilon$" +log_normal: + plot: + file : log_normal_aft.pdf + title : Log Normal AFT Model + qq_title : Log Normal QQ Plot + t0: 2 + model: + penalizer: .5 + labels: + "Intercept: sigma_": "$\\sigma$" + "Intercept: mu_": "$\\mu$" + "atk_value: mu_": "Attack Strength" + "def_value: mu_": "Defence Strength" + "model_layers: mu_": "Layers" + "train_time: mu_": "$T_{train}$" + "predict_time: mu_": "$T_{predict}$" + "adv_accuracy: mu_": "Adv. Accuracy" + "accuracy: mu_": "Ben. Accuracy" + "adv_fit_time: mu_": "$T_{attack}$" + "adv_failure_rate: mu_": "$h_{adv.}(t;\\theta)$" + "failure_rate: mu_": "$h_{ben.}(t;\\theta)$" + "Epochs: mu_": "No. of Epochs" + "model.trainer.batch_size: mu_": "Batch Size" + "def_gen": "Defence" + "attack.init.eps: mu_": "$\\varepsilon$" + "data.sample.random_state: mu_": "Random State" +dummies: + "atk_gen": "Atk:" + "def_gen": "Def:" + "id" : "Data:" diff --git a/examples/pytorch/old_plots/dvc.lock b/examples/pytorch/old_plots/dvc.lock new file mode 100644 index 00000000..a15ff82e --- /dev/null +++ b/examples/pytorch/old_plots/dvc.lock @@ -0,0 +1,686 @@ +schema: '2.0' +stages: + clean@cifar: + cmd: python -m deckard.layers.clean_data -i data/attack_cifar.csv -o data/cifar.csv -c + ../conf/clean_cifar.yaml --drop_if_empty adv_fit_time accuracy train_time predict_time + adv_accuracy Epochs + deps: + - path: ../conf/clean_cifar.yaml + hash: md5 + md5: 4e2abc093db66b77b424854549b80497 + size: 961 + - path: data/attack_cifar.csv + hash: md5 + md5: 5a430aa13b88dff6a8fdf0277c9cd53d + size: 23267442 + outs: + - path: data/cifar.csv + hash: md5 + md5: 9e7500a8d5c6a0dbf7faa08b9d8314d4 + size: 20392296 + clean@mnist: + cmd: python -m deckard.layers.clean_data -i data/attack_mnist.csv -o data/mnist.csv -c + ../conf/clean_mnist.yaml --drop_if_empty adv_fit_time accuracy train_time predict_time + adv_accuracy Epochs + deps: + - path: ../conf/clean_mnist.yaml + hash: md5 + md5: bb112947b87ca42a244135a52cc5e7d5 + size: 1003 + - path: data/attack_mnist.csv + hash: md5 + md5: c4db49ae4fc1a0e6fa4c3d52b03d650a + size: 93006845 + outs: + - path: data/mnist.csv + hash: md5 + md5: a9e63ec1d18b35133c7ee6795997c2ee + size: 69479203 + clean@cifar100: + cmd: python -m deckard.layers.clean_data -i data/attack_cifar100.csv -o data/cifar100.csv -c + ../conf/clean_cifar100.yaml --drop_if_empty adv_fit_time accuracy train_time + predict_time adv_accuracy Epochs + deps: + - path: ../conf/clean_cifar100.yaml + hash: md5 + md5: 5a8cc7e71f7036e2f590bb882acd34fb + size: 897 + - path: data/attack_cifar100.csv + hash: md5 + md5: ea55355b5b530f5751aea19d13067099 + size: 36035453 + outs: + - path: data/cifar100.csv + hash: md5 + md5: 7e02418daff4407e50fef52d4c8a6ac6 + size: 30006985 + merge: + cmd: python -m deckard.layers.merge --smaller_file data/cifar.csv data/cifar100.csv + data/mnist.csv --output_folder data --output_file merged.csv + deps: + - path: data/cifar.csv + hash: md5 + md5: 9e7500a8d5c6a0dbf7faa08b9d8314d4 + size: 20392296 + - path: data/cifar100.csv + hash: md5 + md5: 7e02418daff4407e50fef52d4c8a6ac6 + size: 30006985 + - path: data/mnist.csv + hash: md5 + md5: a9e63ec1d18b35133c7ee6795997c2ee + size: 69479203 + outs: + - path: data/merged.csv + hash: md5 + md5: 9cc7eb6799f013a2d4259a18d36328d2 + size: 123215063 + afr: + cmd: python -m deckard.layers.afr --data_file data/merged.csv --target adv_failures + --duration_col adv_fit_time --config_file afr.yaml --plots_folder plots/ + deps: + - path: data/merged.csv + hash: md5 + md5: 9cc7eb6799f013a2d4259a18d36328d2 + size: 123215063 + params: + afr.yaml: + covariates: + - adv_fit_time + - accuracy + - train_time + - atk_value + - def_value + - data.sample.random_state + - Epochs + - model_layers + - id + - atk_gen + - def_gen + - adv_failures + - adv_accuracy + - predict_time + cox: + plot: + file: cox_aft.pdf + title: Cox Model + qq_title: Cox QQ Plot + t0: 0.3 + model: + penalizer: 0.1 + labels: + data.sample.random_state: Random State + atk_value: Attack Strength + train_time: $T_{train}$ + predict_time: $T_{predict}$ + adv_accuracy: Adv. Accuracy + def_value: Defence Strength + accuracy: Ben. Accuracy + model_layers: Layers + adv_fit_time: $T_{attack}$ + adv_failure_rate: $f_{adv.}(t;\theta)$ + failure_rate: $f_{ben.}(t;\theta)$ + Epochs: No. of Epochs + model.trainer.batch_size: Batch Size + def_gen: Defence + dummies: + atk_gen: 'Atk:' + def_gen: 'Def:' + id: 'Data:' + exponential: + plot: + file: exponential_aft.pdf + title: Exponential Model + qq_title: Exponential QQ Plot + t0: 0.1 + model: + breakpoints: + - 0.1 + labels: + 'Intercept: rho_': $\rho$ + 'Intercept: lambda_': $\lambda$ + 'data.sample.random_state: lambda_': Random State + 'atk_value: lambda_': Attack Strength + 'def_value: lambda_': Defence Strength + 'model_layers: lambda_': Layers + 'train_time: lambda_': $T_{train}$ + 'predict_time: lambda_': $T_{predict}$ + 'adv_accuracy: lambda_': Adv. Accuracy + 'accuracy: lambda_': Ben. Accuracy + 'adv_fit_time: lambda_': $T_{attack}$ + 'adv_failure_rate: lambda_': $f_{adv.}(t;\theta)$ + 'failure_rate: lambda_': $f_{ben.}(t;\theta)$ + 'Epochs: lambda_': No. of Epochs + 'model.trainer.batch_size: lambda_': Batch Size + def_gen: Defence + ': lambda_': '' + gamma: + plot: + file: gamma_aft.pdf + title: Generalized Gamma Model + qq_title: Gamma QQ Plot + t0: 0.3 + model: + penalizer: 0.4 + labels: + 'Intercept: alpha_': $\alpha$ + 'Intercept: beta_': $\beta$ + 'data.sample.random_state: beta_': Random State + 'def_value: beta_': Defence Strength + 'atk_value: beta_': Attack Strength + 'train_time: beta_': $T_{train}$ + 'model_layers: beta_': Layers + 'predict_time: beta_': $T_{predict}$ + 'adv_accuracy: beta_': Adv. Accuracy + 'accuracy: beta_': Ben. Accuracy + 'adv_fit_time: beta_': $T_{attack}$ + 'adv_failure_rate: beta_': $h_{adv.}(t;\theta)$ + 'failure_rate: beta_': $h_{ben.}(t;\theta)$ + 'Epochs: beta_': No. of Epochs + 'model.trainer.batch_size: beta_': Batch Size + def_gen: Defence + 'attack.init.eps: beta_': $\varepsilon$ + log_logistic: + plot: + file: log_logistic_aft.pdf + title: Log logistic AFT Model + qq_title: Log Logistic QQ Plot + t0: 1 + model: + penalizer: 0.2 + labels: + 'Intercept: beta_': $\beta$ + 'Intercept: alpha_': $\alpha$ + 'data.sample.random_state: alpha_': Random State + 'atk_value: alpha_': Attack Strength + 'def_value: alpha_': Defence Strength + 'model_layers: alpha_': Layers + 'train_time: alpha_': $T_{train}$ + 'predict_time: alpha_': $T_{predict}$ + 'adv_accuracy: alpha_': Adv. Accuracy + 'accuracy: alpha_': Ben. Accuracy + 'adv_fit_time: alpha_': $T_{attack}$ + 'adv_failure_rate: alpha_': $h_{adv.}(t;\theta)$ + 'failure_rate: alpha_': $h_{ben.}(t;\theta)$ + 'Epochs: alpha_': No. of Epochs + 'model.trainer.batch_size: alpha_': Batch Size + def_gen: Defence + 'attack.init.eps: alpha_': $\varepsilon$ + log_normal: + plot: + file: log_normal_aft.pdf + title: Log Normal AFT Model + qq_title: Log Normal QQ Plot + t0: 2 + model: + penalizer: 0.5 + labels: + 'Intercept: sigma_': $\sigma$ + 'Intercept: mu_': $\mu$ + 'atk_value: mu_': Attack Strength + 'def_value: mu_': Defence Strength + 'model_layers: mu_': Layers + 'train_time: mu_': $T_{train}$ + 'predict_time: mu_': $T_{predict}$ + 'adv_accuracy: mu_': Adv. Accuracy + 'accuracy: mu_': Ben. Accuracy + 'adv_fit_time: mu_': $T_{attack}$ + 'adv_failure_rate: mu_': $h_{adv.}(t;\theta)$ + 'failure_rate: mu_': $h_{ben.}(t;\theta)$ + 'Epochs: mu_': No. of Epochs + 'model.trainer.batch_size: mu_': Batch Size + def_gen: Defence + 'attack.init.eps: mu_': $\varepsilon$ + 'data.sample.random_state: mu_': Random State + weibull: + plot: + file: weibull_aft.pdf + title: Weibull AFT Model + qq_title: Weibull QQ Plot + t0: 0.3 + model: + penalizer: 0.1 + labels: + 'Intercept: rho_': $\rho$ + 'Intercept: lambda_': $\lambda$ + 'data.sample.random_state: lambda_': Random State + 'atk_value: lambda_': Attack Strength + 'model_layers: lambda_': Layers + 'train_time: lambda_': $T_{train}$ + 'predict_time: lambda_': $T_{predict}$ + 'adv_accuracy: lambda_': Adv. Accuracy + 'accuracy: lambda_': Ben. Accuracy + 'adv_fit_time: lambda_': $T_{attack}$ + 'adv_failure_rate: lambda_': $f_{adv.}(t;\theta)$ + 'failure_rate: lambda_': $f_{ben.}(t;\theta)$ + 'Epochs: lambda_': No. of Epochs + 'model.trainer.batch_size: lambda_': Batch Size + def_gen: Defence + 'def_value: lambda_': Defence Strength + ': lambda_': '' + outs: + - path: plots/aft_comparison.csv + hash: md5 + md5: becda69f88bca7b3aea88f9d03f8d6b2 + size: 372 + - path: plots/aft_comparison.tex + hash: md5 + md5: 573048966254e5c14b42eeceb8bcdcef + size: 640 + - path: plots/cox_aft.pdf + hash: md5 + md5: bf2e1bfe07b049890b2305e73124249d + size: 31094 + - path: plots/cox_aft_dummies.pdf + hash: md5 + md5: 1c788a6e5bd4d70a9eb587dc86ae9361 + size: 29292 + - path: plots/cox_qq.pdf + hash: md5 + md5: 9c211ae31847d09b7017cb4ba12daa2f + size: 19012 + - path: plots/cox_summary.csv + hash: md5 + md5: d03ee02cb0d19a22d3817d71b35a980a + size: 4689 + - path: plots/exponential_aft.pdf + hash: md5 + md5: 49765ceb038d8030bff7b40ec4470a01 + size: 33405 + - path: plots/exponential_aft_dummies.pdf + hash: md5 + md5: 4451784f013d2410fc27568cf9fdbebe + size: 31645 + - path: plots/exponential_qq.pdf + hash: md5 + md5: 1917e0e492bdab785f9ab3104d626725 + size: 21568 + - path: plots/exponential_summary.csv + hash: md5 + md5: 1d080ad56ab9db2ba023ace48f1e755b + size: 5886 + - path: plots/gamma_aft.pdf + hash: md5 + md5: 80ebccdc533086b3184ac75a5dc32762 + size: 29660 + - path: plots/gamma_aft_dummies.pdf + hash: md5 + md5: f04b41b92436317e3d45c74470702825 + size: 35881 + - path: plots/gamma_qq.pdf + hash: md5 + md5: 060ea3a2e5a3153d0977ccd2b47e3056 + size: 20856 + - path: plots/gamma_summary.csv + hash: md5 + md5: 5794ccf0fc040f8f1765b98b34c6a74d + size: 14378 + - path: plots/log_logistic_aft.pdf + hash: md5 + md5: 1eba1aea7ea126e6721ea4178d829cd8 + size: 32396 + - path: plots/log_logistic_aft_dummies.pdf + hash: md5 + md5: 5503a9d6e9b6e377f1ea8fca3d50f34b + size: 32187 + - path: plots/log_logistic_qq.pdf + hash: md5 + md5: 3580ab59b1da38fe38cbe4ccfa70071d + size: 19062 + - path: plots/log_logistic_summary.csv + hash: md5 + md5: ae75911feb1d2d83dbc0a9c5e125420f + size: 5183 + - path: plots/log_normal_aft.pdf + hash: md5 + md5: 656cea7713fe17375dd78fa67db6cb3e + size: 32568 + - path: plots/log_normal_aft_dummies.pdf + hash: md5 + md5: 2812690f988e2cc8a8451f9d0199d563 + size: 32559 + - path: plots/log_normal_qq.pdf + hash: md5 + md5: 437b6932517a099a203eb9c7487255ee + size: 21726 + - path: plots/log_normal_summary.csv + hash: md5 + md5: d48ff310a60a8ad40313cf3b69ab5e58 + size: 5149 + - path: plots/weibull_aft.pdf + hash: md5 + md5: 3c7233e6138fea0d7ce234ea9c1ebace + size: 32594 + - path: plots/weibull_aft_dummies.pdf + hash: md5 + md5: 25f9c4aef4cba8ad79f6a90a52cefdef + size: 31752 + - path: plots/weibull_qq.pdf + hash: md5 + md5: 1c239b3ae7e7f9e3a6acac95d692b913 + size: 18990 + - path: plots/weibull_summary.csv + hash: md5 + md5: 8ee7e7d66bcf24eb8424460826ab7421 + size: 5217 + predict_survival_time: + cmd: python predict_with_best.py --data data/merged.csv --config_file afr.yaml --model + weibull --target adv_failures --duration_col adv_fit_time --output data/merged_afr.csv + deps: + - path: afr.yaml + hash: md5 + md5: ae87070c3ab701c6be86d1098f7b5b23 + size: 6090 + - path: data/merged.csv + hash: md5 + md5: 9cc7eb6799f013a2d4259a18d36328d2 + size: 123215063 + - path: plots/aft_comparison.tex + hash: md5 + md5: 573048966254e5c14b42eeceb8bcdcef + size: 640 + - path: predict_with_best.py + hash: md5 + md5: e60a437ab37d2ee22256268a207f2431 + size: 2571 + outs: + - path: data/merged_afr.csv + hash: md5 + md5: 3a9b42ec41f4b281681899c27c91c05e + size: 124150010 + plot: + cmd: python -m deckard.layers.plots --path plots/ --file data/merged_afr.csv -c + plots.yaml + deps: + - path: data/merged_afr.csv + hash: md5 + md5: 3a9b42ec41f4b281681899c27c91c05e + size: 124150010 + - path: plots.yaml + hash: md5 + md5: a0c5c100248543bb5f0de8949b459bc5 + size: 3815 + params: + afr.yaml: + covariates: + - adv_fit_time + - accuracy + - train_time + - atk_value + - def_value + - data.sample.random_state + - Epochs + - model_layers + - id + - atk_gen + - def_gen + - adv_failures + - adv_accuracy + - predict_time + weibull: + plot: + file: weibull_aft.pdf + title: Weibull AFT Model + qq_title: Weibull QQ Plot + t0: 0.3 + model: + penalizer: 0.1 + labels: + 'Intercept: rho_': $\rho$ + 'Intercept: lambda_': $\lambda$ + 'data.sample.random_state: lambda_': Random State + 'atk_value: lambda_': Attack Strength + 'model_layers: lambda_': Layers + 'train_time: lambda_': $T_{train}$ + 'predict_time: lambda_': $T_{predict}$ + 'adv_accuracy: lambda_': Adv. Accuracy + 'accuracy: lambda_': Ben. Accuracy + 'adv_fit_time: lambda_': $T_{attack}$ + 'adv_failure_rate: lambda_': $f_{adv.}(t;\theta)$ + 'failure_rate: lambda_': $f_{ben.}(t;\theta)$ + 'Epochs: lambda_': No. of Epochs + 'model.trainer.batch_size: lambda_': Batch Size + def_gen: Defence + 'def_value: lambda_': Defence Strength + ': lambda_': '' + plots.yaml: + cat_plot: + - file: adv_accuracy_vs_defence_type.pdf + hue: model_name + kind: boxen + set: + yscale: linear + x: def_gen + xlabels: Defence Type + y: adv_accuracy + ylabels: Adv. Accuracy + rotation: 90 + hue_order: + - ResNet18 + - ResNet34 + - ResNet50 + - ResNet101 + - ResNet152 + legend_title: Model + - file: ben_accuracy_vs_defence_type.pdf + hue: model_name + kind: boxen + x: def_gen + xlabels: Defence Type + y: accuracy + ylabels: Ben. Accuracy + rotation: 90 + hue_order: + - ResNet18 + - ResNet34 + - ResNet50 + - ResNet101 + - ResNet152 + legend_title: Model + - file: trash_score_vs_defence_type.pdf + hue: model_name + kind: boxen + set: + yscale: log + x: def_gen + xlabels: Defence Type + y: c_adv + ylabels: $\bar{C}_{adv.}$ + rotation: 90 + hue_order: + - ResNet18 + - ResNet34 + - ResNet50 + - ResNet101 + - ResNet152 + legend_title: Model + - file: trash_score_vs_attack_type.pdf + hue: model_name + kind: boxen + set: + yscale: log + x: atk_gen + xlabels: Attack Type + y: c_adv + ylabels: $\bar{C}_{adv.}$ + rotation: 90 + hue_order: + - ResNet18 + - ResNet34 + - ResNet50 + - ResNet101 + - ResNet152 + legend_title: Model + - file: adv_accuracy_vs_attack_type.pdf + hue: model_name + kind: boxen + x: atk_gen + xlabels: Attack Type + y: adv_accuracy + ylabels: Adv. Accuracy + rotation: 90 + hue_order: + - ResNet18 + - ResNet34 + - ResNet50 + - ResNet101 + - ResNet152 + legend_title: Model + line_plot: + - file: def_param_vs_accuracy.pdf + hue: def_gen + legend: + bbox_to_anchor: + - 1.05 + - 1 + title: Defence + title: Ben. Accuracy vs Defence Strength + x: def_value + x_scale: linear + xlabel: Defence Control Parameter + y: accuracy + y_scale: + ylabel: Ben. Accuracy + hue_order: + - Control + - Conf + - Gauss-in + - Gauss-out + - FSQ + errorbar: se + err_style: bars + - file: def_param_vs_adv_accuracy.pdf + hue: def_gen + legend: + bbox_to_anchor: + - 1.05 + - 1 + title: Defence + title: Adv. Accuracy vs Defence Strength + x: def_value + x_scale: linear + xlabel: Defence Control Parameter + y: adv_accuracy + y_scale: + ylabel: Adv. Accuracy + hue_order: + - Control + - Conf + - Gauss-in + - Gauss-out + - FSQ + errorbar: se + err_style: bars + - file: def_param_vs_adv_failure_rate.pdf + hue: def_gen + legend: + bbox_to_anchor: + - 1.05 + - 1 + title: Defence + title: $f_{adv}$ vs Defence Strength + x: def_value + x_scale: linear + xlabel: Defence Control Parameter + y: adv_failure_rate + y_scale: linear + ylabel: $f_{adv.}$ + hue_order: + - Control + - Conf + - Gauss-in + - Gauss-out + - FSQ + errorbar: se + err_style: bars + - file: atk_param_vs_accuracy.pdf + hue: atk_gen + legend: + bbox_to_anchor: + - 1.05 + - 1 + title: Adv. Accuracy vs Attack Strength + x: atk_value + x_scale: linear + xlabel: Attack Control Parameter + y: adv_accuracy + y_scale: + ylabel: Adv. Accuracy + hue_order: + - FGM + - PGD + - Deep + - HSJ + - Pixel + - Thresh + errorbar: se + err_style: bars + scatter_plot: + - x: train_time_per_sample + y: adv_failure_rate + hue: model_name + xlabel: $t_{train}$ + ylabel: $f_{adv}$ + title: $f_{adv}$ vs $t_{train}$ + file: adv_failure_rate_vs_train_time.pdf + x_scale: log + legend: + title: Model Name + bbox_to_anchor: + - 1.05 + - 1 + hue_order: + - ResNet18 + - ResNet34 + - ResNet50 + - ResNet101 + - ResNet152 + outs: + - path: plots/adv_accuracy_vs_attack_type.pdf + hash: md5 + md5: 9bd169743b91d14f5d30212cbda4287e + size: 36004 + - path: plots/adv_accuracy_vs_defence_type.pdf + hash: md5 + md5: 38b1f98d9c54b5d33804f5ebb4728fd9 + size: 33276 + - path: plots/adv_failure_rate_vs_train_time.pdf + hash: md5 + md5: 885d2b52acf4be40f8bafc83b375ab02 + size: 274080 + - path: plots/atk_param_vs_accuracy.pdf + hash: md5 + md5: 16268b2860b047140de9266b99cc0472 + size: 21458 + - path: plots/ben_accuracy_vs_defence_type.pdf + hash: md5 + md5: 6d67c55146f0c45c5b760e84c984a491 + size: 34713 + - path: plots/def_param_vs_accuracy.pdf + hash: md5 + md5: 1f4467da61460335aabd287ecdc466dc + size: 19239 + - path: plots/def_param_vs_adv_accuracy.pdf + hash: md5 + md5: b5cf5be5f27f02f5c835b5c8529fd694 + size: 18808 + - path: plots/def_param_vs_adv_failure_rate.pdf + hash: md5 + md5: 1113bf660baf47ed897ae16364315843 + size: 22070 + - path: plots/trash_score_vs_attack_type.pdf + hash: md5 + md5: 8bce1a0e763d7636cbbe71dca655ff7c + size: 45329 + - path: plots/trash_score_vs_defence_type.pdf + hash: md5 + md5: 53678b64ea8db5cf0e727cd3cfd67382 + size: 41417 + copy_results: + cmd: mkdir -p ~/ml_afr/plots && cp -r plots/* ~/ml_afr/plots/ + deps: + - path: plots/ + hash: md5 + md5: b95f4289dd66fc205b9813aa1aa479fd.dir + size: 1094175 + nfiles: 37 diff --git a/examples/pytorch/old_plots/dvc.yaml b/examples/pytorch/old_plots/dvc.yaml new file mode 100644 index 00000000..958ea2e5 --- /dev/null +++ b/examples/pytorch/old_plots/dvc.yaml @@ -0,0 +1,130 @@ +vars: + - plots.yaml:cat_plot + - plots.yaml:line_plot + - plots.yaml:scatter_plot + - afr.yaml:covariates + - afr.yaml:weibull + - afr.yaml:log_logistic + - afr.yaml:log_normal + - afr.yaml:gamma + - afr.yaml:exponential + - afr.yaml:cox + - afr.yaml:dummies +stages: + clean: + foreach: + - cifar + - cifar100 + - mnist + do: + cmd: >- + python -m deckard.layers.clean_data + -i data/attack_${item}.csv + -o data/${item}.csv + -c ../conf/clean_${item}.yaml + --drop_if_empty adv_fit_time accuracy train_time predict_time adv_accuracy Epochs + deps: + - data/attack_${item}.csv + - ../conf/clean_${item}.yaml + outs: + - data/${item}.csv + merge: + deps: + - data/mnist.csv + - data/cifar.csv + - data/cifar100.csv + cmd: >- + python -m deckard.layers.merge + --smaller_file data/cifar.csv data/cifar100.csv data/mnist.csv + --output_folder data + --output_file merged.csv + outs: + - data/merged.csv + + afr: + cmd: python -m deckard.layers.afr --data_file data/merged.csv --target adv_failures --duration_col adv_fit_time --config_file afr.yaml --plots_folder plots/ + deps: + - data/merged.csv + plots: + - plots/weibull_aft.pdf + - plots/weibull_aft_dummies.pdf + - plots/weibull_qq.pdf + - plots/log_logistic_aft.pdf + - plots/log_logistic_aft_dummies.pdf + - plots/log_logistic_qq.pdf + - plots/log_normal_aft.pdf + - plots/log_normal_aft_dummies.pdf + - plots/log_normal_qq.pdf + - plots/gamma_aft.pdf + - plots/gamma_aft_dummies.pdf + - plots/gamma_qq.pdf + - plots/exponential_aft.pdf + - plots/exponential_aft_dummies.pdf + - plots/exponential_qq.pdf + - plots/cox_aft.pdf + - plots/cox_aft_dummies.pdf + - plots/cox_qq.pdf + params: + - afr.yaml: + - dummies + - covariates + - weibull + - log_logistic + - log_normal + - gamma + - exponential + - cox + metrics: + - plots/aft_comparison.csv + outs: + - plots/aft_comparison.tex + - plots/weibull_summary.csv + - plots/cox_summary.csv + - plots/log_logistic_summary.csv + - plots/log_normal_summary.csv + - plots/gamma_summary.csv + - plots/exponential_summary.csv + predict_survival_time: + cmd: >- + python predict_with_best.py + --data data/merged.csv + --config_file afr.yaml + --model weibull + --target adv_failures + --duration_col adv_fit_time + --output data/merged_afr.csv + deps: + - data/merged.csv + - afr.yaml + - plots/aft_comparison.tex + - predict_with_best.py + outs: + - data/merged_afr.csv + plot: + cmd : python -m deckard.layers.plots --path plots/ --file data/merged_afr.csv -c plots.yaml + deps: + - data/merged_afr.csv + - plots.yaml + plots: + - plots/${cat_plot[0].file} + - plots/${cat_plot[1].file} + - plots/${cat_plot[2].file} + - plots/${cat_plot[3].file} + - plots/${cat_plot[4].file} + - plots/${line_plot[0].file} + - plots/${line_plot[1].file} + - plots/${line_plot[2].file} + - plots/${line_plot[3].file} + - plots/${scatter_plot[0].file} + params: + - plots.yaml: + - line_plot + - scatter_plot + - cat_plot + - afr.yaml: + - covariates + - weibull + copy_results: + cmd: mkdir -p ~/ml_afr/plots && cp -r plots/* ~/ml_afr/plots/ + deps: + - plots/ diff --git a/examples/pytorch/mnist/conf/plots.yaml b/examples/pytorch/old_plots/plots.yaml similarity index 57% rename from examples/pytorch/mnist/conf/plots.yaml rename to examples/pytorch/old_plots/plots.yaml index 68401dee..7bc30e78 100644 --- a/examples/pytorch/mnist/conf/plots.yaml +++ b/examples/pytorch/old_plots/plots.yaml @@ -1,16 +1,13 @@ cat_plot: -- file: adv_accuracy_vs_defence_type.eps +- file: adv_accuracy_vs_defence_type.pdf hue: model_name kind: boxen set: yscale: linear - titles: Adv. Accuracy vs Defence Type - titles: Adv. Accuracy vs Defence Type x: def_gen xlabels: Defence Type y: adv_accuracy ylabels: Adv. Accuracy - ylabels: Adv. Accuracy rotation : 90 hue_order: - ResNet18 @@ -18,16 +15,16 @@ cat_plot: - ResNet50 - ResNet101 - ResNet152 -- file: ben_accuracy_vs_defence_type.eps + # titles: "{col_name} dataset" + legend_title: "Model" + # col: 'id' +- file: ben_accuracy_vs_defence_type.pdf hue: model_name kind: boxen - titles: Ben. Accuracy vs Defence Type - titles: Ben. Accuracy vs Defence Type x: def_gen xlabels: Defence Type y: accuracy ylabels: Ben. Accuracy - ylabels: Ben. Accuracy rotation : 90 hue_order: - ResNet18 @@ -35,32 +32,21 @@ cat_plot: - ResNet50 - ResNet101 - ResNet152 -- file: ben_failures_per_train_time_vs_defence_type.eps + # titles: "{col_name} dataset" + legend_title: "Model" + # col: 'id' + # col_order: + # - mnist + # - cifar + # - cifar100 +- file: trash_score_vs_defence_type.pdf hue: model_name kind: boxen set: yscale: log - titles: $\bar{C}_{ben.}$ vs Defence Type x: def_gen xlabels: Defence Type - y: training_time_per_failure - ylabels: $\bar{C}_{ben.}$ - rotation : 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 -- file: adv_failures_per_train_time_vs_defence_type.eps - hue: model_name - kind: boxen - set: - yscale: log - titles: $\bar{C}_{adv.}$ vs Defence Type - x: def_gen - xlabels: Defence Type - y: training_time_per_adv_failure + y: c_adv ylabels: $\bar{C}_{adv.}$ rotation : 90 hue_order: @@ -69,16 +55,21 @@ cat_plot: - ResNet50 - ResNet101 - ResNet152 -- file: adv_failures_per_train_time_vs_attack_type.eps + # titles: "{col_name} dataset" + legend_title: "Model" + # col: 'id' + # col_order: + # - mnist + # - cifar + # - cifar100 +- file: trash_score_vs_attack_type.pdf hue: model_name kind: boxen - legend_title: Model Name set: yscale: log - titles: $\bar{C}_{adv.}$ vs Attack Type x: atk_gen xlabels: Attack Type - y: training_time_per_adv_failure + y: c_adv ylabels: $\bar{C}_{adv.}$ rotation : 90 hue_order: @@ -87,28 +78,16 @@ cat_plot: - ResNet50 - ResNet101 - ResNet152 -- file: adv_failures_per_test_time_vs_defence_type.eps + # titles: "{col_name} dataset" + legend_title: "Model" + # col: 'id' + # col_order: + # - mnist + # - cifar + # - cifar100 +- file: adv_accuracy_vs_attack_type.pdf hue: model_name kind: boxen - legend_title: Model Name - titles: $f_{adv}$ vs Defence Type - x: def_gen - xlabels: Defence Type - y: adv_failure_rate - ylabels: $f_{adv.}$ - rotation : 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 - -- file: adv_accuracy_vs_attack_type.eps - hue: model_name - kind: boxen - legend_title: Model Name - titles: Adv. Accuracy vs Attack Type x: atk_gen xlabels: Attack Type y: adv_accuracy @@ -120,26 +99,15 @@ cat_plot: - ResNet50 - ResNet101 - ResNet152 -- file: ben_failure_rate_vs_defence_type.eps - hue: model_name - kind: boxen - legend_title: Model Name - set: - yscale: log - titles: $f_{ben}(t; \theta)$ vs Defence Type - x: def_gen - xlabels: Defence Type - y: failure_rate - ylabels: $f_{ben}(t; \theta)$ - rotation : 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 + # titles: "{col_name} dataset" + legend_title: "Model" + # col: 'id' + # col_order: + # - mnist + # - cifar + # - cifar100 line_plot: -- file: def_param_vs_accuracy.eps +- file: def_param_vs_accuracy.pdf hue: def_gen legend: {"bbox_to_anchor": [1.05, 1], "title": "Defence"} title: Ben. Accuracy vs Defence Strength @@ -152,14 +120,12 @@ line_plot: hue_order: - Control - Conf - - Epochs - Gauss-in - Gauss-out - - Conf - FSQ errorbar: se err_style: bars -- file: def_param_vs_adv_accuracy.eps +- file: def_param_vs_adv_accuracy.pdf hue: def_gen legend: {"bbox_to_anchor": [1.05, 1], "title": "Defence"} title: Adv. Accuracy vs Defence Strength @@ -172,14 +138,13 @@ line_plot: hue_order: - Control - Conf - - Epochs - Gauss-in - Gauss-out - - Conf - FSQ + # style : id errorbar: se err_style: bars -- file: def_param_vs_adv_failure_rate.eps +- file: def_param_vs_adv_failure_rate.pdf hue: def_gen legend: {"bbox_to_anchor": [1.05, 1], "title": "Defence"} title: $f_{adv}$ vs Defence Strength @@ -189,28 +154,25 @@ line_plot: y: adv_failure_rate y_scale: linear ylabel: $f_{adv.}$ + # style: id hue_order: - Control - Conf - - Epochs - Gauss-in - Gauss-out - - Conf - FSQ errorbar: se err_style: bars -- file: atk_param_vs_accuracy.eps +- file: atk_param_vs_accuracy.pdf hue: atk_gen legend: {bbox_to_anchor: [1.05, 1]} title: Adv. Accuracy vs Attack Strength - title: Adv. Accuracy vs Attack Strength x: atk_value x_scale: linear xlabel: Attack Control Parameter y: adv_accuracy y_scale: ylabel: Adv. Accuracy - ylabel: Adv. Accuracy hue_order: - FGM - PGD @@ -227,7 +189,7 @@ scatter_plot: xlabel: $t_{train}$ ylabel: $f_{adv}$ title: $f_{adv}$ vs $t_{train}$ - file: adv_failure_rate_vs_train_time.eps + file: adv_failure_rate_vs_train_time.pdf # y_scale: log x_scale: log legend: diff --git a/examples/pytorch/old_plots/plots/.gitignore b/examples/pytorch/old_plots/plots/.gitignore new file mode 100644 index 00000000..46da6840 --- /dev/null +++ b/examples/pytorch/old_plots/plots/.gitignore @@ -0,0 +1 @@ +/aft_comparison.tex diff --git a/examples/pytorch/old_plots/predict_with_best.py b/examples/pytorch/old_plots/predict_with_best.py new file mode 100644 index 00000000..c42217d2 --- /dev/null +++ b/examples/pytorch/old_plots/predict_with_best.py @@ -0,0 +1,70 @@ +import argparse +from pathlib import Path +import pandas as pd +import yaml +from deckard.layers.afr import fit_aft, clean_data_for_aft, calculate_raw_failures + + +parser = argparse.ArgumentParser() +parser.add_argument("--data", type=str, required=True) +parser.add_argument("--config_file", type=str, required=True) +parser.add_argument("--target", type=str, required=True, help="Target column name") +parser.add_argument( + "--duration_col", + type=str, + required=True, + help="Duration column name", +) +parser.add_argument("--model", type=str, required=True) +parser.add_argument("--output", type=str, required=True) +parser.add_argument("--training_samples", type=int, default=48000) +parser.add_argument("--attack_samples", type=int, default=100) +args = parser.parse_args() + +suffix = Path(args.data).suffix +if suffix == ".csv": + df = pd.read_csv(args.data) +else: + raise NotImplementedError(f"Unknown file format: {suffix}") + +suffix = Path(args.config_file).suffix +assert Path(args.config_file).exists(), f"Model file not found: {args.config_file}" + +if suffix == ".yaml": + with open(args.config_file, "r") as f: + config = yaml.safe_load(f) +else: + raise NotImplementedError(f"Unknown file format: {suffix}") +assert args.model in config, f"Model not found in the model file: {args.model}" +# Config +dummies = config.pop("dummies", {"atk_gen": "Atk:", "def_gen": "Def:", "id": "Data:"}) +covariate_list = config.get("covariates", []) +model_config = config[args.model].get("model", {}) +# Will generate a subset used for fitting +data = df.copy() +# Calculate raw failures from accuracy +data = calculate_raw_failures(args, df, config) +# Clean data by removing columns that are not in the covariate list and creating dummies for the categorical variables +data = clean_data_for_aft(data, covariate_list, target=args.target, dummy_dict=dummies) +# Fit the model +model = fit_aft( + data, + event_col=args.target, + duration_col=args.duration_col, + **model_config, + mtype=args.model, +) +# Predict the adversarial survival time +adv_survival_time = model.predict_expectation(data) +df = df.assign(adv_survival_time=adv_survival_time) +train_rate = df["train_time_per_sample"] +normalized_survival_time = df["adv_survival_time"] / args.attack_samples +c_adv = train_rate / normalized_survival_time +df = df.assign(c_adv=c_adv) +assert "c_adv" in df.columns, "c_adv column not found in the dataframe" +suffix = Path(args.output).suffix +if suffix == ".csv": + df.to_csv(args.output, index=False) +else: + raise NotImplementedError(f"Unknown file format: {suffix}") +print(f"Output file: {args.output}") diff --git a/examples/pytorch/mnist/params.yaml b/examples/pytorch/params.yaml similarity index 70% rename from examples/pytorch/mnist/params.yaml rename to examples/pytorch/params.yaml index 7a9ccdc6..1f0afa20 100644 --- a/examples/pytorch/mnist/params.yaml +++ b/examples/pytorch/params.yaml @@ -1,7 +1,8 @@ _target_: deckard.base.experiment.Experiment +atk_name: hsj attack: _target_: deckard.base.attack.Attack - attack_size: 10 + attack_size: 100 data: _target_: deckard.base.data.Data generate: @@ -11,18 +12,20 @@ attack: _target_: deckard.base.data.sampler.SklearnDataSampler random_state: 0 stratify: true - sklearn_pipeline: - _target_: deckard.base.data.sklearn_pipeline.SklearnDataPipeline - preprocessor: - name: sklearn.preprocessing.StandardScaler - with_mean: true - with_std: true + test_size: 12000 + train_size: 48000 init: _target_: deckard.base.attack.AttackInitializer + batch_size: 1024 model: _target_: deckard.base.model.Model art: _target_: deckard.base.model.art_pipeline.ArtPipeline + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss data: _target_: deckard.base.data.Data generate: @@ -32,12 +35,8 @@ attack: _target_: deckard.base.data.sampler.SklearnDataSampler random_state: 0 stratify: true - sklearn_pipeline: - _target_: deckard.base.data.sklearn_pipeline.SklearnDataPipeline - preprocessor: - name: sklearn.preprocessing.StandardScaler - with_mean: true - with_std: true + test_size: 12000 + train_size: 48000 initialize: clip_values: - 0 @@ -49,6 +48,10 @@ attack: momentum: 0.9 name: torch.optim.SGD library: pytorch + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD data: _target_: deckard.base.data.Data generate: @@ -58,26 +61,29 @@ attack: _target_: deckard.base.data.sampler.SklearnDataSampler random_state: 0 stratify: true - sklearn_pipeline: - _target_: deckard.base.data.sklearn_pipeline.SklearnDataPipeline - preprocessor: - name: sklearn.preprocessing.StandardScaler - with_mean: true - with_std: true + test_size: 12000 + train_size: 48000 init: _target_: deckard.base.model.ModelInitializer name: torch_example.ResNet18 num_channels: 1 + num_classes: 10 library: pytorch trainer: batch_size: 1024 - nb_epoch: 100 + nb_epochs: 1 + verbose: true name: art.attacks.evasion.HopSkipJump method: evasion model: _target_: deckard.base.model.Model art: _target_: deckard.base.model.art_pipeline.ArtPipeline + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss data: _target_: deckard.base.data.Data generate: @@ -87,12 +93,8 @@ attack: _target_: deckard.base.data.sampler.SklearnDataSampler random_state: 0 stratify: true - sklearn_pipeline: - _target_: deckard.base.data.sklearn_pipeline.SklearnDataPipeline - preprocessor: - name: sklearn.preprocessing.StandardScaler - with_mean: true - with_std: true + test_size: 12000 + train_size: 48000 initialize: clip_values: - 0 @@ -104,6 +106,10 @@ attack: momentum: 0.9 name: torch.optim.SGD library: pytorch + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD data: _target_: deckard.base.data.Data generate: @@ -113,20 +119,18 @@ attack: _target_: deckard.base.data.sampler.SklearnDataSampler random_state: 0 stratify: true - sklearn_pipeline: - _target_: deckard.base.data.sklearn_pipeline.SklearnDataPipeline - preprocessor: - name: sklearn.preprocessing.StandardScaler - with_mean: true - with_std: true + test_size: 12000 + train_size: 48000 init: _target_: deckard.base.model.ModelInitializer name: torch_example.ResNet18 num_channels: 1 + num_classes: 10 library: pytorch trainer: batch_size: 1024 - nb_epoch: 100 + nb_epochs: 1 + verbose: true data: _target_: deckard.base.data.Data generate: @@ -136,22 +140,19 @@ data: _target_: deckard.base.data.sampler.SklearnDataSampler random_state: 0 stratify: true - sklearn_pipeline: - _target_: deckard.base.data.sklearn_pipeline.SklearnDataPipeline - preprocessor: - name: sklearn.preprocessing.StandardScaler - with_mean: true - with_std: true -direction: maximize + test_size: 12000 + train_size: 48000 +dataset: mnist +def_name: control +device_id: gpu +direction: +- maximize files: _target_: deckard.base.files.FileConfig adv_predictions_file: adv_predictions.json attack_dir: attacks attack_file: attack attack_type: .pkl - data_dir: data - data_file: data - data_type: .pkl directory: mnist model_dir: models model_file: model @@ -165,6 +166,11 @@ model: _target_: deckard.base.model.Model art: _target_: deckard.base.model.art_pipeline.ArtPipeline + clip_values: + - 0 + - 255 + criterion: + name: torch.nn.CrossEntropyLoss data: _target_: deckard.base.data.Data generate: @@ -174,12 +180,8 @@ model: _target_: deckard.base.data.sampler.SklearnDataSampler random_state: 0 stratify: true - sklearn_pipeline: - _target_: deckard.base.data.sklearn_pipeline.SklearnDataPipeline - preprocessor: - name: sklearn.preprocessing.StandardScaler - with_mean: true - with_std: true + test_size: 12000 + train_size: 48000 initialize: clip_values: - 0 @@ -191,6 +193,10 @@ model: momentum: 0.9 name: torch.optim.SGD library: pytorch + optimizer: + lr: 0.01 + momentum: 0.9 + name: torch.optim.SGD data: _target_: deckard.base.data.Data generate: @@ -200,21 +206,21 @@ model: _target_: deckard.base.data.sampler.SklearnDataSampler random_state: 0 stratify: true - sklearn_pipeline: - _target_: deckard.base.data.sklearn_pipeline.SklearnDataPipeline - preprocessor: - name: sklearn.preprocessing.StandardScaler - with_mean: true - with_std: true + test_size: 12000 + train_size: 48000 init: _target_: deckard.base.model.ModelInitializer name: torch_example.ResNet18 num_channels: 1 + num_classes: 10 library: pytorch trainer: batch_size: 1024 - nb_epoch: 100 -optimizers: accuracy + nb_epochs: 1 + verbose: true +model_name: ResNet18 +optimizers: +- accuracy scorers: _target_: deckard.base.scorer.ScorerDict accuracy: diff --git a/examples/pytorch/plots/afr.yaml b/examples/pytorch/plots/afr.yaml new file mode 100644 index 00000000..3c0842c4 --- /dev/null +++ b/examples/pytorch/plots/afr.yaml @@ -0,0 +1,177 @@ +covariates: + - adv_fit_time_per_sample + - accuracy + - train_time + - atk_value + - def_value + - Epochs + - model_layers + - id + - atk_gen + - def_gen + - adv_failures + - adv_accuracy + - predict_time +fillna: + Epochs: 20 + data.sample.train_size : 48000 + data.sample.test_size: 12000 +cox: + plot: + file : cox_aft.pdf + title : Cox Model + qq_title : Cox QQ Plot + t0: .3 + model: + penalizer: .2 + labels: + "data.sample.random_state": "Random State" + "atk_value": "Attack Strength" + "train_time": "$T_{train}$" + "predict_time": "$T_{predict}$" + "adv_accuracy": "Adv. Accuracy" + "def_value": "Defence Strength" + "accuracy": "Ben. Accuracy" + "model_layers": "Layers" + "Epochs": "No. of Epochs" + "def_gen": "Defence" +# aalen: +# plot: +# file : aalen_aft.pdf +# title : Aalen Model +# qq_title : Aalen QQ Plot +# t0: 1 +# model: +# alpha: 1 +# coef_penalizer: .1 +# smoothing_penalizer: .1 +# labels: +# "data.sample.random_state": "Random State" +# "atk_value": "Attack Strength" +# "train_time": "$T_{train}$" +# "predict_time": "$T_{predict}$" +# "adv_accuracy": "Adv. Accuracy" +# "accuracy": "Ben. Accuracy" +# "adv_fit_time_per_sample": "$T_{attack}$" +# "adv_failure_rate": "$f_{adv.}(t;\\theta)$" +# "failure_rate": "$f_{ben.}(t;\\theta)$" +# "Epochs": "No. of Epochs" +# "model.trainer.batch_size": "Batch Size" +# "def_gen": "Defence" +gamma: + plot: + file : gamma_aft.pdf + title : Generalized Gamma Model + qq_title : Gamma QQ Plot + t0: .3 + model: + penalizer : .4 + labels: + "Intercept: alpha_": "$\\alpha$" + "Intercept: beta_": "$\\beta$" + "data.sample.random_state: beta_": "Random State" + "def_value: beta_": "Defence Strength" + "atk_value: beta_": "Attack Strength" + "train_time: beta_": "$T_{train}$" + "model_layers: beta_": "Layers" + "predict_time: beta_": "$T_{predict}$" + "adv_accuracy: beta_": "Adv. Accuracy" + "accuracy: beta_": "Ben. Accuracy" + "Epochs: beta_": "No. of Epochs" + "def_gen": "Defence" + "attack.init.eps: beta_": "$\\varepsilon$" +weibull: + plot: + file : weibull_aft.pdf + title : Weibull AFT Model + qq_title : Weibull QQ Plot + t0: .3 + model: + penalizer: .1 + labels: + "Intercept: rho_": "$\\rho$" + "Intercept: lambda_": "$\\lambda$" + "data.sample.random_state: lambda_": "Random State" + "atk_value: lambda_": "Attack Strength" + "model_layers: lambda_": "Layers" + "train_time: lambda_": "$T_{train}$" + "predict_time: lambda_": "$T_{predict}$" + "adv_accuracy: lambda_": "Adv. Accuracy" + "accuracy: lambda_": "Ben. Accuracy" + "Epochs: lambda_": "No. of Epochs" + "model.trainer.batch_size: lambda_": "Batch Size" + "def_gen": "Defence" + "def_value: lambda_" : "Defence Strength" + ": lambda_" : "" +exponential: + plot: + file : exponential_aft.pdf + title : Exponential Model + qq_title : Exponential QQ Plot + t0: .1 + model: + breakpoints: + - .1 + labels: + "Intercept: rho_": "$\\rho$" + "Intercept: lambda_": "$\\lambda$" + "atk_value: lambda_": "Attack Strength" + "def_value: lambda_": "Defence Strength" + "model_layers: lambda_": "Layers" + "train_time: lambda_": "$T_{train}$" + "predict_time: lambda_": "$T_{predict}$" + "adv_accuracy: lambda_": "Adv. Accuracy" + "accuracy: lambda_": "Ben. Accuracy" + "Epochs: lambda_": "No. of Epochs" + "def_gen": "Defence" + ": lambda_" : "" +log_logistic: + plot: + file : log_logistic_aft.pdf + title : Log logistic AFT Model + qq_title : Log Logistic QQ Plot + t0: 1 + model: + penalizer: .2 + labels: + "Intercept: beta_": "$\\beta$" + "Intercept: alpha_": "$\\alpha$" + "data.sample.random_state: alpha_": "Random State" + "atk_value: alpha_": "Attack Strength" + "def_value: alpha_": "Defence Strength" + "model_layers: alpha_": "Layers" + "train_time: alpha_": "$T_{train}$" + "predict_time: alpha_": "$T_{predict}$" + "adv_accuracy: alpha_": "Adv. Accuracy" + "accuracy: alpha_": "Ben. Accuracy" + "Epochs: alpha_": "No. of Epochs" + "def_gen": "Defence" +log_normal: + plot: + file : log_normal_aft.pdf + title : Log Normal AFT Model + qq_title : Log Normal QQ Plot + t0: 2 + model: + penalizer: .5 + labels: + "Intercept: sigma_": "$\\sigma$" + "Intercept: mu_": "$\\mu$" + "atk_value: mu_": "Attack Strength" + "def_value: mu_": "Defence Strength" + "model_layers: mu_": "Layers" + "train_time: mu_": "$T_{train}$" + "predict_time: mu_": "$T_{predict}$" + "adv_accuracy: mu_": "Adv. Accuracy" + "accuracy: mu_": "Ben. Accuracy" + "adv_failure_rate: mu_": "$h_{adv.}(t;\\theta)$" + "failure_rate: mu_": "$h_{ben.}(t;\\theta)$" + "Epochs: mu_": "No. of Epochs" + "model.trainer.batch_size: mu_": "Batch Size" + "def_gen": "Defence" + "attack.init.eps: mu_": "$\\varepsilon$" + "data.sample.random_state: mu_": "Random State" +dummies: + "atk_gen": "Atk:" + "def_gen": "Def:" + "id" : "Data:" diff --git a/examples/pytorch/plots/dvc.lock b/examples/pytorch/plots/dvc.lock new file mode 100644 index 00000000..62c8ff3c --- /dev/null +++ b/examples/pytorch/plots/dvc.lock @@ -0,0 +1,688 @@ +schema: '2.0' +stages: + clean@cifar: + cmd: python -m deckard.layers.clean_data -i data/attack_cifar.csv -o data/cifar.csv -c + ../conf/clean_cifar.yaml --drop_if_empty adv_fit_time accuracy train_time predict_time + adv_accuracy + deps: + - path: ../conf/clean_cifar.yaml + hash: md5 + md5: 64150a388252f623fd67838ed36ca670 + size: 873 + - path: data/attack_cifar.csv + hash: md5 + md5: 9fb240676d1ba7faea366fad55931a01 + size: 26090605 + outs: + - path: data/cifar.csv + hash: md5 + md5: 93572521018214433674bcc3256bba0c + size: 20644834 + clean@mnist: + cmd: python -m deckard.layers.clean_data -i data/attack_mnist.csv -o data/mnist.csv -c + ../conf/clean_mnist.yaml --drop_if_empty adv_fit_time accuracy train_time predict_time + adv_accuracy + deps: + - path: ../conf/clean_mnist.yaml + hash: md5 + md5: b5ee8305cb0c9daa6ff09d6a096c5a34 + size: 869 + - path: data/attack_mnist.csv + hash: md5 + md5: 5b422f124272c3b012f023a354ba2737 + size: 189644858 + outs: + - path: data/mnist.csv + hash: md5 + md5: 994d1bcab57756d6d7aa112c464cd13d + size: 23238114 + clean@cifar100: + cmd: python -m deckard.layers.clean_data -i data/attack_cifar100.csv -o data/cifar100.csv -c + ../conf/clean_cifar100.yaml --drop_if_empty adv_fit_time accuracy train_time + predict_time adv_accuracy + deps: + - path: ../conf/clean_cifar100.yaml + hash: md5 + md5: c5e8689321f99fd2f691ef7f9d8b5e4f + size: 897 + - path: data/attack_cifar100.csv + hash: md5 + md5: 8f508ea00d964ebb838f5ad4e678dc44 + size: 36167717 + outs: + - path: data/cifar100.csv + hash: md5 + md5: 898bb3154def96e0ee20fe3041b2bbff + size: 30522799 + merge: + cmd: python -m deckard.layers.merge --smaller_file data/cifar.csv data/cifar100.csv + data/mnist.csv --output_folder data --output_file merged.csv + deps: + - path: data/cifar.csv + hash: md5 + md5: 93572521018214433674bcc3256bba0c + size: 20644834 + - path: data/cifar100.csv + hash: md5 + md5: 898bb3154def96e0ee20fe3041b2bbff + size: 30522799 + - path: data/mnist.csv + hash: md5 + md5: 994d1bcab57756d6d7aa112c464cd13d + size: 23238114 + outs: + - path: data/merged.csv + hash: md5 + md5: 8d1038b925e7229ed5756e5f8df972a2 + size: 75922957 + afr: + cmd: python -m deckard.layers.afr --data_file data/merged.csv --target adv_failures + --duration_col adv_fit_time_per_sample --config_file afr.yaml --plots_folder + plots/ + deps: + - path: data/merged.csv + hash: md5 + md5: ebca17142f7fbfc920b37824ea1c480e + size: 85713970 + params: + afr.yaml: + covariates: + - adv_fit_time_per_sample + - accuracy + - train_time_per_sample + - atk_value + - def_value + - data.sample.random_state + - Epochs + - model_layers + - id + - atk_gen + - def_gen + - adv_failures + - adv_accuracy + - predict_time_per_sample + cox: + plot: + file: cox_aft.pdf + title: Cox Model + qq_title: Cox QQ Plot + t0: 0.3 + model: + penalizer: 0.1 + labels: + data.sample.random_state: Random State + atk_value: Attack Strength + train_time_per_sample: $t_{train}$ + predict_time_per_sample: $t_{predict}$ + adv_accuracy: Adv. Accuracy + def_value: Defence Strength + accuracy: Ben. Accuracy + model_layers: Layers + adv_fit_time_per_sample: $t_{attack}$ + adv_failure_rate: $f_{adv.}(t;\theta)$ + failure_rate: $f_{ben.}(t;\theta)$ + Epochs: No. of Epochs + model.trainer.batch_size: Batch Size + def_gen: Defence + dummies: + atk_gen: 'Atk:' + def_gen: 'Def:' + id: 'Data:' + exponential: + plot: + file: exponential_aft.pdf + title: Exponential Model + qq_title: Exponential QQ Plot + t0: 0.1 + model: + breakpoints: + - 0.1 + labels: + 'Intercept: rho_': $\rho$ + 'Intercept: lambda_': $\lambda$ + 'data.sample.random_state: lambda_': Random State + 'atk_value: lambda_': Attack Strength + 'def_value: lambda_': Defence Strength + 'model_layers: lambda_': Layers + 'train_time_per_sample: lambda_': $t_{train}$ + 'predict_time_per_sample: lambda_': $t_{predict}$ + 'adv_accuracy: lambda_': Adv. Accuracy + 'accuracy: lambda_': Ben. Accuracy + 'adv_fit_time_per_sample: lambda_': $t_{attack}$ + 'adv_failure_rate: lambda_': $f_{adv.}(t;\theta)$ + 'failure_rate: lambda_': $f_{ben.}(t;\theta)$ + 'Epochs: lambda_': No. of Epochs + 'model.trainer.batch_size: lambda_': Batch Size + def_gen: Defence + ': lambda_': '' + gamma: + plot: + file: gamma_aft.pdf + title: Generalized Gamma Model + qq_title: Gamma QQ Plot + t0: 0.3 + model: + penalizer: 0.4 + labels: + 'Intercept: alpha_': $\alpha$ + 'Intercept: beta_': $\beta$ + 'data.sample.random_state: beta_': Random State + 'def_value: beta_': Defence Strength + 'atk_value: beta_': Attack Strength + 'train_time_per_sample: beta_': $t_{train}$ + 'model_layers: beta_': Layers + 'predict_time_per_sample: beta_': $t_{predict}$ + 'adv_accuracy: beta_': Adv. Accuracy + 'accuracy: beta_': Ben. Accuracy + 'adv_fit_time_per_sample: beta_': $t_{attack}$ + 'adv_failure_rate: beta_': $h_{adv.}(t;\theta)$ + 'failure_rate: beta_': $h_{ben.}(t;\theta)$ + 'Epochs: beta_': No. of Epochs + 'model.trainer.batch_size: beta_': Batch Size + def_gen: Defence + 'attack.init.eps: beta_': $\varepsilon$ + log_logistic: + plot: + file: log_logistic_aft.pdf + title: Log logistic AFR Model + qq_title: Log Logistic QQ Plot + t0: 1 + model: + penalizer: 0.2 + labels: + 'Intercept: beta_': $\beta$ + 'Intercept: alpha_': $\alpha$ + 'data.sample.random_state: alpha_': Random State + 'atk_value: alpha_': Attack Strength + 'def_value: alpha_': Defence Strength + 'model_layers: alpha_': Layers + 'train_time_per_sample: alpha_': $t_{train}$ + 'predict_time_per_sample: alpha_': $t_{predict}$ + 'adv_accuracy: alpha_': Adv. Accuracy + 'accuracy: alpha_': Ben. Accuracy + 'adv_fit_time_per_sample: alpha_': $t_{attack}$ + 'adv_failure_rate: alpha_': $h_{adv.}(t;\theta)$ + 'failure_rate: alpha_': $h_{ben.}(t;\theta)$ + 'Epochs: alpha_': No. of Epochs + 'model.trainer.batch_size: alpha_': Batch Size + def_gen: Defence + 'attack.init.eps: alpha_': $\varepsilon$ + log_normal: + plot: + file: log_normal_aft.pdf + title: Log Normal AFR Model + qq_title: Log Normal QQ Plot + t0: 2 + model: + penalizer: 0.5 + labels: + 'Intercept: sigma_': $\sigma$ + 'Intercept: mu_': $\mu$ + 'atk_value: mu_': Attack Strength + 'def_value: mu_': Defence Strength + 'model_layers: mu_': Layers + 'train_time_per_sample: mu_': $t_{train}$ + 'predict_time_per_sample: mu_': $t_{predict}$ + 'adv_accuracy: mu_': Adv. Accuracy + 'accuracy: mu_': Ben. Accuracy + 'adv_fit_time_per_sample: mu_': $t_{attack}$ + 'adv_failure_rate: mu_': $h_{adv.}(t;\theta)$ + 'failure_rate: mu_': $h_{ben.}(t;\theta)$ + 'Epochs: mu_': No. of Epochs + 'model.trainer.batch_size: mu_': Batch Size + def_gen: Defence + 'attack.init.eps: mu_': $\varepsilon$ + 'data.sample.random_state: mu_': Random State + weibull: + plot: + file: weibull_aft.pdf + title: Weibull AFR Model + qq_title: Weibull QQ Plot + t0: 0.3 + model: + penalizer: 0.1 + labels: + 'Intercept: rho_': $\rho$ + 'Intercept: lambda_': $\lambda$ + 'data.sample.random_state: lambda_': Random State + 'atk_value: lambda_': Attack Strength + 'model_layers: lambda_': Layers + 'train_time_per_sample: lambda_': $t_{train}$ + 'predict_time_per_sample: lambda_': $t_{predict}$ + 'adv_accuracy: lambda_': Adv. Accuracy + 'accuracy: lambda_': Ben. Accuracy + 'adv_fit_time_per_sample: lambda_': $t_{attack}$ + 'adv_failure_rate: lambda_': $f_{adv.}(t;\theta)$ + 'failure_rate: lambda_': $f_{ben.}(t;\theta)$ + 'Epochs: lambda_': No. of Epochs + 'model.trainer.batch_size: lambda_': Batch Size + def_gen: Defence + 'def_value: lambda_': Defence Strength + ': lambda_': '' + outs: + - path: plots/aft_comparison.csv + hash: md5 + md5: d73ecadc5fa94fef2549e63280343bcd + size: 339 + - path: plots/aft_comparison.tex + hash: md5 + md5: 91fb7cb7394fd71f7e817a34af87315a + size: 631 + - path: plots/cox_aft.pdf + hash: md5 + md5: ed27d1978e06ae05dd5d35ddecdd6dc4 + size: 30876 + - path: plots/cox_aft_dummies.pdf + hash: md5 + md5: 3124eb6beac82d84772d5bd49ba9e503 + size: 28856 + - path: plots/cox_qq.pdf + hash: md5 + md5: 5d1a38624404554f04677a952d03a839 + size: 20181 + - path: plots/cox_summary.csv + hash: md5 + md5: 39903e136f766f9840cbe1cc0154a39b + size: 4529 + - path: plots/exponential_aft.pdf + hash: md5 + md5: 66a13486cd114e4cc8267b69a4dd2f04 + size: 33155 + - path: plots/exponential_aft_dummies.pdf + hash: md5 + md5: 6b506006f830ce108dde579d640e1d37 + size: 32185 + - path: plots/exponential_qq.pdf + hash: md5 + md5: 4093a29d709ad92aedf846b50dcab965 + size: 20240 + - path: plots/exponential_summary.csv + hash: md5 + md5: 90b7e50b371d5f54a5fa7b0c4c5743b3 + size: 9022 + - path: plots/gamma_aft.pdf + hash: md5 + md5: 4c0368cb06e211c7d0c2905481701901 + size: 29572 + - path: plots/gamma_aft_dummies.pdf + hash: md5 + md5: 48381e11d6eb87065cccc2179d0e19e6 + size: 35152 + - path: plots/gamma_qq.pdf + hash: md5 + md5: 2c7a027149f292d78f7d4886751413e0 + size: 11343 + - path: plots/gamma_summary.csv + hash: md5 + md5: 1150bbfaebb81f5ea9651766a1f868f7 + size: 13975 + - path: plots/log_logistic_aft.pdf + hash: md5 + md5: 31260174c6af90e4b748d7e06a36f81e + size: 32733 + - path: plots/log_logistic_aft_dummies.pdf + hash: md5 + md5: dd120a0385f2d4341c54f2a8476b7d2e + size: 31885 + - path: plots/log_logistic_qq.pdf + hash: md5 + md5: cd2f80c1444b3420c1c7dfdc066e4fab + size: 18830 + - path: plots/log_logistic_summary.csv + hash: md5 + md5: 55ff092ab00e5cc45860c11caee59024 + size: 4898 + - path: plots/log_normal_aft.pdf + hash: md5 + md5: 11fa430feb2cf96730b85d930541d0ab + size: 32716 + - path: plots/log_normal_aft_dummies.pdf + hash: md5 + md5: 9031ebb8918e71be9c68e7db68c59497 + size: 32241 + - path: plots/log_normal_qq.pdf + hash: md5 + md5: 4410d657b6fe927cb0ca47909f6504b1 + size: 19875 + - path: plots/log_normal_summary.csv + hash: md5 + md5: 6d49d9f09a9aeca15846bcc74a0906fc + size: 5002 + - path: plots/weibull_aft.pdf + hash: md5 + md5: 91ec661dd0ef73b03dbac81292522c92 + size: 32493 + - path: plots/weibull_aft_dummies.pdf + hash: md5 + md5: 54de898f6abcd858bfa0da07d8db002a + size: 31439 + - path: plots/weibull_qq.pdf + hash: md5 + md5: 07b89dc4f873a61bd45a9f4ac580f276 + size: 18007 + - path: plots/weibull_summary.csv + hash: md5 + md5: c77df236abf393dbf50fc58629886dbd + size: 4952 + predict_survival_time: + cmd: python predict_with_best.py --data data/merged.csv --config_file afr.yaml --model + weibull --target adv_failures --duration_col adv_fit_time_per_sample --output + data/merged_afr.csv + deps: + - path: afr.yaml + hash: md5 + md5: 1e39b01abdaf8c823a25379f6ce391d4 + size: 6349 + - path: data/merged.csv + hash: md5 + md5: ebca17142f7fbfc920b37824ea1c480e + size: 85713970 + - path: plots/aft_comparison.tex + hash: md5 + md5: 91fb7cb7394fd71f7e817a34af87315a + size: 631 + - path: predict_with_best.py + hash: md5 + md5: e60a437ab37d2ee22256268a207f2431 + size: 2571 + outs: + - path: data/merged_afr.csv + hash: md5 + md5: f366cd694109920b498c8cf876440050 + size: 86609777 + plot: + cmd: python -m deckard.layers.plots --path plots/ --file data/merged_afr.csv -c + plots.yaml + deps: + - path: data/merged_afr.csv + hash: md5 + md5: f366cd694109920b498c8cf876440050 + size: 86609777 + - path: plots.yaml + hash: md5 + md5: a0c5c100248543bb5f0de8949b459bc5 + size: 3815 + params: + afr.yaml: + covariates: + - adv_fit_time_per_sample + - accuracy + - train_time_per_sample + - atk_value + - def_value + - data.sample.random_state + - Epochs + - model_layers + - id + - atk_gen + - def_gen + - adv_failures + - adv_accuracy + - predict_time_per_sample + weibull: + plot: + file: weibull_aft.pdf + title: Weibull AFR Model + qq_title: Weibull QQ Plot + t0: 0.3 + model: + penalizer: 0.1 + labels: + 'Intercept: rho_': $\rho$ + 'Intercept: lambda_': $\lambda$ + 'data.sample.random_state: lambda_': Random State + 'atk_value: lambda_': Attack Strength + 'model_layers: lambda_': Layers + 'train_time_per_sample: lambda_': $t_{train}$ + 'predict_time_per_sample: lambda_': $t_{predict}$ + 'adv_accuracy: lambda_': Adv. Accuracy + 'accuracy: lambda_': Ben. Accuracy + 'adv_fit_time_per_sample: lambda_': $t_{attack}$ + 'adv_failure_rate: lambda_': $f_{adv.}(t;\theta)$ + 'failure_rate: lambda_': $f_{ben.}(t;\theta)$ + 'Epochs: lambda_': No. of Epochs + 'model.trainer.batch_size: lambda_': Batch Size + def_gen: Defence + 'def_value: lambda_': Defence Strength + ': lambda_': '' + plots.yaml: + cat_plot: + - file: adv_accuracy_vs_defence_type.pdf + hue: model_name + kind: boxen + set: + yscale: linear + x: def_gen + xlabels: Defence Type + y: adv_accuracy + ylabels: Adv. Accuracy + rotation: 90 + hue_order: + - ResNet18 + - ResNet34 + - ResNet50 + - ResNet101 + - ResNet152 + legend_title: Model + - file: ben_accuracy_vs_defence_type.pdf + hue: model_name + kind: boxen + x: def_gen + xlabels: Defence Type + y: accuracy + ylabels: Ben. Accuracy + rotation: 90 + hue_order: + - ResNet18 + - ResNet34 + - ResNet50 + - ResNet101 + - ResNet152 + legend_title: Model + - file: trash_score_vs_defence_type.pdf + hue: model_name + kind: boxen + set: + yscale: log + x: def_gen + xlabels: Defence Type + y: c_adv + ylabels: $\bar{C}_{adv.}$ + rotation: 90 + hue_order: + - ResNet18 + - ResNet34 + - ResNet50 + - ResNet101 + - ResNet152 + legend_title: Model + - file: trash_score_vs_attack_type.pdf + hue: model_name + kind: boxen + set: + yscale: log + x: atk_gen + xlabels: Attack Type + y: c_adv + ylabels: $\bar{C}_{adv.}$ + rotation: 90 + hue_order: + - ResNet18 + - ResNet34 + - ResNet50 + - ResNet101 + - ResNet152 + legend_title: Model + - file: adv_accuracy_vs_attack_type.pdf + hue: model_name + kind: boxen + x: atk_gen + xlabels: Attack Type + y: adv_accuracy + ylabels: Adv. Accuracy + rotation: 90 + hue_order: + - ResNet18 + - ResNet34 + - ResNet50 + - ResNet101 + - ResNet152 + legend_title: Model + line_plot: + - file: def_param_vs_accuracy.pdf + hue: def_gen + legend: + bbox_to_anchor: + - 1.05 + - 1 + title: Defence + title: Ben. Accuracy vs Defence Strength + x: def_value + x_scale: linear + xlabel: Defence Control Parameter + y: accuracy + y_scale: + ylabel: Ben. Accuracy + hue_order: + - Control + - Conf + - Gauss-in + - Gauss-out + - FSQ + errorbar: se + err_style: bars + - file: def_param_vs_adv_accuracy.pdf + hue: def_gen + legend: + bbox_to_anchor: + - 1.05 + - 1 + title: Defence + title: Adv. Accuracy vs Defence Strength + x: def_value + x_scale: linear + xlabel: Defence Control Parameter + y: adv_accuracy + y_scale: + ylabel: Adv. Accuracy + hue_order: + - Control + - Conf + - Gauss-in + - Gauss-out + - FSQ + errorbar: se + err_style: bars + - file: def_param_vs_adv_failure_rate.pdf + hue: def_gen + legend: + bbox_to_anchor: + - 1.05 + - 1 + title: Defence + title: $f_{adv}$ vs Defence Strength + x: def_value + x_scale: linear + xlabel: Defence Control Parameter + y: adv_failure_rate + y_scale: linear + ylabel: $f_{adv.}$ + hue_order: + - Control + - Conf + - Gauss-in + - Gauss-out + - FSQ + errorbar: se + err_style: bars + - file: atk_param_vs_accuracy.pdf + hue: atk_gen + legend: + bbox_to_anchor: + - 1.05 + - 1 + title: Adv. Accuracy vs Attack Strength + x: atk_value + x_scale: linear + xlabel: Attack Control Parameter + y: adv_accuracy + y_scale: + ylabel: Adv. Accuracy + hue_order: + - FGM + - PGD + - Deep + - HSJ + - Pixel + - Thresh + errorbar: se + err_style: bars + scatter_plot: + - x: train_time_per_sample + y: adv_failure_rate + hue: model_name + xlabel: $t_{train}$ + ylabel: $f_{adv}$ + title: $f_{adv}$ vs $t_{train}$ + file: adv_failure_rate_vs_train_time.pdf + x_scale: log + legend: + title: Model Name + bbox_to_anchor: + - 1.05 + - 1 + hue_order: + - ResNet18 + - ResNet34 + - ResNet50 + - ResNet101 + - ResNet152 + outs: + - path: plots/adv_accuracy_vs_attack_type.pdf + hash: md5 + md5: d98e39efcf123c924edb8d76afd9d9ad + size: 36004 + - path: plots/adv_accuracy_vs_defence_type.pdf + hash: md5 + md5: 0eb6d612c6bc0c7dd9843922b7a85cf5 + size: 32374 + - path: plots/adv_failure_rate_vs_train_time.pdf + hash: md5 + md5: b4045f90c0849d5f28cd3519872c636e + size: 211056 + - path: plots/atk_param_vs_accuracy.pdf + hash: md5 + md5: e27401d8b9ade96e516c954cc5ca916c + size: 20341 + - path: plots/ben_accuracy_vs_defence_type.pdf + hash: md5 + md5: 323231eb1f498228dbd22caf2efc4c86 + size: 35108 + - path: plots/def_param_vs_accuracy.pdf + hash: md5 + md5: 5640f125ea5212ed7018cd2535c20206 + size: 18670 + - path: plots/def_param_vs_adv_accuracy.pdf + hash: md5 + md5: 64404302bc5acf53d54a6e1ec50a8085 + size: 18499 + - path: plots/def_param_vs_adv_failure_rate.pdf + hash: md5 + md5: 4a9e2f937c5a0c5fada95d8e5eb0144c + size: 22357 + - path: plots/trash_score_vs_attack_type.pdf + hash: md5 + md5: 7c976e25527e965c383d8cbe1379e151 + size: 43833 + - path: plots/trash_score_vs_defence_type.pdf + hash: md5 + md5: 9043e42e1a1af3b4823197c69e67625b + size: 39732 + copy_results: + cmd: mkdir -p ~/ml_afr/plots && cp -r plots/* ~/ml_afr/plots/ + deps: + - path: plots/ + hash: md5 + md5: e533f85afb78e6228b2e11037932a9c5.dir + size: 1013121 + nfiles: 37 diff --git a/examples/pytorch/plots/dvc.yaml b/examples/pytorch/plots/dvc.yaml new file mode 100644 index 00000000..fe1588b0 --- /dev/null +++ b/examples/pytorch/plots/dvc.yaml @@ -0,0 +1,131 @@ +vars: + - plots.yaml:cat_plot + - plots.yaml:line_plot + - plots.yaml:scatter_plot + - afr.yaml:covariates + - afr.yaml:weibull + - afr.yaml:log_logistic + - afr.yaml:log_normal + - afr.yaml:gamma + - afr.yaml:exponential + - afr.yaml:cox + - afr.yaml:dummies +stages: + clean: + foreach: + - cifar + - cifar100 + - mnist + do: + cmd: >- + python -m deckard.layers.clean_data + -i data/raw_${item}.csv + -o data/${item}.csv + -c ../conf/clean_${item}.yaml + --drop_if_empty adv_fit_time accuracy train_time predict_time adv_accuracy def_value atk_value model_layers + deps: + + - data/raw_${item}.csv + - ../conf/clean_${item}.yaml + outs: + - data/${item}.csv + merge: + deps: + - data/mnist.csv + - data/cifar.csv + - data/cifar100.csv + cmd: >- + python -m deckard.layers.merge + --smaller_file data/cifar.csv data/cifar100.csv data/mnist.csv + --output_folder data + --output_file merged.csv + outs: + - data/merged.csv + + afr: + cmd: python -m deckard.layers.afr --data_file data/merged.csv --target adv_accuracy --duration_col adv_fit_time_per_sample --config_file afr.yaml --plots_folder plots/ + deps: + - data/merged.csv + plots: + - plots/weibull_aft.pdf + - plots/weibull_aft_dummies.pdf + - plots/weibull_qq.pdf + - plots/log_logistic_aft.pdf + - plots/log_logistic_aft_dummies.pdf + - plots/log_logistic_qq.pdf + - plots/log_normal_aft.pdf + - plots/log_normal_aft_dummies.pdf + - plots/log_normal_qq.pdf + - plots/gamma_aft.pdf + - plots/gamma_aft_dummies.pdf + - plots/gamma_qq.pdf + - plots/exponential_aft.pdf + - plots/exponential_aft_dummies.pdf + - plots/exponential_qq.pdf + - plots/cox_aft.pdf + - plots/cox_aft_dummies.pdf + - plots/cox_qq.pdf + params: + - afr.yaml: + - dummies + - covariates + - weibull + - log_logistic + - log_normal + - gamma + - exponential + - cox + metrics: + - plots/aft_comparison.csv + outs: + - plots/aft_comparison.tex + - plots/weibull_summary.csv + - plots/cox_summary.csv + - plots/log_logistic_summary.csv + - plots/log_normal_summary.csv + - plots/gamma_summary.csv + - plots/exponential_summary.csv + predict_survival_time: + cmd: >- + python predict_with_best.py + --data data/merged.csv + --config_file afr.yaml + --model weibull + --target adv_failures + --duration_col adv_fit_time_per_sample + --output data/merged_afr.csv + deps: + - data/merged.csv + - afr.yaml + - plots/aft_comparison.tex + - predict_with_best.py + outs: + - data/merged_afr.csv + plot: + cmd : python -m deckard.layers.plots --path plots/ --file data/merged_afr.csv -c plots.yaml + deps: + - data/merged_afr.csv + - plots.yaml + plots: + - plots/${cat_plot[0].file} + - plots/${cat_plot[1].file} + - plots/${cat_plot[2].file} + - plots/${cat_plot[3].file} + - plots/${cat_plot[4].file} + - plots/${line_plot[0].file} + - plots/${line_plot[1].file} + - plots/${line_plot[2].file} + - plots/${line_plot[3].file} + - plots/${scatter_plot[0].file} + params: + - plots.yaml: + - line_plot + - scatter_plot + - cat_plot + - afr.yaml: + - covariates + - weibull + copy_results: + cmd: mkdir -p ~/ml_afr/plots && cp -r plots/* ~/ml_afr/plots/ + deps: + - plots/ diff --git a/examples/pytorch/cifar100/conf/plots.yaml b/examples/pytorch/plots/plots.yaml similarity index 56% rename from examples/pytorch/cifar100/conf/plots.yaml rename to examples/pytorch/plots/plots.yaml index 0b94a148..7bc30e78 100644 --- a/examples/pytorch/cifar100/conf/plots.yaml +++ b/examples/pytorch/plots/plots.yaml @@ -1,16 +1,13 @@ cat_plot: -- file: adv_accuracy_vs_defence_type.eps +- file: adv_accuracy_vs_defence_type.pdf hue: model_name kind: boxen set: yscale: linear - titles: Adv. Accuracy vs Defence Type - titles: Adv. Accuracy vs Defence Type x: def_gen xlabels: Defence Type y: adv_accuracy ylabels: Adv. Accuracy - ylabels: Adv. Accuracy rotation : 90 hue_order: - ResNet18 @@ -18,33 +15,16 @@ cat_plot: - ResNet50 - ResNet101 - ResNet152 -- file: ben_accuracy_vs_defence_type.eps + # titles: "{col_name} dataset" + legend_title: "Model" + # col: 'id' +- file: ben_accuracy_vs_defence_type.pdf hue: model_name kind: boxen - titles: Ben. Accuracy vs Defence Type - titles: Ben. Accuracy vs Defence Type x: def_gen xlabels: Defence Type y: accuracy ylabels: Ben. Accuracy - ylabels: Ben. Accuracy - rotation : 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 -- file: ben_failures_per_train_time_vs_defence_type.eps - hue: model_name - kind: boxen - set: - yscale: log - titles: $\bar{C}_{ben.}$ vs Defence Type - x: def_gen - xlabels: Defence Type - y: training_time_per_failure - ylabels: $\bar{C}_{ben.}$ rotation : 90 hue_order: - ResNet18 @@ -52,15 +32,21 @@ cat_plot: - ResNet50 - ResNet101 - ResNet152 -- file: adv_failures_per_train_time_vs_defence_type.eps + # titles: "{col_name} dataset" + legend_title: "Model" + # col: 'id' + # col_order: + # - mnist + # - cifar + # - cifar100 +- file: trash_score_vs_defence_type.pdf hue: model_name kind: boxen set: yscale: log - titles: $\bar{C}_{adv.}$ vs Defence Type x: def_gen xlabels: Defence Type - y: training_time_per_adv_failure + y: c_adv ylabels: $\bar{C}_{adv.}$ rotation : 90 hue_order: @@ -69,16 +55,21 @@ cat_plot: - ResNet50 - ResNet101 - ResNet152 -- file: adv_failures_per_train_time_vs_attack_type.eps + # titles: "{col_name} dataset" + legend_title: "Model" + # col: 'id' + # col_order: + # - mnist + # - cifar + # - cifar100 +- file: trash_score_vs_attack_type.pdf hue: model_name kind: boxen - legend_title: Model Name set: yscale: log - titles: $\bar{C}_{adv.}$ vs Attack Type x: atk_gen xlabels: Attack Type - y: training_time_per_adv_failure + y: c_adv ylabels: $\bar{C}_{adv.}$ rotation : 90 hue_order: @@ -87,51 +78,20 @@ cat_plot: - ResNet50 - ResNet101 - ResNet152 -- file: adv_failures_per_test_time_vs_defence_type.eps - hue: model_name - kind: boxen - legend_title: Model Name - titles: $f_{adv}$ vs Defence Type - x: def_gen - xlabels: Defence Type - y: adv_failure_rate - ylabels: $f_{adv.}$ - rotation : 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 -- file: adv_accuracy_vs_attack_type.eps + # titles: "{col_name} dataset" + legend_title: "Model" + # col: 'id' + # col_order: + # - mnist + # - cifar + # - cifar100 +- file: adv_accuracy_vs_attack_type.pdf hue: model_name kind: boxen - legend_title: Model Name - titles: Adv. Accuracy vs Attack Type - titles: Adv. Accuracy vs Attack Type x: atk_gen xlabels: Attack Type y: adv_accuracy ylabels: Adv. Accuracy - ylabels: Adv. Accuracy - rotation : 90 - hue_order: - - ResNet18 - - ResNet34 - - ResNet50 - - ResNet101 - - ResNet152 -- file: ben_failure_rate_vs_defence_type.eps - hue: model_name - kind: boxen - legend_title: Model Name - set: - yscale: log - titles: $f_{ben}(t; \theta)$ vs Defence Type - x: def_gen - xlabels: Defence Type - y: failure_rate - ylabels: $f_{ben}(t; \theta)$ rotation : 90 hue_order: - ResNet18 @@ -139,8 +99,15 @@ cat_plot: - ResNet50 - ResNet101 - ResNet152 + # titles: "{col_name} dataset" + legend_title: "Model" + # col: 'id' + # col_order: + # - mnist + # - cifar + # - cifar100 line_plot: -- file: def_param_vs_accuracy.eps +- file: def_param_vs_accuracy.pdf hue: def_gen legend: {"bbox_to_anchor": [1.05, 1], "title": "Defence"} title: Ben. Accuracy vs Defence Strength @@ -153,14 +120,12 @@ line_plot: hue_order: - Control - Conf - - Epochs - Gauss-in - Gauss-out - - Conf - FSQ errorbar: se err_style: bars -- file: def_param_vs_adv_accuracy.eps +- file: def_param_vs_adv_accuracy.pdf hue: def_gen legend: {"bbox_to_anchor": [1.05, 1], "title": "Defence"} title: Adv. Accuracy vs Defence Strength @@ -173,45 +138,41 @@ line_plot: hue_order: - Control - Conf - - Epochs - Gauss-in - Gauss-out - - Conf - FSQ + # style : id errorbar: se err_style: bars -- file: def_param_vs_adv_failure_rate.eps +- file: def_param_vs_adv_failure_rate.pdf hue: def_gen legend: {"bbox_to_anchor": [1.05, 1], "title": "Defence"} title: $f_{adv}$ vs Defence Strength x: def_value - x_scale: log + x_scale: linear xlabel: Defence Control Parameter y: adv_failure_rate - y_scale: log + y_scale: linear ylabel: $f_{adv.}$ + # style: id hue_order: - Control - Conf - - Epochs - Gauss-in - Gauss-out - - Conf - FSQ errorbar: se err_style: bars -- file: atk_param_vs_accuracy.eps +- file: atk_param_vs_accuracy.pdf hue: atk_gen legend: {bbox_to_anchor: [1.05, 1]} title: Adv. Accuracy vs Attack Strength - title: Adv. Accuracy vs Attack Strength x: atk_value x_scale: linear xlabel: Attack Control Parameter y: adv_accuracy y_scale: ylabel: Adv. Accuracy - ylabel: Adv. Accuracy hue_order: - FGM - PGD @@ -228,8 +189,8 @@ scatter_plot: xlabel: $t_{train}$ ylabel: $f_{adv}$ title: $f_{adv}$ vs $t_{train}$ - file: adv_failure_rate_vs_train_time.eps - y_scale: linear + file: adv_failure_rate_vs_train_time.pdf + # y_scale: log x_scale: log legend: title: Model Name diff --git a/examples/pytorch/plots/predict_with_best.py b/examples/pytorch/plots/predict_with_best.py new file mode 100644 index 00000000..c42217d2 --- /dev/null +++ b/examples/pytorch/plots/predict_with_best.py @@ -0,0 +1,70 @@ +import argparse +from pathlib import Path +import pandas as pd +import yaml +from deckard.layers.afr import fit_aft, clean_data_for_aft, calculate_raw_failures + + +parser = argparse.ArgumentParser() +parser.add_argument("--data", type=str, required=True) +parser.add_argument("--config_file", type=str, required=True) +parser.add_argument("--target", type=str, required=True, help="Target column name") +parser.add_argument( + "--duration_col", + type=str, + required=True, + help="Duration column name", +) +parser.add_argument("--model", type=str, required=True) +parser.add_argument("--output", type=str, required=True) +parser.add_argument("--training_samples", type=int, default=48000) +parser.add_argument("--attack_samples", type=int, default=100) +args = parser.parse_args() + +suffix = Path(args.data).suffix +if suffix == ".csv": + df = pd.read_csv(args.data) +else: + raise NotImplementedError(f"Unknown file format: {suffix}") + +suffix = Path(args.config_file).suffix +assert Path(args.config_file).exists(), f"Model file not found: {args.config_file}" + +if suffix == ".yaml": + with open(args.config_file, "r") as f: + config = yaml.safe_load(f) +else: + raise NotImplementedError(f"Unknown file format: {suffix}") +assert args.model in config, f"Model not found in the model file: {args.model}" +# Config +dummies = config.pop("dummies", {"atk_gen": "Atk:", "def_gen": "Def:", "id": "Data:"}) +covariate_list = config.get("covariates", []) +model_config = config[args.model].get("model", {}) +# Will generate a subset used for fitting +data = df.copy() +# Calculate raw failures from accuracy +data = calculate_raw_failures(args, df, config) +# Clean data by removing columns that are not in the covariate list and creating dummies for the categorical variables +data = clean_data_for_aft(data, covariate_list, target=args.target, dummy_dict=dummies) +# Fit the model +model = fit_aft( + data, + event_col=args.target, + duration_col=args.duration_col, + **model_config, + mtype=args.model, +) +# Predict the adversarial survival time +adv_survival_time = model.predict_expectation(data) +df = df.assign(adv_survival_time=adv_survival_time) +train_rate = df["train_time_per_sample"] +normalized_survival_time = df["adv_survival_time"] / args.attack_samples +c_adv = train_rate / normalized_survival_time +df = df.assign(c_adv=c_adv) +assert "c_adv" in df.columns, "c_adv column not found in the dataframe" +suffix = Path(args.output).suffix +if suffix == ".csv": + df.to_csv(args.output, index=False) +else: + raise NotImplementedError(f"Unknown file format: {suffix}") +print(f"Output file: {args.output}") diff --git a/examples/pytorch/mnist/torch_example.py b/examples/pytorch/torch_example.py similarity index 98% rename from examples/pytorch/mnist/torch_example.py rename to examples/pytorch/torch_example.py index 2007d9d0..3fff2080 100644 --- a/examples/pytorch/mnist/torch_example.py +++ b/examples/pytorch/torch_example.py @@ -4,6 +4,7 @@ __all__ = [ "ResNet18", "ResNet50", + "ResNet34", "ResNet101", "ResNet152", ] diff --git a/examples/pytorch/cifar100/wait.sh b/examples/pytorch/wait.sh similarity index 53% rename from examples/pytorch/cifar100/wait.sh rename to examples/pytorch/wait.sh index 7acc5e65..5270709e 100644 --- a/examples/pytorch/cifar100/wait.sh +++ b/examples/pytorch/wait.sh @@ -2,12 +2,16 @@ rm -rf waiting.log || true echo "Trying to allocate gpu" start=$(date +%s) -until gpu-allocate-cli allocate --duration 72h --wait && echo "Elapsed time: $(( $( date +%s ) - $start )) seconds" >| waiting.log && dvc repro +echo "Start time (seconds): $start" >| waiting.log +until gpu-allocate-cli allocate --duration 72h --wait && echo "Elapsed time: $(( $( date +%s ) - $start )) seconds" >| waiting.log #&& dvc repro do - echo "Waiting 30 mins" - sleep 1800 echo "Trying to allocate gpu" echo "Elapsed time: $(( $( date +%s ) - $start )) seconds" + echo "Elapsed time: $(( $( date +%s ) - $start )) seconds" >> waiting.log echo "Elapsed time in hours: $(( ($(date +%s) - $start) / 3600 )) hours" + echo "Elapsed time in hours: $(( ($(date +%s) - $start) / 3600 )) hours" >> waiting.log echo "Elapsed time in days: $(( ($(date +%s) - $start) / 86400 )) days" + echo "Elapsed time in days: $(( ($(date +%s) - $start) / 86400 )) days" >> waiting.log + echo "Waiting 10 minutes before trying to allocate gpu again" + sleep 10m done diff --git a/hardcopy.0 b/hardcopy.0 new file mode 100644 index 00000000..45f34325 --- /dev/null +++ b/hardcopy.0 @@ -0,0 +1,40 @@ + File "/home/cmeyers/deckard/.venv/lib/python3.10/site-packages/dvc/fs/dvc.py", line 263, in _datafss + datafss[key] = DataFileSystem(index=self.repo.index.data["repo"]) + File "/home/cmeyers/deckard/.venv/lib/python3.10/site-packages/funcy/objects.py", line 25, in __get__ + res = instance.__dict__[self.fget.__name__] = self.fget(instance) + File "/home/cmeyers/deckard/.venv/lib/python3.10/site-packages/dvc/repo/__init__.py", line 280, in index + return Index.from_repo(self) + File "/home/cmeyers/deckard/.venv/lib/python3.10/site-packages/dvc/repo/index.py", line 321, in from_repo + for _, idx in collect_files(repo, onerror=onerror): + File "/home/cmeyers/deckard/.venv/lib/python3.10/site-packages/dvc/repo/index.py", line 90, in collect_files + index = Index.from_file(repo, file_path) + File "/home/cmeyers/deckard/.venv/lib/python3.10/site-packages/dvc/repo/index.py", line 345, in from_file + stages=list(dvcfile.stages.values()), + File "/usr/lib/python3.10/_collections_abc.py", line 925, in __iter__ + yield self._mapping[key] + File "/home/cmeyers/deckard/.venv/lib/python3.10/site-packages/dvc/stage/loader.py", line 134, in __getitem__ + resolved_data = self.resolver.resolve_one(name) + File "/home/cmeyers/deckard/.venv/lib/python3.10/site-packages/dvc/parsing/__init__.py", line 178, in resolve_one + return definition.resolve() + File "/home/cmeyers/deckard/.venv/lib/python3.10/site-packages/dvc/parsing/__init__.py", line 245, in resolve + return self.resolve_stage(**kwargs) + File "/home/cmeyers/deckard/.venv/lib/python3.10/site-packages/dvc/parsing/__init__.py", line 286, in resolve_stage + resolved = { + File "/home/cmeyers/deckard/.venv/lib/python3.10/site-packages/dvc/parsing/__init__.py", line 287, in + key: self._resolve(context, value, key, skip_checks) + File "/home/cmeyers/deckard/.venv/lib/python3.10/site-packages/dvc/parsing/__init__.py", line 305, in _resolve + format_and_raise(exc, f"'{self.where}.{self.name}.{key}'", self.relpath) + File "/home/cmeyers/deckard/.venv/lib/python3.10/site-packages/dvc/parsing/__init__.py", line 74, in format_and_raise + _reraise_err(ResolveError, message, from_exc=exc) + File "/home/cmeyers/deckard/.venv/lib/python3.10/site-packages/dvc/parsing/__init__.py", line 83, in _reraise_err + raise err +dvc.parsing.ResolveError: failed to parse 'stages.attack.deps' in 'cifar_old/reports/attack/d025e4f73c458b14c5cb7a552e85b031/dvc.yaml': Could not find 'files.directory' +(base) cmeyers@cmeyers:~/deckard/examples/pytorch$ python -m deckard.layers.parse --config_file mnist.yaml && python -m deckard.layers.experiment train@mnist --config_file mnist.yaml^C +(base) cmeyers@cmeyers:~/deckard/examples/pytorch$ dvc repro --forceogd -ng/__init__.py", line 83, in _reraise_err^C +(base) cmeyers@cmeyers:~/deckard/examples/pytorch$ dvc repro +^CERROR: interrupted by the user +(base) cmeyers@cmeyers:~/deckard/examples/pytorch$ ^C +(base) cmeyers@cmeyers:~/deckard/examples/pytorch$ pkill python +(base) cmeyers@cmeyers:~/deckard/examples/pytorch$ pkill dvc +(base) cmeyers@cmeyers:~/deckard/examples/pytorch$ pkill wait.sh +(base) cmeyers@cmeyers:~/deckard/examples/pytorch$ diff --git a/ml-afr.workspace.code-workspace b/ml-afr.workspace.code-workspace new file mode 100644 index 00000000..40bbfbd3 --- /dev/null +++ b/ml-afr.workspace.code-workspace @@ -0,0 +1,11 @@ +{ + "folders": [ + { + "path": "." + }, + { + "path": "../ml_afr" + } + ], + "settings": {} +}