Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

timing bug #145

Merged
merged 4 commits into from
Nov 29, 2023
Merged

timing bug #145

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0294c21
fix timing bug
simplymathematics Nov 29, 2023
2cab18d
fix timing bug
simplymathematics Nov 29, 2023
3eb34d7
linting
simplymathematics Nov 29, 2023
cf64d90
fix timing bug
simplymathematics Nov 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 42 additions & 21 deletions deckard/base/attack/attack.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from pathlib import Path
import logging
from copy import deepcopy
from time import process_time_ns
from time import process_time_ns, time
from omegaconf import DictConfig, OmegaConf
from hydra.utils import instantiate
from art.utils import to_categorical, compute_success
Expand Down Expand Up @@ -154,6 +154,7 @@ def __call__(
kwargs.update({"y": data[2][: self.attack_size]})
if "AdversarialPatch" in self.name:
start = process_time_ns()
start_timestamp = time()
patches, _ = atk.generate(ben_samples, **kwargs)
samples = atk.apply_patch(
ben_samples,
Expand All @@ -162,14 +163,16 @@ def __call__(
)
else:
start = process_time_ns()
start_timestamp = time()
samples = atk.generate(ben_samples, **kwargs)
end = process_time_ns()
end_timestamp = time()
time_dict.update(
{
"adv_fit_time_per_sample": (end - start) / (len(samples) * 1e9),
"adv_fit_time": (end - start) / 1e9,
"adv_fit_start_time": start,
"adv_fit_stop_time": end,
"adv_fit_start_time": start_timestamp,
"adv_fit_stop_time": end_timestamp,
},
)
device = str(model.device) if hasattr(model, "device") else "cpu"
Expand Down Expand Up @@ -205,31 +208,34 @@ def __call__(
"predict_proba",
):
start = process_time_ns()
start_timestamp = time()
adv_probabilities = model.model.predict_proba(samples)
end = process_time_ns()
try:
start = process_time_ns()
start_timestamp = time()
adv_probabilities = model.predict_proba(samples)
end = process_time_ns()

end_timestamp = time()
except AttributeError:
start = process_time_ns()
start_timestamp = time()
adv_probabilities = model.predict(samples)
end = process_time_ns()
end_timestamp = time()
time_dict.update(
{
"adv_predict_time_per_sample": (end - start) / (len(samples) * 1e9),
"adv_predict_time": (end - start) / 1e9,
"adv_predict_start_time": start,
"adv_predict_stop_time": end,
"adv_predict_start_time": start_timestamp,
"adv_predict_stop_time": end_timestamp,
},
)
device = str(model.device) if hasattr(model, "device") else "cpu"
time_dict.update({"adv_predict_device": device})
results["adv_probabilities"] = np.array(adv_probabilities)
if adv_probabilities_file is not None:
self.data.save(adv_probabilities, adv_probabilities_file)

if adv_losses_file is not None and Path(adv_losses_file).exists():
adv_loss = self.data.load(adv_losses_file)
results["adv_losses"] = np.array(adv_loss)
Expand Down Expand Up @@ -324,12 +330,15 @@ def __call__(
)
try:
start = process_time_ns()
start_timestamp = time()
samples, _ = atk.poison(
x_trigger=x_trigger,
y_trigger=y_trigger,
x_train=x_train,
y_train=y_train,
)
end = process_time_ns()
end_timestamp = time()
except RuntimeError as e:
if "expected scalar type Long" in str(e):
# if hasattr(y_train, "type"):
Expand Down Expand Up @@ -359,21 +368,24 @@ def __call__(
attack_size=self.attack_size,
)
start = process_time_ns()
start_timestamp = time()
samples, _ = atk.poison(
x_trigger=x_trigger,
y_trigger=y_trigger,
x_train=x_train,
y_train=y_train,
)
end = process_time_ns()
end_timestamp = time()
else:
raise e
end = process_time_ns()

time_dict.update(
{
"adv_fit_time_per_sample": (end - start) / (len(samples) * 1e9),
"adv_fit_time": (end - start) / 1e9,
"adv_fit_start_time": start,
"adv_fit_stop_time": end,
"adv_fit_start_time": start_timestamp,
"adv_fit_stop_time": end_timestamp,
},
)
device = str(model.device) if hasattr(model, "device") else "cpu"
Expand All @@ -397,22 +409,28 @@ def __call__(
"predict_proba",
):
start = process_time_ns()
start_timestamp = time()
adv_probabilities = model.model.predict_proba(samples)
end = process_time_ns()
end_timestamp = time()
try:
start = process_time_ns()
start_timestamp = time()
adv_probabilities = model.predict_proba(samples)
end = process_time_ns()
end_timestamp = time()
except AttributeError:
start = process_time_ns()
start_timestamp = time()
adv_probabilities = model.predict(samples)
end = process_time_ns()
end_timestamp = time()
time_dict.update(
{
"adv_predict_time_per_sample": (end - start) / (len(samples) * 1e9),
"adv_predict_time": (end - start) / 1e9,
"adv_predict_start_time": start,
"adv_predict_stop_time": end,
"adv_predict_start_time": start_timestamp,
"adv_predict_stop_time": end_timestamp,
},
)
device = str(model.device) if hasattr(model, "device") else "cpu"
Expand Down Expand Up @@ -514,25 +532,24 @@ def __call__(
if "MembershipInferenceBlackBox" in self.name:
infer = self.kwargs.pop("infer", {})
fit = self.kwargs.pop("fit", {})
start = process_time_ns()
atk.fit(x=x_train, y=y_train, test_x=x_test, test_y=y_test, **fit)
end = process_time_ns()

x_train = data[0][: self.attack_size]
y_train = data[2][: self.attack_size]
x_test = data[1][: self.attack_size]
y_test = data[3][: self.attack_size]
start = process_time_ns()
start_timestamp = time()
preds = atk.infer(x_test, y_test, **infer)
end = process_time_ns()
end_timestamp = time()
else:
raise NotImplementedError(f"Attack {self.name} not implemented.")
time_dict.update(
{
"adv_fit_time_per_sample": (end - start) / (len(preds) * 1e9),
"adv_fit_time": (end - start) / 1e9,
"adv_fit_start_time": start,
"adv_fit_stop_time": end,
"adv_fit_start_time": start_timestamp,
"adv_fit_stop_time": end_timestamp,
},
)
device = str(model.device) if hasattr(model, "device") else "cpu"
Expand Down Expand Up @@ -606,20 +623,22 @@ def __call__(
attack = deepcopy(self.init)
attack = attack(model=model, attack_size=self.attack_size)
start = process_time_ns()
start_timestamp = time()
attacked_model = attack.extract(
x=data[0][: self.attack_size],
y=data[2][: self.attack_size],
thieved_classifier=thieved_classifier,
**kwargs,
)
end = process_time_ns()
end_timestamp = time()
time_dict.update(
{
"adv_fit_time_per_sample": (end - start)
/ (self.attack_size * 1e9),
"adv_fit_time": (end - start) / 1e9,
"adv_fit_start_time": start,
"adv_fit_stop_time": end,
"adv_fit_start_time": start_timestamp,
"adv_fit_stop_time": end_timestamp,
},
)
device = str(model.device) if hasattr(model, "device") else "cpu"
Expand All @@ -634,14 +653,16 @@ def __call__(
preds = self.data.load(adv_predictions_file)
else:
start = process_time_ns()
start_timestamp = time()
preds = attacked_model.predict(data[1][: self.attack_size])
end = process_time_ns()
end_timestamp = time()
time_dict.update(
{
"adv_predict_time_per_sample": (end - start) / (len(preds) * 1e9),
"adv_predict_time": (end - start) / 1e9,
"adv_predict_start_time": start,
"adv_predict_stop_time": end,
"adv_predict_start_time": start_timestamp,
"adv_predict_stop_time": end_timestamp,
},
)
device = str(model.device) if hasattr(model, "device") else "cpu"
Expand Down
46 changes: 36 additions & 10 deletions deckard/base/model/model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import pickle
from dataclasses import dataclass, field, asdict, is_dataclass
from pathlib import Path
from time import process_time_ns
from time import process_time_ns, time
from typing import Union, Dict
from omegaconf import OmegaConf, DictConfig, ListConfig
from copy import deepcopy
Expand Down Expand Up @@ -131,24 +131,30 @@ def __call__(self, data: list, model: object, library=None):
raise NotImplementedError(f"Training library {library} not implemented")
try:
start = process_time_ns()
start_timestamp = time()
model.fit(data[0], data[2], **trainer)
end = process_time_ns() - start
end_timestamp = time()
except np.AxisError: # pragma: no cover
from art.utils import to_categorical

data[2] = to_categorical(data[2])
start = process_time_ns()
start_timestamp = time()
model.fit(data[0], data[2], **trainer)
end = process_time_ns() - start
end_timestamp = time()
except ValueError as e: # pragma: no cover
if "Shape of labels" in str(e):
from art.utils import to_categorical

nb_classes = len(np.unique(data[2]))
data[2] = to_categorical(data[2], nb_classes=nb_classes)
start = process_time_ns()
start_timestamp = time()
model.fit(data[0], data[2], **trainer)
end = process_time_ns() - start
end_timestamp = time()
else:
raise e
except AttributeError as e: # pragma: no cover
Expand All @@ -157,8 +163,10 @@ def __call__(self, data: list, model: object, library=None):
data[0] = np.array(data[0])
data[2] = np.array(data[2])
start = process_time_ns()
start_timestamp = time()
model.fit(data[0], data[2], **trainer)
end = process_time_ns() - start
end_timestamp = time()
except Exception as e:
raise e
except RuntimeError as e: # pragma: no cover
Expand All @@ -167,8 +175,10 @@ def __call__(self, data: list, model: object, library=None):

tf.config.run_functions_eagerly(True)
start = process_time_ns()
start_timestamp = time()
model.fit(data[0], data[2], **trainer)
end = process_time_ns() - start
end_timestamp = time()
elif "should be the same" in str(e).lower():
import torch

Expand All @@ -185,15 +195,17 @@ def __call__(self, data: list, model: object, library=None):
data[2].to(device)
model.model.to(device) if hasattr(model, "model") else model.to(device)
start = process_time_ns()
start_timestamp = time()
model.fit(data[0], data[2], **trainer)
end = process_time_ns() - start
end_timestamp = time()
else:
raise e
time_dict = {
"train_time": (start - end) / 1e9,
"train_time_per_sample": end / (len(data[0]) * 1e9),
"train_time_start": start,
"train_time_end": end,
"train_start_time": start_timestamp,
"train_end_time": end_timestamp,
"train_device": device,
}

Expand Down Expand Up @@ -550,7 +562,10 @@ def predict(self, data=None, model=None, predictions_file=None):
device = str(model.device) if hasattr(model, "device") else "cpu"
try:
start = process_time_ns()
start_timestamp = time()
predictions = model.predict(data[1])
end = process_time_ns() - start
end_timestamp = time()
except NotFittedError as e: # pragma: no cover
logger.warning(e)
logger.warning(f"Model {model} is not fitted. Fitting now.")
Expand All @@ -568,15 +583,16 @@ def predict(self, data=None, model=None, predictions_file=None):
logger.error(e)
raise e
end = process_time_ns() - start
end_timestamp = time()
if predictions_file is not None:
self.data.save(predictions, predictions_file)
return (
predictions,
{
"predict_time": (start - end) / 1e9,
"predict_time_per_sample": start - end / (len(data[0]) * 1e9),
"predict_start_time": start,
"predict_stop_time": end,
"predict_time_per_sample": (start - end) / (len(data[0]) * 1e9),
"predict_start_time": start_timestamp,
"predict_stop_time": end_timestamp,
"predict_device": device,
},
)
Expand Down Expand Up @@ -612,21 +628,25 @@ def predict_proba(self, data=None, model=None, probabilities_file=None):
)
elif hasattr(model, "predict_proba"):
start = process_time_ns()
start_timestamp = time()
predictions = model.predict_proba(data[1])
end = process_time_ns() - start
end_timestamp = time()
else:
start = process_time_ns()
start_timestamp = time()
predictions = model.predict(data[1])
end = process_time_ns() - start
end_timestamp = time()
if probabilities_file is not None:
self.data.save(predictions, probabilities_file)
return (
predictions,
{
"predict_proba_time": (start - end) / 1e9,
"predict_proba_time_per_sample": end / (len(data[0]) * 1e9),
"predict_proba_start_time": start,
"predict_proba_stop_time": end,
"predict_proba_start_time": start_timestamp,
"predict_proba_stop_time": end_timestamp,
"predict_proba_device": device,
},
)
Expand Down Expand Up @@ -661,16 +681,22 @@ def predict_log_loss(self, data, model, losses_file=None):
)
if hasattr(model, "predict_log_proba"):
start = process_time_ns()
start_timestamp = time()
predictions = model.predict_log_proba(data[1])
end = process_time_ns() - start
end_timestamp = time()
elif hasattr(model, "predict_proba"):
start = process_time_ns()
start_timestamp = time()
predictions = model.predict_proba(data[1])
end = process_time_ns() - start
end_timestamp = time()
elif hasattr(model, "predict"):
start = process_time_ns()
start_timestamp = time()
predictions = model.predict(data[1])
end = process_time_ns() - start
end_timestamp = time()
else: # pragma: no cover
raise ValueError(
f"Model {model} does not have a predict_log_proba or predict_proba method.",
Expand All @@ -682,8 +708,8 @@ def predict_log_loss(self, data, model, losses_file=None):
{
"predict_log_proba_time": (start - end) / 1e9,
"predict_log_proba_time_per_sample": end / (len(data[0]) * 1e9),
"predict_log_proba_start_time": start,
"predict_log_proba_stop_time": end,
"predict_log_proba_start_time": start_timestamp,
"predict_log_proba_stop_time": end_timestamp,
"predict_log_device": device,
},
)
Expand Down
Loading
Loading