Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Simplify BlackBox Experiments for FeaturePerfPrecision #887

Merged
merged 7 commits into from
Aug 5, 2024
Merged

Simplify BlackBox Experiments for FeaturePerfPrecision #887

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
932910f
* Unified compilation flags
May 16, 2024
60b0085
* Refactored black box experiment steps to fit pattern of remaining e…
May 16, 2024
d421d94
Merge branch 'vara-dev' into f-FeaturePerfPrecision
LuAbelt May 22, 2024
0e15889
Merge branch 'vara-dev' into f-FeaturePerfPrecision
LuAbelt Jun 13, 2024
5165a76
Merge branch 'vara-dev' into f-FeaturePerfPrecision
vulder Jul 21, 2024
b0326e0
Merge branch 'vara-dev' into f-FeaturePerfPrecision
vulder Aug 2, 2024
0578c1a
* Add CLI option to tables to group synthetic case studies
Aug 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion varats/varats/data/databases/feature_perf_precision_database.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -427,7 +427,7 @@ class EbpfTraceTEF(Profiler):
def __init__(self) -> None:
super().__init__(
"eBPFTrace", fpp.EbpfTraceTEFProfileRunner,
fpp.TEFProfileOverheadRunner, fpp.MPRTEFAggregate
fpp.EbpfTraceTEFOverheadRunner, fpp.MPRTEFAggregate
)

def is_regression(
Expand Down
177 changes: 32 additions & 145 deletions varats/varats/experiments/vara/feature_perf_precision.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,11 +102,16 @@ def select_project_binaries(project: VProject) -> tp.List[ProjectBinaryWrapper]:

def get_extra_cflags(project: VProject) -> tp.List[str]:
"""Get additional cflags for some projects."""
extra_flags = []

if project.name == "DunePerfRegression":
extra_flags += ["-pthread"]

if project.name in ["DunePerfRegression", "HyTeg"]:
# Disable phasar for dune as the analysis cannot handle dunes size
return ["-fvara-disable-phasar"]
extra_flags += ["-fvara-disable-phasar"]

return []
return extra_flags


def get_threshold(project: VProject) -> int:
Expand Down Expand Up @@ -399,9 +404,11 @@ def attach_usdt_bcc(report_file: Path, binary: Path) -> Future:


def setup_actions_for_vara_experiment(
experiment: FeatureExperiment, project: VProject,
experiment: FeatureExperiment,
project: VProject,
instr_type: FeatureInstrType,
analysis_step: tp.Type[AnalysisProjectStepBaseTy]
analysis_step: tp.Type[AnalysisProjectStepBaseTy],
report_type=MultiPatchReport
) -> tp.MutableSequence[actions.Step]:
"""Sets up actions for a given perf precision experiment."""

Expand Down Expand Up @@ -458,7 +465,7 @@ def setup_actions_for_vara_experiment(
analysis_step(
project,
binary,
file_name=MultiPatchReport.create_patched_report_name(
file_name=report_type.create_patched_report_name(
patch, "rep_measurements"
)
)
Expand All @@ -474,7 +481,7 @@ def setup_actions_for_vara_experiment(
analysis_step(
project,
binary,
file_name=MultiPatchReport.
file_name=report_type.
create_baseline_report_name("rep_measurements")
)
] + patch_steps
Expand Down Expand Up @@ -587,7 +594,7 @@ def actions_for_project(
)


class RunBlackBoxBaseline(OutputFolderStep): # type: ignore
class RunBlackBoxBaseline(AnalysisProjectStepBase): # type: ignore
"""Executes the traced project binaries on the specified workloads."""

NAME = "VaRARunTracedBinaries"
Expand All @@ -603,11 +610,7 @@ def __init__(
report_file_ending: str = "txt",
reps: int = REPS
) -> None:
super().__init__(project=project)
self.__binary = binary
self.__report_file_ending = report_file_ending
self.__reps = reps
self.__file_name = file_name
super().__init__(project, binary, file_name, report_file_ending, reps)

def call_with_output_folder(self, tmp_dir: Path) -> StepResult:
return self.run_traced_code(tmp_dir)
Expand All @@ -620,15 +623,15 @@ def __str__(self, indent: int = 0) -> str:
def run_traced_code(self, tmp_dir: Path) -> StepResult:
"""Runs the binary with the embedded tracing code."""
with local.cwd(local.path(self.project.builddir)):
zip_tmp_dir = tmp_dir / self.__file_name
zip_tmp_dir = tmp_dir / self._file_name
with ZippedReportFolder(zip_tmp_dir) as reps_tmp_dir:
for rep in range(0, self.__reps):
for rep in range(0, self._reps):
for prj_command in perf_prec_workload_commands(
self.project, self.__binary
self.project, self._binary
):
time_report_file = Path(reps_tmp_dir) / (
f"baseline_{prj_command.command.label}_{rep}"
f".{self.__report_file_ending}"
f".{self._report_file_ending}"
)

print(f"Running example {prj_command.command.label}")
Expand All @@ -639,7 +642,7 @@ def run_traced_code(self, tmp_dir: Path) -> StepResult:
time["-v", "-o", time_report_file],
project=self.project
)
pb_cmd(retcode=self.__binary.valid_exit_codes)
pb_cmd(retcode=self._binary.valid_exit_codes)

return StepResult.OK

Expand All @@ -661,78 +664,10 @@ def actions_for_project(
Args:
project: to analyze
"""
project.cflags += ["-flto", "-fuse-ld=lld", "-fno-omit-frame-pointer"]

project.cflags += get_extra_cflags(project)

project.ldflags += self.get_vara_tracing_ldflags()

# Add the required runtime extensions to the project(s).
project.runtime_extension = bb_ext.run.RuntimeExtension(
project, self
) << bb_ext.time.RunWithTime()

# Add the required compiler extensions to the project(s).
project.compiler_extension = bb_ext.compiler.RunCompiler(
project, self
) << WithUnlimitedStackSize()

# Add own error handler to compile step.
project.compile = get_default_compile_error_wrapped(
self.get_handle(), project, self.REPORT_SPEC.main_report
)

# TODO: change to multiple binaries
binary = select_project_binaries(project)[0]
if binary.type != BinaryType.EXECUTABLE:
raise AssertionError("Experiment only works with executables.")

result_filepath = create_new_success_result_filepath(
self.get_handle(),
self.get_handle().report_spec().main_report, project, binary,
get_current_config_id(project)
)

patch_provider = PatchProvider.get_provider_for_project(project)
patches = patch_provider.get_patches_for_revision(
ShortCommitHash(project.version_of_primary)
)[IDENTIFIER_PATCH_TAG]
print(f"{patches=}")

patch_steps = []
for patch in patches:
print(f"Got patch with path: {patch.path}")
patch_steps.append(ApplyPatch(project, patch))
patch_steps.append(ReCompile(project))
patch_steps.append(
RunBlackBoxBaseline(
project,
binary,
file_name=MPRTimeReportAggregate.create_patched_report_name(
patch, "rep_measurements"
)
)
)
patch_steps.append(RevertPatch(project, patch))

analysis_actions = get_config_patch_steps(project)

analysis_actions.append(actions.Compile(project))
analysis_actions.append(
ZippedExperimentSteps(
result_filepath, [
RunBlackBoxBaseline(
project,
binary,
file_name=MPRTimeReportAggregate.
create_baseline_report_name("rep_measurements")
)
] + patch_steps
)
return setup_actions_for_vara_experiment(
self, project, FeatureInstrType.NONE, RunBlackBoxBaseline,
MPRTimeReportAggregate
)
analysis_actions.append(actions.Clean(project))

return analysis_actions


################################################################################
Expand Down Expand Up @@ -1097,7 +1032,7 @@ def actions_for_project(
)


class RunBlackBoxBaselineOverhead(OutputFolderStep): # type: ignore
class RunBlackBoxBaselineOverhead(AnalysisProjectStepBase): # type: ignore
"""Executes the traced project binaries on the specified workloads."""

NAME = "VaRARunTracedBinaries"
Expand All @@ -1109,13 +1044,11 @@ def __init__(
self,
project: VProject,
binary: ProjectBinaryWrapper,
file_name: str,
report_file_ending: str = "txt",
reps: int = REPS
) -> None:
super().__init__(project=project)
self.__binary = binary
self.__report_file_ending = report_file_ending
self.__reps = reps
super().__init__(project, binary, file_name, report_file_ending, reps)

def call_with_output_folder(self, tmp_dir: Path) -> StepResult:
return self.run_traced_code(tmp_dir)
Expand All @@ -1128,13 +1061,13 @@ def __str__(self, indent: int = 0) -> str:
def run_traced_code(self, tmp_dir: Path) -> StepResult:
"""Runs the binary with the embedded tracing code."""
with local.cwd(local.path(self.project.builddir)):
for rep in range(0, self.__reps):
for rep in range(0, self._reps):
for prj_command in perf_prec_workload_commands(
self.project, self.__binary
self.project, self._binary
):
time_report_file = tmp_dir / (
f"overhead_{prj_command.command.label}_{rep}"
f".{self.__report_file_ending}"
f".{self._report_file_ending}"
)

with cleanup(prj_command):
Expand All @@ -1144,7 +1077,7 @@ def run_traced_code(self, tmp_dir: Path) -> StepResult:
project=self.project
)

pb_cmd(retcode=self.__binary.valid_exit_codes)
pb_cmd(retcode=self._binary.valid_exit_codes)

return StepResult.OK

Expand All @@ -1166,52 +1099,6 @@ def actions_for_project(
Args:
project: to analyze
"""
project.cflags += ["-flto", "-fuse-ld=lld", "-fno-omit-frame-pointer"]

project.cflags += get_extra_cflags(project)

project.ldflags += self.get_vara_tracing_ldflags()

# Add the required runtime extensions to the project(s).
project.runtime_extension = bb_ext.run.RuntimeExtension(
project, self
) << bb_ext.time.RunWithTime()

# Add the required compiler extensions to the project(s).
project.compiler_extension = bb_ext.compiler.RunCompiler(
project, self
) << WithUnlimitedStackSize()

# Add own error handler to compile step.
project.compile = get_default_compile_error_wrapped(
self.get_handle(), project, self.REPORT_SPEC.main_report
)

# TODO: change to multiple binaries
binary = select_project_binaries(project)[0]
if binary.type != BinaryType.EXECUTABLE:
raise AssertionError("Experiment only works with executables.")

result_filepath = create_new_success_result_filepath(
self.get_handle(),
self.get_handle().report_spec().main_report, project, binary,
get_current_config_id(project)
)

analysis_actions = get_config_patch_steps(project)

analysis_actions.append(actions.Compile(project))
analysis_actions.append(
ZippedExperimentSteps(
result_filepath,
[
RunBlackBoxBaselineOverhead( # type: ignore
project,
binary
),
]
)
return setup_actions_for_vara_overhead_experiment(
self, project, FeatureInstrType.NONE, RunBlackBoxBaselineOverhead
)
analysis_actions.append(actions.Clean(project))

return analysis_actions
25 changes: 18 additions & 7 deletions varats/varats/tables/feature_perf_precision.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,9 @@
from varats.table.table import Table
from varats.table.table_utils import dataframe_to_table
from varats.table.tables import TableFormat, TableGenerator
from varats.ts_utils.cli_util import make_cli_option
from varats.utils.git_util import calc_repo_loc, ChurnConfig

GROUP_SYNTHETIC_CATEGORIES = True

SYNTH_CATEGORIES = [
"Static Analysis", "Dynamic Analysis", "Configurability",
"Implementation Pattern"
Expand Down Expand Up @@ -251,8 +250,18 @@ def add_extras(doc: Document) -> None:
)


GROUP_SYNTHETIC_OPTION = make_cli_option(
"--group-synth",
type=bool,
default=False,
help="Group synthetic case studies in tables."
)


class FeaturePerfPrecisionTableGenerator(
TableGenerator, generator_name="fperf-precision", options=[]
TableGenerator,
generator_name="fperf-precision",
options=[GROUP_SYNTHETIC_OPTION]
):
"""Generator for `FeaturePerfPrecisionTable`."""

Expand Down Expand Up @@ -331,7 +340,7 @@ def tabulate(self, table_format: TableFormat, wrap_table: bool) -> str:
precision_df, overhead_df, on=["CaseStudy", "Profiler"]
)

if GROUP_SYNTHETIC_CATEGORIES:
if self.table_kwargs["group_synth"]:

merged_df["CaseStudy"] = merged_df["CaseStudy"].apply(
compute_cs_category_grouping
Expand Down Expand Up @@ -368,7 +377,7 @@ def tabulate(self, table_format: TableFormat, wrap_table: bool) -> str:

# All means need to be computed before they are added as rows
overall_mean = pivot_df.mean()
if GROUP_SYNTHETIC_CATEGORIES:
if self.table_kwargs["group_synth"]:
synth_mean = pivot_df.loc[pivot_df.index.isin(SYNTH_CATEGORIES)
].mean()
real_world_mean = pivot_df.loc[~pivot_df.index.
Expand Down Expand Up @@ -464,7 +473,9 @@ def add_extras(doc: Document) -> None:


class FeaturePerfOverheadComparisionTableGenerator(
TableGenerator, generator_name="fperf-overhead-comp", options=[]
TableGenerator,
generator_name="fperf-overhead-comp",
options=[GROUP_SYNTHETIC_OPTION]
):
"""Generator for `FeaturePerfOverheadTable`."""

Expand Down Expand Up @@ -571,7 +582,7 @@ def tabulate(self, table_format: TableFormat, wrap_table: bool) -> str:
df = pd.concat(cs_data).sort_index()
df.index.name = 'CaseStudy'

if GROUP_SYNTHETIC_CATEGORIES:
if self.table_kwargs["group_synth"]:
df.index = df.index.map(compute_cs_category_grouping)

df = df.groupby(df.index.name, as_index=True).agg({
Expand Down
Loading