Skip to content

Commit

Permalink
Merge pull request #802 from douglasjacobsen/hpl-mxp
Browse files Browse the repository at this point in the history
Add support for HPL-MxP to nvidia-hpl
  • Loading branch information
rfbgo authored Dec 17, 2024
2 parents b6f4fc3 + 37bc23f commit 39345d8
Show file tree
Hide file tree
Showing 2 changed files with 78 additions and 15 deletions.
38 changes: 35 additions & 3 deletions var/ramble/repos/builtin/applications/nvidia-hpl/application.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,12 +38,29 @@ class NvidiaHpl(HplBase):
"execute", "./hpl.sh --dat {experiment_run_dir}/HPL.dat", use_mpi=True
)

executable(
"execute-mxp",
'./hpl-mxp.sh --gpu-affinity "{gpu_affinity}" --n {Ns} --nb {block_size} --nprow {Ps} --npcol {Qs} --nporder {nporder}',
use_mpi=True,
)

workload("standard", executables=["execute"])
workload("calculator", executables=["execute"])

workload_group("standard", workloads=["standard"], mode="append")
workload_group("calculator", workloads=["calculator"], mode="append")
workload_group("all_workloads", workloads=["standard", "calculator"])
workload("standard-mxp", executables=["execute-mxp"])
workload("calculator-mxp", executables=["execute-mxp"])

workload_group(
"standard", workloads=["standard", "standard-mxp"], mode="append"
)
workload_group(
"calculator", workloads=["calculator", "calculator-mxp"], mode="append"
)
workload_group(
"all_workloads",
workloads=["standard", "standard-mxp", "calculator", "calculator-mxp"],
)
workload_group("mxp", workloads=["standard-mxp", "calculator-mxp"])

workload_variable(
"nvshmem_disable_cuda_vmm",
Expand Down Expand Up @@ -168,3 +185,18 @@ class NvidiaHpl(HplBase):
description="Size of each block",
workload_group="calculator",
)

workload_variable(
"nporder",
default="row",
description="Major order to use for matrix",
values=["row", "column"],
workload_group="mxp",
)

workload_variable(
"gpu_affinity",
default="0:1:2:3:4:5:6:7",
description="Colon delimited list of GPU IDs",
workload_group="mxp",
)
55 changes: 43 additions & 12 deletions var/ramble/repos/builtin/base_applications/hpl/base_application.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@

import os
from ramble.appkit import *
from ramble.expander import Expander

import math

Expand Down Expand Up @@ -262,14 +261,8 @@ class Hpl(ExecutableApplication):
)

# FOMs:
log_str = os.path.join(
Expander.expansion_str("experiment_run_dir"),
Expander.expansion_str("experiment_name") + ".out",
)

figure_of_merit(
"Time",
log_file=log_str,
fom_regex=r".*\s+(?P<N>[0-9]+)\s+(?P<NB>[0-9]+)\s+(?P<P>[0-9]+)\s+(?P<Q>[0-9]+)\s+(?P<time>[0-9]+\.[0-9]+)\s+(?P<gflops>[0-9].*)\n",
group_name="time",
units="s",
Expand All @@ -278,7 +271,6 @@ class Hpl(ExecutableApplication):

figure_of_merit(
"GFlops",
log_file=log_str,
fom_regex=r".*\s+(?P<N>[0-9]+)\s+(?P<NB>[0-9]+)\s+(?P<P>[0-9]+)\s+(?P<Q>[0-9]+)\s+(?P<time>[0-9]+\.[0-9]+)\s+(?P<gflops>[0-9].*)\n",
group_name="gflops",
units="GFLOP/s",
Expand All @@ -291,6 +283,39 @@ class Hpl(ExecutableApplication):
output_format="N-NB-P-Q = {N}-{NB}-{P}-{Q}",
)

# MxP FOMs
gflops_regex = (
r"\s+GFLOPS = (?P<gflops>\S+), per GPU =\s+(?P<per_gflops>\S+)"
)
lu_gflops_regex = (
r"\s+LU GFLOPS = (?P<gflops>\S+), per GPU =\s+(?P<per_gflops>\S+)"
)
figure_of_merit(
"Total GFLOPs",
fom_regex=gflops_regex,
group_name="gflops",
units="GFLOPs",
)
figure_of_merit(
"Per GPU GFLOPs",
fom_regex=gflops_regex,
group_name="per_gflops",
units="GFLOPs",
)

figure_of_merit(
"Total LU GFLOPs",
fom_regex=lu_gflops_regex,
group_name="gflops",
units="GFLOPs",
)
figure_of_merit(
"Per GPU LU GFLOPs",
fom_regex=lu_gflops_regex,
group_name="per_gflops",
units="GFLOPs",
)

# ( setting_name, setting_description )
hpl_settings = [
("output_file", "output file name (if any)"),
Expand Down Expand Up @@ -341,10 +366,14 @@ def _isqrt(self, n):
else:
return hi

def _calculate_values(self, workspace):
register_phase(
"calculate_values", pipeline="setup", run_before=["make_experiments"]
)

def _calculate_values(self, workspace, app_inst):
expander = self.expander
calculated_settings = {}
if expander.workload_name == "calculator":
if "calculator" in expander.workload_name:
# Find the best P and Q whose product is the number of available
# cores, with P less than Q
nNodes = int(expander.expand_var_name("n_nodes"))
Expand Down Expand Up @@ -425,11 +454,13 @@ def _calculate_values(self, workspace):
" " + calculated_settings[setting]["comment"]
)

self.define_variable(setting, pad_value(value, pad_comment))
if "mxp" in self.expander.workload_name:
self.define_variable(setting, value)
else:
self.define_variable(setting, pad_value(value, pad_comment))

def _make_experiments(self, workspace, app_inst=None):
super()._make_experiments(workspace)
self._calculate_values(workspace)

input_path = os.path.join(
self.expander.expand_var_name("experiment_run_dir"), "HPL.dat"
Expand Down

0 comments on commit 39345d8

Please sign in to comment.