From 5cbe333af66b3a4c270cc13b54ae487660b00606 Mon Sep 17 00:00:00 2001
From: Douglas Jacobsen <dwjacobsen@google.com>
Date: Tue, 17 Dec 2024 11:50:54 -0700
Subject: [PATCH 1/2] Update HPL base_application for MxP

This commit makes changes to the HPL base_application definition to
define variables before templates are rendered, and also prevent padding
values if the workload contains `mxp`.
---
 .../base_applications/hpl/base_application.py | 55 +++++++++++++++----
 1 file changed, 43 insertions(+), 12 deletions(-)

diff --git a/var/ramble/repos/builtin/base_applications/hpl/base_application.py b/var/ramble/repos/builtin/base_applications/hpl/base_application.py
index 8613e3a63..e8ad4aeb5 100644
--- a/var/ramble/repos/builtin/base_applications/hpl/base_application.py
+++ b/var/ramble/repos/builtin/base_applications/hpl/base_application.py
@@ -8,7 +8,6 @@
 
 import os
 from ramble.appkit import *
-from ramble.expander import Expander
 
 import math
 
@@ -262,14 +261,8 @@ class Hpl(ExecutableApplication):
     )
 
     # FOMs:
-    log_str = os.path.join(
-        Expander.expansion_str("experiment_run_dir"),
-        Expander.expansion_str("experiment_name") + ".out",
-    )
-
     figure_of_merit(
         "Time",
-        log_file=log_str,
         fom_regex=r".*\s+(?P<N>[0-9]+)\s+(?P<NB>[0-9]+)\s+(?P<P>[0-9]+)\s+(?P<Q>[0-9]+)\s+(?P<time>[0-9]+\.[0-9]+)\s+(?P<gflops>[0-9].*)\n",
         group_name="time",
         units="s",
@@ -278,7 +271,6 @@ class Hpl(ExecutableApplication):
 
     figure_of_merit(
         "GFlops",
-        log_file=log_str,
         fom_regex=r".*\s+(?P<N>[0-9]+)\s+(?P<NB>[0-9]+)\s+(?P<P>[0-9]+)\s+(?P<Q>[0-9]+)\s+(?P<time>[0-9]+\.[0-9]+)\s+(?P<gflops>[0-9].*)\n",
         group_name="gflops",
         units="GFLOP/s",
@@ -291,6 +283,39 @@ class Hpl(ExecutableApplication):
         output_format="N-NB-P-Q = {N}-{NB}-{P}-{Q}",
     )
 
+    # MxP FOMs
+    gflops_regex = (
+        r"\s+GFLOPS = (?P<gflops>\S+), per GPU =\s+(?P<per_gflops>\S+)"
+    )
+    lu_gflops_regex = (
+        r"\s+LU GFLOPS = (?P<gflops>\S+), per GPU =\s+(?P<per_gflops>\S+)"
+    )
+    figure_of_merit(
+        "Total GFLOPs",
+        fom_regex=gflops_regex,
+        group_name="gflops",
+        units="GFLOPs",
+    )
+    figure_of_merit(
+        "Per GPU GFLOPs",
+        fom_regex=gflops_regex,
+        group_name="per_gflops",
+        units="GFLOPs",
+    )
+
+    figure_of_merit(
+        "Total LU GFLOPs",
+        fom_regex=lu_gflops_regex,
+        group_name="gflops",
+        units="GFLOPs",
+    )
+    figure_of_merit(
+        "Per GPU LU GFLOPs",
+        fom_regex=lu_gflops_regex,
+        group_name="per_gflops",
+        units="GFLOPs",
+    )
+
     # ( setting_name, setting_description )
     hpl_settings = [
         ("output_file", "output file name (if any)"),
@@ -341,10 +366,14 @@ def _isqrt(self, n):
             else:
                 return hi
 
-    def _calculate_values(self, workspace):
+    register_phase(
+        "calculate_values", pipeline="setup", run_before=["make_experiments"]
+    )
+
+    def _calculate_values(self, workspace, app_inst):
         expander = self.expander
         calculated_settings = {}
-        if expander.workload_name == "calculator":
+        if "calculator" in expander.workload_name:
             # Find the best P and Q whose product is the number of available
             # cores, with P less than Q
             nNodes = int(expander.expand_var_name("n_nodes"))
@@ -425,11 +454,13 @@ def _calculate_values(self, workspace):
                         " " + calculated_settings[setting]["comment"]
                     )
 
-            self.define_variable(setting, pad_value(value, pad_comment))
+            if "mxp" in self.expander.workload_name:
+                self.define_variable(setting, value)
+            else:
+                self.define_variable(setting, pad_value(value, pad_comment))
 
     def _make_experiments(self, workspace, app_inst=None):
         super()._make_experiments(workspace)
-        self._calculate_values(workspace)
 
         input_path = os.path.join(
             self.expander.expand_var_name("experiment_run_dir"), "HPL.dat"

From 37bc23f9332b59cb9c2640d9beb138acfd6cf92b Mon Sep 17 00:00:00 2001
From: Douglas Jacobsen <dwjacobsen@google.com>
Date: Tue, 17 Dec 2024 11:51:50 -0700
Subject: [PATCH 2/2] Add `mxp` versions of nvidia-hpl workloads

---
 .../applications/nvidia-hpl/application.py    | 38 +++++++++++++++++--
 1 file changed, 35 insertions(+), 3 deletions(-)

diff --git a/var/ramble/repos/builtin/applications/nvidia-hpl/application.py b/var/ramble/repos/builtin/applications/nvidia-hpl/application.py
index 85b0b7c2f..ed4e0d6b8 100644
--- a/var/ramble/repos/builtin/applications/nvidia-hpl/application.py
+++ b/var/ramble/repos/builtin/applications/nvidia-hpl/application.py
@@ -38,12 +38,29 @@ class NvidiaHpl(HplBase):
         "execute", "./hpl.sh --dat {experiment_run_dir}/HPL.dat", use_mpi=True
     )
 
+    executable(
+        "execute-mxp",
+        './hpl-mxp.sh --gpu-affinity "{gpu_affinity}" --n {Ns} --nb {block_size} --nprow {Ps} --npcol {Qs} --nporder {nporder}',
+        use_mpi=True,
+    )
+
     workload("standard", executables=["execute"])
     workload("calculator", executables=["execute"])
 
-    workload_group("standard", workloads=["standard"], mode="append")
-    workload_group("calculator", workloads=["calculator"], mode="append")
-    workload_group("all_workloads", workloads=["standard", "calculator"])
+    workload("standard-mxp", executables=["execute-mxp"])
+    workload("calculator-mxp", executables=["execute-mxp"])
+
+    workload_group(
+        "standard", workloads=["standard", "standard-mxp"], mode="append"
+    )
+    workload_group(
+        "calculator", workloads=["calculator", "calculator-mxp"], mode="append"
+    )
+    workload_group(
+        "all_workloads",
+        workloads=["standard", "standard-mxp", "calculator", "calculator-mxp"],
+    )
+    workload_group("mxp", workloads=["standard-mxp", "calculator-mxp"])
 
     workload_variable(
         "nvshmem_disable_cuda_vmm",
@@ -168,3 +185,18 @@ class NvidiaHpl(HplBase):
         description="Size of each block",
         workload_group="calculator",
     )
+
+    workload_variable(
+        "nporder",
+        default="row",
+        description="Major order to use for matrix",
+        values=["row", "column"],
+        workload_group="mxp",
+    )
+
+    workload_variable(
+        "gpu_affinity",
+        default="0:1:2:3:4:5:6:7",
+        description="Colon delimited list of GPU IDs",
+        workload_group="mxp",
+    )