Price performance

quisby/benchmarks/coremark/compare.py

@@ -8,11 +8,32 @@
     get_sheet,
     create_sheet, clear_sheet_data, clear_sheet_charts,
 )
-from quisby.util import combine_two_array_alternating, merge_lists_alternately
+from quisby.util import combine_two_array_alternating, merge_lists_alternately, read_config
 from quisby.benchmarks.coremark.graph import graph_coremark_data
+import re
 
 
-def compare_coremark_results(spreadsheets, spreadsheetId, test_name, table_name=["System name"]):
+def extract_prefix_and_number(input_string):
+    match = re.search(r'^(.*?)(\d+)(.*?)$', input_string)
+    if match:
+        prefix = match.group(1)
+        return prefix
+    return None
+
+
+def compare_inst(item1, item2):
+    cloud_type = read_config("cloud", "cloud_type")
+    if cloud_type == "localhost":
+        return True
+    elif cloud_type == "aws":
+        return item1.split(".")[0] == item2.split(".")[0]
+    elif cloud_type == "gcp":
+        return item1.split("-")[0], item2.split("-")[0]
+    elif cloud_type == "azure":
+        return extract_prefix_and_number(item1) == extract_prefix_and_number(item2)
+
+
+def compare_coremark_results(spreadsheets, spreadsheetId, test_name, table_name=["System name","Price/perf"]):
     values = []
     results = []
     spreadsheet_name = []
@@ -29,14 +50,23 @@ def compare_coremark_results(spreadsheets, spreadsheetId, test_name, table_name=
     for value in list_1:
         for ele in list_2:
             # Check max throughput
-            if value[0][0] in table_name and ele[0][0] in table_name:
+            if value[0][0] in table_name and ele[0][0] in table_name and value[0][0] == ele[0][0]:
                 results.append([""])
                 for item1 in value:
                     for item2 in ele:
                         if item1[0] == item2[0]:
                             results = merge_lists_alternately(results, item1, item2)
                 break
 
+            elif value[0][0] == "Cost/Hr" and ele[0][0] == "Cost/Hr":
+                if compare_inst(value[1][0], ele[1][0]):
+                    results.append([""])
+                    for item1 in value:
+                        for item2 in ele:
+                            if item1[0] == item2[0]:
+                                results.append(item1)
+                    break
+
             elif value[1][0] == ele[1][0]:
                 if value[0][0] == ele[0][0]:
                     results.append([""])

quisby/benchmarks/coremark/coremark.py

@@ -4,6 +4,8 @@
 import re
 
 from quisby.util import read_config
+from quisby.pricing.cloud_pricing import get_cloud_pricing
+
 
 def extract_prefix_and_number(input_string):
     match = re.search(r'^(.*?)(\d+)(.*?)$', input_string)
@@ -15,47 +17,75 @@ def extract_prefix_and_number(input_string):
     return None, None, None
 
 
-
 def custom_key(item):
     cloud_type = read_config("cloud","cloud_type")
     if item[1][0] == "localhost":
-        return (item[1][0])
+        return item[1][0]
     elif cloud_type == "aws":
         instance_type =item[1][0].split(".")[0]
         instance_number = item[1][0].split(".")[1]
-        return (instance_type, instance_number)
+        return instance_type, instance_number
     elif cloud_type == "gcp":
          instance_type = item[1][0].split("-")[0]
          instance_number = int(item[1][0].split('-')[-1])
-         return (instance_type, instance_number)
+         return instance_type, instance_number
     elif cloud_type == "azure":
         instance_type, instance_number, version=extract_prefix_and_number(item[1][0])
-        return (instance_type, instance_number)
+        return instance_type, instance_number
+
 
+def calc_price_performance(inst, avg):
+    region = read_config("cloud", "region")
+    cloud_type = read_config("cloud", "cloud_type")
+    os_type = read_config("test", "os_type")
+    cost_per_hour = None
+    try:
+        cost_per_hour = get_cloud_pricing(
+            inst, region, cloud_type.lower(), os_type)
+        price_perf = float(avg)/float(cost_per_hour)
+    except Exception as exc:
+        custom_logger.debug(str(exc))
+        custom_logger.error("Error calculating value !")
+    return cost_per_hour, price_perf
 
-def create_summary_coremark_data(results,OS_RELEASE):
+
+def create_summary_coremark_data(results, OS_RELEASE):
     final_results = []
     cal_data = [["System name", "test passes_"+OS_RELEASE]]
 
     # Sort data based on instance name
     sorted_data = sorted(results, key=custom_key)
+    cost_per_hour, price_per_perf = [], []
 
     # Add summary data
     for item in sorted_data:
         sum = 0
         avg = 0
         iterations = 0
-        for index in range(3,len(item)):
+        for index in range(3, len(item)):
             sum = sum + float(item[index][1])
             iterations = iterations + 1
         avg = float(sum/iterations)
-        cal_data.append([item[1][0],avg])
+        try:
+            cph, pp = calc_price_performance(item[1][0], avg)
+        except Exception as exc:
+            custom_logger.error(str(exc))
+            break
+        cal_data.append([item[1][0], avg])
+        price_per_perf.append([item[1][0], pp])
+        cost_per_hour.append([item[1][0], cph])
 
-        final_results += item
     final_results += [[""]]
     final_results += cal_data
+    final_results.append([""])
+    final_results.append(["Cost/Hr"])
+    final_results += cost_per_hour
+    final_results.append([""])
+    final_results.append(["Price/perf", f"system-{OS_RELEASE}"])
+    final_results += price_per_perf
     return final_results
 
+
 def extract_coremark_data(path, system_name, OS_RELEASE):
     """"""
     results = []
@@ -90,3 +120,6 @@ def extract_coremark_data(path, system_name, OS_RELEASE):
 
     return results
 
+
+
+

quisby/benchmarks/coremark/graph.py

@@ -101,13 +101,21 @@ def graph_coremark_data(spreadsheetId, range, action):
 
     header_row = []
     sheetId = -1
+
     for index, row in enumerate(data):
         if "System name" in row:
             start_index = index
             header_row.extend(row)
+            title = "%s : %s" % (range, "Test Passes")
+            subtitle = ""
+        elif "Price/perf" in row:
+            start_index = index
+            header_row.extend(row)
+            title = "%s : %s" % (range, "Price-Performance")
+            subtitle = "Passes/$"
         if start_index:
             if not row:
-                end_index = index - 1
+                end_index = index
             if index + 1 == len(data):
                 end_index = index + 1
 
@@ -124,18 +132,15 @@ def graph_coremark_data(spreadsheetId, range, action):
                 "addChart": {
                     "chart": {
                         "spec": {
-                            "title": "%s : %s" % (range, "Test passes"),
+                            "title": title,
+                            "subtitle": subtitle,
                             "basicChart": {
                                 "chartType": "COMBO",
                                 "legendPosition": "BOTTOM_LEGEND",
                                 "axis": [
                                     {
                                         "position": "LEFT_AXIS",
-                                        "title": "Test passes"
-                                    },
-                                    {
-                                        "position": "BOTTOM_AXIS",
-                                        "title": "Machine types",
+                                        "title": graph_data[0][1],
                                     },
                                     {
                                         "position": "RIGHT_AXIS",
@@ -168,7 +173,7 @@ def graph_coremark_data(spreadsheetId, range, action):
                                 "anchorCell": {
                                     "sheetId": sheetId,
                                     "rowIndex": GRAPH_ROW_INDEX,
-                                    "columnIndex": column_count + 1,
+                                    "columnIndex": column_count + GRAPH_COL_INDEX,
                                 }
                             }
                         },

quisby/benchmarks/coremark_pro/compare.py

@@ -8,7 +8,28 @@
     get_sheet,
     create_sheet, clear_sheet_data, clear_sheet_charts,
 )
-from quisby.util import merge_lists_alternately
+from quisby.util import merge_lists_alternately,read_config
+import re
+
+
+def extract_prefix_and_number(input_string):
+    match = re.search(r'^(.*?)(\d+)(.*?)$', input_string)
+    if match:
+        prefix = match.group(1)
+        return prefix
+    return None
+
+
+def compare_inst(item1, item2):
+    cloud_type = read_config("cloud", "cloud_type")
+    if cloud_type == "localhost":
+        return True
+    elif cloud_type == "aws":
+        return item1.split(".")[0] == item2.split(".")[0]
+    elif cloud_type == "gcp":
+        return item1.split("-")[0], item2.split("-")[0]
+    elif cloud_type == "azure":
+        return extract_prefix_and_number(item1) == extract_prefix_and_number(item2)
 
 
 def compare_coremark_pro_results(spreadsheets, spreadsheetId, test_name, table_name=["System name"]):
@@ -37,6 +58,16 @@ def compare_coremark_pro_results(spreadsheets, spreadsheetId, test_name, table_n
                                 results = merge_lists_alternately(results, item1, item2)
                     break
 
+            elif value[0][0] == "Cost/Hr" and ele[0][0] == "Cost/Hr":
+                if compare_inst(value[1][0], ele[1][0]):
+                    results.append([""])
+                    for item1 in value:
+                        for item2 in ele:
+                            if item1[0] == item2[0]:
+                                results.append(item1)
+                    break
+
+
             elif value[1][0] == ele[1][0]:
                 if value[0][0] == ele[0][0]:
                     results.append([""])

quisby/benchmarks/coremark_pro/coremark_pro.py

@@ -2,6 +2,7 @@
 
 from quisby import custom_logger
 from quisby.util import read_config
+from quisby.pricing.cloud_pricing import get_cloud_pricing
 
 
 def extract_prefix_and_number(input_string):
@@ -35,22 +36,56 @@ def custom_key(item):
         return "", ""
 
 
+def calc_price_performance(inst, avg):
+    region = read_config("cloud", "region")
+    cloud_type = read_config("cloud", "cloud_type")
+    os_type = read_config("test", "os_type")
+    cost_per_hour = None
+    try:
+        cost_per_hour = get_cloud_pricing(
+            inst, region, cloud_type.lower(), os_type)
+        price_perf = float(avg)/float(cost_per_hour)
+    except Exception as exc:
+        custom_logger.debug(str(exc))
+        custom_logger.error("Error calculating value !")
+    return cost_per_hour, price_perf
+
+
 def create_summary_coremark_pro_data(results, OS_RELEASE):
     final_results = []
     multi_iter = [["Multi Iterations"], ["System name", "Score_" + OS_RELEASE]]
     single_iter = [["Single Iterations"], ["System name", "Score_" + OS_RELEASE]]
     # Sort data based on instance name
     sorted_data = sorted(results, key=custom_key)
     # Add summary data
+    cost_per_hour, price_perf_single, price_perf_multi = [], [],[]
     for item in sorted_data:
         for index in range(3, len(item)):
             multi_iter.append([item[1][0], item[index][1]])
             single_iter.append([item[1][0], item[index][2]])
+            try:
+                cph, ppm = calc_price_performance(item[1][0], item[index][1])
+                cph, pps = calc_price_performance(item[1][0], item[index][2])
+            except Exception as exc:
+                custom_logger.error(str(exc))
+                break
+            price_perf_multi.append([item[1][0], ppm])
+            price_perf_single.append([item[1][0], pps])
+            cost_per_hour.append([item[1][0], cph])
             # final_results += item
     final_results += [[""]]
-    final_results += multi_iter
-    final_results += [[""]]
     final_results += single_iter
+    final_results.append([""])
+    final_results.append(["Cost/Hr"])
+    final_results += cost_per_hour
+    final_results.extend([[""], ["Single Iterations"]])
+    final_results.append(["Price/perf", f"single-iter-{OS_RELEASE}"])
+    final_results += price_perf_single
+    final_results += [[""]]
+    final_results += multi_iter
+    final_results.extend([[""], ["Multi Iterations"]])
+    final_results.append(["Price/perf", f"multi-iter-{OS_RELEASE}"])
+    final_results += price_perf_multi
     return final_results
 
 

quisby/benchmarks/coremark_pro/graph.py

@@ -103,12 +103,19 @@ def graph_coremark_pro_data(spreadsheetId, range, action):
 
     header = []
     sheetId = -1
-
     for index, row in enumerate(data):
         if "System name" in row:
             start_index = index
+            header.extend(row)
             iteration = data[index - 1][0]
+            title = "%s : %s" % (range, "Score")
+            subtitle = iteration
+        elif "Price/perf" in row:
+            start_index = index
             header.extend(row)
+            iteration = data[index - 1][0]
+            title = "%s : %s" % (range, "Price-Performance")
+            subtitle = "%s : %s" % ("Score/$", iteration)
         if start_index:
             if not row:
                 end_index = index
@@ -130,7 +137,8 @@ def graph_coremark_pro_data(spreadsheetId, range, action):
                 "addChart": {
                     "chart": {
                         "spec": {
-                            "title": "%s : %s : %s" % (range, "score", iteration),
+                            "title": title,
+                            "subtitle": subtitle,
                             "basicChart": {
                                 "chartType": "COMBO",
                                 "legendPosition": "BOTTOM_LEGEND",
@@ -183,7 +191,7 @@ def graph_coremark_pro_data(spreadsheetId, range, action):
                 GRAPH_ROW_INDEX += 20
                 GRAPH_COL_INDEX = 1
             else:
-                GRAPH_ROW_INDEX = end_index
+                GRAPH_COL_INDEX += 6
 
             body = {"requests": requests}