From 3052c3072a865a97a2545522a983eb76dda97465 Mon Sep 17 00:00:00 2001 From: Jackie Lian <86454986+Jackie-Lian@users.noreply.github.com> Date: Thu, 31 Oct 2024 00:47:33 -0700 Subject: [PATCH] Update monitor.py to add two price columns Two old columns are removed: License and Knowledge Cutoff Two new columns are added: Input Token Price and Output Token Price. The two new columns will be merged into one column in a later PR (ie. merged into a single column called Price, which takes the average between the two) --- fastchat/serve/monitor/monitor.py | 61 +++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 20 deletions(-) diff --git a/fastchat/serve/monitor/monitor.py b/fastchat/serve/monitor/monitor.py index 683a40b54..6915c7035 100644 --- a/fastchat/serve/monitor/monitor.py +++ b/fastchat/serve/monitor/monitor.py @@ -191,6 +191,9 @@ def load_leaderboard_table_csv(filename, add_hyperlink=True): v = round(ast.literal_eval(v), 2) else: v = np.nan + elif h == "input_token_price" or h == "output_token_price": + if v == "null": + v = "-" item[h] = v if add_hyperlink: item["Model"] = model_hyperlink(item["Model"], item["Link"]) @@ -242,7 +245,11 @@ def get_full_table(arena_df, model_table_df, model_to_score): row.append(model_table_df.iloc[i]["Organization"]) # license row.append(model_table_df.iloc[i]["License"]) - + # Input Price + row.append(model_table_df.iloc[i]["input_token_price"]) + # Output Price + row.append(model_table_df.iloc[i]["output_token_price"]) + values.append(row) values.sort(key=lambda x: -x[1] if not np.isnan(x[1]) else 1e9) return values @@ -350,12 +357,14 @@ def process_row(row): f"+{round(row['rating_q975'] - row['rating'])}/-{round(row['rating'] - row['rating_q025'])}", round(row["num_battles"]), model_info.get("Organization", "Unknown"), - model_info.get("License", "Unknown"), - ( - "Unknown" - if model_info.get("Knowledge cutoff date", "-") == "-" - else model_info.get("Knowledge cutoff date", "Unknown") - ), + # model_info.get("License", "Unknown"), + # ( + # "Unknown" + # if model_info.get("Knowledge cutoff date", "-") == "-" + # else model_info.get("Knowledge cutoff date", "Unknown") + # ), + model_info.get("input_token_price", "-"), + model_info.get("output_token_price", "-"), ] ) return result @@ -378,8 +387,10 @@ def update_leaderboard_df(arena_table_vals): "95% CI", "Votes", "Organization", - "License", - "Knowledge Cutoff", + # "License", + # "Knowledge Cutoff", + "Input Token Price", + "Output Token Price", ] elo_dataframe = pd.DataFrame(arena_table_vals, columns=columns) @@ -407,8 +418,10 @@ def update_overall_leaderboard_df(arena_table_vals): "95% CI", "Votes", "Organization", - "License", - "Knowledge Cutoff", + # "License", + # "Knowledge Cutoff", + "Input Token Price", + "Output Token Price", ] elo_dataframe = pd.DataFrame(arena_table_vals, columns=columns) @@ -503,6 +516,8 @@ def update_leaderboard_and_plots(category, filters): ), is_overall=category == "Overall", ) + + if category != "Overall": arena_values = update_leaderboard_df(arena_values) arena_values = gr.Dataframe( @@ -514,8 +529,10 @@ def update_leaderboard_and_plots(category, filters): "95% CI", "Votes", "Organization", - "License", - "Knowledge Cutoff", + # "License", + # "Knowledge Cutoff", + "Input Token Price", + "Output Token Price", ], datatype=[ "number", @@ -531,7 +548,7 @@ def update_leaderboard_and_plots(category, filters): value=arena_values, elem_id="arena_leaderboard_dataframe", height=1000, - column_widths=[75, 75, 180, 60, 60, 60, 70, 80, 60], + column_widths=[75, 75, 180, 60, 60, 60, 70, 55, 55], wrap=True, ) else: @@ -545,8 +562,10 @@ def update_leaderboard_and_plots(category, filters): "95% CI", "Votes", "Organization", - "License", - "Knowledge Cutoff", + # "License", + # "Knowledge Cutoff", + "Input Token Price", + "Output Token Price", ], datatype=[ "number", @@ -562,7 +581,7 @@ def update_leaderboard_and_plots(category, filters): value=arena_values, elem_id="arena_leaderboard_dataframe", height=1000, - column_widths=[75, 75, 180, 60, 60, 60, 70, 80, 60], + column_widths=[75, 75, 180, 60, 60, 60, 70, 55, 55], wrap=True, ) @@ -631,8 +650,10 @@ def update_leaderboard_and_plots(category, filters): "95% CI", "Votes", "Organization", - "License", - "Knowledge Cutoff", + # "License", + # "Knowledge Cutoff", + "Input Token Price", + "Output Token Price", ], datatype=[ "number", @@ -648,7 +669,7 @@ def update_leaderboard_and_plots(category, filters): value=arena_vals, elem_id="arena_leaderboard_dataframe", height=1000, - column_widths=[75, 75, 180, 60, 60, 60, 70, 80, 60], + column_widths=[75, 75, 180, 60, 60, 60, 70, 55, 55], wrap=True, )