From 20b9435f41bcacaad72ad2399787f7d0e7464c43 Mon Sep 17 00:00:00 2001 From: Andrew Zhu Date: Wed, 24 Apr 2024 13:22:23 -0400 Subject: [PATCH 1/2] leaderboard: add info on function calling finetunes --- README.md | 2 ++ leaderboard-submissions/metadata/gemma-7b-it.json | 2 ++ leaderboard-submissions/metadata/llama-3.json | 4 +++- leaderboard-submissions/results/claude.json | 3 ++- leaderboard-submissions/results/gpt-3.5-turbo.json | 4 +++- leaderboard-submissions/results/gpt-4-turbo.json | 4 +++- leaderboard-submissions/results/gpt-4.json | 4 +++- leaderboard-submissions/results/llama-chat.json | 5 +++-- leaderboard-submissions/results/mistral-chat.json | 6 ++++-- leaderboard-submissions/results/mixtral.json | 6 ++++-- 10 files changed, 29 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index ce8ea29..491669b 100644 --- a/README.md +++ b/README.md @@ -193,6 +193,8 @@ You will also need to write a metadata file for your model. Your metadata file s "citation": "The list of authors and year, in citation format (e.g. `Zhu et al., 2024`)", "type": "FOUNDATION | FINETUNE | PROMPT | OTHER", "context": "The context length of the model your system uses (as an int)", + "is_trained_for_function_calling": "Whether your model was trained for function calling specifically (true/false)", + "details": "Additional model details (e.g. API model revision or Hugging Face model ID) - optional", "closedbook_generations": "YOUR-SYSTEM-NAME.jsonl", "openbook_generations": "YOUR-SYSTEM-NAME.jsonl", "evidenceprovided_generations": "YOUR-SYSTEM-NAME.jsonl" diff --git a/leaderboard-submissions/metadata/gemma-7b-it.json b/leaderboard-submissions/metadata/gemma-7b-it.json index d0eb34e..f183163 100644 --- a/leaderboard-submissions/metadata/gemma-7b-it.json +++ b/leaderboard-submissions/metadata/gemma-7b-it.json @@ -5,6 +5,8 @@ "citation": "Gemma Team, 2024", "type": "FOUNDATION", "context": 8192, + "is_trained_for_function_calling": false, + "details": "google/gemma-1.1-7b-it", "closedbook_generations": "results-closedbook-gemma.jsonl", "openbook_generations": "results-openbook-gemma.jsonl", "evidenceprovided_generations": "results-wiki-provided-gemma.jsonl" diff --git a/leaderboard-submissions/metadata/llama-3.json b/leaderboard-submissions/metadata/llama-3.json index e35cbf5..a31cdfd 100644 --- a/leaderboard-submissions/metadata/llama-3.json +++ b/leaderboard-submissions/metadata/llama-3.json @@ -1,10 +1,12 @@ { - "name": "Llama 3 70B-Instruct", + "name": "Llama 3 70B Instruct", "authors": "Meta", "url": "https://ai.meta.com/blog/meta-llama-3/", "citation": "Meta, 2024", "type": "FOUNDATION", "context": 8192, + "is_trained_for_function_calling": false, + "details": "meta-llama/Meta-Llama-3-70B-Instruct", "closedbook_generations": "results-closedbook-llama3.jsonl", "openbook_generations": "results-openbook-llama3.jsonl", "evidenceprovided_generations": "results-wiki-provided-llama3.jsonl" diff --git a/leaderboard-submissions/results/claude.json b/leaderboard-submissions/results/claude.json index 675be13..c69536a 100644 --- a/leaderboard-submissions/results/claude.json +++ b/leaderboard-submissions/results/claude.json @@ -7,7 +7,8 @@ "url": "https://www.anthropic.com/news/claude-2-1", "citation": "Anthropic, 2023", "type": "FOUNDATION", - "context": 200000 + "context": 200000, + "is_trained_for_function_calling": false }, "closedbook": { "acc": { diff --git a/leaderboard-submissions/results/gpt-3.5-turbo.json b/leaderboard-submissions/results/gpt-3.5-turbo.json index ea1e9dd..bbda945 100644 --- a/leaderboard-submissions/results/gpt-3.5-turbo.json +++ b/leaderboard-submissions/results/gpt-3.5-turbo.json @@ -7,7 +7,9 @@ "url": "https://platform.openai.com/docs/models/gpt-3-5-turbo", "citation": "OpenAI, 2023", "type": "FOUNDATION", - "context": 16384 + "context": 16384, + "is_trained_for_function_calling": true, + "details": "gpt-3.5-turbo-1106 (February 2024)" }, "closedbook": { "acc": { diff --git a/leaderboard-submissions/results/gpt-4-turbo.json b/leaderboard-submissions/results/gpt-4-turbo.json index 9e3297c..f8372ba 100644 --- a/leaderboard-submissions/results/gpt-4-turbo.json +++ b/leaderboard-submissions/results/gpt-4-turbo.json @@ -7,7 +7,9 @@ "url": "https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo", "citation": "OpenAI, 2023", "type": "FOUNDATION", - "context": 128000 + "context": 128000, + "is_trained_for_function_calling": true, + "details": "gpt-4-0125-preview (February 2024)" }, "closedbook": { "acc": { diff --git a/leaderboard-submissions/results/gpt-4.json b/leaderboard-submissions/results/gpt-4.json index 305e62a..5134292 100644 --- a/leaderboard-submissions/results/gpt-4.json +++ b/leaderboard-submissions/results/gpt-4.json @@ -7,7 +7,9 @@ "url": "https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo", "citation": "OpenAI, 2023", "type": "FOUNDATION", - "context": 8192 + "context": 8192, + "is_trained_for_function_calling": true, + "details": "gpt-4-0613 (February 2024)" }, "closedbook": { "acc": { diff --git a/leaderboard-submissions/results/llama-chat.json b/leaderboard-submissions/results/llama-chat.json index f5209b3..bfb54c1 100644 --- a/leaderboard-submissions/results/llama-chat.json +++ b/leaderboard-submissions/results/llama-chat.json @@ -2,12 +2,13 @@ "_submission_hash": "", "_results_hash": "", "metadata": { - "name": "LLaMA 2 70B", + "name": "LLaMA 2 Chat 70B", "authors": "Meta", "url": "https://ai.meta.com/research/publications/llama-2-open-foundation-and-fine-tuned-chat-models/", "citation": "Touvron et al., 2023", "type": "FOUNDATION", - "context": 4096 + "context": 4096, + "is_trained_for_function_calling": false }, "closedbook": { "acc": { diff --git a/leaderboard-submissions/results/mistral-chat.json b/leaderboard-submissions/results/mistral-chat.json index f2f0b1f..454850b 100644 --- a/leaderboard-submissions/results/mistral-chat.json +++ b/leaderboard-submissions/results/mistral-chat.json @@ -2,12 +2,14 @@ "_submission_hash": "", "_results_hash": "", "metadata": { - "name": "Mistral-7B", + "name": "Mistral-7B Instruct", "authors": "Mistral AI", "url": "https://mistral.ai/news/announcing-mistral-7b/", "citation": "Jiang et al., 2023", "type": "FOUNDATION", - "context": 32000 + "context": 32000, + "is_trained_for_function_calling": false, + "details": "mistralai/Mistral-7B-Instruct-v0.2" }, "closedbook": { "acc": { diff --git a/leaderboard-submissions/results/mixtral.json b/leaderboard-submissions/results/mixtral.json index c708cc5..7d5a9d9 100644 --- a/leaderboard-submissions/results/mixtral.json +++ b/leaderboard-submissions/results/mixtral.json @@ -2,12 +2,14 @@ "_submission_hash": "", "_results_hash": "", "metadata": { - "name": "Mixtral-8x7B", + "name": "Mixtral-8x7B Instruct", "authors": "Mistral AI", "url": "https://mistral.ai/news/mixtral-of-experts/", "citation": "Jiang et al., 2024", "type": "FOUNDATION", - "context": 32000 + "context": 32000, + "is_trained_for_function_calling": false, + "details": "mistralai/Mixtral-8x7B-Instruct-v0.1" }, "closedbook": { "acc": { From 510019f4df9392030cb8aa1a29f595d2bf6faee4 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 24 Apr 2024 23:13:34 +0000 Subject: [PATCH 2/2] leaderboard: add eval results --- leaderboard-submissions/results/gemma-7b-it.json | 14 ++++++++------ leaderboard-submissions/results/llama-3.json | 16 +++++++++------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/leaderboard-submissions/results/gemma-7b-it.json b/leaderboard-submissions/results/gemma-7b-it.json index 0b4c035..c403d5e 100644 --- a/leaderboard-submissions/results/gemma-7b-it.json +++ b/leaderboard-submissions/results/gemma-7b-it.json @@ -1,13 +1,15 @@ { - "_submission_hash": "1305a2b1c69400ce2a3d23f6faead21a2e44cb9e1934a8c887ee6cc28b8e9878", - "_results_hash": "6b80d6d21303b16c04af772854de01666bfc11814dbcdbec20be3d8c8221beb2", + "_submission_hash": "0b1d7a2b2d3477a66f69963324a9cb3b4f62bafb2bb6c797eed5bf065387bf0d", + "_results_hash": "6c8f812e898656aae077b588104118e6d86f3337353ef1e79cc6f83d142d7a0e", "metadata": { "name": "Gemma 1.1 7B IT", "authors": "Thomas Mesnard, Cassidy Hardin, Robert Dadashi, Surya Bhupatiraju, Shreya Pathak, Laurent Sifre, Morgane Rivi\u00e8re, Mihir Sanjay Kale, Juliette Love, Pouya Tafti, L\u00e9onard Hussenot, Pier Giuseppe Sessa, Aakanksha Chowdhery, Adam Roberts, Aditya Barua, Alex Botev, Alex Castro-Ros, Ambrose Slone, Am\u00e9lie H\u00e9liou, Andrea Tacchetti, Anna Bulanova, Antonia Paterson, Beth Tsai, Bobak Shahriari, Charline Le Lan, Christopher A. Choquette-Choo, Cl\u00e9ment Crepy, Daniel Cer, Daphne Ippolito, David Reid, Elena Buchatskaya, Eric Ni, Eric Noland, Geng Yan, George Tucker, George-Christian Muraru, Grigory Rozhdestvenskiy, Henryk Michalewski, Ian Tenney, Ivan Grishchenko, Jacob Austin, James Keeling, Jane Labanowski, Jean-Baptiste Lespiau, Jeff Stanway, Jenny Brennan, Jeremy Chen, Johan Ferret, Justin Chiu, Justin Mao-Jones, Katherine Lee, Kathy Yu, Katie Millican, Lars Lowe Sjoesund, Lisa Lee, Lucas Dixon, Machel Reid, Maciej Miku\u0142a, Mateo Wirth, Michael Sharman, Nikolai Chinaev, Nithum Thain, Olivier Bachem, Oscar Chang, Oscar Wahltinez, Paige Bailey, Paul Michel, Petko Yotov, Rahma Chaabouni, Ramona Comanescu, Reena Jana, Rohan Anil, Ross McIlroy, Ruibo Liu, Ryan Mullins, Samuel L Smith, Sebastian Borgeaud, Sertan Girgin, Sholto Douglas, Shree Pandya, Siamak Shakeri, Soham De, Ted Klimenko, Tom Hennigan, Vlad Feinberg, Wojciech Stokowiec, Yu-hui Chen, Zafarali Ahmed, Zhitao Gong, Tris Warkentin, Ludovic Peran, Minh Giang, Cl\u00e9ment Farabet, Oriol Vinyals, Jeff Dean, Koray Kavukcuoglu, Demis Hassabis, Zoubin Ghahramani, Douglas Eck , Joelle Barral, Fernando Pereira, Eli Collins, Armand Joulin, Noah Fiedel, Evan Senter, Alek Andreev, Kathleen Kenealy", "url": "https://ai.google.dev/gemma/docs", "citation": "Gemma Team, 2024", "type": "FOUNDATION", - "context": 8192 + "context": 8192, + "is_trained_for_function_calling": false, + "details": "google/gemma-1.1-7b-it" }, "closedbook": { "acc": { @@ -32,7 +34,7 @@ } }, "bleurt": 0.36741639172545126, - "gpt": 0.03729281767955801 + "gpt": 0.04834254143646409 }, "openbook": { "acc": { @@ -57,7 +59,7 @@ } }, "bleurt": 0.3456440573733156, - "gpt": 0.024861878453038673 + "gpt": 0.022099447513812154 }, "evidenceprovided": { "acc": { @@ -82,6 +84,6 @@ } }, "bleurt": 0.3787077568697831, - "gpt": 0.031767955801104975 + "gpt": 0.03591160220994475 } } \ No newline at end of file diff --git a/leaderboard-submissions/results/llama-3.json b/leaderboard-submissions/results/llama-3.json index abf2e4c..3ba9249 100644 --- a/leaderboard-submissions/results/llama-3.json +++ b/leaderboard-submissions/results/llama-3.json @@ -1,13 +1,15 @@ { - "_submission_hash": "5f7a3733ec45981f2a9d8387ef7fa618c96187f9e61d91c0d6c13372a4ccbfac", - "_results_hash": "149438b76bcb138f7b7798d57a62d60a2fce23548401d78a36d13aeeeef8acdb", + "_submission_hash": "55567660c1534ca3d368b2872e302aab1b26cb83d3f764b65a77c0e05119beb8", + "_results_hash": "52f3142a1f80fcbd17101bef14179180e28dead7c448c1867faa157c9c7abacc", "metadata": { - "name": "Llama 3 70B-Instruct", + "name": "Llama 3 70B Instruct", "authors": "Meta", "url": "https://ai.meta.com/blog/meta-llama-3/", "citation": "Meta, 2024", "type": "FOUNDATION", - "context": 8192 + "context": 8192, + "is_trained_for_function_calling": false, + "details": "meta-llama/Meta-Llama-3-70B-Instruct" }, "closedbook": { "acc": { @@ -32,7 +34,7 @@ } }, "bleurt": 0.47776114855229196, - "gpt": 0.15883977900552487 + "gpt": 0.1574585635359116 }, "openbook": { "acc": { @@ -57,7 +59,7 @@ } }, "bleurt": 0.4730032659881846, - "gpt": 0.21685082872928177 + "gpt": 0.22099447513812154 }, "evidenceprovided": { "acc": { @@ -82,6 +84,6 @@ } }, "bleurt": 0.5209101250508378, - "gpt": 0.27486187845303867 + "gpt": 0.27071823204419887 } } \ No newline at end of file