From e5defca92451efb03efd89b70ee17f16f46e0a8b Mon Sep 17 00:00:00 2001 From: Arid Hasan Date: Fri, 22 Dec 2023 19:54:20 -0400 Subject: [PATCH 1/3] Add JAIS13b ZeroShot for ArEmotion asset --- .../emotion/Emotion_JAIS13b_ZeroShot.py | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 assets/ar/sentiment_emotion_others/emotion/Emotion_JAIS13b_ZeroShot.py diff --git a/assets/ar/sentiment_emotion_others/emotion/Emotion_JAIS13b_ZeroShot.py b/assets/ar/sentiment_emotion_others/emotion/Emotion_JAIS13b_ZeroShot.py new file mode 100644 index 00000000..c88fbf42 --- /dev/null +++ b/assets/ar/sentiment_emotion_others/emotion/Emotion_JAIS13b_ZeroShot.py @@ -0,0 +1,93 @@ +from llmebench.datasets import EmotionDataset +from llmebench.models import FastChatModel +from llmebench.tasks import EmotionTask + + +def metadata(): + return { + "author": "Arabic Language Technologies, QCRI, HBKU", + "model": "JAIS-13b", + "description": "Locally hosted JAIS-13b-chat model using FastChat.", + "scores": {"Macro-F1": ""}, + } + + +def config(): + return { + "dataset": EmotionDataset, + "task": EmotionTask, + "model": FastChatModel, + "model_args": { + "class_labels": [ + "anger", + "disgust", + "fear", + "joy", + "love", + "optimism", + "pessimism", + "sadness", + "surprise", + "trust", + ], + "max_tries": 30, + }, + } + + +def prompt(input_sample): + base_prompt = ( + f"Predict all the possible emotions in the following Arabic sentence without explanation and put them in a Python list. List of emotions is: anger, anticipation, disgust, fear, joy, love, optimism, pessimism, sadness, surprise, and trust.\n " + f"Sentence: {input_sample}\n" + f"label: \n" + ) + return [ + { + "role": "user", + "content": base_prompt, + }, + ] + + +emotions_positions = { + "anger": 0, + "anticipation": 1, + "disgust": 2, + "fear": 3, + "joy": 4, + "love": 5, + "optimism": 6, + "pessimism": 7, + "sadness": 8, + "surprise": 9, + "trust": 10, +} + + +def emotions_array(labels): + labels_arr = [] + for x, y in emotions_positions.items(): + v = 0 + if x in labels: + v = 1 + labels_arr.append(v) + return labels_arr + + +def post_process(response): + #emotions_array(response["choices"][0]["message"]["content"]) + out = emotions_array(response["choices"][0]["message"]["content"]) + out = out.strip().lower() + + if "i apologize" in out: + return None + + j = out.find("label:") + if j > 0: + out = out[j + len("label:") :] + else: + j = out.find(" is:\n\n") + if j > 0: + out = out[j + len(" is:\n\n") :] + out = out.strip().title().lower() + return out From cc3e3ef36a37d2d34f94e783b7074492ddc35b70 Mon Sep 17 00:00:00 2001 From: Firoj Alam Date: Mon, 29 Jan 2024 11:43:02 +0300 Subject: [PATCH 2/3] feature: Emotion/Jais-13b --- .../emotion/Emotion_Jais13b_FewShot.py | 92 +++++++++++++++++++ ...eroShot.py => Emotion_Jais13b_ZeroShot.py} | 17 +--- 2 files changed, 94 insertions(+), 15 deletions(-) create mode 100644 assets/ar/sentiment_emotion_others/emotion/Emotion_Jais13b_FewShot.py rename assets/ar/sentiment_emotion_others/emotion/{Emotion_JAIS13b_ZeroShot.py => Emotion_Jais13b_ZeroShot.py} (80%) diff --git a/assets/ar/sentiment_emotion_others/emotion/Emotion_Jais13b_FewShot.py b/assets/ar/sentiment_emotion_others/emotion/Emotion_Jais13b_FewShot.py new file mode 100644 index 00000000..8edfa03c --- /dev/null +++ b/assets/ar/sentiment_emotion_others/emotion/Emotion_Jais13b_FewShot.py @@ -0,0 +1,92 @@ +from llmebench.datasets import EmotionDataset +from llmebench.models import FastChatModel +from llmebench.tasks import EmotionTask + + +def metadata(): + return { + "author": "Arabic Language Technologies, QCRI, HBKU", + "model": "JAIS-13b", + "description": "Locally hosted JAIS-13b-chat model using FastChat. 3-shot results.", + "scores": {"Jaccard similarity": "0.1001005"}, + } + + +def config(): + return { + "dataset": EmotionDataset, + "task": EmotionTask, + "model": FastChatModel, + "model_args": { + "class_labels": [ + "anger", + "disgust", + "fear", + "joy", + "love", + "optimism", + "pessimism", + "sadness", + "surprise", + "trust", + ], + "max_tries": 30, + }, + } + + +def prompt(input_sample, examples): + base_prompt = f"Predict all the possible emotions in the following Arabic sentence without explanation and put them in a Python list. List of emotions is: anger, anticipation, disgust, fear, joy, love, optimism, pessimism, sadness, surprise, and trust.\n " + + return [ + { + "role": "user", + "content": few_shot_prompt(input_sample, base_prompt, examples), + }, + ] + + +def few_shot_prompt(input_sample, base_prompt, examples): + out_prompt = base_prompt + "\n" + for example in examples: + # Found chatgpt confused when using 0 and 1 in the prompt + label_list = ", ".join(map(str, example["label"])) + out_prompt = ( + out_prompt + "Sentence: " + example["input"] + "\n" + label_list + "\n\n" + ) + + # Append the sentence we want the model to predict for but leave the Label blank + out_prompt = out_prompt + "Sentence: " + input_sample + "\nlabel: \n" + + return out_prompt + + +emotions_positions = { + "anger": 0, + "anticipation": 1, + "disgust": 2, + "fear": 3, + "joy": 4, + "love": 5, + "optimism": 6, + "pessimism": 7, + "sadness": 8, + "surprise": 9, + "trust": 10, +} + + +def emotions_array(labels): + labels_arr = [] + for x, y in emotions_positions.items(): + v = 0 + if x.lower() in labels: + v = 1 + labels_arr.append(v) + return labels_arr + + +def post_process(response): + out = emotions_array(response["choices"][0]["message"]["content"]) + + return out diff --git a/assets/ar/sentiment_emotion_others/emotion/Emotion_JAIS13b_ZeroShot.py b/assets/ar/sentiment_emotion_others/emotion/Emotion_Jais13b_ZeroShot.py similarity index 80% rename from assets/ar/sentiment_emotion_others/emotion/Emotion_JAIS13b_ZeroShot.py rename to assets/ar/sentiment_emotion_others/emotion/Emotion_Jais13b_ZeroShot.py index c88fbf42..899f8635 100644 --- a/assets/ar/sentiment_emotion_others/emotion/Emotion_JAIS13b_ZeroShot.py +++ b/assets/ar/sentiment_emotion_others/emotion/Emotion_Jais13b_ZeroShot.py @@ -8,7 +8,7 @@ def metadata(): "author": "Arabic Language Technologies, QCRI, HBKU", "model": "JAIS-13b", "description": "Locally hosted JAIS-13b-chat model using FastChat.", - "scores": {"Macro-F1": ""}, + "scores": {"Jaccard similarity": "0.16779"}, } @@ -68,26 +68,13 @@ def emotions_array(labels): labels_arr = [] for x, y in emotions_positions.items(): v = 0 - if x in labels: + if x.lower() in labels: v = 1 labels_arr.append(v) return labels_arr def post_process(response): - #emotions_array(response["choices"][0]["message"]["content"]) out = emotions_array(response["choices"][0]["message"]["content"]) - out = out.strip().lower() - if "i apologize" in out: - return None - - j = out.find("label:") - if j > 0: - out = out[j + len("label:") :] - else: - j = out.find(" is:\n\n") - if j > 0: - out = out[j + len(" is:\n\n") :] - out = out.strip().title().lower() return out From 585d13e2ebda67ea06612f6c194ef50344e9cf5b Mon Sep 17 00:00:00 2001 From: Fahim Imaduddin Dalvi Date: Wed, 31 Jan 2024 16:03:30 +0300 Subject: [PATCH 3/3] Fix minor metadata issue --- .../emotion/Emotion_Jais13b_FewShot.py | 4 ++-- .../emotion/Emotion_Jais13b_ZeroShot.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/assets/ar/sentiment_emotion_others/emotion/Emotion_Jais13b_FewShot.py b/assets/ar/sentiment_emotion_others/emotion/Emotion_Jais13b_FewShot.py index 8edfa03c..461b1636 100644 --- a/assets/ar/sentiment_emotion_others/emotion/Emotion_Jais13b_FewShot.py +++ b/assets/ar/sentiment_emotion_others/emotion/Emotion_Jais13b_FewShot.py @@ -6,8 +6,8 @@ def metadata(): return { "author": "Arabic Language Technologies, QCRI, HBKU", - "model": "JAIS-13b", - "description": "Locally hosted JAIS-13b-chat model using FastChat. 3-shot results.", + "model": "Jais-13b-chat", + "description": "Locally hosted Jais-13b-chat model using FastChat. 3-shot results.", "scores": {"Jaccard similarity": "0.1001005"}, } diff --git a/assets/ar/sentiment_emotion_others/emotion/Emotion_Jais13b_ZeroShot.py b/assets/ar/sentiment_emotion_others/emotion/Emotion_Jais13b_ZeroShot.py index 899f8635..cfddddb1 100644 --- a/assets/ar/sentiment_emotion_others/emotion/Emotion_Jais13b_ZeroShot.py +++ b/assets/ar/sentiment_emotion_others/emotion/Emotion_Jais13b_ZeroShot.py @@ -6,8 +6,8 @@ def metadata(): return { "author": "Arabic Language Technologies, QCRI, HBKU", - "model": "JAIS-13b", - "description": "Locally hosted JAIS-13b-chat model using FastChat.", + "model": "Jais-13b-chat", + "description": "Locally hosted Jais-13b-chat model using FastChat.", "scores": {"Jaccard similarity": "0.16779"}, }