From f4c9d3b84f3b56645258fcb757382138896f9209 Mon Sep 17 00:00:00 2001 From: Arid Hasan Date: Fri, 22 Dec 2023 01:35:52 -0400 Subject: [PATCH 1/2] Add JAIS ZeroShot for ArSarcasm asset --- .../sarcasm/ArSarcasm2_JAIS13b_ZeroShot.py | 61 +++++++++++++++++++ .../sarcasm/ArSarcasm_JAIS13b_ZeroShot.py | 61 +++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 assets/ar/sentiment_emotion_others/sarcasm/ArSarcasm2_JAIS13b_ZeroShot.py create mode 100644 assets/ar/sentiment_emotion_others/sarcasm/ArSarcasm_JAIS13b_ZeroShot.py diff --git a/assets/ar/sentiment_emotion_others/sarcasm/ArSarcasm2_JAIS13b_ZeroShot.py b/assets/ar/sentiment_emotion_others/sarcasm/ArSarcasm2_JAIS13b_ZeroShot.py new file mode 100644 index 00000000..651660ab --- /dev/null +++ b/assets/ar/sentiment_emotion_others/sarcasm/ArSarcasm2_JAIS13b_ZeroShot.py @@ -0,0 +1,61 @@ +from llmebench.datasets import ArSarcasm2Dataset +from llmebench.models import FastChatModel +from llmebench.tasks import SarcasmTask + + +def metadata(): + return { + "author": "Arabic Language Technologies, QCRI, HBKU", + "model": "JAIS-13b", + "description": "Locally hosted JAIS-13b-chat model using FastChat.", + "scores": {"Macro-F1": ""}, + } + + +def config(): + return { + "dataset": ArSarcasm2Dataset, + "task": SarcasmTask, + "model": FastChatModel, + "model_args": { + "class_labels": ["TRUE", "FALSE"], + "max_tries": 3, + }, + } + + +def prompt(input_sample): + base_prompt = ( + f'Predict whether the following "tweet" is sarcastic. Return "yes" if the tweet is sarcastic ' + f'and "no" if the tweet is not sarcastic. Provide only label.\n\ntweet: {input_sample} \n' + f"label: \n" + ) + + return [ + { + "role": "user", + "content": base_prompt, + }, + ] + + +def post_process(response): + out = response["choices"][0]["message"]["content"] + out = out.strip().lower() + + if "i apologize" in out: + return None + + j = out.find("label:") + if j > 0: + out = out[j + len("label:") :] + else: + j = out.find(" is:\n\n") + if j > 0: + out = out[j + len(" is:\n\n") :] + out = out.strip().title() + if out.lower() == 'yes': + return "TRUE" + elif out.lower() == 'no': + return "FALSE" + return None diff --git a/assets/ar/sentiment_emotion_others/sarcasm/ArSarcasm_JAIS13b_ZeroShot.py b/assets/ar/sentiment_emotion_others/sarcasm/ArSarcasm_JAIS13b_ZeroShot.py new file mode 100644 index 00000000..68c1c4b7 --- /dev/null +++ b/assets/ar/sentiment_emotion_others/sarcasm/ArSarcasm_JAIS13b_ZeroShot.py @@ -0,0 +1,61 @@ +from llmebench.datasets import ArSarcasmDataset +from llmebench.models import FastChatModel +from llmebench.tasks import SarcasmTask + + +def metadata(): + return { + "author": "Arabic Language Technologies, QCRI, HBKU", + "model": "JAIS-13b", + "description": "Locally hosted JAIS-13b-chat model using FastChat.", + "scores": {"Macro-F1": ""}, + } + + +def config(): + return { + "dataset": ArSarcasmDataset, + "task": SarcasmTask, + "model": FastChatModel, + "model_args": { + "class_labels": ["TRUE", "FALSE"], + "max_tries": 3, + }, + } + + +def prompt(input_sample): + base_prompt = ( + f'Predict whether the following "tweet" is sarcastic. Return "yes" if the tweet is sarcastic ' + f'and "no" if the tweet is not sarcastic. Provide only label.\n\ntweet: {input_sample} \n' + f"label: \n" + ) + + return [ + { + "role": "user", + "content": base_prompt, + }, + ] + + +def post_process(response): + out = response["choices"][0]["message"]["content"] + out = out.strip().lower() + + if "i apologize" in out: + return None + + j = out.find("label:") + if j > 0: + out = out[j + len("label:") :] + else: + j = out.find(" is:\n\n") + if j > 0: + out = out[j + len(" is:\n\n") :] + out = out.strip().title() + if out.lower() == 'yes': + return "TRUE" + elif out.lower() == 'no': + return "FALSE" + return None From 9558858d7426b418a46a0589fc113243adafddb1 Mon Sep 17 00:00:00 2001 From: Fahim Imaduddin Dalvi Date: Wed, 31 Jan 2024 11:50:22 +0300 Subject: [PATCH 2/2] Format code and minor metadata changes --- .../sarcasm/ArSarcasm2_JAIS13b_ZeroShot.py | 9 ++++----- .../sarcasm/ArSarcasm_JAIS13b_ZeroShot.py | 9 ++++----- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/assets/ar/sentiment_emotion_others/sarcasm/ArSarcasm2_JAIS13b_ZeroShot.py b/assets/ar/sentiment_emotion_others/sarcasm/ArSarcasm2_JAIS13b_ZeroShot.py index 651660ab..63f6a77f 100644 --- a/assets/ar/sentiment_emotion_others/sarcasm/ArSarcasm2_JAIS13b_ZeroShot.py +++ b/assets/ar/sentiment_emotion_others/sarcasm/ArSarcasm2_JAIS13b_ZeroShot.py @@ -6,9 +6,8 @@ def metadata(): return { "author": "Arabic Language Technologies, QCRI, HBKU", - "model": "JAIS-13b", - "description": "Locally hosted JAIS-13b-chat model using FastChat.", - "scores": {"Macro-F1": ""}, + "model": "Jais-13b-chat", + "description": "Locally hosted Jais-13b-chat model using FastChat.", } @@ -54,8 +53,8 @@ def post_process(response): if j > 0: out = out[j + len(" is:\n\n") :] out = out.strip().title() - if out.lower() == 'yes': + if out.lower() == "yes": return "TRUE" - elif out.lower() == 'no': + elif out.lower() == "no": return "FALSE" return None diff --git a/assets/ar/sentiment_emotion_others/sarcasm/ArSarcasm_JAIS13b_ZeroShot.py b/assets/ar/sentiment_emotion_others/sarcasm/ArSarcasm_JAIS13b_ZeroShot.py index 68c1c4b7..1bd4f1f3 100644 --- a/assets/ar/sentiment_emotion_others/sarcasm/ArSarcasm_JAIS13b_ZeroShot.py +++ b/assets/ar/sentiment_emotion_others/sarcasm/ArSarcasm_JAIS13b_ZeroShot.py @@ -6,9 +6,8 @@ def metadata(): return { "author": "Arabic Language Technologies, QCRI, HBKU", - "model": "JAIS-13b", - "description": "Locally hosted JAIS-13b-chat model using FastChat.", - "scores": {"Macro-F1": ""}, + "model": "Jais-13b-chat", + "description": "Locally hosted Jais-13b-chat model using FastChat.", } @@ -54,8 +53,8 @@ def post_process(response): if j > 0: out = out[j + len(" is:\n\n") :] out = out.strip().title() - if out.lower() == 'yes': + if out.lower() == "yes": return "TRUE" - elif out.lower() == 'no': + elif out.lower() == "no": return "FALSE" return None