diff --git a/assets/ar/sentiment_emotion_others/stance_detection/ANSStance_Jais13b_FewShot.py b/assets/ar/sentiment_emotion_others/stance_detection/ANSStance_Jais13b_FewShot.py new file mode 100644 index 00000000..815fef8b --- /dev/null +++ b/assets/ar/sentiment_emotion_others/stance_detection/ANSStance_Jais13b_FewShot.py @@ -0,0 +1,95 @@ +from llmebench.datasets import ANSStanceDataset +from llmebench.models import FastChatModel +from llmebench.tasks import StanceTask + + +def metadata(): + return { + "author": "Arabic Language Technologies, QCRI, HBKU", + "model": "Jais-13b", + "description": "Locally hosted Jais-13b-chat model using FastChat. 5-shot", + "scores": {"Macro-F1": "0.5272154156628485"}, + } + + +def config(): + return { + "dataset": ANSStanceDataset, + "task": StanceTask, + "model": FastChatModel, + "model_args": { + "class_labels": ["agree", "disagree"], + "max_tries": 30, + }, + } + + +def prompt(input_sample, examples): + ref_s = input_sample["sentence_1"] + claim = input_sample["sentence_2"] + base_prompt = ( + f"Given a reference sentence and a claim, predict whether the claim agrees or disagrees with the reference sentence. Reply only using 'agree', 'disagree', or use 'other' if the sentence and claim are unrelated." + f"\n\n" + f"reference sentence: {ref_s}" + f"\nclaim: {claim}" + f"\nlabel: \n" + ) + + return [ + { + "role": "user", + "content": few_shot_prompt(input_sample, base_prompt, examples), + }, + ] + + +def few_shot_prompt(input_sample, base_prompt, examples): + out_prompt = base_prompt + for example in examples: + ref_s = example["input"]["sentence_1"] + claim = example["input"]["sentence_2"] + + out_prompt = ( + out_prompt + + "reference sentence: " + + ref_s + + "\nclaim: " + + claim + + "\nlabel: " + + example["label"] + + "\n\n" + ) + + # Append the sentence we want the model to predict for but leave the label blank + + ref_s = input_sample["sentence_1"] + claim = input_sample["sentence_2"] + + out_prompt = ( + out_prompt + + "reference sentence: " + + ref_s + + "\nclaim: " + + claim + + "\nlabel: \n" + ) + + return out_prompt + + +def post_process(response): + out = response["choices"][0]["message"]["content"] + out = out.strip().lower() + + if "i apologize" in out: + return None + + j = out.find("label:") + if j > 0: + out = out[j + len("label:") :] + else: + j = out.find(" is:\n\n") + if j > 0: + out = out[j + len(" is:\n\n") :] + out = out.strip().title() + return out.lower() diff --git a/assets/ar/sentiment_emotion_others/stance_detection/ANSStance_JAIS13b_ZeroShot.py b/assets/ar/sentiment_emotion_others/stance_detection/ANSStance_Jais13b_ZeroShot.py similarity index 96% rename from assets/ar/sentiment_emotion_others/stance_detection/ANSStance_JAIS13b_ZeroShot.py rename to assets/ar/sentiment_emotion_others/stance_detection/ANSStance_Jais13b_ZeroShot.py index c14b22e5..89e3c00c 100644 --- a/assets/ar/sentiment_emotion_others/stance_detection/ANSStance_JAIS13b_ZeroShot.py +++ b/assets/ar/sentiment_emotion_others/stance_detection/ANSStance_Jais13b_ZeroShot.py @@ -8,6 +8,7 @@ def metadata(): "author": "Arabic Language Technologies, QCRI, HBKU", "model": "Jais-13b-chat", "description": "Locally hosted Jais-13b-chat model using FastChat.", + "scores": {"Macro-F1": "0.465626102292769"}, }