diff --git a/assets/en/sentiment_emotion_others/sentiment/SST2_Llama_7b_chat_ZeroShot.py b/assets/en/sentiment_emotion_others/sentiment/SST2_Llama_7b_chat_ZeroShot.py index 258c83be..ba5691a6 100644 --- a/assets/en/sentiment_emotion_others/sentiment/SST2_Llama_7b_chat_ZeroShot.py +++ b/assets/en/sentiment_emotion_others/sentiment/SST2_Llama_7b_chat_ZeroShot.py @@ -8,7 +8,7 @@ def metadata(): "author": "Arabic Language Technologies, QCRI, HBKU", "model": "Llama-2-13b-chat-hf", "description": "Locally hosted Llama-2-13b-chat hf model using FastChat.", - "scores": {"Accuracy": "0.861"}, + "scores": {"Accuracy": "0.924"}, } @@ -30,29 +30,43 @@ def config(): def prompt(input_sample): + prompt_string = ( + f"You are tasked with analyzing the sentiment of the given sentence. " + f"Please read it carefully and determine whether the sentiment expressed is positive or negative. Provide only label.\n\n" + f"sentence: {input_sample.strip()}\n" + f"label:\n" + ) return [ { "role": "system", - "content": "You are an AI assistant that helps people find information.", - }, - { - "role": "user", - "content": f'Classify the sentiment of the following sentence as "Positive" or "Negative". Output only the label and nothing else.\nSentence: {input_sample}\nLabel: ', + "content": "You are a data annotation expert specializing in sentiment analysis.", }, + {"role": "user", "content": prompt_string}, ] def post_process(response): - out = response["choices"][0]["message"]["content"] - out = out.strip().lower() - j = out.find("label:") - if j > 0: - out = out[j + len("label:") :] - out = out.strip().lower() - - if out == "positive": - return 1 - elif out == "negative": - return 0 + mapping = {"positive": 1, "negative": 0} + + pred_label = response["choices"][0]["message"]["content"].lower() + + if "\n\nlabel: negative" in pred_label: + pred_label = "negative" + elif "\n\nlabel: positive" in pred_label: + pred_label = "positive" + elif "\n\nlabel:" in pred_label: + pred_label = pred_label.split("\n\nlabel:")[1] + pred_label = pred_label.strip().lower() + if pred_label == "positive" or pred_label == "negative": + return mapping[pred_label] + elif "\n\nnegative" in pred_label: + pred_label = "negative" + elif "\n\npositive" in pred_label: + pred_label = "positive" + else: + pred_label = None + + if pred_label is not None: + return mapping[pred_label] else: return None