From a7864cc82d4ef4b7922a0cb4f6e6e8915bb90d5e Mon Sep 17 00:00:00 2001
From: Fahim Imaduddin Dalvi <faimaduddin@hbku.edu.qa>
Date: Sun, 29 Oct 2023 08:54:36 +0300
Subject: [PATCH] Add SST2 asset for RedPajama-INCITE chat model

---
 .../SST2_RedPajama_INCITE_3b_chat_ZeroShot.py | 62 +++++++++++++++++++
 1 file changed, 62 insertions(+)
 create mode 100644 assets/en/sentiment_emotion_others/sentiment/SST2_RedPajama_INCITE_3b_chat_ZeroShot.py

diff --git a/assets/en/sentiment_emotion_others/sentiment/SST2_RedPajama_INCITE_3b_chat_ZeroShot.py b/assets/en/sentiment_emotion_others/sentiment/SST2_RedPajama_INCITE_3b_chat_ZeroShot.py
new file mode 100644
index 00000000..b7b2ef43
--- /dev/null
+++ b/assets/en/sentiment_emotion_others/sentiment/SST2_RedPajama_INCITE_3b_chat_ZeroShot.py
@@ -0,0 +1,62 @@
+from llmebench.datasets import HuggingFaceDataset
+from llmebench.models import FastChatModel
+from llmebench.tasks import ClassificationTask
+
+
+def metadata():
+    return {
+        "author": "Arabic Language Technologies, QCRI, HBKU",
+        "model": "togethercomputer/RedPajama-INCITE-Chat-3B-v1",
+        "description": "Locally hosted RedPajama-INCITE-Chat 3b parameters model using FastChat.",
+        "scores": {"Accuracy": "0.681"},
+    }
+
+
+def config():
+    return {
+        "dataset": HuggingFaceDataset,
+        "dataset_args": {
+            "huggingface_dataset_name": "sst2",
+            "column_mapping": {
+                "input": "sentence",
+                "label": "label",
+                "input_id": "idx",
+            },
+        },
+        "task": ClassificationTask,
+        "model": FastChatModel,
+        "general_args": {"custom_test_split": "validation"},
+    }
+
+
+def prompt(input_sample):
+    prompt_string = (
+        f"You are tasked with analyzing the sentiment of the given sentence. "
+        f"Please read it carefully and determine whether the sentiment expressed is positive or negative. Provide only label.\n\n"
+        f"sentence: {input_sample.strip()}\n"
+        f"label:\n"
+    )
+    return [
+        {
+            "role": "system",
+            "content": "You are a data annotation expert specializing in sentiment analysis.",
+        },
+        {"role": "user", "content": prompt_string},
+    ]
+
+
+def post_process(response):
+    response = response["choices"][0]["message"]["content"].lower()
+    response = response.replace(".", "")
+    response = response.strip()
+
+    if response.endswith("positive"):
+        return 1
+    elif response.endswith("negative"):
+        return 0
+    elif "positive" in response and "negative" not in response:
+        return 1
+    elif "negative" in response and "positive" not in response:
+        return 0
+    else:
+        return None