updating the chatbot to 2024

.gitignore

@@ -169,4 +169,5 @@ src/scraping/data/*
 src/evals/parenting_chatbot/*
 src/genai/parenting_chatbot/prodigy_eval/_scrap/*
 !src/genai/parenting_chatbot/prodigy_eval/data/
-!src/genai/sandbox/signals/data/
+!src/genai/sandbox/signals/data/
+src/genai/sandbox/signals/data/signals_2024.json

src/genai/sandbox/signals/data/00_system.jsonl

@@ -1 +1 @@
-{"role": "user", "content": "###Instructions###\nYou are a helpful, kind, intelligent and polite futurist. You work for the United Kingdom's innovation agency Nesta, and your task is to engage the user about the future signals and trends that Nesta has researched, by helping the user imagine and appreciate how the signals will impact their life. You will personalise the user experience by taking the information provided by the user and tailoring your explanation to the user background. You are also expert in equity, diversity and inclusion and your answers will be inclusive, the answers will never be based on negative stereotypes, you will never offend and you will exercise sensitivity about topics such as ethnicity and gender. Here are the future signals that you can talk about: {signals}. Do not discuss other future signals as this is not part of this year's Nesta's Signals edition."}
+{"role": "system", "content": "###Instructions###\nYou are a helpful, kind, intelligent and polite futurist. You work for the United Kingdom's innovation agency Nesta, and your task is to engage the user about the future signals and trends that Nesta has researched, by helping the user imagine and appreciate how the signals will impact their life. You will personalise the user experience by taking the information provided by the user and tailoring your explanation to the user background. You are also expert in equity, diversity and inclusion and your answers will be inclusive, the answers will never be based on negative stereotypes, you will never offend and you will exercise sensitivity about topics such as ethnicity and gender. Here are the future signals that you can talk about: {signals}. Do not discuss other future signals as this is not part of this year's Nesta's Signals edition."}

src/genai/sandbox/signals/data/01_intro.jsonl

@@ -1,3 +1,3 @@
 {"role": "assistant", "content": "Hi, I’m Scout, Discovery Hub’s experimental AI assistant which helps people explore and interpret signals about the future. ✨"}
-{"role": "assistant", "content": "This year we have collected signals about a variety of topics, from 🤖 robochefs and ⚡ abundant energy to 👶 pronatalism and 🌱 having green neighbours."}
+{"role": "assistant", "content": "This year we have collected signals about a variety of topics, from 🏥 virtual hospital wards and 🧠 neuroprivacy to ⚡ space solar power and 🧪 data poisoning."}
 {"role": "assistant", "content": "Tell me one or two things about you and your interests, so that I can suggest which future signals might be the most relevant to you!"}

src/genai/sandbox/signals/data/02_signal_impact.jsonl

@@ -1,2 +1,2 @@
-{"role": "user", "content": "Start your answer by explaining in one clear sentence how the selected future signal might be relevant to the user, given the user information and conversation history. Then describe three ways how the selected future signal might impact them. Keep these descriptions short, two-three sentences at most. Finish your answer by encouraging the user to ask questions about this signal (note that you will try your best to answer them) or suggest to ask about the other future signals. Remember that you must be patient and never offend or be aggressive.   \n\n###Future signal###{signal}\n\n###User information### Here is what the user told you about themselves: {user_input}.\n\n###Answer###"
-}
+{"role": "user", "content": "Start your answer by summarising the signal in one clear, concise sentence and then follow by explaining in another clear sentence how the selected future signal might be relevant to the user, given the user information and conversation history. Then describe three ways how the selected future signal might impact them. Keep these descriptions short, two-three sentences at most. Finish your answer by encouraging the user to ask questions about this signal (note that you will try your best to answer them) or suggest to ask about the other future signals. Remember that you must be patient and never offend or be aggressive.   \n\n###Future signal###{signal}\n\n###User information### Here is what the user told you about themselves: {user_input}.\n\n###Answer###"
+}

src/genai/sandbox/signals/data/intent_actions.json

@@ -1,7 +1,7 @@
 [
     {
         "name": "new_signal",
-        "description": "User wishes to change the topic and talk about an new future signal. Alternatively, the user has been just presented with a set of future signal options by the assistant, and the user has now chosen which signal to talk about more."
+        "description": "User wishes to change the topic and talk about an different future signal. Alternatively, the user has been just presented with a set of future signal options by the assistant, and the user has now chosen which signal to talk about more."
     },
     {
         "name": "more_signals",
@@ -11,4 +11,4 @@
         "name": "following_up",
         "description": "User is following up with another question about the signal that's being discussed just now."
     }
-]
+]

src/genai/sandbox/signals/signals_test.ipynb

@@ -17,7 +17,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 233,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -27,9 +27,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 42,
+   "execution_count": 3,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/karlis.kanders/Documents/code/discovery_generative_ai/.venv/lib/python3.9/site-packages/pinecone/index.py:4: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from tqdm.autonotebook import tqdm\n"
+     ]
+    }
+   ],
    "source": [
     "from genai.eyfs import (\n",
     "    TextGenerator,\n",
@@ -39,7 +48,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 184,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -75,7 +84,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 170,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -88,7 +97,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 135,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -258,17 +267,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 103,
+   "execution_count": 25,
    "metadata": {},
    "outputs": [],
    "source": [
-    "path_func_top_signals = \"data/func_top_signals.json\"\n",
-    "path_prompt_top_signals = \"data/prompt_top_signals.jsonl\""
+    "path_func_top_signals = \"data/func_top_three_signals.json\"\n",
+    "path_prompt_top_signals = \"data/prompt_top_three_signals.jsonl\""
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 104,
+   "execution_count": 26,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -278,7 +287,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 105,
+   "execution_count": 27,
    "metadata": {},
    "outputs": [
     {
@@ -288,7 +297,7 @@
        "  'content': '###Instructions### Predict which three of the following future signals are the most relevant to user input. You have to choose three of these signals. \\n\\n###Future signal summaries###\\n{signals}\\n\\n###User input:\\n{user_input}'}]"
       ]
      },
-     "execution_count": 105,
+     "execution_count": 27,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -300,7 +309,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 106,
+   "execution_count": 28,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -310,16 +319,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 116,
+   "execution_count": 32,
    "metadata": {},
    "outputs": [],
    "source": [
-    "user_input = \"I like burgers\""
+    "user_input = \"I am a parent\""
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 117,
+   "execution_count": 33,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -336,20 +345,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 118,
+   "execution_count": 34,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "{'prediction': ['robochefs',\n",
-       "  'abundant_energy',\n",
-       "  'baby_boom',\n",
-       "  'hidden_figures',\n",
-       "  'green_neighbours']}"
+       "{'prediction': ['Pronatalism vs pro-family',\n",
+       "  'Hidden Figures',\n",
+       "  'Green neighbours']}"
       ]
      },
-     "execution_count": 118,
+     "execution_count": 34,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -359,6 +366,26 @@
     "answer"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 35,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['Pronatalism vs pro-family', 'Hidden Figures', 'Green neighbours']"
+      ]
+     },
+     "execution_count": 35,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "answer['prediction'][0:3]"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -577,7 +604,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 211,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [],
    "source": [