optimized_similarity_calculation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f3825813-b2d0-48f5-bca7-4468d9b9bf01",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading model...\n",
      "model loaded\n",
      "                              Title                  Author  \\\n",
      "0   The Bear Came Over the Mountain             Alice Munro   \n",
      "1                          The Boss           Robert Coover   \n",
      "2         The Hairless Are Careless           Colin Barrett   \n",
      "3                       Smithereens        Aleksandar Hemon   \n",
      "4                    Where Are You?       Joyce Carol Oates   \n",
      "..                              ...                     ...   \n",
      "79                        The Mouse             H. H. Munro   \n",
      "80          Mrs. Packletide's Tiger             H. H. Munro   \n",
      "81                One of These Days  Gabriel Garcia Marquez   \n",
      "82          A RETRIEVED REFORMATION                O. Henry   \n",
      "83                      The Hostage             Amelia Gray   \n",
      "\n",
      "                                                story  \\\n",
      "0   Fiona lived in her parents’ house, in the town...   \n",
      "1   The gunman lights a cigarette, watches despond...   \n",
      "2   “Prejudicial of me, I know,” said Tommy, noddi...   \n",
      "3   Near our mountain cabin, in Jahorina, there wa...   \n",
      "4   The husband had got into the habit of calling ...   \n",
      "..                                                ...   \n",
      "79  Theodoric Voler had been brought up, from infa...   \n",
      "80  It was Mrs. Packletide's pleasure and intentio...   \n",
      "81  Monday dawned warm and rainless. Aurelio Escov...   \n",
      "82  A guard came to the prison shoe-shop, where Ji...   \n",
      "83  Nobody knew what to do at first. Deposit slips...   \n",
      "\n",
      "                                      summary_gpt_3_5  \\\n",
      "0   Fiona and Grant are a married couple. Fiona's ...   \n",
      "1   The story follows a gunman who is waiting in a...   \n",
      "2   Tommy, Ernie, and Luke are discussing bald peo...   \n",
      "3   The story is set near a mountain cabin in Jaho...   \n",
      "4   In this story, a husband and wife have develop...   \n",
      "..                                                ...   \n",
      "79  Theodoric Voler, a man who has been sheltered ...   \n",
      "80  Mrs. Packletide, motivated by jealousy of Loon...   \n",
      "81  Aurelio Escovar, a dentist without a degree, o...   \n",
      "82  Jimmy Valentine, a skilled safecracker, is rel...   \n",
      "83  In this story, a bank robber attempts to rob a...   \n",
      "\n",
      "                                   summary_gpt_3_5_t1  sid  \n",
      "0   Fiona and Grant are a married couple who live ...   s0  \n",
      "1   The story follows a gunman who finds himself a...   s1  \n",
      "2   Tommy and his friends, Ernie and Luke, are at ...   s2  \n",
      "3   The narrator recalls a summer from their child...   s3  \n",
      "4   In this story, a husband and wife have a pecul...   s4  \n",
      "..                                                ...  ...  \n",
      "79  Theodoric Voler, a man who had been sheltered ...  s79  \n",
      "80  Mrs. Packletide is determined to shoot a tiger...  s80  \n",
      "81  Aurelio Escovar, a dentist with no formal educ...  s81  \n",
      "82  Jimmy Valentine, a skilled safecracker, is rel...  s82  \n",
      "83  In this story, a bank robber enters a bank dur...  s83  \n",
      "\n",
      "[84 rows x 6 columns]\n",
      "read csvs\n",
      "(186898, 512)\n",
      "embeddings matrix ready\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "tqdm.pandas()\n",
    "import scipy.spatial\n",
    "import tensorflow_hub as hub\n",
    "import tensorflow as tf\n",
    "# from numpy import dot\n",
    "# from numpy.linalg import norm\n",
    "print(\"loading model...\")\n",
    "model_path = \"universal-sentence-encoder-large_5\"\n",
    "encoder = hub.load(model_path)\n",
    "print(\"model loaded\")\n",
    "stories = pd.read_csv('stories_with_summary.csv')\n",
    "print(stories)\n",
    "\n",
    "matrix = np.load('euler/trope_examples_embeddings.npy')\n",
    "print(matrix.shape)\n",
    "print(\"embeddings matrix ready\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "10191673-a8f8-40b5-9142-ead497b5deae",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "read csvs\n"
     ]
    }
   ],
   "source": [
    "trope_examples = pd.read_csv('dataset/trope_examples.csv')#, dtype={\"embedding\": })\n",
    "trope_examples = trope_examples.dropna()\n",
    "print(\"read csvs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "52a1fa73-2e7a-4f40-b9ce-b6bfe9527b82",
   "metadata": {},
   "outputs": [],
   "source": [
    "task = \"summary_gpt_3_5\"\n",
    "task = \"story\"\n",
    "task = \"summary_gpt_3_5_t1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "01c57db9-e8b5-4e6a-bfe3-2bf729b0faa2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1it [00:00,  3.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2it [00:00,  3.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3it [00:00,  3.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "4it [00:01,  3.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "5it [00:01,  3.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:01,  3.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "7it [00:02,  3.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "8it [00:02,  3.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "9it [00:02,  3.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "10it [00:02,  3.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11it [00:03,  3.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "12it [00:03,  3.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s12\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "13it [00:03,  3.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s13\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "14it [00:04,  3.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s14\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "15it [00:04,  3.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:04,  3.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s16\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "17it [00:04,  3.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s17\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "18it [00:05,  3.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s18\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "19it [00:05,  3.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s19\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "20it [00:05,  3.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "21it [00:05,  3.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s21\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "22it [00:06,  3.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s22\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23it [00:06,  3.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s23\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "24it [00:06,  3.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s24\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "25it [00:07,  3.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s25\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "26it [00:07,  3.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s26\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "27it [00:07,  3.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s27\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "28it [00:07,  3.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s28\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "29it [00:08,  3.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s29\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "30it [00:08,  3.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "31it [00:08,  3.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s31\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "32it [00:09,  3.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s32\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "33it [00:09,  3.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s33\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "34it [00:09,  3.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s34\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "35it [00:09,  3.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s35\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "36it [00:10,  3.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s36\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "37it [00:10,  3.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s37\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "38it [00:10,  3.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s38\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "39it [00:11,  3.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s39\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "40it [00:11,  3.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s40\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "41it [00:11,  3.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s41\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "42it [00:11,  3.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s42\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "43it [00:12,  3.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s43\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "44it [00:12,  3.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s44\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45it [00:12,  3.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s45\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "46it [00:12,  3.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s46\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "47it [00:13,  3.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s47\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "48it [00:13,  3.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s48\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "49it [00:13,  3.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s49\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "50it [00:14,  3.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "51it [00:14,  3.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s51\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "52it [00:14,  3.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s52\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "53it [00:14,  3.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s53\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "54it [00:15,  3.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s54\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "55it [00:15,  3.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s55\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "56it [00:15,  3.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s56\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "57it [00:16,  3.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s57\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "58it [00:16,  3.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s58\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "59it [00:16,  3.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s59\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "60it [00:17,  3.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s60\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "61it [00:17,  3.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s61\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "62it [00:17,  3.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s62\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "63it [00:17,  3.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s63\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "64it [00:18,  3.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "65it [00:18,  3.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s65\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "66it [00:18,  3.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s66\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "67it [00:19,  3.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s67\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "68it [00:19,  3.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s68\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "69it [00:19,  3.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s69\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "70it [00:19,  3.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s70\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "71it [00:20,  3.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s71\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "72it [00:20,  3.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s72\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "73it [00:20,  3.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s73\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "74it [00:21,  3.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s74\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "75it [00:21,  3.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s75\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "76it [00:21,  3.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s76\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "77it [00:21,  3.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s77\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "78it [00:22,  3.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s78\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "79it [00:22,  3.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s79\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "80it [00:22,  3.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s80\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "81it [00:22,  3.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s81\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "82it [00:23,  3.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s82\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "83it [00:23,  3.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing story - s83\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "84it [00:23,  3.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saving csv\n"
     ]
    }
   ],
   "source": [
    "# loop through stories/summaries\n",
    "for story in tqdm(stories.itertuples()):\n",
    "    print(f\"processing story - {story.sid}\")\n",
    "    # make story/summary matrix\n",
    "    story_embed = encoder([story.summary_gpt_3_5_t1])\n",
    "    story_embed = np.tile(story_embed, (matrix.shape[0],1))\n",
    "    # print(story_embed.shape)\n",
    "    # similarity = 1 - scipy.spatial.distance.cdist(matrix, story_embed, 'cosine')\n",
    "    # trope_examples[story.sid] = similarity[:,0]\n",
    "    # also works\n",
    "    cosine_similarities = tf.reduce_sum(tf.multiply(matrix, story_embed), axis=1)\n",
    "    # cosine_similarities\n",
    "    trope_examples[story.sid] = cosine_similarities\n",
    "\n",
    "print(\"saving csv\")\n",
    "# drop the trope_embeddings column and then save the df\n",
    "# trope_examples = trope_examples.drop('embed', axis=1)\n",
    "trope_examples.to_csv(f'./new/similarity_{task}.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "62836fc3-8bc2-41e2-8d60-3b9b96e91577",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}