data_generate.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_sample = pd.read_json('./development.json',lines=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>input_field</th>\n",
       "      <th>output_field</th>\n",
       "      <th>task_name</th>\n",
       "      <th>task_type</th>\n",
       "      <th>metric</th>\n",
       "      <th>is_multiple_choice</th>\n",
       "      <th>track</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Instructions: Tell me what this product catego...</td>\n",
       "      <td>A toggle switch is an electric switch operated...</td>\n",
       "      <td>task1</td>\n",
       "      <td>generation</td>\n",
       "      <td>sent-transformer</td>\n",
       "      <td>False</td>\n",
       "      <td>amazon-kdd-cup-24-understanding-shopping-concepts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Instructions: Explain the product category nam...</td>\n",
       "      <td>A watch band is a bracelet that straps a wrist...</td>\n",
       "      <td>task1</td>\n",
       "      <td>generation</td>\n",
       "      <td>sent-transformer</td>\n",
       "      <td>False</td>\n",
       "      <td>amazon-kdd-cup-24-understanding-shopping-concepts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Input: Explain the product type Handbag\\nOutput:</td>\n",
       "      <td>Handbags are bags carried close to the body th...</td>\n",
       "      <td>task1</td>\n",
       "      <td>generation</td>\n",
       "      <td>sent-transformer</td>\n",
       "      <td>False</td>\n",
       "      <td>amazon-kdd-cup-24-understanding-shopping-concepts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Input: Explain the product type Water Purifica...</td>\n",
       "      <td>A water purification unit removes impurities b...</td>\n",
       "      <td>task1</td>\n",
       "      <td>generation</td>\n",
       "      <td>sent-transformer</td>\n",
       "      <td>False</td>\n",
       "      <td>amazon-kdd-cup-24-understanding-shopping-concepts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Which of the following product categories may ...</td>\n",
       "      <td>0</td>\n",
       "      <td>task2</td>\n",
       "      <td>multiple-choice</td>\n",
       "      <td>accuracy</td>\n",
       "      <td>True</td>\n",
       "      <td>amazon-kdd-cup-24-understanding-shopping-concepts</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         input_field  \\\n",
       "0  Instructions: Tell me what this product catego...   \n",
       "1  Instructions: Explain the product category nam...   \n",
       "2   Input: Explain the product type Handbag\\nOutput:   \n",
       "3  Input: Explain the product type Water Purifica...   \n",
       "4  Which of the following product categories may ...   \n",
       "\n",
       "                                        output_field task_name  \\\n",
       "0  A toggle switch is an electric switch operated...     task1   \n",
       "1  A watch band is a bracelet that straps a wrist...     task1   \n",
       "2  Handbags are bags carried close to the body th...     task1   \n",
       "3  A water purification unit removes impurities b...     task1   \n",
       "4                                                  0     task2   \n",
       "\n",
       "         task_type            metric  is_multiple_choice  \\\n",
       "0       generation  sent-transformer               False   \n",
       "1       generation  sent-transformer               False   \n",
       "2       generation  sent-transformer               False   \n",
       "3       generation  sent-transformer               False   \n",
       "4  multiple-choice          accuracy                True   \n",
       "\n",
       "                                               track  \n",
       "0  amazon-kdd-cup-24-understanding-shopping-concepts  \n",
       "1  amazon-kdd-cup-24-understanding-shopping-concepts  \n",
       "2  amazon-kdd-cup-24-understanding-shopping-concepts  \n",
       "3  amazon-kdd-cup-24-understanding-shopping-concepts  \n",
       "4  amazon-kdd-cup-24-understanding-shopping-concepts  "
      ]
     },
     "execution_count": 201,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_sample.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'generation',\n",
       " 'multiple-choice',\n",
       " 'named_entity_recognition',\n",
       " 'ranking',\n",
       " 'retrieval'}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(list(data_sample['task_type']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy',\n",
       " 'bleu',\n",
       " 'hit rate@3',\n",
       " 'jp-bleu',\n",
       " 'micro f1',\n",
       " 'ndcg',\n",
       " 'rougel',\n",
       " 'sent-transformer'}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(list(data_sample['metric']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_instruct = pd.read_json('./dataset.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>split</th>\n",
       "      <th>task</th>\n",
       "      <th>setting</th>\n",
       "      <th>instruction</th>\n",
       "      <th>input</th>\n",
       "      <th>options</th>\n",
       "      <th>output</th>\n",
       "      <th>few_shot_example</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>train</td>\n",
       "      <td>Multiclass_Product_Classification</td>\n",
       "      <td>IND_Single_Instruction</td>\n",
       "      <td>What is the relevance between the query and th...</td>\n",
       "      <td>{\"query\": \"fathers christmas gift\", \"product t...</td>\n",
       "      <td>[\"A: The product is relevant to the query, and...</td>\n",
       "      <td>A: The product is relevant to the query, and s...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>train</td>\n",
       "      <td>Answerability_Prediction</td>\n",
       "      <td>IND_Diverse_Instruction</td>\n",
       "      <td>Output yes if the supporting document can answ...</td>\n",
       "      <td>{\"question\": \"can i place a few ashes of my la...</td>\n",
       "      <td>None</td>\n",
       "      <td>no</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>train</td>\n",
       "      <td>Product_Relation_Prediction</td>\n",
       "      <td>IND_Diverse_Instruction</td>\n",
       "      <td>Analyze the titles of Product 1 and Product 2 ...</td>\n",
       "      <td>{\"Product 1:\": \"Duralex Amalfi Glasses 1004AC0...</td>\n",
       "      <td>[\"A: Users who buy product 1 may also buy prod...</td>\n",
       "      <td>B</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>train</td>\n",
       "      <td>Sequential_Recommendation</td>\n",
       "      <td>IND_Single_Instruction</td>\n",
       "      <td>Given the products the user has purchased in h...</td>\n",
       "      <td>['1st: UTG Shooter\\'s Competition Bipod, Rubbe...</td>\n",
       "      <td>['A: Queen Size - Otis Haley 110 Futon Mattres...</td>\n",
       "      <td>H</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>train</td>\n",
       "      <td>Multiclass_Product_Classification</td>\n",
       "      <td>IND_Diverse_Instruction</td>\n",
       "      <td>Analyze the query and product title to assess ...</td>\n",
       "      <td>{\"query\": \"sloth not today\", \"product title\": ...</td>\n",
       "      <td>[\"A: The product is relevant to the query, and...</td>\n",
       "      <td>B: The product is somewhat relevant. It fails ...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   split                               task                  setting  \\\n",
       "0  train  Multiclass_Product_Classification   IND_Single_Instruction   \n",
       "1  train           Answerability_Prediction  IND_Diverse_Instruction   \n",
       "2  train        Product_Relation_Prediction  IND_Diverse_Instruction   \n",
       "3  train          Sequential_Recommendation   IND_Single_Instruction   \n",
       "4  train  Multiclass_Product_Classification  IND_Diverse_Instruction   \n",
       "\n",
       "                                         instruction  \\\n",
       "0  What is the relevance between the query and th...   \n",
       "1  Output yes if the supporting document can answ...   \n",
       "2  Analyze the titles of Product 1 and Product 2 ...   \n",
       "3  Given the products the user has purchased in h...   \n",
       "4  Analyze the query and product title to assess ...   \n",
       "\n",
       "                                               input  \\\n",
       "0  {\"query\": \"fathers christmas gift\", \"product t...   \n",
       "1  {\"question\": \"can i place a few ashes of my la...   \n",
       "2  {\"Product 1:\": \"Duralex Amalfi Glasses 1004AC0...   \n",
       "3  ['1st: UTG Shooter\\'s Competition Bipod, Rubbe...   \n",
       "4  {\"query\": \"sloth not today\", \"product title\": ...   \n",
       "\n",
       "                                             options  \\\n",
       "0  [\"A: The product is relevant to the query, and...   \n",
       "1                                               None   \n",
       "2  [\"A: Users who buy product 1 may also buy prod...   \n",
       "3  ['A: Queen Size - Otis Haley 110 Futon Mattres...   \n",
       "4  [\"A: The product is relevant to the query, and...   \n",
       "\n",
       "                                              output few_shot_example  \n",
       "0  A: The product is relevant to the query, and s...             None  \n",
       "1                                                 no             None  \n",
       "2                                                  B             None  \n",
       "3                                                  H             None  \n",
       "4  B: The product is somewhat relevant. It fails ...             None  "
      ]
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_instruct.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>split</th>\n",
       "      <th>task</th>\n",
       "      <th>setting</th>\n",
       "      <th>instruction</th>\n",
       "      <th>input</th>\n",
       "      <th>options</th>\n",
       "      <th>output</th>\n",
       "      <th>few_shot_example</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>train</td>\n",
       "      <td>Multiclass_Product_Classification</td>\n",
       "      <td>IND_Single_Instruction</td>\n",
       "      <td>What is the relevance between the query and th...</td>\n",
       "      <td>{\"query\": \"fathers christmas gift\", \"product t...</td>\n",
       "      <td>[\"A: The product is relevant to the query, and...</td>\n",
       "      <td>A: The product is relevant to the query, and s...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>train</td>\n",
       "      <td>Answerability_Prediction</td>\n",
       "      <td>IND_Diverse_Instruction</td>\n",
       "      <td>Output yes if the supporting document can answ...</td>\n",
       "      <td>{\"question\": \"can i place a few ashes of my la...</td>\n",
       "      <td>None</td>\n",
       "      <td>no</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>train</td>\n",
       "      <td>Product_Relation_Prediction</td>\n",
       "      <td>IND_Diverse_Instruction</td>\n",
       "      <td>Analyze the titles of Product 1 and Product 2 ...</td>\n",
       "      <td>{\"Product 1:\": \"Duralex Amalfi Glasses 1004AC0...</td>\n",
       "      <td>[\"A: Users who buy product 1 may also buy prod...</td>\n",
       "      <td>B</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>train</td>\n",
       "      <td>Sequential_Recommendation</td>\n",
       "      <td>IND_Single_Instruction</td>\n",
       "      <td>Given the products the user has purchased in h...</td>\n",
       "      <td>['1st: UTG Shooter\\'s Competition Bipod, Rubbe...</td>\n",
       "      <td>['A: Queen Size - Otis Haley 110 Futon Mattres...</td>\n",
       "      <td>H</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>train</td>\n",
       "      <td>Multiclass_Product_Classification</td>\n",
       "      <td>IND_Diverse_Instruction</td>\n",
       "      <td>Analyze the query and product title to assess ...</td>\n",
       "      <td>{\"query\": \"sloth not today\", \"product title\": ...</td>\n",
       "      <td>[\"A: The product is relevant to the query, and...</td>\n",
       "      <td>B: The product is somewhat relevant. It fails ...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202545</th>\n",
       "      <td>val</td>\n",
       "      <td>Sequential_Recommendation</td>\n",
       "      <td>IND_Single_Instruction</td>\n",
       "      <td>Given the products the user has purchased in h...</td>\n",
       "      <td>['1st: American Weigh Scales GEMINI-20 Portabl...</td>\n",
       "      <td>['A: Oster 6812-001 Core 16-Speed Blender with...</td>\n",
       "      <td>G</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202546</th>\n",
       "      <td>val</td>\n",
       "      <td>Answer_Generation</td>\n",
       "      <td>IND_Diverse_Instruction</td>\n",
       "      <td>Generate an answer to the question by utilizin...</td>\n",
       "      <td>{\"question\": \"Is it compatible with IPhone 5?\"...</td>\n",
       "      <td>None</td>\n",
       "      <td>No, it's the wider iPhone 4 type connector</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202547</th>\n",
       "      <td>val</td>\n",
       "      <td>Sentiment_Analysis</td>\n",
       "      <td>IND_Single_Instruction</td>\n",
       "      <td>Given the user's review, identify the user's s...</td>\n",
       "      <td>I bought this item to hook up my phone to play...</td>\n",
       "      <td>[\"A: very positive\", \"B: positive\", \"C: neutra...</td>\n",
       "      <td>A: very positive</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202548</th>\n",
       "      <td>val</td>\n",
       "      <td>Answer_Generation</td>\n",
       "      <td>IND_Single_Instruction</td>\n",
       "      <td>Given a question and the related document, and...</td>\n",
       "      <td>{\"question\": \"Does this lens come with the sun...</td>\n",
       "      <td>None</td>\n",
       "      <td>I received my lens about a month ago and yes i...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202549</th>\n",
       "      <td>val</td>\n",
       "      <td>Sequential_Recommendation</td>\n",
       "      <td>IND_Diverse_Instruction</td>\n",
       "      <td>The user's purchase history implies her prefer...</td>\n",
       "      <td>['1st: eForCity Leather Hand Grip Strap Compat...</td>\n",
       "      <td>['A: NIUTOP® 7.0 Touch Screen Digitizer For Sa...</td>\n",
       "      <td>H</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>202550 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        split                               task                  setting  \\\n",
       "0       train  Multiclass_Product_Classification   IND_Single_Instruction   \n",
       "1       train           Answerability_Prediction  IND_Diverse_Instruction   \n",
       "2       train        Product_Relation_Prediction  IND_Diverse_Instruction   \n",
       "3       train          Sequential_Recommendation   IND_Single_Instruction   \n",
       "4       train  Multiclass_Product_Classification  IND_Diverse_Instruction   \n",
       "...       ...                                ...                      ...   \n",
       "202545    val          Sequential_Recommendation   IND_Single_Instruction   \n",
       "202546    val                  Answer_Generation  IND_Diverse_Instruction   \n",
       "202547    val                 Sentiment_Analysis   IND_Single_Instruction   \n",
       "202548    val                  Answer_Generation   IND_Single_Instruction   \n",
       "202549    val          Sequential_Recommendation  IND_Diverse_Instruction   \n",
       "\n",
       "                                              instruction  \\\n",
       "0       What is the relevance between the query and th...   \n",
       "1       Output yes if the supporting document can answ...   \n",
       "2       Analyze the titles of Product 1 and Product 2 ...   \n",
       "3       Given the products the user has purchased in h...   \n",
       "4       Analyze the query and product title to assess ...   \n",
       "...                                                   ...   \n",
       "202545  Given the products the user has purchased in h...   \n",
       "202546  Generate an answer to the question by utilizin...   \n",
       "202547  Given the user's review, identify the user's s...   \n",
       "202548  Given a question and the related document, and...   \n",
       "202549  The user's purchase history implies her prefer...   \n",
       "\n",
       "                                                    input  \\\n",
       "0       {\"query\": \"fathers christmas gift\", \"product t...   \n",
       "1       {\"question\": \"can i place a few ashes of my la...   \n",
       "2       {\"Product 1:\": \"Duralex Amalfi Glasses 1004AC0...   \n",
       "3       ['1st: UTG Shooter\\'s Competition Bipod, Rubbe...   \n",
       "4       {\"query\": \"sloth not today\", \"product title\": ...   \n",
       "...                                                   ...   \n",
       "202545  ['1st: American Weigh Scales GEMINI-20 Portabl...   \n",
       "202546  {\"question\": \"Is it compatible with IPhone 5?\"...   \n",
       "202547  I bought this item to hook up my phone to play...   \n",
       "202548  {\"question\": \"Does this lens come with the sun...   \n",
       "202549  ['1st: eForCity Leather Hand Grip Strap Compat...   \n",
       "\n",
       "                                                  options  \\\n",
       "0       [\"A: The product is relevant to the query, and...   \n",
       "1                                                    None   \n",
       "2       [\"A: Users who buy product 1 may also buy prod...   \n",
       "3       ['A: Queen Size - Otis Haley 110 Futon Mattres...   \n",
       "4       [\"A: The product is relevant to the query, and...   \n",
       "...                                                   ...   \n",
       "202545  ['A: Oster 6812-001 Core 16-Speed Blender with...   \n",
       "202546                                               None   \n",
       "202547  [\"A: very positive\", \"B: positive\", \"C: neutra...   \n",
       "202548                                               None   \n",
       "202549  ['A: NIUTOP® 7.0 Touch Screen Digitizer For Sa...   \n",
       "\n",
       "                                                   output few_shot_example  \n",
       "0       A: The product is relevant to the query, and s...             None  \n",
       "1                                                      no             None  \n",
       "2                                                       B             None  \n",
       "3                                                       H             None  \n",
       "4       B: The product is somewhat relevant. It fails ...             None  \n",
       "...                                                   ...              ...  \n",
       "202545                                                  G             None  \n",
       "202546         No, it's the wider iPhone 4 type connector             None  \n",
       "202547                                   A: very positive             None  \n",
       "202548  I received my lens about a month ago and yes i...             None  \n",
       "202549                                                  H             None  \n",
       "\n",
       "[202550 rows x 8 columns]"
      ]
     },
     "execution_count": 204,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_instruct[data_instruct['split']!='test']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "task = 'Product_Relation_Prediction'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2         [\"A: Users who buy product 1 may also buy prod...\n",
       "5         [\"A: Users who buy product 1 may also buy prod...\n",
       "6         [\"A: Users who buy product 1 may also buy prod...\n",
       "12        [\"A: Users who buy product 1 may also buy prod...\n",
       "31        [\"A: Users who buy product 1 may also buy prod...\n",
       "                                ...                        \n",
       "210545    [\"A: Users who buy product 1 may also buy prod...\n",
       "210546    [\"A: Users who buy product 1 may also buy prod...\n",
       "210547    [\"A: Users who buy product 1 may also buy prod...\n",
       "210548    [\"A: Users who buy product 1 may also buy prod...\n",
       "210549    [\"A: Users who buy product 1 may also buy prod...\n",
       "Name: options, Length: 30000, dtype: object"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_instruct[data_instruct['task']==task].options"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>split</th>\n",
       "      <th>task</th>\n",
       "      <th>setting</th>\n",
       "      <th>instruction</th>\n",
       "      <th>input</th>\n",
       "      <th>options</th>\n",
       "      <th>output</th>\n",
       "      <th>few_shot_example</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>203550</th>\n",
       "      <td>test</td>\n",
       "      <td>Product_Relation_Prediction</td>\n",
       "      <td>IND_Diverse_Instruction_1-shot</td>\n",
       "      <td>Evaluate the titles of Product 1 and Product 2...</td>\n",
       "      <td>{\"Product 1:\": \"Cerwin-Vega XED52 Speaker 275 ...</td>\n",
       "      <td>None</td>\n",
       "      <td>B</td>\n",
       "      <td>{\"input\": \"{\\\"Product 1:\\\": \\\"Cooler Master HA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205550</th>\n",
       "      <td>test</td>\n",
       "      <td>Product_Relation_Prediction</td>\n",
       "      <td>IND_Diverse_Instruction</td>\n",
       "      <td>Evaluate the titles of Product 1 and Product 2...</td>\n",
       "      <td>{\"Product 1:\": \"Cerwin-Vega XED52 Speaker 275 ...</td>\n",
       "      <td>[\"A: Users who buy product 1 may also buy prod...</td>\n",
       "      <td>B</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207550</th>\n",
       "      <td>test</td>\n",
       "      <td>Product_Relation_Prediction</td>\n",
       "      <td>IND_Unseen_Instruction</td>\n",
       "      <td>Analyze the titles of Product 1 and Product 2 ...</td>\n",
       "      <td>{\"Product 1:\": \"Cerwin-Vega XED52 Speaker 275 ...</td>\n",
       "      <td>[\"A: Users who buy product 1 may also buy prod...</td>\n",
       "      <td>B</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>209550</th>\n",
       "      <td>test</td>\n",
       "      <td>Product_Relation_Prediction</td>\n",
       "      <td>IND_Single_Instruction</td>\n",
       "      <td>Given the title of two products, predict if th...</td>\n",
       "      <td>{\"Product 1:\": \"Cerwin-Vega XED52 Speaker 275 ...</td>\n",
       "      <td>[\"A: Users who buy product 1 may also buy prod...</td>\n",
       "      <td>B</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       split                         task                         setting  \\\n",
       "203550  test  Product_Relation_Prediction  IND_Diverse_Instruction_1-shot   \n",
       "205550  test  Product_Relation_Prediction         IND_Diverse_Instruction   \n",
       "207550  test  Product_Relation_Prediction          IND_Unseen_Instruction   \n",
       "209550  test  Product_Relation_Prediction          IND_Single_Instruction   \n",
       "\n",
       "                                              instruction  \\\n",
       "203550  Evaluate the titles of Product 1 and Product 2...   \n",
       "205550  Evaluate the titles of Product 1 and Product 2...   \n",
       "207550  Analyze the titles of Product 1 and Product 2 ...   \n",
       "209550  Given the title of two products, predict if th...   \n",
       "\n",
       "                                                    input  \\\n",
       "203550  {\"Product 1:\": \"Cerwin-Vega XED52 Speaker 275 ...   \n",
       "205550  {\"Product 1:\": \"Cerwin-Vega XED52 Speaker 275 ...   \n",
       "207550  {\"Product 1:\": \"Cerwin-Vega XED52 Speaker 275 ...   \n",
       "209550  {\"Product 1:\": \"Cerwin-Vega XED52 Speaker 275 ...   \n",
       "\n",
       "                                                  options output  \\\n",
       "203550                                               None      B   \n",
       "205550  [\"A: Users who buy product 1 may also buy prod...      B   \n",
       "207550  [\"A: Users who buy product 1 may also buy prod...      B   \n",
       "209550  [\"A: Users who buy product 1 may also buy prod...      B   \n",
       "\n",
       "                                         few_shot_example  \n",
       "203550  {\"input\": \"{\\\"Product 1:\\\": \\\"Cooler Master HA...  \n",
       "205550                                               None  \n",
       "207550                                               None  \n",
       "209550                                               None  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inp = '{\"Product 1:\": \"Cerwin-Vega XED52 Speaker 275 W PMPO 2-Way, 2 Count, Black\", \"Product 2:\": \"Rockford R169X2 6 x 9 Inches Full Range Coaxial Speaker, Set of 2\"}'\n",
    "data_instruct[data_instruct['input']==inp]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Predict whether two products are similar, whether two products are likely to be purchased or viewed together based on their titles. Choose your answer from the provided options.'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_instruct[data_instruct['task']==task]['instruction'].values[5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[\"A: Users who buy product 1 may also buy product 2.\", \"B: Users who view product 1 may also view product 2.\", \"C: The product 1 is similar with the product 2.\"]'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_instruct[data_instruct['task']==task]['options'].values[5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'A'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_instruct[data_instruct['task']==task]['output'].values[5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Answer_Generation',\n",
       " 'Answerability_Prediction',\n",
       " 'Attribute_Value_Extraction',\n",
       " 'Multiclass_Product_Classification',\n",
       " 'Product_Matching',\n",
       " 'Product_Relation_Prediction',\n",
       " 'Product_Substitute_Identification',\n",
       " 'Query_Product_Rank',\n",
       " 'Sentiment_Analysis',\n",
       " 'Sequential_Recommendation'}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(list(data_instruct['task']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'test', 'train', 'val'}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(list(data_instruct['split']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_test = data_instruct[data_instruct['split']=='test']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_test = data_test[data_test['task'].isin(['Answer_Generation','Product_Relation_Prediction','Multiclass_Product_Classification','Query_Product_Rank','Sentiment_Analysis','Sequential_Recommendation'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(40000, 8)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def replace_letter(str_inp):\n",
    "    ls = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'\n",
    "    for idx,item in enumerate(ls):\n",
    "        str_inp = str_inp.replace(item+': ',str(idx)+'. ')\n",
    "    return str_inp\n",
    "def replace_out(str_inp):\n",
    "    ls = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'\n",
    "    for idx,item in enumerate(ls):\n",
    "        str_inp = str_inp.replace(item,str(idx))\n",
    "    return str_inp\n",
    "# replace_letter(ques['product list'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = []\n",
    "sam_num = 0\n",
    "for idx, row in data_test.iterrows():\n",
    "    temp_res = {}\n",
    "    if row['setting'] != 'IND_Diverse_Instruction':\n",
    "        continue\n",
    "    task = row['task']\n",
    "    inst = row['instruction']\n",
    "    inp = row['input']\n",
    "    optins = row['options']\n",
    "    out = row['output']\n",
    "    inst_res = 'Instruction: '+ inst\n",
    "    if task == 'Answer_Generation':\n",
    "        ques = eval(inp)\n",
    "        # docu = [str(i)+'. ' + ques['document'][i] for i in range(len(ques['document']))]\n",
    "        inp_res = 'Question: ' + ques['question'] + '\\n' + 'Document: ' +' '.join(ques['document'])\n",
    "        temp_res['input_field'] = inst_res+'\\n' + inp_res + '\\nAnswer: '\n",
    "        temp_res['output_field'] = out\n",
    "        temp_res['task_name'] = 'task1'\n",
    "        temp_res['task_type'] = 'generation'\n",
    "        temp_res['metric'] = 'sent-transformer'\n",
    "        temp_res['is_multiple_choice'] = 'False'\n",
    "        temp_res['track'] = 'amazon-kdd-cup-24-user-behavior-alignment'\n",
    "        # ques = inp.replace('\\\",','.').replace('\\\"','').replace('{','').replace('}','').replace('[','').replace(']','')\n",
    "    elif task == 'Product_Relation_Prediction':\n",
    "        ques = eval(inp)\n",
    "        opt = eval(optins)\n",
    "        inp_res = \"Product 1: \" + ques['Product 1:'] + '\\n' + \"Product 2: \" + ques['Product 2:'] \n",
    "        opt = replace_letter('\\n'.join(opt))\n",
    "        inp_res = inp_res+'\\n' + opt\n",
    "        temp_res['input_field'] = inst_res+'\\n' + inp_res + '\\nAnswer: '\n",
    "        temp_res['output_field'] =int(out.replace('A','0').replace('B','1').replace('C','2').replace('D','3').replace('E','4'))\n",
    "        temp_res['task_name'] = 'MCQ'\n",
    "        temp_res['task_type'] = 'multiple-choice'\n",
    "        temp_res['metric'] = 'accuracy'\n",
    "        temp_res['is_multiple_choice'] = 'True'\n",
    "        temp_res['track'] = 'amazon-kdd-cup-24-user-behavior-alignment'\n",
    "        \n",
    "        \n",
    "    elif task == 'Multiclass_Product_Classification':\n",
    "        ques = eval(inp)\n",
    "        opt = eval(optins)\n",
    "        inp_res = \"Query: \" + ques['query'] + '\\n' + \"Product: \" + ques['product title'] \n",
    "        opt = replace_letter('\\n'.join(opt))\n",
    "        inp_res = inp_res+'\\n' + opt\n",
    "        temp_res['input_field'] = inst_res+'\\n' + inp_res + '\\nAnswer: '\n",
    "        temp_res['output_field'] = int(replace_letter(out)[0])\n",
    "        temp_res['task_name'] = 'MCQ'\n",
    "        temp_res['task_type'] = 'multiple-choice'\n",
    "        temp_res['metric'] = 'accuracy'\n",
    "        temp_res['is_multiple_choice'] = True\n",
    "        temp_res['track'] = 'amazon-kdd-cup-24-user-behavior-alignment'\n",
    "        \n",
    "    elif task == 'Query_Product_Rank':\n",
    "        ques = eval(inp)\n",
    "        prod_ls = [replace_letter(ques['product list'][i]) for i in range(len(ques['product list']))]\n",
    "        inp_res = \"Query: \" + ques['query'] + '\\n' + \"Candidate product list:\\n\" + '\\n'.join(prod_ls)\n",
    "        out = replace_out(out).split(',')\n",
    "        out_score_ls = [0]*len(out)\n",
    "        for idx,item in enumerate(out):\n",
    "            if idx<2:\n",
    "                out_score_ls[int(item)] = 1\n",
    "            elif idx<3:\n",
    "                out_score_ls[int(item)] = 0.1\n",
    "            elif idx<4:\n",
    "                out_score_ls[int(item)] = 0.01\n",
    "            else:\n",
    "                out_score_ls[int(item)] = 0\n",
    "        \n",
    "        temp_res['input_field'] = inst_res+'\\n' + inp_res + '\\nAnswer (Only response oprion numbers and separated by a comma): '\n",
    "        temp_res['output_field'] = out_score_ls\n",
    "        temp_res['task_name'] = 'task4'\n",
    "        temp_res['task_type'] = 'ranking'\n",
    "        temp_res['metric'] = 'ndcg'\n",
    "        temp_res['is_multiple_choice'] = False\n",
    "        temp_res['track'] = 'amazon-kdd-cup-24-user-behavior-alignment'\n",
    "    elif task == 'Sentiment_Analysis':\n",
    "        inp_res = inp\n",
    "        opt = eval(optins)\n",
    "        opt = replace_letter('\\n'.join(opt))\n",
    "        inp_res = inp_res+'\\n' + opt\n",
    "        temp_res['input_field'] = inst_res+'\\n' + inp_res + '\\nAnswer: '\n",
    "        temp_res['output_field'] = int(replace_letter(out)[0])\n",
    "        temp_res['task_name'] = 'MCQ'\n",
    "        temp_res['task_type'] = 'multiple-choice'\n",
    "        temp_res['metric'] = 'accuracy'\n",
    "        temp_res['is_multiple_choice'] = True\n",
    "        temp_res['track'] = 'amazon-kdd-cup-24-user-behavior-alignment'\n",
    "    elif task == 'Sequential_Recommendation':\n",
    "        ques = eval(inp)\n",
    "        inp_res = 'User history behavior:\\n' + '\\n'.join(ques)\n",
    "        opt = eval(optins)\n",
    "        opt = replace_letter('\\n'.join(opt))\n",
    "        inp_res = inp_res +  \"\\n\\nCandidate product list: \\n\" +  opt\n",
    "        out = replace_out(out)\n",
    "        \n",
    "        temp_res['input_field'] = inst_res+'\\n' + inp_res + '\\nAnswer (choice 3 numbers and separated by a comma): '\n",
    "        temp_res['output_field'] = [int(out)]\n",
    "        temp_res['task_name'] = 'task6'\n",
    "        temp_res['task_type'] = 'retrieval'\n",
    "        temp_res['metric'] = 'hit rate@3'\n",
    "        temp_res['is_multiple_choice'] = False\n",
    "        temp_res['track'] = 'amazon-kdd-cup-24-user-behavior-alignment'\n",
    "        \n",
    "    else:\n",
    "        pass\n",
    "    res.append(temp_res)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "metadata": {},
   "outputs": [],
   "source": [
    "finla_df=pd.DataFrame(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "metadata": {},
   "outputs": [],
   "source": [
    "ecellm_df = finla_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([\"Instruction: Answer the given question using the supporting document.\\nQuestion: What are the dimensions of this griddle?\\nDocument: It's a wonderful griddle, with a great new idea. love this griddle! easy to use! makes breakfast a snap! I can't think of anything I'd change about this griddle. There is no end to what you can do with this griddle. It has become our favorite appliance and we find excuses to use it. the griddle did what I thought it would....... it is perfect for cooking pancakes and eggs for breakfast. You will love it. This is a pretty good griddle. Used it for few weeks and does a great job. I recommend this Griddle. Love it, cooks everything fast and even. Highly recommended if you're looking for a electric griddle. This griddle was purchased as a gift and works great. Easy to clean up and very easy to store. Heats evenly. Something never achieved in a pan in this house. If you're in the market for a griddle this one should fit the bill nicely. My wife is thrilled and very happy with this griddle! It stores away with no hassle and can fit into the side cabinit with no problem...\\nAnswer: \",\n",
       "       '22\" x 11 1/2\"  (that\\'s the actual griddle surface [I didn\\'t include the handle measurement]).  I can easily cook 8 pancakes at a time.  I love this griddle.  I had an older version and i lost the plug and decided just to buy the new one.  I am so happy that I lost the plug since the newer version of a great griddle is just so amazing.',\n",
       "       'task1', 'generation', 'sent-transformer', 'False',\n",
       "       'amazon-kdd-cup-24-user-behavior-alignment'], dtype=object)"
      ]
     },
     "execution_count": 340,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ecellm_df[ecellm_df['task_type']=='generation'].iloc[4].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([['Instruction: Evaluate the titles of Product 1 and Product 2 to assess their similarity and whether they are likely to be purchased or viewed together. Then, select the appropriate option.\\nProduct 1: Cooplay 4 Sets of 40pcs Mini Circle Sock Clip Holders Stockings Ring Locks Washing Sorters Laundry Storage Organization Soft Rubber with 5 Multicolor (40)\\nProduct 2: Cooplay Sock Holders Stockings Ring Clip Locks Washing Sorters Organizer Tool for House (30)\\n0. Users who buy product 1 may also buy product 2.\\n1. Users who view product 1 may also view product 2.\\n2. The product 1 is similar with the product 2.\\nAnswer: ',\n",
       "        1, 'MCQ', 'multiple-choice', 'accuracy', 'True',\n",
       "        'amazon-kdd-cup-24-user-behavior-alignment']], dtype=object)"
      ]
     },
     "execution_count": 266,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "finla_df[856:857].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A user has made a query with keyword '$0.00 not kindleunlimited'. Given the following numbered list of 5 products, please rank the products according their relevance with the query. \n",
      "Product List: \n",
      "1. Wild Irish Rose (The Merriams Book 1)\n",
      "2. Everything Is F*cked: A Book About Hope (The Subtle Art of Not Giving a F*ck (2 Book Series))\n",
      "3. The School Mistress (Emerson Pass Historicals Book 1)\n",
      "4. A Family Affair: A Small Town Family Saga (Truth In Lies Book 1)\n",
      "5. Blood in the Bayou: A Bone-Chilling FBI Thriller (FBI Agent Jade Monroe Live or Die Series Book 1)\n",
      "You should output a permutation of 1 to 5. There should be a comma separating two numbers. Each product and its number should appear only once in the output. Only respond with the ranking results. Do not say any word or explanations.\n",
      "Answer (Only response oprion numbers and separated by a comma):  \n"
     ]
    }
   ],
   "source": [
    "print(finla_df[finla_df.task_type=='ranking'].values[0][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "multiple-choice    3000\n",
       "ranking            1000\n",
       "generation         1000\n",
       "retrieval          1000\n",
       "Name: task_type, dtype: int64"
      ]
     },
     "execution_count": 250,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "finla_df.task_type.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>input_field</th>\n",
       "      <th>output_field</th>\n",
       "      <th>task_name</th>\n",
       "      <th>task_type</th>\n",
       "      <th>metric</th>\n",
       "      <th>is_multiple_choice</th>\n",
       "      <th>track</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Instruction: Evaluate the titles of Product 1 ...</td>\n",
       "      <td>1</td>\n",
       "      <td>MCQ</td>\n",
       "      <td>multiple-choice</td>\n",
       "      <td>accuracy</td>\n",
       "      <td>True</td>\n",
       "      <td>amazon-kdd-cup-24-user-behavior-alignment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Instruction: Given the title of two products, ...</td>\n",
       "      <td>1</td>\n",
       "      <td>MCQ</td>\n",
       "      <td>multiple-choice</td>\n",
       "      <td>accuracy</td>\n",
       "      <td>True</td>\n",
       "      <td>amazon-kdd-cup-24-user-behavior-alignment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Instruction: Analyze the titles of Product 1 a...</td>\n",
       "      <td>0</td>\n",
       "      <td>MCQ</td>\n",
       "      <td>multiple-choice</td>\n",
       "      <td>accuracy</td>\n",
       "      <td>True</td>\n",
       "      <td>amazon-kdd-cup-24-user-behavior-alignment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Instruction: Evaluate the titles of Product 1 ...</td>\n",
       "      <td>0</td>\n",
       "      <td>MCQ</td>\n",
       "      <td>multiple-choice</td>\n",
       "      <td>accuracy</td>\n",
       "      <td>True</td>\n",
       "      <td>amazon-kdd-cup-24-user-behavior-alignment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Instruction: Predict whether two products are ...</td>\n",
       "      <td>1</td>\n",
       "      <td>MCQ</td>\n",
       "      <td>multiple-choice</td>\n",
       "      <td>accuracy</td>\n",
       "      <td>True</td>\n",
       "      <td>amazon-kdd-cup-24-user-behavior-alignment</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         input_field output_field task_name  \\\n",
       "0  Instruction: Evaluate the titles of Product 1 ...            1       MCQ   \n",
       "1  Instruction: Given the title of two products, ...            1       MCQ   \n",
       "2  Instruction: Analyze the titles of Product 1 a...            0       MCQ   \n",
       "3  Instruction: Evaluate the titles of Product 1 ...            0       MCQ   \n",
       "4  Instruction: Predict whether two products are ...            1       MCQ   \n",
       "\n",
       "         task_type    metric is_multiple_choice  \\\n",
       "0  multiple-choice  accuracy               True   \n",
       "1  multiple-choice  accuracy               True   \n",
       "2  multiple-choice  accuracy               True   \n",
       "3  multiple-choice  accuracy               True   \n",
       "4  multiple-choice  accuracy               True   \n",
       "\n",
       "                                       track  \n",
       "0  amazon-kdd-cup-24-user-behavior-alignment  \n",
       "1  amazon-kdd-cup-24-user-behavior-alignment  \n",
       "2  amazon-kdd-cup-24-user-behavior-alignment  \n",
       "3  amazon-kdd-cup-24-user-behavior-alignment  \n",
       "4  amazon-kdd-cup-24-user-behavior-alignment  "
      ]
     },
     "execution_count": 214,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "finla_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "finla_df=pd.DataFrame(res)\n",
    "outfilepath=os.path.join('./','ecellm_test.json')\n",
    "a=finla_df.to_json(path_or_buf=outfilepath,orient = 'records',force_ascii=False,lines='orient')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = pd.read_json('./ecellm_test.json',lines=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>input_field</th>\n",
       "      <th>output_field</th>\n",
       "      <th>task_name</th>\n",
       "      <th>task_type</th>\n",
       "      <th>metric</th>\n",
       "      <th>is_multiple_choice</th>\n",
       "      <th>track</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Instruction: Evaluate the titles of Product 1 ...</td>\n",
       "      <td>1</td>\n",
       "      <td>MCQ</td>\n",
       "      <td>multiple-choice</td>\n",
       "      <td>accuracy</td>\n",
       "      <td>True</td>\n",
       "      <td>amazon-kdd-cup-24-user-behavior-alignment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Instruction: Given the title of two products, ...</td>\n",
       "      <td>1</td>\n",
       "      <td>MCQ</td>\n",
       "      <td>multiple-choice</td>\n",
       "      <td>accuracy</td>\n",
       "      <td>True</td>\n",
       "      <td>amazon-kdd-cup-24-user-behavior-alignment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Instruction: Analyze the titles of Product 1 a...</td>\n",
       "      <td>0</td>\n",
       "      <td>MCQ</td>\n",
       "      <td>multiple-choice</td>\n",
       "      <td>accuracy</td>\n",
       "      <td>True</td>\n",
       "      <td>amazon-kdd-cup-24-user-behavior-alignment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Instruction: Evaluate the titles of Product 1 ...</td>\n",
       "      <td>0</td>\n",
       "      <td>MCQ</td>\n",
       "      <td>multiple-choice</td>\n",
       "      <td>accuracy</td>\n",
       "      <td>True</td>\n",
       "      <td>amazon-kdd-cup-24-user-behavior-alignment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Instruction: Predict whether two products are ...</td>\n",
       "      <td>1</td>\n",
       "      <td>MCQ</td>\n",
       "      <td>multiple-choice</td>\n",
       "      <td>accuracy</td>\n",
       "      <td>True</td>\n",
       "      <td>amazon-kdd-cup-24-user-behavior-alignment</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         input_field output_field task_name  \\\n",
       "0  Instruction: Evaluate the titles of Product 1 ...            1       MCQ   \n",
       "1  Instruction: Given the title of two products, ...            1       MCQ   \n",
       "2  Instruction: Analyze the titles of Product 1 a...            0       MCQ   \n",
       "3  Instruction: Evaluate the titles of Product 1 ...            0       MCQ   \n",
       "4  Instruction: Predict whether two products are ...            1       MCQ   \n",
       "\n",
       "         task_type    metric is_multiple_choice  \\\n",
       "0  multiple-choice  accuracy               True   \n",
       "1  multiple-choice  accuracy               True   \n",
       "2  multiple-choice  accuracy               True   \n",
       "3  multiple-choice  accuracy               True   \n",
       "4  multiple-choice  accuracy               True   \n",
       "\n",
       "                                       track  \n",
       "0  amazon-kdd-cup-24-user-behavior-alignment  \n",
       "1  amazon-kdd-cup-24-user-behavior-alignment  \n",
       "2  amazon-kdd-cup-24-user-behavior-alignment  \n",
       "3  amazon-kdd-cup-24-user-behavior-alignment  \n",
       "4  amazon-kdd-cup-24-user-behavior-alignment  "
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Instruction: Analyze the query and each product title. Sort the products in descending order based on their relevance to the query. The most relevant product should be at the top of the list, and output the ranked list.\n",
      "Query: cabinet twin murphy bed\n",
      "Candidate product list:\n",
      "0. Full Wall Bed with Desk\n",
      "1. Mellow Platform Bedswith Headboard andBedside Storage-Ottoman Blue, Twin\n",
      "2. Heavy Duty Bed Support Hardware DIY Kit for Queen Twin Size Bed Horizontal Mounting Wall Bed Springs Mechanism Heavy Duty Bed Support Hardware DIY Kit 23cm/35cm (Small)\n",
      "Answer (Only response oprion numbers and separated by a comma): \n"
     ]
    }
   ],
   "source": [
    "print(list(df_test[df_test['task_name']=='task4'].values[30])[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22\" x 11 1/2\"  (that's the actual griddle surface [I didn't include the handle measurement]).  I can easily cook 8 pancakes at a time.  I love this griddle.  I had an older version and i lost the plug and decided just to buy the new one.  I am so happy that I lost the plug since the newer version of a great griddle is just so amazing.\n"
     ]
    }
   ],
   "source": [
    "print(list(df_test[df_test['task_name']=='task1'].values[4])[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# EC-Guide data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import json\n",
    "from random import sample\n",
    "root = './EC-Guide'\n",
    "listdir = os.listdir(root)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Generation_ERK.csv',\n",
       " 'Retrieval_CR.csv',\n",
       " 'Generation_EPF.csv',\n",
       " 'Ranking_QPR.csv',\n",
       " 'Retrieval_IPP.csv',\n",
       " 'Mcq_PRP.csv',\n",
       " 'Mcq_QPRP.csv',\n",
       " 'Generation_PKS.csv',\n",
       " 'Generation_EPK.csv',\n",
       " 'Generation_EBT.csv',\n",
       " 'Generation_PQA.csv',\n",
       " 'Retrieval_RRA.csv',\n",
       " 'Retrieval_RRS.csv',\n",
       " 'Generation_CR.csv',\n",
       " 'Mcq_SA.csv',\n",
       " 'Generation_PE.csv',\n",
       " 'Mcq_PKS.csv',\n",
       " 'Mcq_QPRJ.csv',\n",
       " 'Generation_RTS.csv',\n",
       " 'Retrieval_PR.csv',\n",
       " 'Mcq_SAP.csv',\n",
       " 'Mcq_SPA.csv']"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "listdir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_all = []\n",
    "for path in listdir:\n",
    "    res = []\n",
    "    sam_num = 0\n",
    "    single_data = pd.read_csv(root+ '/' + path)\n",
    "    # print(single_data.head())\n",
    "    df_all.append(single_data)\n",
    "df_ec = pd.concat(df_all)\n",
    "df_ec = df_ec[df_ec.metric.isin(['hit rate@3', 'sent-transformer', 'ndcg', 'accuracy'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 276,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>input_field</th>\n",
       "      <th>output_field</th>\n",
       "      <th>task_type</th>\n",
       "      <th>metric</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>For the product 'Krusell 75516 Orbit Flex Leat...</td>\n",
       "      <td>[3, 4, 5]</td>\n",
       "      <td>retrieval</td>\n",
       "      <td>hit rate@3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>For a product \"Silverstone Tek Mini-ITX DTX Co...</td>\n",
       "      <td>[1, 4, 7, 13]</td>\n",
       "      <td>retrieval</td>\n",
       "      <td>hit rate@3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Assess the categories for product 'Trident Cas...</td>\n",
       "      <td>[0, 1, 8]</td>\n",
       "      <td>retrieval</td>\n",
       "      <td>hit rate@3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Examine the list of categories for the product...</td>\n",
       "      <td>[2, 3, 8, 13, 14]</td>\n",
       "      <td>retrieval</td>\n",
       "      <td>hit rate@3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Look at the possible categories for 'Natrabio ...</td>\n",
       "      <td>[6, 7, 9, 12]</td>\n",
       "      <td>retrieval</td>\n",
       "      <td>hit rate@3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         input_field       output_field  \\\n",
       "0  For the product 'Krusell 75516 Orbit Flex Leat...          [3, 4, 5]   \n",
       "1  For a product \"Silverstone Tek Mini-ITX DTX Co...      [1, 4, 7, 13]   \n",
       "2  Assess the categories for product 'Trident Cas...          [0, 1, 8]   \n",
       "3  Examine the list of categories for the product...  [2, 3, 8, 13, 14]   \n",
       "4  Look at the possible categories for 'Natrabio ...      [6, 7, 9, 12]   \n",
       "\n",
       "   task_type      metric  \n",
       "0  retrieval  hit rate@3  \n",
       "1  retrieval  hit rate@3  \n",
       "2  retrieval  hit rate@3  \n",
       "3  retrieval  hit rate@3  \n",
       "4  retrieval  hit rate@3  "
      ]
     },
     "execution_count": 276,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ec.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "int"
      ]
     },
     "execution_count": 277,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(eval('11'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 278,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = []\n",
    "for index, row in df_ec.iterrows():\n",
    "    task = row['task_type']\n",
    "    temp_res = {}\n",
    "    if task == 'multiple-choice':\n",
    "        temp_res['input_field'] = row['input_field']+' '\n",
    "        temp_res['output_field'] = int(row['output_field'])\n",
    "        temp_res['task_name'] = row['task_type']\n",
    "        temp_res['task_type'] = row['task_type']\n",
    "        temp_res['metric'] = row['metric']\n",
    "        temp_res['is_multiple_choice'] = True\n",
    "        temp_res['track'] = 'amazon-kdd-cup-24-user-behavior-alignment'\n",
    "    elif task == 'retrieval':\n",
    "        temp_res['input_field'] = row['input_field'].replace('Output:','Answer (choice 3 numbers and separated by a comma): ')\n",
    "        if type(eval(row['output_field'])) == type([]):\n",
    "            temp_res['output_field'] = [int(eval(row['output_field'])[0])]\n",
    "        elif type(eval(row['output_field'])) == type(0):\n",
    "            temp_res['output_field'] = [eval(row['output_field'])]\n",
    "        temp_res['task_name'] = row['task_type']\n",
    "        temp_res['task_type'] = row['task_type']\n",
    "        temp_res['metric'] = row['metric']\n",
    "        temp_res['is_multiple_choice'] = False\n",
    "        temp_res['track'] = 'amazon-kdd-cup-24-user-behavior-alignment'\n",
    "    elif task == 'ranking':\n",
    "        try:\n",
    "            out = eval(row['output_field'])\n",
    "            out_score_ls = [0]*len(out)\n",
    "            for idx,item in enumerate(out):\n",
    "                if idx<2:\n",
    "                    out_score_ls[int(item)-1] = 1\n",
    "                elif idx<3:\n",
    "                    out_score_ls[int(item)-1] = 0.1\n",
    "                elif idx<4:\n",
    "                    out_score_ls[int(item)-1] = 0.01\n",
    "                else:\n",
    "                    out_score_ls[int(item)-1] = 0\n",
    "        except:\n",
    "            print(out,out_score_ls)\n",
    "        temp_res['input_field'] = row['input_field'].replace('Output:','Answer (Only response oprion numbers and separated by a comma): ')\n",
    "        temp_res['output_field'] = out_score_ls\n",
    "        temp_res['task_name'] = row['task_type']\n",
    "        temp_res['task_type'] = row['task_type']\n",
    "        temp_res['metric'] = row['metric']\n",
    "        temp_res['is_multiple_choice'] = False\n",
    "        temp_res['track'] = 'amazon-kdd-cup-24-user-behavior-alignment'\n",
    "    elif task == 'generation':\n",
    "        temp_res['input_field'] = row['input_field'].replace('Output:','Answer: ')\n",
    "        temp_res['output_field'] = row['output_field']\n",
    "        temp_res['task_name'] = row['task_type']\n",
    "        temp_res['task_type'] = row['task_type']\n",
    "        temp_res['metric'] = row['metric']\n",
    "        temp_res['is_multiple_choice'] = False\n",
    "        temp_res['track'] = 'amazon-kdd-cup-24-user-behavior-alignment'\n",
    "    res.append(temp_res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 279,
   "metadata": {},
   "outputs": [],
   "source": [
    "finla_df=pd.DataFrame(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 280,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "retrieval          22381\n",
       "generation         19086\n",
       "multiple-choice     8825\n",
       "ranking             4008\n",
       "Name: task_type, dtype: int64"
      ]
     },
     "execution_count": 280,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "finla_df.task_type.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['retrieval', 'generation', 'ranking', 'multiple-choice'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 283,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "finla_df.task_type.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [],
   "source": [
    "finla_df = finla_df[~finla_df.output_field.isna()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are an intelligent shopping assistant that can rank products based on their relevance to the query. The following numbered list contains 5 products. Please rank the products according to their relevance with the query 'succionador de cltoris'. \n",
      "Product List: \n",
      "1. Cupping Set Massage Cups with Pump Self Cupping Vacuum Cupping Set Massage Cupping Set,24PC\n",
      "2. VibeMax Mini Massager: Compact - Lightweight - USB Rechargeable - for Muscle Tension Relief in Neck, Back, Shoulders, Legs and Feet - Black\n",
      "3. Ben Wa Balls Kegel Balls for Women Kegel Exercise Weights Balls, Doctor Recommended for Bladder Control & Pelvic Floor Exercises (Purple)\n",
      "4. Helisun Cordless Electric Wand Massagers 8 Frequencies + 20 Vibration Modes USB Rechargeable Personal Massagers Handheld Magic Massages for Neck and Back Muscle Pain Quiet\n",
      "5. Wearable Women Sucking Toy, Remote Suction Toy for Women Pleasures, Waterproof and Rechargeable Sucker\n",
      "You should output a permutation of 1 to 5. There should be a comma separating two numbers. Each product and its number should appear only once in the output. Only respond with the ranking results. Do not say any word or explanations.\n",
      "Answer (Only response oprion numbers and separated by a comma):  \n"
     ]
    }
   ],
   "source": [
    "print(finla_df[finla_df.task_type=='ranking'].values[3000][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "outfilepath=os.path.join('./','ec-guide.json')\n",
    "a=finla_df.to_json(path_or_buf=outfilepath,orient = 'records',force_ascii=False,lines='orient')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_df = finla_df\n",
    "task_grouped_df = data_df.groupby(by=[\"task_type\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['input_field', 'output_field', 'task_name', 'task_type', 'metric',\n",
       "       'is_multiple_choice', 'track'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 258,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generation\n",
      "multiple-choice\n",
      "ranking\n",
      "retrieval\n"
     ]
    }
   ],
   "source": [
    "for task_type, task_group_data_df in task_grouped_df:\n",
    "        task_group_data_df = task_group_data_df.reset_index(drop=True)\n",
    "        \n",
    "        is_multiple_choice = task_type == \"multiple-choice\"\n",
    "        print(task_type)\n",
    "        # batch_size = model.get_batch_size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['retrieval', 'generation', 'ranking', 'multiple-choice'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ec['task_type'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A user has made a query with keyword 'leapfrog learning friends 100 words book'. Given the following numbered list of 5 products, please rank the products according their relevance with the query. \n",
      "Product List: \n",
      "1. Osmo - Coding Starter Kit for Fire Tablet - 3 Educational Learning Games - Ages 5-10+ - Learn to Code, Coding Basics & Coding Puzzles - STEM Toy (Osmo Fire Tablet Base Included) (Amazon Exclusive)\n",
      "2. BEST LEARNING Stack & Learn - Educational Activity Toy for Infants Babies Toddlers for 6 Month and up - Ideal Baby Toy Gifts\n",
      "3. VTech Musical Rhymes Book, Red\n",
      "4. LEARNING DYNAMICS 4 Weeks to Read | Reading System for 4 to 7 Years Old Kindergartners | Build Confidence with Their Own Personal Library, Includes 50 Books, Teaching Manual, Activities and Workbooks\n",
      "5. LeapFrog RockIt Twist Handheld Learning Game System, Green\n",
      "You should output a permutation of 1 to 5. There should be a comma separating two numbers. Each product and its number should appear only once in the output. Only respond with the ranking results. Do not say any word or explanations.\n",
      "Output: \n"
     ]
    }
   ],
   "source": [
    "task = 'ranking'\n",
    "print(df_ec[df_ec['task_type'] == task].values[2000][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3, 4, 2, 1, 5]\n"
     ]
    }
   ],
   "source": [
    "print(df_ec[df_ec['task_type'] == task].values[652][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/killer/anaconda3/envs/py39/lib/python3.9/site-packages/pandas/core/arrays/masked.py:60: UserWarning: Pandas requires version '1.3.6' or newer of 'bottleneck' (version '1.3.2' currently installed).\n",
      "  from pandas.core import (\n"
     ]
    },
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: './EC-Guide'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_2057162/4099464865.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mlistdir\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlistdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"./EC-Guide\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m \u001b[0;31m# {\"input_field\":\"Instructions: Tell me what this product category is about\\nInput: Toggle Switch\\nOutput:\",\"output_field\":\"A toggle switch is an electric switch operated by means of a projecting lever that is moved up and down.\",\"task_name\":\"task1\",\"task_type\":\"generation\",\"metric\":\"sent-transformer\",\"is_multiple_choice\":false,\"track\":\"amazon-kdd-cup-24-understanding-shopping-concepts\"}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: './EC-Guide'"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import json\n",
    "from random import sample\n",
    "\n",
    "\n",
    "\n",
    "listdir = os.listdir(\"./EC-Guide\")\n",
    "# {\"input_field\":\"Instructions: Tell me what this product category is about\\nInput: Toggle Switch\\nOutput:\",\"output_field\":\"A toggle switch is an electric switch operated by means of a projecting lever that is moved up and down.\",\"task_name\":\"task1\",\"task_type\":\"generation\",\"metric\":\"sent-transformer\",\"is_multiple_choice\":false,\"track\":\"amazon-kdd-cup-24-understanding-shopping-concepts\"}\n",
    "\n",
    "# input_field,output_field,task_type,metric\n",
    "\n",
    "convert_data = []\n",
    "gen_data = []\n",
    "mcp_data = []\n",
    "ranking_data = []\n",
    "retrieval_data = []\n",
    "for path in listdir:\n",
    "    single_data = pd.read_csv(\"/amazon-kdd-cup-2024-starter-kit/data/EC-Guide/EC-Guide/\" + path)\n",
    "    for index, row in single_data.iterrows():\n",
    "        data = {}\n",
    "        data['input_field'] = row['input_field'] + ' '\n",
    "        data['task_type'] = row['task_type']\n",
    "        data['output_field'] = row['output_field']\n",
    "        \n",
    "        data['metric'] = row['metric']\n",
    "        data['task_name'] = row['task_type']\n",
    "        if row['task_type'] == 'multiple-choice':\n",
    "            data['is_multiple_choice'] = True\n",
    "        else:\n",
    "            data['is_multiple_choice'] = False\n",
    "\n",
    "        if data['output_field'] == \"\" or data['metric'] == 'bool':\n",
    "            continue\n",
    "        \n",
    "        if path.startswith('Generation'):\n",
    "            gen_data.append(data)\n",
    "        elif path.startswith('Mcq'):\n",
    "            if type(data['output_field']) == str:\n",
    "                data['output_field'] = eval(data['output_field'])\n",
    "            mcp_data.append(data)\n",
    "        elif path.startswith('Ranking'):\n",
    "            if type(data['output_field']) == str:\n",
    "                data['output_field'] = eval(data['output_field'])\n",
    "            t = [1,1,1,1,1]\n",
    "            t[data['output_field'][0]-1] = 1\n",
    "            t[data['output_field'][1]-1] = 1\n",
    "            t[data['output_field'][2]-1] = 0.1\n",
    "            t[data['output_field'][3]-1] = 0.1\n",
    "            t[data['output_field'][4]-1] = 0\n",
    "            data['output_field'] = t\n",
    "            ranking_data.append(data)\n",
    "        elif path.startswith('Retrieval'):\n",
    "            if type(data['output_field']) == str:\n",
    "                data['output_field'] = eval(data['output_field'])\n",
    "            if type(data['output_field']) != list:\n",
    "                data['output_field'] = [data['output_field']]\n",
    "            retrieval_data.append(data)\n",
    "        else:\n",
    "            print(path)\n",
    "\n",
    "convert_data.extend(sample(gen_data, len(gen_data)//6))\n",
    "convert_data.extend(sample(mcp_data, len(mcp_data)//6))\n",
    "convert_data.extend(sample(ranking_data, len(ranking_data)//6))\n",
    "convert_data.extend(sample(retrieval_data, len(retrieval_data)//6))\n",
    "print(len(convert_data))\n",
    "json.dump(convert_data, open(\"/amazon-kdd-cup-2024-starter-kit/data/EC-Guide_small.json\", mode='w'), indent=4)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}