Advanced_features.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "fa3e6d03",
   "metadata": {},
   "source": [
    "More Feature Engineering Feature\n",
    "\n",
    "1-> Token Features\n",
    "\n",
    "Ex -> Sachin is a great batsman\n",
    "Tokens -> Sachin,is,a,great,batsman (total 5 in this sentence)\n",
    "Words  -> Normal word except for the stop words are words \n",
    "Stopwords -> words like is,of etc they don't have semantic meaning only help in sentence formation \n",
    "\n",
    "cwc min -> Ratio of number of common words to the length of smaller question \n",
    "cwc max -> Ratio of number of common words to the length of larger question\n",
    "csc min -> Number of common stopwords/min of stopwords in the both question\n",
    "csc max -> Number of common stopwords/max of stopwords in the both question\n",
    "ctc min -> Number of common tokens/min of tokens in both question\n",
    "ctc max -> Number of common tokens/max of tokens in both question\n",
    "last word equal -> check for last word in the both sentence (0,1)\n",
    "first word equal -> check for first word in the both sentence (0,1)\n",
    "\n",
    "\n",
    "2 -> Length based feature \n",
    "mean length -> mean of the length of both questions (tokens mean)\n",
    "absolute mean difference -> tokens absolute difference\n",
    "largest substr ratio -> ratio of length of the longest substring almong the 2 questions to the length of the smaller question\n",
    "\n",
    "3 -> Fuzzy Features\n",
    "fuzzy ratio ->  \n",
    "fuzzy partical ratio ->   \n",
    "token sort ratio ->  \n",
    "token set ratio ->  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f6741384",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "import re\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5cdfd0c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.read_csv('train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "73bfbbee",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df=df.sample(30000,random_state=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6733dc76",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>398782</th>\n",
       "      <td>398782</td>\n",
       "      <td>496695</td>\n",
       "      <td>532029</td>\n",
       "      <td>What is the best marketing automation tool for...</td>\n",
       "      <td>What is the best marketing automation tool for...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115086</th>\n",
       "      <td>115086</td>\n",
       "      <td>187729</td>\n",
       "      <td>187730</td>\n",
       "      <td>I am poor but I want to invest. What should I do?</td>\n",
       "      <td>I am quite poor and I want to be very rich. Wh...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>327711</th>\n",
       "      <td>327711</td>\n",
       "      <td>454161</td>\n",
       "      <td>454162</td>\n",
       "      <td>I am from India and live abroad. I met a guy f...</td>\n",
       "      <td>T.I.E.T to Thapar University to Thapar Univers...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>367788</th>\n",
       "      <td>367788</td>\n",
       "      <td>498109</td>\n",
       "      <td>491396</td>\n",
       "      <td>Why do so many people in the U.S. hate the sou...</td>\n",
       "      <td>My boyfriend doesnt feel guilty when he hurts ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151235</th>\n",
       "      <td>151235</td>\n",
       "      <td>237843</td>\n",
       "      <td>50930</td>\n",
       "      <td>Consequences of Bhopal gas tragedy?</td>\n",
       "      <td>What was the reason behind the Bhopal gas trag...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            id    qid1    qid2  \\\n",
       "398782  398782  496695  532029   \n",
       "115086  115086  187729  187730   \n",
       "327711  327711  454161  454162   \n",
       "367788  367788  498109  491396   \n",
       "151235  151235  237843   50930   \n",
       "\n",
       "                                                question1  \\\n",
       "398782  What is the best marketing automation tool for...   \n",
       "115086  I am poor but I want to invest. What should I do?   \n",
       "327711  I am from India and live abroad. I met a guy f...   \n",
       "367788  Why do so many people in the U.S. hate the sou...   \n",
       "151235                Consequences of Bhopal gas tragedy?   \n",
       "\n",
       "                                                question2  is_duplicate  \n",
       "398782  What is the best marketing automation tool for...             1  \n",
       "115086  I am quite poor and I want to be very rich. Wh...             0  \n",
       "327711  T.I.E.T to Thapar University to Thapar Univers...             0  \n",
       "367788  My boyfriend doesnt feel guilty when he hurts ...             0  \n",
       "151235  What was the reason behind the Bhopal gas trag...             0  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d6420690",
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocessor(q):\n",
    "    q=str(q).lower().strip() #convert the sentence into lower and strip the spaces\n",
    "    \n",
    "    #replaces certain special characters with their string equivalent\n",
    "    \n",
    "    q=q.replace('%','percent')\n",
    "    q = q.replace('$', ' dollar ')\n",
    "    q = q.replace('₹', ' rupee ')\n",
    "    q = q.replace('€', ' euro ')\n",
    "    q = q.replace('@', ' at ')\n",
    "    \n",
    "    q = q.replace('[math]', '')\n",
    "    \n",
    "    q = q.replace(',000,000,000 ', 'b ')\n",
    "    q = q.replace(',000,000 ', 'm ')\n",
    "    q = q.replace(',000 ', 'k ')\n",
    "    q = re.sub(r'([0-9]+)000000000', r'\\1b', q)\n",
    "    q = re.sub(r'([0-9]+)000000', r'\\1m', q)\n",
    "    q = re.sub(r'([0-9]+)000', r'\\1k', q)\n",
    "    \n",
    "    \n",
    "    # chainging word to thier actual form\n",
    "    \n",
    "    contractions = { \n",
    "    \"ain't\": \"am not\",\n",
    "    \"aren't\": \"are not\",\n",
    "    \"can't\": \"can not\",\n",
    "    \"can't've\": \"can not have\",\n",
    "    \"'cause\": \"because\",\n",
    "    \"could've\": \"could have\",\n",
    "    \"couldn't\": \"could not\",\n",
    "    \"couldn't've\": \"could not have\",\n",
    "    \"didn't\": \"did not\",\n",
    "    \"doesn't\": \"does not\",\n",
    "    \"don't\": \"do not\",\n",
    "    \"hadn't\": \"had not\",\n",
    "    \"hadn't've\": \"had not have\",\n",
    "    \"hasn't\": \"has not\",\n",
    "    \"haven't\": \"have not\",\n",
    "    \"he'd\": \"he would\",\n",
    "    \"he'd've\": \"he would have\",\n",
    "    \"he'll\": \"he will\",\n",
    "    \"he'll've\": \"he will have\",\n",
    "    \"he's\": \"he is\",\n",
    "    \"how'd\": \"how did\",\n",
    "    \"how'd'y\": \"how do you\",\n",
    "    \"how'll\": \"how will\",\n",
    "    \"how's\": \"how is\",\n",
    "    \"i'd\": \"i would\",\n",
    "    \"i'd've\": \"i would have\",\n",
    "    \"i'll\": \"i will\",\n",
    "    \"i'll've\": \"i will have\",\n",
    "    \"i'm\": \"i am\",\n",
    "    \"i've\": \"i have\",\n",
    "    \"isn't\": \"is not\",\n",
    "    \"it'd\": \"it would\",\n",
    "    \"it'd've\": \"it would have\",\n",
    "    \"it'll\": \"it will\",\n",
    "    \"it'll've\": \"it will have\",\n",
    "    \"it's\": \"it is\",\n",
    "    \"let's\": \"let us\",\n",
    "    \"ma'am\": \"madam\",\n",
    "    \"mayn't\": \"may not\",\n",
    "    \"might've\": \"might have\",\n",
    "    \"mightn't\": \"might not\",\n",
    "    \"mightn't've\": \"might not have\",\n",
    "    \"must've\": \"must have\",\n",
    "    \"mustn't\": \"must not\",\n",
    "    \"mustn't've\": \"must not have\",\n",
    "    \"needn't\": \"need not\",\n",
    "    \"needn't've\": \"need not have\",\n",
    "    \"o'clock\": \"of the clock\",\n",
    "    \"oughtn't\": \"ought not\",\n",
    "    \"oughtn't've\": \"ought not have\",\n",
    "    \"shan't\": \"shall not\",\n",
    "    \"sha'n't\": \"shall not\",\n",
    "    \"shan't've\": \"shall not have\",\n",
    "    \"she'd\": \"she would\",\n",
    "    \"she'd've\": \"she would have\",\n",
    "    \"she'll\": \"she will\",\n",
    "    \"she'll've\": \"she will have\",\n",
    "    \"she's\": \"she is\",\n",
    "    \"should've\": \"should have\",\n",
    "    \"shouldn't\": \"should not\",\n",
    "    \"shouldn't've\": \"should not have\",\n",
    "    \"so've\": \"so have\",\n",
    "    \"so's\": \"so as\",\n",
    "    \"that'd\": \"that would\",\n",
    "    \"that'd've\": \"that would have\",\n",
    "    \"that's\": \"that is\",\n",
    "    \"there'd\": \"there would\",\n",
    "    \"there'd've\": \"there would have\",\n",
    "    \"there's\": \"there is\",\n",
    "    \"they'd\": \"they would\",\n",
    "    \"they'd've\": \"they would have\",\n",
    "    \"they'll\": \"they will\",\n",
    "    \"they'll've\": \"they will have\",\n",
    "    \"they're\": \"they are\",\n",
    "    \"they've\": \"they have\",\n",
    "    \"to've\": \"to have\",\n",
    "    \"wasn't\": \"was not\",\n",
    "    \"we'd\": \"we would\",\n",
    "    \"we'd've\": \"we would have\",\n",
    "    \"we'll\": \"we will\",\n",
    "    \"we'll've\": \"we will have\",\n",
    "    \"we're\": \"we are\",\n",
    "    \"we've\": \"we have\",\n",
    "    \"weren't\": \"were not\",\n",
    "    \"what'll\": \"what will\",\n",
    "    \"what'll've\": \"what will have\",\n",
    "    \"what're\": \"what are\",\n",
    "    \"what's\": \"what is\",\n",
    "    \"what've\": \"what have\",\n",
    "    \"when's\": \"when is\",\n",
    "    \"when've\": \"when have\",\n",
    "    \"where'd\": \"where did\",\n",
    "    \"where's\": \"where is\",\n",
    "    \"where've\": \"where have\",\n",
    "    \"who'll\": \"who will\",\n",
    "    \"who'll've\": \"who will have\",\n",
    "    \"who's\": \"who is\",\n",
    "    \"who've\": \"who have\",\n",
    "    \"why's\": \"why is\",\n",
    "    \"why've\": \"why have\",\n",
    "    \"will've\": \"will have\",\n",
    "    \"won't\": \"will not\",\n",
    "    \"won't've\": \"will not have\",\n",
    "    \"would've\": \"would have\",\n",
    "    \"wouldn't\": \"would not\",\n",
    "    \"wouldn't've\": \"would not have\",\n",
    "    \"y'all\": \"you all\",\n",
    "    \"y'all'd\": \"you all would\",\n",
    "    \"y'all'd've\": \"you all would have\",\n",
    "    \"y'all're\": \"you all are\",\n",
    "    \"y'all've\": \"you all have\",\n",
    "    \"you'd\": \"you would\",\n",
    "    \"you'd've\": \"you would have\",\n",
    "    \"you'll\": \"you will\",\n",
    "    \"you'll've\": \"you will have\",\n",
    "    \"you're\": \"you are\",\n",
    "    \"you've\": \"you have\"\n",
    "    }\n",
    "    \n",
    "    q_decontracted=[]\n",
    "    \n",
    "    for word in q.split():\n",
    "        if word in contractions:\n",
    "            word=contractions[word]\n",
    "            \n",
    "        q_decontracted.append(word)\n",
    "        \n",
    "        \n",
    "        \n",
    "    q = ' '.join(q_decontracted)\n",
    "    q = q.replace(\"'ve\", \" have\")\n",
    "    q = q.replace(\"n't\", \" not\")\n",
    "    q = q.replace(\"'re\", \" are\")\n",
    "    q = q.replace(\"'ll\", \" will\")\n",
    "    \n",
    "    q = BeautifulSoup(q)\n",
    "    q = q.get_text()\n",
    "    \n",
    "    # Remove punctuations\n",
    "    pattern = re.compile('\\W')\n",
    "    q = re.sub(pattern, ' ', q).strip()\n",
    "    \n",
    "    \n",
    "    return q\n",
    "\n",
    "    \n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "10541e6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df['question1']=new_df['question1'].apply(preprocessor)\n",
    "new_df['question2']=new_df['question2'].apply(preprocessor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d1fde899",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>398782</th>\n",
       "      <td>398782</td>\n",
       "      <td>496695</td>\n",
       "      <td>532029</td>\n",
       "      <td>what is the best marketing automation tool for...</td>\n",
       "      <td>what is the best marketing automation tool for...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115086</th>\n",
       "      <td>115086</td>\n",
       "      <td>187729</td>\n",
       "      <td>187730</td>\n",
       "      <td>i am poor but i want to invest  what should i do</td>\n",
       "      <td>i am quite poor and i want to be very rich  wh...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>327711</th>\n",
       "      <td>327711</td>\n",
       "      <td>454161</td>\n",
       "      <td>454162</td>\n",
       "      <td>i am from india and live abroad  i met a guy f...</td>\n",
       "      <td>t i e t to thapar university to thapar univers...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>367788</th>\n",
       "      <td>367788</td>\n",
       "      <td>498109</td>\n",
       "      <td>491396</td>\n",
       "      <td>why do so many people in the u s  hate the sou...</td>\n",
       "      <td>my boyfriend doesnt feel guilty when he hurts ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151235</th>\n",
       "      <td>151235</td>\n",
       "      <td>237843</td>\n",
       "      <td>50930</td>\n",
       "      <td>consequences of bhopal gas tragedy</td>\n",
       "      <td>what was the reason behind the bhopal gas tragedy</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            id    qid1    qid2  \\\n",
       "398782  398782  496695  532029   \n",
       "115086  115086  187729  187730   \n",
       "327711  327711  454161  454162   \n",
       "367788  367788  498109  491396   \n",
       "151235  151235  237843   50930   \n",
       "\n",
       "                                                question1  \\\n",
       "398782  what is the best marketing automation tool for...   \n",
       "115086   i am poor but i want to invest  what should i do   \n",
       "327711  i am from india and live abroad  i met a guy f...   \n",
       "367788  why do so many people in the u s  hate the sou...   \n",
       "151235                 consequences of bhopal gas tragedy   \n",
       "\n",
       "                                                question2  is_duplicate  \n",
       "398782  what is the best marketing automation tool for...             1  \n",
       "115086  i am quite poor and i want to be very rich  wh...             0  \n",
       "327711  t i e t to thapar university to thapar univers...             0  \n",
       "367788  my boyfriend doesnt feel guilty when he hurts ...             0  \n",
       "151235  what was the reason behind the bhopal gas tragedy             0  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4872ec2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df['q1_len']=new_df['question1'].str.len()\n",
    "new_df['q2_len']=new_df['question2'].str.len()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9f31c09b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>q1_len</th>\n",
       "      <th>q2_len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>398782</th>\n",
       "      <td>398782</td>\n",
       "      <td>496695</td>\n",
       "      <td>532029</td>\n",
       "      <td>what is the best marketing automation tool for...</td>\n",
       "      <td>what is the best marketing automation tool for...</td>\n",
       "      <td>1</td>\n",
       "      <td>75</td>\n",
       "      <td>76</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115086</th>\n",
       "      <td>115086</td>\n",
       "      <td>187729</td>\n",
       "      <td>187730</td>\n",
       "      <td>i am poor but i want to invest  what should i do</td>\n",
       "      <td>i am quite poor and i want to be very rich  wh...</td>\n",
       "      <td>0</td>\n",
       "      <td>48</td>\n",
       "      <td>56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>327711</th>\n",
       "      <td>327711</td>\n",
       "      <td>454161</td>\n",
       "      <td>454162</td>\n",
       "      <td>i am from india and live abroad  i met a guy f...</td>\n",
       "      <td>t i e t to thapar university to thapar univers...</td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>367788</th>\n",
       "      <td>367788</td>\n",
       "      <td>498109</td>\n",
       "      <td>491396</td>\n",
       "      <td>why do so many people in the u s  hate the sou...</td>\n",
       "      <td>my boyfriend doesnt feel guilty when he hurts ...</td>\n",
       "      <td>0</td>\n",
       "      <td>58</td>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151235</th>\n",
       "      <td>151235</td>\n",
       "      <td>237843</td>\n",
       "      <td>50930</td>\n",
       "      <td>consequences of bhopal gas tragedy</td>\n",
       "      <td>what was the reason behind the bhopal gas tragedy</td>\n",
       "      <td>0</td>\n",
       "      <td>34</td>\n",
       "      <td>49</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            id    qid1    qid2  \\\n",
       "398782  398782  496695  532029   \n",
       "115086  115086  187729  187730   \n",
       "327711  327711  454161  454162   \n",
       "367788  367788  498109  491396   \n",
       "151235  151235  237843   50930   \n",
       "\n",
       "                                                question1  \\\n",
       "398782  what is the best marketing automation tool for...   \n",
       "115086   i am poor but i want to invest  what should i do   \n",
       "327711  i am from india and live abroad  i met a guy f...   \n",
       "367788  why do so many people in the u s  hate the sou...   \n",
       "151235                 consequences of bhopal gas tragedy   \n",
       "\n",
       "                                                question2  is_duplicate  \\\n",
       "398782  what is the best marketing automation tool for...             1   \n",
       "115086  i am quite poor and i want to be very rich  wh...             0   \n",
       "327711  t i e t to thapar university to thapar univers...             0   \n",
       "367788  my boyfriend doesnt feel guilty when he hurts ...             0   \n",
       "151235  what was the reason behind the bhopal gas tragedy             0   \n",
       "\n",
       "        q1_len  q2_len  \n",
       "398782      75      76  \n",
       "115086      48      56  \n",
       "327711     104     119  \n",
       "367788      58     145  \n",
       "151235      34      49  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "be288cc1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>q1_len</th>\n",
       "      <th>q2_len</th>\n",
       "      <th>q1_num_words</th>\n",
       "      <th>q2_num_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>398782</th>\n",
       "      <td>398782</td>\n",
       "      <td>496695</td>\n",
       "      <td>532029</td>\n",
       "      <td>what is the best marketing automation tool for...</td>\n",
       "      <td>what is the best marketing automation tool for...</td>\n",
       "      <td>1</td>\n",
       "      <td>75</td>\n",
       "      <td>76</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115086</th>\n",
       "      <td>115086</td>\n",
       "      <td>187729</td>\n",
       "      <td>187730</td>\n",
       "      <td>i am poor but i want to invest  what should i do</td>\n",
       "      <td>i am quite poor and i want to be very rich  wh...</td>\n",
       "      <td>0</td>\n",
       "      <td>48</td>\n",
       "      <td>56</td>\n",
       "      <td>13</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>327711</th>\n",
       "      <td>327711</td>\n",
       "      <td>454161</td>\n",
       "      <td>454162</td>\n",
       "      <td>i am from india and live abroad  i met a guy f...</td>\n",
       "      <td>t i e t to thapar university to thapar univers...</td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>119</td>\n",
       "      <td>28</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>367788</th>\n",
       "      <td>367788</td>\n",
       "      <td>498109</td>\n",
       "      <td>491396</td>\n",
       "      <td>why do so many people in the u s  hate the sou...</td>\n",
       "      <td>my boyfriend doesnt feel guilty when he hurts ...</td>\n",
       "      <td>0</td>\n",
       "      <td>58</td>\n",
       "      <td>145</td>\n",
       "      <td>14</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151235</th>\n",
       "      <td>151235</td>\n",
       "      <td>237843</td>\n",
       "      <td>50930</td>\n",
       "      <td>consequences of bhopal gas tragedy</td>\n",
       "      <td>what was the reason behind the bhopal gas tragedy</td>\n",
       "      <td>0</td>\n",
       "      <td>34</td>\n",
       "      <td>49</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            id    qid1    qid2  \\\n",
       "398782  398782  496695  532029   \n",
       "115086  115086  187729  187730   \n",
       "327711  327711  454161  454162   \n",
       "367788  367788  498109  491396   \n",
       "151235  151235  237843   50930   \n",
       "\n",
       "                                                question1  \\\n",
       "398782  what is the best marketing automation tool for...   \n",
       "115086   i am poor but i want to invest  what should i do   \n",
       "327711  i am from india and live abroad  i met a guy f...   \n",
       "367788  why do so many people in the u s  hate the sou...   \n",
       "151235                 consequences of bhopal gas tragedy   \n",
       "\n",
       "                                                question2  is_duplicate  \\\n",
       "398782  what is the best marketing automation tool for...             1   \n",
       "115086  i am quite poor and i want to be very rich  wh...             0   \n",
       "327711  t i e t to thapar university to thapar univers...             0   \n",
       "367788  my boyfriend doesnt feel guilty when he hurts ...             0   \n",
       "151235  what was the reason behind the bhopal gas tragedy             0   \n",
       "\n",
       "        q1_len  q2_len  q1_num_words  q2_num_words  \n",
       "398782      75      76            13            13  \n",
       "115086      48      56            13            16  \n",
       "327711     104     119            28            21  \n",
       "367788      58     145            14            32  \n",
       "151235      34      49             5             9  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df['q1_num_words']=new_df['question1'].apply(lambda row:len(row.split(\" \")))\n",
    "new_df['q2_num_words']=new_df['question2'].apply(lambda row:len(row.split(\" \")))\n",
    "new_df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "03c232db",
   "metadata": {},
   "outputs": [],
   "source": [
    "def common_words(row):\n",
    "    w1=set(map(lambda word:word.lower().strip(),row['question1'].split(\" \")))\n",
    "    w2=set(map(lambda word:word.lower().strip(),row['question2'].split(\" \")))\n",
    "    return len(w1&w2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "34bca3a1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>q1_len</th>\n",
       "      <th>q2_len</th>\n",
       "      <th>q1_num_words</th>\n",
       "      <th>q2_num_words</th>\n",
       "      <th>word_common</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>398782</th>\n",
       "      <td>398782</td>\n",
       "      <td>496695</td>\n",
       "      <td>532029</td>\n",
       "      <td>what is the best marketing automation tool for...</td>\n",
       "      <td>what is the best marketing automation tool for...</td>\n",
       "      <td>1</td>\n",
       "      <td>75</td>\n",
       "      <td>76</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115086</th>\n",
       "      <td>115086</td>\n",
       "      <td>187729</td>\n",
       "      <td>187730</td>\n",
       "      <td>i am poor but i want to invest  what should i do</td>\n",
       "      <td>i am quite poor and i want to be very rich  wh...</td>\n",
       "      <td>0</td>\n",
       "      <td>48</td>\n",
       "      <td>56</td>\n",
       "      <td>13</td>\n",
       "      <td>16</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>327711</th>\n",
       "      <td>327711</td>\n",
       "      <td>454161</td>\n",
       "      <td>454162</td>\n",
       "      <td>i am from india and live abroad  i met a guy f...</td>\n",
       "      <td>t i e t to thapar university to thapar univers...</td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>119</td>\n",
       "      <td>28</td>\n",
       "      <td>21</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>367788</th>\n",
       "      <td>367788</td>\n",
       "      <td>498109</td>\n",
       "      <td>491396</td>\n",
       "      <td>why do so many people in the u s  hate the sou...</td>\n",
       "      <td>my boyfriend doesnt feel guilty when he hurts ...</td>\n",
       "      <td>0</td>\n",
       "      <td>58</td>\n",
       "      <td>145</td>\n",
       "      <td>14</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151235</th>\n",
       "      <td>151235</td>\n",
       "      <td>237843</td>\n",
       "      <td>50930</td>\n",
       "      <td>consequences of bhopal gas tragedy</td>\n",
       "      <td>what was the reason behind the bhopal gas tragedy</td>\n",
       "      <td>0</td>\n",
       "      <td>34</td>\n",
       "      <td>49</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            id    qid1    qid2  \\\n",
       "398782  398782  496695  532029   \n",
       "115086  115086  187729  187730   \n",
       "327711  327711  454161  454162   \n",
       "367788  367788  498109  491396   \n",
       "151235  151235  237843   50930   \n",
       "\n",
       "                                                question1  \\\n",
       "398782  what is the best marketing automation tool for...   \n",
       "115086   i am poor but i want to invest  what should i do   \n",
       "327711  i am from india and live abroad  i met a guy f...   \n",
       "367788  why do so many people in the u s  hate the sou...   \n",
       "151235                 consequences of bhopal gas tragedy   \n",
       "\n",
       "                                                question2  is_duplicate  \\\n",
       "398782  what is the best marketing automation tool for...             1   \n",
       "115086  i am quite poor and i want to be very rich  wh...             0   \n",
       "327711  t i e t to thapar university to thapar univers...             0   \n",
       "367788  my boyfriend doesnt feel guilty when he hurts ...             0   \n",
       "151235  what was the reason behind the bhopal gas tragedy             0   \n",
       "\n",
       "        q1_len  q2_len  q1_num_words  q2_num_words  word_common  \n",
       "398782      75      76            13            13           12  \n",
       "115086      48      56            13            16            8  \n",
       "327711     104     119            28            21            4  \n",
       "367788      58     145            14            32            1  \n",
       "151235      34      49             5             9            3  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df['word_common']=new_df.apply(common_words,axis=1)\n",
    "new_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "19be4655",
   "metadata": {},
   "outputs": [],
   "source": [
    "def total_words(row):\n",
    "    w1=set(map(lambda word:word.lower().strip(),row['question1'].split(\" \")))\n",
    "    w2=set(map(lambda word:word.lower().strip(),row['question2'].split(\" \")))\n",
    "    return len(w1)+len(w2)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "6de00af1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>q1_len</th>\n",
       "      <th>q2_len</th>\n",
       "      <th>q1_num_words</th>\n",
       "      <th>q2_num_words</th>\n",
       "      <th>word_common</th>\n",
       "      <th>word_total</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>398782</th>\n",
       "      <td>398782</td>\n",
       "      <td>496695</td>\n",
       "      <td>532029</td>\n",
       "      <td>what is the best marketing automation tool for...</td>\n",
       "      <td>what is the best marketing automation tool for...</td>\n",
       "      <td>1</td>\n",
       "      <td>75</td>\n",
       "      <td>76</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>12</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115086</th>\n",
       "      <td>115086</td>\n",
       "      <td>187729</td>\n",
       "      <td>187730</td>\n",
       "      <td>i am poor but i want to invest  what should i do</td>\n",
       "      <td>i am quite poor and i want to be very rich  wh...</td>\n",
       "      <td>0</td>\n",
       "      <td>48</td>\n",
       "      <td>56</td>\n",
       "      <td>13</td>\n",
       "      <td>16</td>\n",
       "      <td>8</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>327711</th>\n",
       "      <td>327711</td>\n",
       "      <td>454161</td>\n",
       "      <td>454162</td>\n",
       "      <td>i am from india and live abroad  i met a guy f...</td>\n",
       "      <td>t i e t to thapar university to thapar univers...</td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>119</td>\n",
       "      <td>28</td>\n",
       "      <td>21</td>\n",
       "      <td>4</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>367788</th>\n",
       "      <td>367788</td>\n",
       "      <td>498109</td>\n",
       "      <td>491396</td>\n",
       "      <td>why do so many people in the u s  hate the sou...</td>\n",
       "      <td>my boyfriend doesnt feel guilty when he hurts ...</td>\n",
       "      <td>0</td>\n",
       "      <td>58</td>\n",
       "      <td>145</td>\n",
       "      <td>14</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151235</th>\n",
       "      <td>151235</td>\n",
       "      <td>237843</td>\n",
       "      <td>50930</td>\n",
       "      <td>consequences of bhopal gas tragedy</td>\n",
       "      <td>what was the reason behind the bhopal gas tragedy</td>\n",
       "      <td>0</td>\n",
       "      <td>34</td>\n",
       "      <td>49</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            id    qid1    qid2  \\\n",
       "398782  398782  496695  532029   \n",
       "115086  115086  187729  187730   \n",
       "327711  327711  454161  454162   \n",
       "367788  367788  498109  491396   \n",
       "151235  151235  237843   50930   \n",
       "\n",
       "                                                question1  \\\n",
       "398782  what is the best marketing automation tool for...   \n",
       "115086   i am poor but i want to invest  what should i do   \n",
       "327711  i am from india and live abroad  i met a guy f...   \n",
       "367788  why do so many people in the u s  hate the sou...   \n",
       "151235                 consequences of bhopal gas tragedy   \n",
       "\n",
       "                                                question2  is_duplicate  \\\n",
       "398782  what is the best marketing automation tool for...             1   \n",
       "115086  i am quite poor and i want to be very rich  wh...             0   \n",
       "327711  t i e t to thapar university to thapar univers...             0   \n",
       "367788  my boyfriend doesnt feel guilty when he hurts ...             0   \n",
       "151235  what was the reason behind the bhopal gas tragedy             0   \n",
       "\n",
       "        q1_len  q2_len  q1_num_words  q2_num_words  word_common  word_total  \n",
       "398782      75      76            13            13           12          26  \n",
       "115086      48      56            13            16            8          24  \n",
       "327711     104     119            28            21            4          38  \n",
       "367788      58     145            14            32            1          34  \n",
       "151235      34      49             5             9            3          13  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df['word_total']=new_df.apply(total_words,axis=1)\n",
    "new_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ab479c5a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>q1_len</th>\n",
       "      <th>q2_len</th>\n",
       "      <th>q1_num_words</th>\n",
       "      <th>q2_num_words</th>\n",
       "      <th>word_common</th>\n",
       "      <th>word_total</th>\n",
       "      <th>word_share</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>398782</th>\n",
       "      <td>398782</td>\n",
       "      <td>496695</td>\n",
       "      <td>532029</td>\n",
       "      <td>what is the best marketing automation tool for...</td>\n",
       "      <td>what is the best marketing automation tool for...</td>\n",
       "      <td>1</td>\n",
       "      <td>75</td>\n",
       "      <td>76</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>12</td>\n",
       "      <td>26</td>\n",
       "      <td>0.46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115086</th>\n",
       "      <td>115086</td>\n",
       "      <td>187729</td>\n",
       "      <td>187730</td>\n",
       "      <td>i am poor but i want to invest  what should i do</td>\n",
       "      <td>i am quite poor and i want to be very rich  wh...</td>\n",
       "      <td>0</td>\n",
       "      <td>48</td>\n",
       "      <td>56</td>\n",
       "      <td>13</td>\n",
       "      <td>16</td>\n",
       "      <td>8</td>\n",
       "      <td>24</td>\n",
       "      <td>0.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>327711</th>\n",
       "      <td>327711</td>\n",
       "      <td>454161</td>\n",
       "      <td>454162</td>\n",
       "      <td>i am from india and live abroad  i met a guy f...</td>\n",
       "      <td>t i e t to thapar university to thapar univers...</td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>119</td>\n",
       "      <td>28</td>\n",
       "      <td>21</td>\n",
       "      <td>4</td>\n",
       "      <td>38</td>\n",
       "      <td>0.11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>367788</th>\n",
       "      <td>367788</td>\n",
       "      <td>498109</td>\n",
       "      <td>491396</td>\n",
       "      <td>why do so many people in the u s  hate the sou...</td>\n",
       "      <td>my boyfriend doesnt feel guilty when he hurts ...</td>\n",
       "      <td>0</td>\n",
       "      <td>58</td>\n",
       "      <td>145</td>\n",
       "      <td>14</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "      <td>34</td>\n",
       "      <td>0.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151235</th>\n",
       "      <td>151235</td>\n",
       "      <td>237843</td>\n",
       "      <td>50930</td>\n",
       "      <td>consequences of bhopal gas tragedy</td>\n",
       "      <td>what was the reason behind the bhopal gas tragedy</td>\n",
       "      <td>0</td>\n",
       "      <td>34</td>\n",
       "      <td>49</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "      <td>13</td>\n",
       "      <td>0.23</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            id    qid1    qid2  \\\n",
       "398782  398782  496695  532029   \n",
       "115086  115086  187729  187730   \n",
       "327711  327711  454161  454162   \n",
       "367788  367788  498109  491396   \n",
       "151235  151235  237843   50930   \n",
       "\n",
       "                                                question1  \\\n",
       "398782  what is the best marketing automation tool for...   \n",
       "115086   i am poor but i want to invest  what should i do   \n",
       "327711  i am from india and live abroad  i met a guy f...   \n",
       "367788  why do so many people in the u s  hate the sou...   \n",
       "151235                 consequences of bhopal gas tragedy   \n",
       "\n",
       "                                                question2  is_duplicate  \\\n",
       "398782  what is the best marketing automation tool for...             1   \n",
       "115086  i am quite poor and i want to be very rich  wh...             0   \n",
       "327711  t i e t to thapar university to thapar univers...             0   \n",
       "367788  my boyfriend doesnt feel guilty when he hurts ...             0   \n",
       "151235  what was the reason behind the bhopal gas tragedy             0   \n",
       "\n",
       "        q1_len  q2_len  q1_num_words  q2_num_words  word_common  word_total  \\\n",
       "398782      75      76            13            13           12          26   \n",
       "115086      48      56            13            16            8          24   \n",
       "327711     104     119            28            21            4          38   \n",
       "367788      58     145            14            32            1          34   \n",
       "151235      34      49             5             9            3          13   \n",
       "\n",
       "        word_share  \n",
       "398782        0.46  \n",
       "115086        0.33  \n",
       "327711        0.11  \n",
       "367788        0.03  \n",
       "151235        0.23  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df['word_share'] = round(new_df['word_common']/new_df['word_total'],2)\n",
    "new_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "de453c9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "\n",
    "def fetch_token_features(row):\n",
    "    \n",
    "    q1 = row['question1']\n",
    "    q2 = row['question2']\n",
    "    \n",
    "    SAFE_DIV = 0.0001 \n",
    "\n",
    "    STOP_WORDS = stopwords.words(\"english\")\n",
    "    \n",
    "    token_features = [0.0]*8\n",
    "    \n",
    "    # Converting the Sentence into Tokens: \n",
    "    q1_tokens = q1.split()\n",
    "    q2_tokens = q2.split()\n",
    "    \n",
    "    if len(q1_tokens) == 0 or len(q2_tokens) == 0:\n",
    "        return token_features\n",
    "\n",
    "    # Get the non-stopwords in Questions\n",
    "    q1_words = set([word for word in q1_tokens if word not in STOP_WORDS])\n",
    "    q2_words = set([word for word in q2_tokens if word not in STOP_WORDS])\n",
    "    \n",
    "    #Get the stopwords in Questions\n",
    "    q1_stops = set([word for word in q1_tokens if word in STOP_WORDS])\n",
    "    q2_stops = set([word for word in q2_tokens if word in STOP_WORDS])\n",
    "    \n",
    "    # Get the common non-stopwords from Question pair\n",
    "    common_word_count = len(q1_words.intersection(q2_words))\n",
    "    \n",
    "    # Get the common stopwords from Question pair\n",
    "    common_stop_count = len(q1_stops.intersection(q2_stops))\n",
    "    \n",
    "    # Get the common Tokens from Question pair\n",
    "    common_token_count = len(set(q1_tokens).intersection(set(q2_tokens)))\n",
    "    \n",
    "    \n",
    "    token_features[0] = common_word_count / (min(len(q1_words), len(q2_words)) + SAFE_DIV)\n",
    "    token_features[1] = common_word_count / (max(len(q1_words), len(q2_words)) + SAFE_DIV)\n",
    "    token_features[2] = common_stop_count / (min(len(q1_stops), len(q2_stops)) + SAFE_DIV)\n",
    "    token_features[3] = common_stop_count / (max(len(q1_stops), len(q2_stops)) + SAFE_DIV)\n",
    "    token_features[4] = common_token_count / (min(len(q1_tokens), len(q2_tokens)) + SAFE_DIV)\n",
    "    token_features[5] = common_token_count / (max(len(q1_tokens), len(q2_tokens)) + SAFE_DIV)\n",
    "    \n",
    "    # Last word of both question is same or not\n",
    "    token_features[6] = int(q1_tokens[-1] == q2_tokens[-1])\n",
    "    \n",
    "    # First word of both question is same or not\n",
    "    token_features[7] = int(q1_tokens[0] == q2_tokens[0])\n",
    "    \n",
    "    return token_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "bf7759e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "token_features=new_df.apply(fetch_token_features,axis=1)\n",
    "new_df[\"cwc_min\"]       = list(map(lambda x: x[0], token_features))\n",
    "new_df[\"cwc_max\"]       = list(map(lambda x: x[1], token_features))\n",
    "new_df[\"csc_min\"]       = list(map(lambda x: x[2], token_features))\n",
    "new_df[\"csc_max\"]       = list(map(lambda x: x[3], token_features))\n",
    "new_df[\"ctc_min\"]       = list(map(lambda x: x[4], token_features))\n",
    "new_df[\"ctc_max\"]       = list(map(lambda x: x[5], token_features))\n",
    "new_df[\"last_word_eq\"]  = list(map(lambda x: x[6], token_features))\n",
    "new_df[\"first_word_eq\"] = list(map(lambda x: x[7], token_features))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "f636bbb1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>q1_len</th>\n",
       "      <th>q2_len</th>\n",
       "      <th>q1_num_words</th>\n",
       "      <th>q2_num_words</th>\n",
       "      <th>...</th>\n",
       "      <th>word_total</th>\n",
       "      <th>word_share</th>\n",
       "      <th>cwc_min</th>\n",
       "      <th>cwc_max</th>\n",
       "      <th>csc_min</th>\n",
       "      <th>csc_max</th>\n",
       "      <th>ctc_min</th>\n",
       "      <th>ctc_max</th>\n",
       "      <th>last_word_eq</th>\n",
       "      <th>first_word_eq</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>398782</th>\n",
       "      <td>398782</td>\n",
       "      <td>496695</td>\n",
       "      <td>532029</td>\n",
       "      <td>what is the best marketing automation tool for...</td>\n",
       "      <td>what is the best marketing automation tool for...</td>\n",
       "      <td>1</td>\n",
       "      <td>75</td>\n",
       "      <td>76</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>...</td>\n",
       "      <td>26</td>\n",
       "      <td>0.46</td>\n",
       "      <td>0.874989</td>\n",
       "      <td>0.874989</td>\n",
       "      <td>0.999980</td>\n",
       "      <td>0.999980</td>\n",
       "      <td>0.923070</td>\n",
       "      <td>0.923070</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115086</th>\n",
       "      <td>115086</td>\n",
       "      <td>187729</td>\n",
       "      <td>187730</td>\n",
       "      <td>i am poor but i want to invest  what should i do</td>\n",
       "      <td>i am quite poor and i want to be very rich  wh...</td>\n",
       "      <td>0</td>\n",
       "      <td>48</td>\n",
       "      <td>56</td>\n",
       "      <td>13</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>24</td>\n",
       "      <td>0.33</td>\n",
       "      <td>0.666644</td>\n",
       "      <td>0.499988</td>\n",
       "      <td>0.714276</td>\n",
       "      <td>0.624992</td>\n",
       "      <td>0.583328</td>\n",
       "      <td>0.466664</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>327711</th>\n",
       "      <td>327711</td>\n",
       "      <td>454161</td>\n",
       "      <td>454162</td>\n",
       "      <td>i am from india and live abroad  i met a guy f...</td>\n",
       "      <td>t i e t to thapar university to thapar univers...</td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>119</td>\n",
       "      <td>28</td>\n",
       "      <td>21</td>\n",
       "      <td>...</td>\n",
       "      <td>38</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.428565</td>\n",
       "      <td>0.272725</td>\n",
       "      <td>0.149999</td>\n",
       "      <td>0.115384</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>367788</th>\n",
       "      <td>367788</td>\n",
       "      <td>498109</td>\n",
       "      <td>491396</td>\n",
       "      <td>why do so many people in the u s  hate the sou...</td>\n",
       "      <td>my boyfriend doesnt feel guilty when he hurts ...</td>\n",
       "      <td>0</td>\n",
       "      <td>58</td>\n",
       "      <td>145</td>\n",
       "      <td>14</td>\n",
       "      <td>32</td>\n",
       "      <td>...</td>\n",
       "      <td>34</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151235</th>\n",
       "      <td>151235</td>\n",
       "      <td>237843</td>\n",
       "      <td>50930</td>\n",
       "      <td>consequences of bhopal gas tragedy</td>\n",
       "      <td>what was the reason behind the bhopal gas tragedy</td>\n",
       "      <td>0</td>\n",
       "      <td>34</td>\n",
       "      <td>49</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>...</td>\n",
       "      <td>13</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0.749981</td>\n",
       "      <td>0.599988</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.599988</td>\n",
       "      <td>0.333330</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            id    qid1    qid2  \\\n",
       "398782  398782  496695  532029   \n",
       "115086  115086  187729  187730   \n",
       "327711  327711  454161  454162   \n",
       "367788  367788  498109  491396   \n",
       "151235  151235  237843   50930   \n",
       "\n",
       "                                                question1  \\\n",
       "398782  what is the best marketing automation tool for...   \n",
       "115086   i am poor but i want to invest  what should i do   \n",
       "327711  i am from india and live abroad  i met a guy f...   \n",
       "367788  why do so many people in the u s  hate the sou...   \n",
       "151235                 consequences of bhopal gas tragedy   \n",
       "\n",
       "                                                question2  is_duplicate  \\\n",
       "398782  what is the best marketing automation tool for...             1   \n",
       "115086  i am quite poor and i want to be very rich  wh...             0   \n",
       "327711  t i e t to thapar university to thapar univers...             0   \n",
       "367788  my boyfriend doesnt feel guilty when he hurts ...             0   \n",
       "151235  what was the reason behind the bhopal gas tragedy             0   \n",
       "\n",
       "        q1_len  q2_len  q1_num_words  q2_num_words  ...  word_total  \\\n",
       "398782      75      76            13            13  ...          26   \n",
       "115086      48      56            13            16  ...          24   \n",
       "327711     104     119            28            21  ...          38   \n",
       "367788      58     145            14            32  ...          34   \n",
       "151235      34      49             5             9  ...          13   \n",
       "\n",
       "        word_share   cwc_min   cwc_max   csc_min   csc_max   ctc_min  \\\n",
       "398782        0.46  0.874989  0.874989  0.999980  0.999980  0.923070   \n",
       "115086        0.33  0.666644  0.499988  0.714276  0.624992  0.583328   \n",
       "327711        0.11  0.000000  0.000000  0.428565  0.272725  0.149999   \n",
       "367788        0.03  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
       "151235        0.23  0.749981  0.599988  0.000000  0.000000  0.599988   \n",
       "\n",
       "         ctc_max  last_word_eq  first_word_eq  \n",
       "398782  0.923070           1.0            1.0  \n",
       "115086  0.466664           1.0            1.0  \n",
       "327711  0.115384           0.0            0.0  \n",
       "367788  0.000000           0.0            0.0  \n",
       "151235  0.333330           1.0            0.0  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "e8e7e70e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import distance\n",
    "\n",
    "def fetch_length_features(row):\n",
    "    \n",
    "    q1 = row['question1']\n",
    "    q2 = row['question2']\n",
    "    \n",
    "    length_features = [0.0]*3\n",
    "    \n",
    "    # Converting the Sentence into Tokens: \n",
    "    q1_tokens = q1.split()\n",
    "    q2_tokens = q2.split()\n",
    "    \n",
    "    if len(q1_tokens) == 0 or len(q2_tokens) == 0:\n",
    "        return length_features\n",
    "    \n",
    "    # Absolute length features\n",
    "    length_features[0] = abs(len(q1_tokens) - len(q2_tokens))\n",
    "    \n",
    "    #Average Token Length of both Questions\n",
    "    length_features[1] = (len(q1_tokens) + len(q2_tokens))/2\n",
    "    \n",
    "    strs = list(distance.lcsubstrings(q1, q2))\n",
    "    length_features[2] = len(strs[0]) / (min(len(q1), len(q2)) + 1)\n",
    "    \n",
    "    return length_features\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "fbfea680",
   "metadata": {},
   "outputs": [],
   "source": [
    "length_features = new_df.apply(fetch_length_features, axis=1)\n",
    "\n",
    "new_df['abs_len_diff'] = list(map(lambda x: x[0], length_features))\n",
    "new_df['mean_len'] = list(map(lambda x: x[1], length_features))\n",
    "new_df['longest_substr_ratio'] = list(map(lambda x: x[2], length_features))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "71625f3f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>q1_len</th>\n",
       "      <th>q2_len</th>\n",
       "      <th>q1_num_words</th>\n",
       "      <th>q2_num_words</th>\n",
       "      <th>...</th>\n",
       "      <th>cwc_max</th>\n",
       "      <th>csc_min</th>\n",
       "      <th>csc_max</th>\n",
       "      <th>ctc_min</th>\n",
       "      <th>ctc_max</th>\n",
       "      <th>last_word_eq</th>\n",
       "      <th>first_word_eq</th>\n",
       "      <th>abs_len_diff</th>\n",
       "      <th>mean_len</th>\n",
       "      <th>longest_substr_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>398782</th>\n",
       "      <td>398782</td>\n",
       "      <td>496695</td>\n",
       "      <td>532029</td>\n",
       "      <td>what is the best marketing automation tool for...</td>\n",
       "      <td>what is the best marketing automation tool for...</td>\n",
       "      <td>1</td>\n",
       "      <td>75</td>\n",
       "      <td>76</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>...</td>\n",
       "      <td>0.874989</td>\n",
       "      <td>0.999980</td>\n",
       "      <td>0.999980</td>\n",
       "      <td>0.923070</td>\n",
       "      <td>0.923070</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.855263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115086</th>\n",
       "      <td>115086</td>\n",
       "      <td>187729</td>\n",
       "      <td>187730</td>\n",
       "      <td>i am poor but i want to invest  what should i do</td>\n",
       "      <td>i am quite poor and i want to be very rich  wh...</td>\n",
       "      <td>0</td>\n",
       "      <td>48</td>\n",
       "      <td>56</td>\n",
       "      <td>13</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>0.499988</td>\n",
       "      <td>0.714276</td>\n",
       "      <td>0.624992</td>\n",
       "      <td>0.583328</td>\n",
       "      <td>0.466664</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>13.5</td>\n",
       "      <td>0.224490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>327711</th>\n",
       "      <td>327711</td>\n",
       "      <td>454161</td>\n",
       "      <td>454162</td>\n",
       "      <td>i am from india and live abroad  i met a guy f...</td>\n",
       "      <td>t i e t to thapar university to thapar univers...</td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>119</td>\n",
       "      <td>28</td>\n",
       "      <td>21</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.428565</td>\n",
       "      <td>0.272725</td>\n",
       "      <td>0.149999</td>\n",
       "      <td>0.115384</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>0.047619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>367788</th>\n",
       "      <td>367788</td>\n",
       "      <td>498109</td>\n",
       "      <td>491396</td>\n",
       "      <td>why do so many people in the u s  hate the sou...</td>\n",
       "      <td>my boyfriend doesnt feel guilty when he hurts ...</td>\n",
       "      <td>0</td>\n",
       "      <td>58</td>\n",
       "      <td>145</td>\n",
       "      <td>14</td>\n",
       "      <td>32</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>21.5</td>\n",
       "      <td>0.050847</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151235</th>\n",
       "      <td>151235</td>\n",
       "      <td>237843</td>\n",
       "      <td>50930</td>\n",
       "      <td>consequences of bhopal gas tragedy</td>\n",
       "      <td>what was the reason behind the bhopal gas tragedy</td>\n",
       "      <td>0</td>\n",
       "      <td>34</td>\n",
       "      <td>49</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>...</td>\n",
       "      <td>0.599988</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.599988</td>\n",
       "      <td>0.333330</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.542857</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            id    qid1    qid2  \\\n",
       "398782  398782  496695  532029   \n",
       "115086  115086  187729  187730   \n",
       "327711  327711  454161  454162   \n",
       "367788  367788  498109  491396   \n",
       "151235  151235  237843   50930   \n",
       "\n",
       "                                                question1  \\\n",
       "398782  what is the best marketing automation tool for...   \n",
       "115086   i am poor but i want to invest  what should i do   \n",
       "327711  i am from india and live abroad  i met a guy f...   \n",
       "367788  why do so many people in the u s  hate the sou...   \n",
       "151235                 consequences of bhopal gas tragedy   \n",
       "\n",
       "                                                question2  is_duplicate  \\\n",
       "398782  what is the best marketing automation tool for...             1   \n",
       "115086  i am quite poor and i want to be very rich  wh...             0   \n",
       "327711  t i e t to thapar university to thapar univers...             0   \n",
       "367788  my boyfriend doesnt feel guilty when he hurts ...             0   \n",
       "151235  what was the reason behind the bhopal gas tragedy             0   \n",
       "\n",
       "        q1_len  q2_len  q1_num_words  q2_num_words  ...   cwc_max   csc_min  \\\n",
       "398782      75      76            13            13  ...  0.874989  0.999980   \n",
       "115086      48      56            13            16  ...  0.499988  0.714276   \n",
       "327711     104     119            28            21  ...  0.000000  0.428565   \n",
       "367788      58     145            14            32  ...  0.000000  0.000000   \n",
       "151235      34      49             5             9  ...  0.599988  0.000000   \n",
       "\n",
       "         csc_max   ctc_min   ctc_max  last_word_eq  first_word_eq  \\\n",
       "398782  0.999980  0.923070  0.923070           1.0            1.0   \n",
       "115086  0.624992  0.583328  0.466664           1.0            1.0   \n",
       "327711  0.272725  0.149999  0.115384           0.0            0.0   \n",
       "367788  0.000000  0.000000  0.000000           0.0            0.0   \n",
       "151235  0.000000  0.599988  0.333330           1.0            0.0   \n",
       "\n",
       "        abs_len_diff  mean_len  longest_substr_ratio  \n",
       "398782           0.0      13.0              0.855263  \n",
       "115086           3.0      13.5              0.224490  \n",
       "327711           6.0      23.0              0.047619  \n",
       "367788          17.0      21.5              0.050847  \n",
       "151235           4.0       7.0              0.542857  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "afffac43",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting fuzzywuzzy\n",
      "  Downloading fuzzywuzzy-0.18.0-py2.py3-none-any.whl (18 kB)Note: you may need to restart the kernel to use updated packages.\n",
      "Installing collected packages: fuzzywuzzy\n",
      "Successfully installed fuzzywuzzy-0.18.0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "pip install fuzzywuzzy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "ba23a880",
   "metadata": {},
   "outputs": [],
   "source": [
    "from fuzzywuzzy import fuzz\n",
    "\n",
    "def fetch_fuzzy_features(row):\n",
    "    \n",
    "    q1 = row['question1']\n",
    "    q2 = row['question2']\n",
    "    \n",
    "    fuzzy_features = [0.0]*4\n",
    "    \n",
    "    # fuzz_ratio\n",
    "    fuzzy_features[0] = fuzz.QRatio(q1, q2)\n",
    "\n",
    "    # fuzz_partial_ratio\n",
    "    fuzzy_features[1] = fuzz.partial_ratio(q1, q2)\n",
    "\n",
    "    # token_sort_ratio\n",
    "    fuzzy_features[2] = fuzz.token_sort_ratio(q1, q2)\n",
    "\n",
    "    # token_set_ratio\n",
    "    fuzzy_features[3] = fuzz.token_set_ratio(q1, q2)\n",
    "\n",
    "    return fuzzy_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "978955ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "fuzzy_features = new_df.apply(fetch_fuzzy_features, axis=1)\n",
    "\n",
    "# Creating new feature columns for fuzzy features\n",
    "new_df['fuzz_ratio'] = list(map(lambda x: x[0], fuzzy_features))\n",
    "new_df['fuzz_partial_ratio'] = list(map(lambda x: x[1], fuzzy_features))\n",
    "new_df['token_sort_ratio'] = list(map(lambda x: x[2], fuzzy_features))\n",
    "new_df['token_set_ratio'] = list(map(lambda x: x[3], fuzzy_features))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "e277bb70",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(30000, 28)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>q1_len</th>\n",
       "      <th>q2_len</th>\n",
       "      <th>q1_num_words</th>\n",
       "      <th>q2_num_words</th>\n",
       "      <th>...</th>\n",
       "      <th>ctc_max</th>\n",
       "      <th>last_word_eq</th>\n",
       "      <th>first_word_eq</th>\n",
       "      <th>abs_len_diff</th>\n",
       "      <th>mean_len</th>\n",
       "      <th>longest_substr_ratio</th>\n",
       "      <th>fuzz_ratio</th>\n",
       "      <th>fuzz_partial_ratio</th>\n",
       "      <th>token_sort_ratio</th>\n",
       "      <th>token_set_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>398782</th>\n",
       "      <td>398782</td>\n",
       "      <td>496695</td>\n",
       "      <td>532029</td>\n",
       "      <td>what is the best marketing automation tool for...</td>\n",
       "      <td>what is the best marketing automation tool for...</td>\n",
       "      <td>1</td>\n",
       "      <td>75</td>\n",
       "      <td>76</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>...</td>\n",
       "      <td>0.923070</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.855263</td>\n",
       "      <td>99</td>\n",
       "      <td>99</td>\n",
       "      <td>99</td>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115086</th>\n",
       "      <td>115086</td>\n",
       "      <td>187729</td>\n",
       "      <td>187730</td>\n",
       "      <td>i am poor but i want to invest  what should i do</td>\n",
       "      <td>i am quite poor and i want to be very rich  wh...</td>\n",
       "      <td>0</td>\n",
       "      <td>48</td>\n",
       "      <td>56</td>\n",
       "      <td>13</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>0.466664</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>13.5</td>\n",
       "      <td>0.224490</td>\n",
       "      <td>69</td>\n",
       "      <td>67</td>\n",
       "      <td>65</td>\n",
       "      <td>74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>327711</th>\n",
       "      <td>327711</td>\n",
       "      <td>454161</td>\n",
       "      <td>454162</td>\n",
       "      <td>i am from india and live abroad  i met a guy f...</td>\n",
       "      <td>t i e t to thapar university to thapar univers...</td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>119</td>\n",
       "      <td>28</td>\n",
       "      <td>21</td>\n",
       "      <td>...</td>\n",
       "      <td>0.115384</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>0.047619</td>\n",
       "      <td>26</td>\n",
       "      <td>29</td>\n",
       "      <td>34</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>367788</th>\n",
       "      <td>367788</td>\n",
       "      <td>498109</td>\n",
       "      <td>491396</td>\n",
       "      <td>why do so many people in the u s  hate the sou...</td>\n",
       "      <td>my boyfriend doesnt feel guilty when he hurts ...</td>\n",
       "      <td>0</td>\n",
       "      <td>58</td>\n",
       "      <td>145</td>\n",
       "      <td>14</td>\n",
       "      <td>32</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>21.5</td>\n",
       "      <td>0.050847</td>\n",
       "      <td>29</td>\n",
       "      <td>41</td>\n",
       "      <td>23</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151235</th>\n",
       "      <td>151235</td>\n",
       "      <td>237843</td>\n",
       "      <td>50930</td>\n",
       "      <td>consequences of bhopal gas tragedy</td>\n",
       "      <td>what was the reason behind the bhopal gas tragedy</td>\n",
       "      <td>0</td>\n",
       "      <td>34</td>\n",
       "      <td>49</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>...</td>\n",
       "      <td>0.333330</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.542857</td>\n",
       "      <td>55</td>\n",
       "      <td>70</td>\n",
       "      <td>48</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            id    qid1    qid2  \\\n",
       "398782  398782  496695  532029   \n",
       "115086  115086  187729  187730   \n",
       "327711  327711  454161  454162   \n",
       "367788  367788  498109  491396   \n",
       "151235  151235  237843   50930   \n",
       "\n",
       "                                                question1  \\\n",
       "398782  what is the best marketing automation tool for...   \n",
       "115086   i am poor but i want to invest  what should i do   \n",
       "327711  i am from india and live abroad  i met a guy f...   \n",
       "367788  why do so many people in the u s  hate the sou...   \n",
       "151235                 consequences of bhopal gas tragedy   \n",
       "\n",
       "                                                question2  is_duplicate  \\\n",
       "398782  what is the best marketing automation tool for...             1   \n",
       "115086  i am quite poor and i want to be very rich  wh...             0   \n",
       "327711  t i e t to thapar university to thapar univers...             0   \n",
       "367788  my boyfriend doesnt feel guilty when he hurts ...             0   \n",
       "151235  what was the reason behind the bhopal gas tragedy             0   \n",
       "\n",
       "        q1_len  q2_len  q1_num_words  q2_num_words  ...   ctc_max  \\\n",
       "398782      75      76            13            13  ...  0.923070   \n",
       "115086      48      56            13            16  ...  0.466664   \n",
       "327711     104     119            28            21  ...  0.115384   \n",
       "367788      58     145            14            32  ...  0.000000   \n",
       "151235      34      49             5             9  ...  0.333330   \n",
       "\n",
       "        last_word_eq  first_word_eq  abs_len_diff  mean_len  \\\n",
       "398782           1.0            1.0           0.0      13.0   \n",
       "115086           1.0            1.0           3.0      13.5   \n",
       "327711           0.0            0.0           6.0      23.0   \n",
       "367788           0.0            0.0          17.0      21.5   \n",
       "151235           1.0            0.0           4.0       7.0   \n",
       "\n",
       "        longest_substr_ratio  fuzz_ratio  fuzz_partial_ratio  \\\n",
       "398782              0.855263          99                  99   \n",
       "115086              0.224490          69                  67   \n",
       "327711              0.047619          26                  29   \n",
       "367788              0.050847          29                  41   \n",
       "151235              0.542857          55                  70   \n",
       "\n",
       "        token_sort_ratio  token_set_ratio  \n",
       "398782                99               99  \n",
       "115086                65               74  \n",
       "327711                34               43  \n",
       "367788                23               30  \n",
       "151235                48               69  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(new_df.shape)\n",
    "new_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "0bee8351",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<seaborn.axisgrid.PairGrid at 0x23f7fbd8430>"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAasAAAFlCAYAAABV88epAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABSKklEQVR4nO3deXxj5XX4/8/RZkveZI/t2TeGGWDYYViyEMhWIBtJsxSyNCFpKW3I0m+z8O3eX9p+k2/aftMkJBQSQraG0oQkJAwQQsOSsM2wzQIzzD7j8XjfF1mydX5/3KsZjy3Zkq0ryfZ5v17C1tXV1bG4o6Pnuc9zHlFVjDHGmFLmK3YAxhhjzHQsWRljjCl5lqyMMcaUPEtWxhhjSp4lK2OMMSXPkpUxxpiSNyeT1VVXXaWA3eyWr9uU7HyzW55vZgbmZLLq6OgodghmAbHzzZjim5PJyhhjzMLiabISkTtEpE1EdmR4XETkqyKyV0S2icgFXsZjTCnrGRpkb0sX3UMDxQ5lwRjqaWGodV+xwzBZCHh8/DuBrwPfy/D41cB693YJ8E33pzELypb9HfzwmSNsPdTNxWvruO6iFVy0tr7YYc1ro3v+h/CTX0V6DpE84xpip72TyKrzih2WycDTZKWqj4nImil2uQb4njoFCp8SkaiILFXVY17GZUwp2dXcxefu2cGBjkEAmrqPsq2pl29cdzYbltYVObr5Kb7/SUJ3fwASQwD4fvf/KB/qYLj27wlX2ZeEUlTsa1bLgSPj7je524xZMA50xo4nqpS9bQMc6IwVKaL5z9e563iiOr5t2134epqKFJGZTrGTlaTZlnZop4jcICJbRWRre3u7x2GdMBQf5frvPMPulv6CvaYpvkKeb+XB9P8Mw0G/p6+7oPnLJm8LVqA+r6+MmJkqdrJqAlaOu78CaE63o6repqqbVHVTQ0NDQYID+OWLx/jN7na+eP/LBXtNU3yFPN/W1Aa58szFJ21769lLWR0Nevq6C1mi4Uy07pSTtiUv+wzly88qUkRmOsX+GnEvcJOI3IUzsKK31K5X3b/zGB9+1Wp++vxRxpKK35euMWjMzK1dvIibLlcu39DA7pZ+Tl9axdlLKli92K5XeSW88lyGr7md0NGnkN4mkisuIdZwDpXFDsxk5GmyEpEfAVcA9SLSBPwdEARQ1VuBzcBbgL3AEHC9l/HMxO6WAd5+9jIe3tXG7pZ+Ni6rLnZIZh46e1U9Z6+yC/uFFF69CVZvApwuJktUpc3r0YDXTfO4Ah/3MobZGBgZpWtwhMXV5aysi7C/Y8CSlTHGFEGxr1mVtD2t/ayojeDzCY2VZRzqHJr+ScYYY/LOktUUjnQPs7jaGTXUWF3OvnarLGCMMcVgyWoKzT3D1EZCACypLps0F8YYY0xhWLKawtHuYRZVOMmqvrKMYz02SdMYY4rBktUUmrqHWFTpdANGIyE6B0dwxoQYY4wpJEtWU2juiR1vWYUCPsJBP91DiSJHZYwxC48lqym09ceoc5MVQF1FiNY+6wo0xphCs2SVwVhS6YuNUlV+ouRNrSUrY4wpCktWGXQNxqkqC5xUXikaDtLWN1LEqIwxZmGyZJVBx8AINZGTC4lWlwdpH7BkZYwxhWbJKoPOgTg14ZOTVWV5gPZ+S1bGGFNolqwy6BgYmZSsqsuDdFrLyhhjCs6SVQYdAyNUl09IVuEAnYPxIkVkjDELlyWrDDoGRqgoO7kofVV5kC5LVsYYU3CWrDLoGkxQVX5ysqq2ZGWMMUVhySqD7qERqia0rKrDAXqGElZyyRhjCsySVQZdgwkqJ7SsygJ+fD4YjI8VKSpjjFmYLFll0DMUp7Js8kLKVeVBeoetPqAxxhSSJasMeoYSJ5VaSqkqC9AzZNetjDGmkDxPViJylYjsFpG9InJzmsdrROQXIvKiiOwUkeu9jmk6qkrv8OQBFuBMDO61yuvGGFNQniYrEfEDtwBXAxuB60Rk44TdPg68pKrnAlcA/yoiIYpoMD5G0C8E/ZPfnoqyAD3WDWiMMQXldcvqYmCvqu5X1ThwF3DNhH0UqBIRASqBLmDU47im1D0YpzJNFyBARchPt3UDGmNMQXmdrJYDR8bdb3K3jfd14AygGdgOfEpVkxMPJCI3iMhWEdna3t7uVbwAThdgmsEVABUhZ/i6md8Keb4ZY6bndbKSNNsmTlK6EngBWAacB3xdRKonPUn1NlXdpKqbGhoa8h3nSfqGE5OqV6REygLWsloACnm+GWOm53WyagJWjru/AqcFNd71wD3q2AscAE73OK4p9Q4nqCjzp32ssixAt1WxMMaYgvI6WW0B1ovIWnfQxLXAvRP2OQy8EUBEFgOnAfs9jmtKvcMJIqEM3YBlfusGNMaYAkv/iZwnqjoqIjcBDwJ+4A5V3SkiN7qP3wp8AbhTRLbjdBt+XlU7vIxrOk6ySt+yqggF6I1ZsjLGmELyNFkBqOpmYPOEbbeO+70Z+D2v48hF73CCcDBDsioL0GdD140xpqCsgkUa3RlKLQFEQn76Y0UdWW+MMQuOJas0eoYyjwasKAtYsjLGmAKzZJXGVKMBI0E/Q/FRxpK2TIgxxhSKJas0+oYTVGQYDejzCeGQnwFrXRljTMFYskqjLzZKJEM3IDhzrWyZEGOMKZysRwOKyAVTPa6qz80+nNLQH0tQkWHoOjjJqs+GrxtjTMHkMnT9G8AFwDac+VBnA88ACZwSSm/Ie3RFMjAymnFSMEAkZC0rY4wppFy6AQ8CF7r10i7ESVx7VfX1qjpvElUs4SxZH/SnK2voiIT8NtfKGGMKKJdkdbqqbk/dUdUdOIVn55W+WILKsgDOiiXpRcpsrpUxxhRSLt2AL4vIt4Af4HT7fRB42ZOoiqhveDTjHKuUcNBv16yMMaaAcklW1wN/CnzKvf8Y8M28R1RkfbHMw9ZTwkHrBjTGmELKOlmpakxEbgU2q+puD2Mqqr7hBJEME4JTbICFMcYUVtbXrETkHTiLJD7g3j9PRCYu9zHn9cVGMxaxTYmE/FZ53RhjCiiXARZ/B1wM9ACo6gvAmrxHVGT9sczLg6RUhAL0D9sAC2OMKZRcktWoqvZ6FkmJ6BuevmUVtpaVMcYUVC4DLHaIyPsBv4isBz4JPOFNWMXTH0sQnmaARUVZwGoDGmNMAeXSsvoEcCYwAvwn0At82oOYimqqVYJTIiEbum6MMYWUdbJS1SFV/StVvci9/bWqxlKPi8jXvAmxsPqyTFbWsjLGmMLJZ9X116TbKCJXichuEdkrIjdn2OcKEXlBRHaKyKN5jClnvbGp6wKCc81qcGQMVVvTyhhjCiGXa1Y5ExE/cAvwZqAJ2CIi96rqS+P2ieIUyb1KVQ+LSKOXMU2nP4uWVcDnIxgQhuJj01a7MMYYM3ter2d1MU6x2/2qGgfuAq6ZsM/7gXtU9TCAqrZ5HNOU+kdGp01WYMvbG2NMIeUzWaWr/LocODLufpO7bbwNQK2IPCIiz4rIH+YxppwNZNENCO5cKxtkYYwxBZHPPqx/T7MtXQKbeKEnAFwIvBEIA0+KyFOq+spJBxK5AbgBYNWqVbOPNoP+kem7AcFGBM53hTrfjDHZmTZZicgvmJxgjlPVd7g/70zzcBOwctz9FUBzmn06VHUQGBSRx4BzgZOSlareBtwGsGnTJk9GNiSTynB8bNpJwZBKVtYNOF8V4nwzxmQvm27AfwH+FTgADAO3u7cBYMc0z90CrBeRtSISAq4FJtYT/DlwmYgERCQCXEKRlh4ZiI9SHvTj82VeyyolErJrVsYYUyjTtqxU9VEAEfmCqr5u3EO/cFtBUz13VERuAh4E/MAdqrpTRG50H79VVV8WkQeAbUAS+Ja7sGPB9cdGqcxydF845LNrVsYYUyC5XLNqEJFTVHU/gIisBRqme5KqbgY2T9h264T7Xwa+nEMsnshmQnBKecBPnxWzNcaYgsglWX0aeERE9rv31+BegJ4v+rMcCQgQttGAxhhTMFl9MouID6gB1gOnu5t3qeqIV4EVQzbLg6RUhPy2AKMxxhRIVvOsVDUJ3KSqI6r6onubV4kKnCXtw1kmq7AlK2OMKZhcJgU/JCKfEZGVIlKXunkWWRH0Z7FKcEokFLCh68YYUyC5XLP6qPvz4+O2KXBK/sIprv7YaNYtq4qQn35rWRljTEFknaxUda2XgZSC3uFE1i2rsFWwMMaYgsk6WYlIEPhTIDXX6hHgP1R13nxi9w4lqCjLvhtwYMS6AY0xphByuWb1TZwaft9wbxe62+aNvlgi66HrkZDfKlgYY0yB5HLN6iJVPXfc/f8RkRfzHVAx9Q1n37IKh/zEEmOMJRV/FuWZjDHGzFwuLasxEVmXuiMipwBj+Q+pePpymBTsEyEc9FtXoDHGFEAuLavPAr9xK1gIsBq43pOoiiSXScEAkbIAfcMJasJBD6MyxhiTy2jAh0VkPXAaTrKahxUssm9ZAVTaasHGGFMQuYwGfBx4DHgc+N18S1QAA1kuaZ9iCzAaY0xh5HLN6sPAbuDdwBMislVE/p83YRXeyOgYo0mlLJD9W1IR8tNnE4ONMcZzuXQD7heRYSDu3l4PnOFVYIXWHxulosyPSPYj+8JWcskYYwoi62aEiOwDfgYsBr4NnKWqV3kUV8H1x0apzOF6FbjdgNayMsYYz+XSDfhV4DBwHfBJ4MPjh7LPdX3DCSJZrhKcEraJwcYYUxBZJytV/XdVfS/wJuBZ4O+BVzyKq+D6YgkqchhcARAJBugZjnsUkTHGmJRcugH/VUSeBp4GzgX+FmcxxnmhdwYtq0iZrWlljDGFkEs34FPAO1T1TFX9I1X9rqqmlrhHRM5M9yQRuUpEdovIXhG5OdPBReQiERkTkffkEFPe9A2P5tyyqggF7JqVMcYUQC7dgP+tqq1T7PL9iRtExA/cAlwNbASuE5GNGfb7EvBgtvHkW18s++VBUiIhPz2WrIwxxnO5tKymk27M98XAXlXdr6px4C7gmjT7fQL4CdCWx3hy0juUfcX1lIqyAP3DNsDCGGO8ls9kpWm2LQeOjLvf5G47TkSWA+8Cbp3q4CJygzsReWt7e/tsY52kZzhOJMuK6ykVVsFi3vL6fDPG5CafySqddK2tiUntK8DnVXXKCu6qepuqblLVTQ0NDfmK77je4QQVuc6zstqA85bX55sxJje5fTpPLd0Y7iZg5bj7K4DmCftsAu5yK0fUA28RkVFV/VkeY5tWbw5rWaVEQn6G47amlTHGeC2XoesPT7VNVS9N87QtwHoRWSsiIeBa4N7xO6jqWlVdo6prgB8Df1boRAXOaMBcr1n5RNyJwdYVaIwxXpr201lEyoEIUC8itZzo2qsGlk31XFUdFZGbcEb5+YE7VHWniNzoPj7ldapCclpWuTc0K8sD9A4niEZCHkRljDEGsusG/BPg0ziJ6VlOJKs+nGHpU1LVzcDmCdvSJilV/UgW8XiiP5agcibJqixgE4ONMcZj0346q+q/A/8uIp9Q1a8VIKaCU1X63KrruaoIWRULY4zxWi6jAVtEpApARP5aRO4RkQs8iqugBkZGCfl9BHy5D46ssJaVMcZ4LpdP579R1X4ReS1wJfBd4JvehFVYvcMJKstnNjAyYi0rY4zxXC7JKjUP6q3AN1X158C8GFXQM5SgagbXq8BpWfUMWbIyxhgv5ZKsjorIfwDvAzaLSFmOzy9ZfbNoWVWEAnQP2TIhxhjjpVySzftwhqBfpao9QB3wWS+CKrSeGVSvSKksC9A1aMnKGGO8lEvV9SFVvQfoFZFVQBDY5VlkBdQ7nCCS4/IgKZXWDWiMMZ7LpYLFO0RkD3AAeNT9eb9XgRVS91B8RsPWASrKA/RYN6Axxngql27ALwCXAq+o6lqc5e1/50lUBdY5EKeiLDij51rLyhhjvJdLskqoaifgExGfqv4GOM+bsAqrezA+49GAVsHCGGO8l8sndI+IVAKPAT8UkTZgXqyP0TUUZ019xYyem0pWqopbOd4YY0ye5dKyugYYBv4ceADYB7zdi6AKrWdw5vOsQgEfAb8wGJ9yOS5jjDGzkPUntKoOjrv7XQ9iKZquoThV5TO7ZgVQXR6kezA+o0K4xhhjppfNEiH9pF+yXgBV1eq8R1Vgsym3BFATDtI1GGdlXSSPURljjEnJpup6VSECKZZkUhmIjc6qVVRVHqDLhq8bY4xn5kW5pNnoHU4QDvlntSx9ZVmAbqtiYYwxnlnwyapzcISa8MyvV4GzWrCVXDLGGO9YshqIzzpZVYQsWeVFz2H48UfRH74Pjj5b7GiMMSXE82QlIleJyG4R2SsiN6d5/AMiss29PSEi53od03hdg3Gqphpc0fQMPPpF2PETSKYfnl5VHqRjwJLVrAx20Hb7e7hy55u5ZPd1bL/zz+HYi8WOyhhTIjxNViLiB24BrgY2AteJyMYJux0ALlfVc3BKOt3mZUwTdQzGqQ5nSFZ7fw2/+ypE6uHg4/DYl0k3MDIaDtLeH/M20Pnuvv/FX439MWcsr+O9Z1byidFPMfqTP4GxeTHv3BgzS163rC4G9qrqflWNA3fhTC4+TlWfUNVu9+5TwAqPYzpJ10CcynR1AQda4Jnb4cKPwMqL4fw/hO4DsPfhSbvWRIJ0Wstq5o4+x559+3k2tox3bQjy2hUBIpEIvxo5C7b/d7GjM8aUAK+T1XLgyLj7Te62TD5GgSu5t/fHqE7XDfjsd2HVpVDZ6Nz3B+CMd8Czd8LYyEm7VpcH6bBrVjP32//HXZUf5HUrAoT8zqjMN6wO8AO9ymnNarppfsaYhcTrZJVuPHjaTx4ReT1Osvp8hsdvEJGtIrK1vb09bwF2DKapXtF/DI4+B6tfc/L26EqoWjqpdeVMCj45gZks9R1D9z3Cfd0refXyE18aLlzs58XeMF1jYTjwaMHD8up8M8bMjNfJqglYOe7+CqB54k4icg7wLeAat7L7JKp6m6puUtVNDQ0NeQuwrS9GbWRCsnr5l7D8QgiWT37CmtfASz9jfM4tD/pQhcERu76Ssxd/xEuL34aIsKLqxHeb8oBwVr2fR6PvclqzBebV+WaMmRmvk9UWYL2IrBWREHAtcO/4HdxVh+8BPqSqr3gczyQdA3GikdCJDWMJ2Pewc50qndq1zkX/1peObxIRaiMhOgasdZUTVXjhhzwavIxzG/2Tqtaf3eDnf0ZOhz0PQayvSEEaY0qBp8lKVUeBm4AHgZeBu1V1p4jcKCI3urv9LbAI+IaIvCAiW72MaaKOgRGi41tWTc9A5WKI1KV/gojT6trz0EmbayJBS1a5at0B8UEe7a7jzPrJp+I5DT5+1+JDG8+CVx4oQoDGmFLh+TwrVd2sqhtUdZ2q/pO77VZVvdX9/Y9UtVZVz3Nvm7yOKWVwZJSxpBIOjlvSft//wNJppnotOQcOPwFjJwZV1EaCtPVZssrJjnsYWXkZ29rH2LjIP+nhhoiPkB/2NbwBtt9dhACNMaViQVewaO8fobYidKL7KT7oTERtPHPqJ4ZroGqJMwjDFQ0Hae2zuVZZU4WXfsa2yteyospHJJi+NuPGRT6e1LPh0BMw0l/gII0xpWJBJ6u2/hFqx5daOroVatdAKDz9kxs3woHHjt+tCYdosWSVvY5XYGSAZ4aXsaE282m4vtbPU21+5wvEhK5XY8zCsaCTVUtfjNqKcYMrDjzmJKFsLD7LSW5jCQBqK4Ic67VklbVd98HKS3j62Bgb6jKfhqfV+djaMgYrLoKXf1HAAI0xpWRhJ6veYWpTIwHHRtwuwDOye3J5NVQ0QMs2AGojIVosWWXv5XtJLr+IF9rG2FA7+XpVypIKYWRMOVp7kTO/zf1yYIxZWBZ0smruGZesjj4P1cshVJH9ARo3OgMtcJKVXbPK0kA7dOxhX/mZVASFaHnmtcREhNPq/DzXVwPVS+HwkwUM1BhTKhZ0sjraM8yiSjdZHX4CGk7P7QCNZ8Dhp0GTLKp0rlmplQaa3p4HYfkFPNsuU16vSlkX9bG1ZdSZMrBrcwECNMaUmgWdrI71xKirCIGOwZEtsDjL61UpFfUQKIOOPURCAQI+H91D1k01LbdCyNaWMU7JIlmtr01dt7oYdt9ntQKNWYAWdrLqjbGoIgRtu6C8CsK1uR+k4YzjXVONVWU09wznOcp5JhFzlltZvolnW0dZP8X1qpRToj729SSJVa+F0ZgzktAYs6As2GQVS4zRHxt1RgMeesJJOjPReMbx61b1VWU0dVuymtLBx6HuFHqopnVQWVWV+XpVSsgvrKr28WJ70mldWVegMQvOgk1WTd1DNFaX4ROcZJPtkPWJapbDyAD0NrGoIkRT91Be45x3dv0SVmzi+bZRTq314fdNn6wATq318VzrqDOEfZcNYTdmoVmwyepw1xCNVeXQddBZrr5qycwOJD63dfUkDVVlHOwYzGuc80oy6bSKVlzClmNjnBrN/vRbX+vj6WNjTqmr9t3OiEJjzIKxYJPVka5hGipDcOh3TqtKsvuGn1bjmXDwcRZXl3PAklVmR591pgbULOfpY2OcVjf99aqUDXU+nm8bI+kLwIpNsNu6Ao1ZSBZsstrfPkB9VRkc/C0snqYW4HTq1kJ/K0sCgxzstG7AjF76Oay8mJEx5aXOMU7NYiRgSl25j0hA2NfjXrfa+VMPAzXGlJoFm6xeaR1geWgA4v3OCsCz4fPD4o00djxNW3+M+GgyP0HOJ27hWla/hm1tYyyvzFy8NpPT6nw81eyWXmraAsPd3sRqjCk5CzZZ7e8YYHnPc7DkbOe602wtPovAwUdprCrnUKd1BU7S/BygULuWp5pHOX2KeoCZnL7Iz5NHRyEYhmXnO/O1jDELwoJMVoMjo/QMJag/8pBzwT4f6tbBUDsrq2B3qy1lMcm2/4Y1l4EIvz06xumLcj/1Ni7y8WTzGElVWP0a2PZfHgRqjClFCzJZ7W7tZ2WV4BN16gHmg88HS85l2WgTu45ZsjrJ2Cjs+DGsfR3DCXUWW6zPfnBFSkPERzgIu7qSTlfgsRehr9mDgI0xpWZBJqudR3tZo82w7LzZjQKcaNn5rOzdws7mnvwdcz7Y9zBUNELNSp4+NsbaGh/hwMze93Ma/Dx6OOGUuVr9GnjhR3kO1hhTihZkstp2qINVQ9udwqj5VL2UU8qH2Haowwrajrfl27DuDQA8fCjBuY25t6pSzm7w8/DhMefOqW+CZ+905m8ZY+Y1z5OViFwlIrtFZK+I3JzmcRGRr7qPbxORC7yO6fm9RzilNgRlVXk/dsPqMxiLx2whxpSew3DkKVh7OarKQ4dGOX8Wyeqseh8vd47RNZyE+g0QLIe9v85jwMVxtLWFrl2P0/3Mj+ja/TuOHDtW7JDmvd6m3SR2/IzRLd8hvu+39PX1FDskMwVPk5WI+IFbgKuBjcB1IjKxrtHVwHr3dgPwTS9j6ukfonlQWXPqDGsBTkOWnsN6XxPPbtvuyfHnnCe/4bSAguW82J4k4IMVWdQDzCTkF86u9/PQoVGnC/e0t8IT/57HgAuvuauLmu3fo+6ut1G7+UbqfvQWorvuoqO3q9ihzVt9R3dR/chfEfzxhwnc92lCP7yGisO/KXZYZgpet6wuBvaq6n5VjQN3AddM2Oca4HvqeAqIishSrwJ65tH72BDsxF+32psX8Ac5oz7E408/7c3x55KBdnjhh3D62wH42Z44lyzxI7O8TnjpMj/3vOIuxbL2ddCxB5q2zjbaogl1H6Dyd//npG1Vj38BOg4VKaL5L9K5A9n70IkNyVH8D/0Ng8d2Fy8oMyWvk9Vy4Mi4+03utlz3QURuEJGtIrK1vX2GdeFGR/j11h2cvSz/3X/jnX3qGh7rqkWPPu/p65S8x74Mp1wOFfWMjCk/3zPKZSsDsz7s+Yv9vNw5xuG+JPiDcPZ74aG/y+s6V3k537KkQ92gE667JcdQm/TsGUn33vYewZ+wOZKlyutkle4r9MRPlGz2QVVvU9VNqrqpoaFhRsGMPf4VHk6cxflrZ/b8bC2PluEPhdl2z5cX7sX/tpedeVDnXAvAvXsTrK4RFlfM/pQL+YXXrQzw3R0jzoZT3wR9R+Hle2d97JR8nG9Zv1bNyslrqVXUozWrPH3dhSxZt27SSGBdewXxyAwLWhvPeZ2smoDxtYxWABMnxmSzz+y1bOfR3z7GospyllZ6+2eLCJeuquDHvafDU7d4+lolaTQO99wA538QymsYTSq3PDfC1acE8/YSV64JcPfuBJ3DSfAF4JI/gfs+A4MdeXuNQmlcdRp97/o+Y4vWAzBWfxp97/wejStPLXJk81d/7Rkk3/41Z7VvQFe/lsQVf011/bIiR2YymX2fzNS2AOtFZC1wFLgWeP+Efe4FbhKRu4BLgF5Vze9QqMEOuOv93B76K65YVZ7XQ2fy+tVB/vfBC/iLRz9PdOm5zrWVhUAV7v+sU119/ZUA/GBnnKqQcHZ9/r4k1Ed8vHZ5gP/7zAhfujwMi8+Cda+H//oQ/OHPnHlYc0j1hstorf4pvuFutHwRjUvzNFndpFVXvxTqP8TQ4vPxJ4aIVy6nqt7e81LmaRNDVUeBm4AHgZeBu1V1p4jcKCI3urttBvYDe4HbgT/LaxB9x+DOt/GbmndxMF7NZStmPmw6F4vCPi5dHuRfon8Nd/8h7H+kIK9bVMkxeOBmp5L9a/4cRHila4yvPDvCh88KzXpgxUTvPi3Ibw6P8tBBd7DFue93igrf9QGIz71rD4uXrKRh7TmWqAoosvwsytZcbIlqDvB8npWqblbVDaq6TlX/yd12q6re6v6uqvpx9/GzVTU/w7qSSace3X9cxpHGK/js4Uv52DkhAlmuTJsP7z0tyK/aa/jJmn+AH38UHvp7GJmnpZjadsF33w6Hn4Q3fwFCEXZ0jPHB+4b44MYQy6vyf6pVBIVPXBDis4/EnITl88Nlf+EUJv6P1zlJ0xgzL8hcrLSwadMm3bo1TU5LJuHAo3DgMacWXbACNn2U77at45EjCW44t/BdQ0f6k/zLMyO8dskYH9D7OKvrV5SdcZUzKGDxWRBd5UxsnUtUIdYLXfvg6HPw8i+gZRuc9W4GTnkb27vgZ3sSPHAgwYfPCvGqZd72Nu/pHuOW5+Kc3eDnAxuDXLHCjxx+Ap7/vnNNYuO74NQ3wtKMRYun/AaT8XwzZmYK9415HpmTyUpE2oFJk1DOW+wrf/7GyuMrKe7vTsZA+bn/9/z/HvpYcGXy6Ez/WCHNCMVsDROmw1e34E7QqPZSpQMzed9yfr/jBGn1NQrAD2M3jSzVdvWJyJqo7/g3FPmHvmczPL1DVa/KGEyG880j9cDcGyXimKuxFzruKc83k96cTFaFJiJbVXVTsePIlcU998zlv32uxl7KcYvIE6r66lk8/yPAJlW9aYbPP+g+v2OmsYjIO4FXVPWlmcSQsiAL2RpjzFwwm0SVb7OI5Z045fZmxZKVMcaUKBEZcH8uFZHHROQFEdkhIpdN8ZzrReQVEXkUeM247XeKyHvSHPsK99g/FZGXRORWkcnLp6f2d3//nIhsF5EXReSL7rY/FpEt7rafiEhERF4NvAP4shv7Ovf2gIg8KyKPi8jp2bwXXs+zmi9uK3YAM2Rxzz1z+W+fq7HPhbjfDzyoqv/kFgiPpNvJrav6D8CFQC/wGyCbum8X47R+DgEPAL8P/DjDa1yN01q6RFWHRKTOfegeVb3d3ecfgY+p6tdE5F7gl6r6Y/exh4EbVXWPiFwCfAN4w3QBWrLKgqrOhZN5Eot77pnLf/tcjX2OxL0FuENEgsDPVPWFDPtdAjyiqu0AIvJfwIYsjv+Mqu53n/Mj4LVkSFbAm4DvqOoQgKqmlgc4y01SUaASZ37tSUSkEng18N/j5l1mNUzbugGNMabEqepjwOtwKgF9X0T+cKrdM2wfxf3MFydThKZ4zlQj7zKN1r0TuElVz8Zp3aWbk+MDelT1vHG3rNZrsmRljDElTkRWA21uN9u3gUyL1D4NXCEii9xW2HvHPXYQp3sQnKWZxhfrvFhE1rrXqv4AmGpG/a+Aj4pIxI0t1Q1YBRxzX/cD4/bvdx9DVfuAAyLyXve5IiLnTvFax1myMsaY0ncF8IKIPA+8G0i74qhbV/XvgSeBXwPPjXv4duByEXkGp7twfE2yJ4EvAjuAA8BPMwWiqg/g1HTdKiIvAJ9xH/obnGT5ELBr3FPuAj4rIs+LyDqcRPYxEXkR2MnkNQ7TmpPzrK666ip94IEHih2GmT+mnLBt55vJs5IqECAiVwCfUdW3FTmUKc3JllVHx1ycJG/mKjvfjCk+Gw1ojDFzkIg8zeSRdB9S1e25HEdVHwEeyVNYnvE0WYnIHcDbcC4MnpXmccHpe30LMAR8RFWfm7hfsfT0tBPu2gOjMeLRU6hqXFPskOa14aM78fUdRSvqKV+V6fqxMQZAVS8pdgyF5HXL6k7g68D3Mjx+NbDevV0CfNP9WXQ9TbuofuE2fM9+BzRJaNkFjFz5ZcpWl2QJsTlvdPevCP/iJhhohbIqkr/3zwxtuJLKqsXFDs0YUwK8XnzxMaBril2uAb7nrmn1FBB1Z2AXXVXHi/i2fhs0CYA0P0fw+e/QP9Rb5Mjmn+GjOwn88pNOogIY6cd3358Tat9b3MCMMSWj2AMslgNHxt1vcrcVnbRO7vb1HXgEetsKH8w85xtogf5jJ29MjiK9h4sTUBHFR5Mkk3NvhK4xXit2sko3hDPtv1QRuUFEtorI1vb2do/DAl20fvK2pechkbo0e5vZ0MgiCNeevFEErSxeI7vQ5xvAWFJ53388yXW3P8XoWLIgr2nMbInIVSKyW0T2isjNXr1OsZNVE7By3P0VQHO6HVX1NlXdpKqbGhoaPA9suPEC9JTXn9hQtYTRi/+MyppFnr/2QlO+8jzGrvwi+N3qLyIkX3czI9G1RYup0OcbwEMvtTIcH6NjYISnD0zVe25MaXCL6t6CM/5gI3CdiMx6OZB0ij10/V7gJhG5C2dgRa87A7voKledTd+b/4Vw9yuQGGZ00XrCKzIui25mKbb2DQQ++At8vYfQyiWM1KyhqmF1scMqqEdfaWPTmlqSqvxyWzOvObW+2CGZeWbNzfe9H/hnYBVwGPjLg19863/O4pAXA3vHFcG9C2cswqwWWkzH66HrP8IpE1IvIk3A3+HWo1LVW4HNOMPW9+IMXb/ey3hyVb30VFh6KnByES2TfxU1jVDTCFwKnFxhc6F4Ym8nN16xjmRS+c4TB4sdjpln3ER1OyeWF1kN3L7m5vuYRcJKN+7AkxHdniYrVb1umscV+LiXMRgzF/THErT2x1hVG2FMlSNdQ8QSY5QH/cUOzcwf/8zkdbAi7vaZJqusxx3MVrGvWRljgD1tA6ysjeDzCUG/j2XRMK+09hc7LDO/rMpxezayHncwW5asjCkBe1r7WV4bPn5/VV2EXccsWZm8yjQXZDZzRLYA693lRULAtThjEfLOkpUxJWB3Sz9La04kq8aqMg52Dk7xDGNy9pc4YwPGG3K3z4iqjgI34awK/DJwt6runHGEU7BkZUwJ2N8xyNLqEwurLq4uZ3+HJSuTP+4gij8GDuFcVzoE/PEsRwOiqptVdYOqrlPVf8pDqGkVe+i6MQZo6h7myjOXHL+/uLqMh3e1FjEiMx+5iWlWyalYrGVlTJGpKs09wzRUnljtYXF1OUe6hpmLi6Ma4wVLVsYUWc9QAhGoKDvR0VFZFiCpSl9stIiRGVM6LFkZU2RHuodYMu56FYCIUF9ZRktvrEhRGVNaLFkZU2TNPTEWVUyu2bGoIsSx3uEiRGRM6bFkZUyRtfbFiEYmJ6vaihDHrGVlDGDJypiiO9Y7TE14cvXJ2kiQYz3WsjKlTUTuEJE2Ednh5etYsjKmyI71xqhL0w1YWxHiqCUrU/ruBK7y+kVsnpUxRdbSG2Pj0upJ2+siIV5u7itCRGbe+vuaSUuE8Pe9s50U/JiIrMlDdFOylpUxRdbaF6M2zTWraCRE+8BIESIy85KTqG7HWRpE3J+3u9tLniUrY4qsfWAkbbKqCQfpGIgXISIzT021REjJs2RlTBHFEmPER5NUlE1et6omHKR3OMFY0qpYmLzwYomQgrFkZUwRdQyMEA2HEJm8hp3fJ1SXB+gctK5AkxdeLBFSMJasjCmijoE40cjkYesp0UiItj5LViYv8r5ECICI/Ah4EjhNRJpE5GOzOV4mnicrEblKRHaLyF4RuTnN4zUi8gsReVFEdorI9V7HZEypaO8fSTvHKiUaCdLeb8nK5IEz6m/SEiF5GA14naouVdWgqq5Q1W/nIdpJPB26LiJ+4BbgzTjLH28RkXtV9aVxu30ceElV3y4iDcBuEfmhqtqVZTPvdQyMUD1FsnIGWViyMnniJCZbIiSNi4G9qrrfTT53AddM2EeBKnE67SuBLsBKTZsFoaN/hOryzN8ZK8sCdA3a9zZjvE5Wy4Ej4+43udvG+zpwBtAMbAc+papJj+MypiS09U/dsqoqD9pcK2PwPllNHuLktKTGuxJ4AVgGnAd8XUQmTecXkRtEZKuIbG1vb893nMacpFDnW/vACNXlU3UDBuyalTF4n6yagJXj7q/AaUGNdz1wjzr2AgeA0yceSFVvU9VNqrqpoaHBs4CNgcKdb53TXLOqKg/SaRODjfE8WW0B1ovIWhEJAdcC907Y5zDwRgARWQycBuz3OC5jSkL3YGLKa1Y14aDNszIGj0cDquqoiNwEPAj4gTtUdaeI3Og+fivwBeBOEdmO0234eVXt8DIuY0pF11Ccqim6AavLgzbAwhgKUHVdVTcDmydsu3Xc783A73kdhzGlJplUeoenbllVhwN0DyZQ1bRVLoxZKKyChTFF0hdLUB70EfBn/mdYFvDj88FQfKyAkS0cR3uGedO/Pco//vKl6Xc2RWXJypgi6RyME51icEWKdQV65z8e3ceaRRHuef4oe9v6ix2OmYIlK2OKpGswPuWw9ZSacJDuIUtW+TaWVH72/FGuOW85l55Sx70vTByobEqJJStjiqRrME5VFi2rqnKrYuGFl5r7iEZC1FeWceHqOn69q63YIZkpWLIypkh6huJUlk0/xslKLnnj6QOdnL6kCoB1DRXsaxsglrBrg6XKkpUxRdI1mKAiNHnRxYkqrWXliWcPdbOuoRJwBrKsrIuws7mvyFGZTLIauu7OgUq3XKkAqqrn5DUqYxaAzsERKrO4ZlURsmTlhV0t/Vy+4UR1krX1FWxv6uHC1bVFjMpkku08q/vdn993f34AZ9Gu7+Y9ImMWiM6BOA2VZdPuZyWX8i+WGONozzDLouHj25bVlPNK60ARozJTybYb8DWq+jlV3e7ebgauVNVDqnrIywCNma+6h+JUTTEhOKW6PEDXkJVcyqe9bQMsrSknOG6O2/LaCK+02vD1UpVtsqoQkdem7ojIq4EKb0IyZmHoHoxTmUWyqix3qliY/NnfMXhSqwpgeTTMvnZrWZWqbLsBPwbcISI1ONeueoGPehaVMQtA91BiyrqAKZVlAZtnlWcHOwZprDq5C7Y2EmRkNEnvUIKayPT/X0xhZZWsVPVZ4Fx3nSlR1V5vwzJm/usZilOVxdD1qvIgPUPWssqn/R0DLK4uP2mbiLC0ppxDXYOcE4kWJzCTUVbdgCKyWES+DfyXqvaKyEYR+ZjHsRkzb40llYGRUSqySlYBeoedYrYmPw62D7FkQrICaKwu53DXUBEiMtPJ9prVnTjLfCxz778CfNqDeIxZEHqHE0RCAfy+6SupB/0+gn5h0IrZ5s3h7qFJLSuA+ooQhzotWZWibJNVvareDSTBWacKsH85xsxQ91CcmnD2K/RUlQfptrlWeRFLjDEQGyWa5rpUQ1U5BzsGixCVmU62yWpQRBbhTgwWkUtxBlkYY2agZyie1YTglGorZps3R3uGaagK4UuzPtji6jIOWTdgScr2q93/wlmOfp2I/A5oAN7jWVTGzHPdg4msBlekVJUF6LZBFnlxtHuY+gyTsRdVltHSGytwRCYb2Y4GfE5ELgdOwymxtFtVj//LEZE3q+pDHsVozLzTPRTPanBFSmV5gB5rWeXF0Z5hFmVKVhUhWvpitjJzCcq6kK2qjqrqTlXdMT5Rub6U57iMmdd6hhI5JauKkN+uWeVJU/cQiypCaR8rD/opD/isFmMJylfV9YxfQUTkKhHZLSJ7ReTmDPtcISIviMhOEXk0TzEZU7K6h+JZVVxPiVg3YN40dQ+zqDJ9sgJoqCqjuce6AktNvpJV2gkgIuIHbgGuBjYC14nIxgn7RIFvAO9Q1TOB9+YpJmNKVudAdnUBU2xNq/w51hujriJzAeFFlWUc7bFBFqXG6/WsLgb2qup+VY0DdwHXTNjn/cA9qnoYQFVtuU4z73UPxaksy340oCWr/GnpjVGXoRsQnLJLx2yQRcnJV7I6mGH7cuDIuPtN7rbxNgC1IvKIiDwrIn+Yp5iMKVnZVlxPqSq3+oD5oKq09ceoi7jJaiwOz/8AjjxzfJ9oOGTJqgRN+a9FRH5/qsdV9R73Z6b90l3LmthlGAAuBN4IhIEnReQpVX1lQiw3ADcArFq1aqqwjJk1r8+37hwHWFSWBaw+YB70DY/iFyGcul74zO3QtRde/gVc/UWoXUttRYij3dYNWGqma1m93b19DPg2zqKLHwC+BXwwi+M3ASvH3V8BNKfZ5wFVHVTVDuAx4NyJB1LV21R1k6puamhomPiwMXnl9fnWN5zI8ZpVkJ5hS1azdaxv3LD1WC8ceATOfh+svQx2/gyAOnf4uiktUyYrVb1eVa/HaQ1tVNV3q+q7gTOzPP4WYL2IrBWREHAtzuTi8X4OXCYiARGJAJcAL+f0Vxgzx/QMJ6jMcZ5V77B1A85WS2/sxLD1Q7+FhtMhVAHLzofDT8Ko00XY2meLXZaabP+1rFHVY+Put+Jca5qSqo6KyE04RXD9wB2qulNEbnQfv1VVXxaRB4BtOLUHv6WqO3L6K4yZQ4bjY6BQFsj+knEk5CcWT5IYS560uq3JTVvfyImagEeegXr3Y6ysCioXQ9tL1NafQ1u/TQwuNdkmq0dE5EHgRzitrGuB32TzRFXdDGyesO3WCfe/DHw5y1iMmdO6h+JUhQM5fRD6RNwqFgkaqjIPuzZTa+2LURMOQnIUWnfAhqtPPFh3CjS/QGTZBQhCX2zU2deUhKy+oqnqTcCtONeSzgNuU9VPeBiXMfNW91Cc6hyK2KZUW8mlWTvWGyMaCUH3QSivg1DkxIN1a+HYiwAsqgzR3m/XrUrJtC0rEfEB21T1LOCn3odkzPzWM5Sgsiz76hUplVbFYtZa+mKcuyIK7bsguuLkB6uXQ88RGEtQGwnS1jfCqY1VRYnTTDZty0pVk8CLImLjxY3Jg1wnBKdU2lyrWWvti1EbCULbLic5jRcog0gd9B6iJhyi1VpWJSXba1ZLgZ0i8gxwfGUyVX2HJ1EZM485RWxn1rKybsDZae8fcboBu/bBGW+fvEPNcmjfQzSykTYbEVhSsk1W/+BpFMYsID05Lg+SUmETg2clmVS6BuNEy4D+Y87ov4kqGqH7ADWRc2m1uVYlJdsBFo8Cu4Aq9/ayu80Yk6OuwXhOc6xSKkJWH3A2uofiREJ+ggPNEK4Df5qu2Kol0H2Q2oiVXCo1WSUrEXkf8AxORfT3AU+LiK0UbMwMdM4wWVWWW7KajfaBEWorQtBz0ElK6VQuge5DRCMB2vqtG7CUZPsv5q+Ai1IV0UWkAfg18GOvAjNmvuoeSnBqY2XOz6sqC3Coc3D6HU1abX0j1EZCzoi/igwltMoqASXqG6bdklVJyXYqvG/C0h2dOTzXGDNOz2CcqhmOBrSW1cy19Y84k3x7DkFFffqdRKByMdFEOx0DlqxKSbYJ5wEReVBEPiIiHwHuY0JVCmNMdnqGE1TmUMQ2pao8aAMsZqG9f4Tq8gD0NjkDKTKpqCcy2MTomDIUHy1cgGZKWf2LUdXPusuFvBZn2Y/bVNUmCBszAz1Dcapmcs2qLGCV12ehtW+YmnAABlqhYlHmHSN1SF8TdRXLaO8fYfWi3P9fmfzL6v+CiHwUeDy1fpUxZmbGkspgfGxGAyyqygP0DieswOoMtfaNsK5qFEJV4M+8UjCRRmjbQTTyGjdZVRQuSJNRtt2Aa4D/EJF9InK3iHxCRM7zLixj5qfe4QQVIT8+X+7JJuj3EfL76B+xrqmZaO8foTbZnfl6VUrFIuhrJhoJ2ojAEpLtPKu/VdU3AGcBvwU+CzzrZWDGzEddg3GqZ1HJuzocoGfQugJnon1ghJpEu1NSaSrhOhjqpKY8YCMCS0i286z+WkTuB34FnAp8BmfVX2NMDmZ6vSqlqjxIl5VcmpGOgRGiI0edZDQVfwBCVVT5Rmiz+oAlI9t/Nb8PjOKMAnwUeEpV7f+iMTnqGozntJz9RFVWzHZGhuNjJMaUyMBhWJzFQucV9UQZoLXPqq6Ximy7AS8A3ohTxeLNwHYR+a2XgRkzHzlFbGeRrKyY7Yx0DIxQFwkhAy3TdwMCROqIJrvtmlUJyXY04FnAZcDlwCbgCPC4h3EZMy91Dc2s1FJKRVmALrtmlbO2/pgzbL23ffpuQIBwlOhIG+0Dy6ff1xREtqMBv4RTwParwBmq+npV/dtsnigiV4nIbhHZKyI3T7HfRSIyZjUHzXzWNTiziusplWUBugbt236u2vtHqC1TCIYhMMWw9ZRwHdGRo3QOWCu2VGQ7KfitUz0uIj9R1Xen2e4HbsHpOmwCtojIvar6Upr9vgQ8mG3gxsxFHQMj1FeUzfj5VeUB+wCdgfb+Eap9IxCZZth6SriOmqEn6RqMk0zqjKYamPzKV32/UzJsvxjYq6r7VTUO3AVck2a/TwA/AdrSPGbMvDH7ARZBqw84A619I9QwCOHa7J4QriMwdIxIyG8DWkpEvpKVZti+HOf6VkqTu+04EVkOvAu4NU+xGFOyugZmOc/KitnOSGtfjBrtgXA0uyeEwiA+asN+2q2gbUnwunJ6urbzxMT2FeDzqjo25YFEbhCRrSKytb29PV/xGZOWV+dbdx7mWdk3/dy19bsTgrNtWQGEF1EbStry9iUiX8kqU4duE7By3P0VQPOEfTYBd4nIQeA9wDdE5J0TD6Sqt6nqJlXd1NCQYS0aY/LEq/OteyhB1SxaVs48KxsNmKu2/hjReEuOyaqWGn/MqliUiGwrWHxqmm2fz/DULcB6EVkrIiHgWuDe8Tuo6lpVXaOqa3AWc/wzVf1ZNnEZM5ckxpIMJ8aIhPwzPkalW8w2mczU827S6RyIE401ZTdsPSUcpYZBm2tVIrJtWX04zbaPpH5R1V+le5KqjgI34Yzyexm4W1V3isiNInJjjrEaM6d1D8apLg/gm0XF9IDPRzjop9eWCslaMql0DcaJJlqhrDr7J4ajVI/10NpnxXpKwZSd5yJyHfB+YK2IjG8RVeOsFjwtVd3MhIUaVTXtYApV/Ug2xzRmLuoamt3gipRoJEjnYJzaiizmCxm6huJEgkIgXAO+HK58hGuJjrazz5JVSZjuSu8TwDGgHvjXcdv7gW1eBWXMfNQ1EKe6fPbJqtqGr+ekrW+EurIkRHK4XgUQrqM2vs26AUvElMlKVQ8Bh0TkTcCwqiZFZANwOrC9EAEaM190DMapDs9+1VlnYrB9gGarrT9GbSAO5bkmqyjRkWbarfJ6Sci2TfwYUO7OiXoYuB6406ugjJmPOgdGqMpHyyocoNNaVllr6x+hmiEor8ntif4Q0dAYHdayKgnZJitR1SGcpUK+pqrvAjZ6F5Yx80/HwMis5lilVJZZN2Au2vtH3AnBObasgHC4kmQyyYCtzlx0WScrEXkV8AGcNa0g+7WwjDFAe39+BlhU2Qq2OWnpjVEz2jmjZCWRKHVlSdpskEXRZZusPgX8b+Cn7tDzU4DfeBeWMfNPx8BI3gZYWAmg7LX0DVOXaMltjlVKeZTawIgNsigB07aO3Irob1fVd6S2qep+4JNeBmbMfOPUBZx9h0RNOGjXUXLQ2jvMa5JdUFaZ+5PLo+6KwdayKrZpW1Zuzb4LCxCLMfNax2B+WlY14SAd1rLKWnvvELXlAZjJZOxwHTXJHut2LQHZfs173p0U/N/AYGqjqt7jSVTGzENdg3GikWmSVdtLID5oOD3jLjURG2CRLVWlY2iM6JIZriEWqSU6dphjvdayKrZsk1UdTsWKN4zbpoAlK2OyMBQfZTSphINT1AV87nuw72FQhQ1XwnkfSLtbZVmAwfgY8dEkoYDXCyfMbd1DCcp9SULhqpkdoDxK3Wgb+3uG8huYyVm2KwVf73UgxsxnHf1x6iIhJFNXVNtLsOdX8KqPgybhya/Dik1Qf9qkXX0iRMNBOgdHWFoT9jjyua21L0ZdYGRGIwEB8AeoDSZp7e7Pb2AmZ9PVBvycqv5fEfkaaRZYVFUbZGFMFtoHYtRM1QX43Pdh3RsgVOHcP+UNzrbf+8e0u0cjQTr645asptHSG6NO+meerIDacIDWvuE8RmVmYro+hNTSH/uAZ9PcjDFZaO+PE800x6r3MPQchGXnn9i2/ALo3At9R9M+xQZZZKelL0at9sxs2LqrtrKctsEkqrYsSzFN1w3YKiKrccorvb4A8RgzL7UPjGQetr7n17D0fPCNu57lDzrJa8+v4cLJK/TUhIO0Wc26aR3rGSY62gHhyd2p2SqPVBKUJL3DCaIRq3RfLNO1rL4JPIBTuHbruNuz7k9jTBba+2IZ6gIqHHgMlp47+aElZ8PBx0nTA091OGjLrWehubOXOt8AhGbRXRquoz4wZCMCi2zKZKWqX1PVM4A7VPWUcbe1qnpKgWI0Zs5r6RshGk7zrbxrn/Ozasnkx6qXw1gcug9NeigaDtFiE1Wndayrl9ryWY6YDNeyiD5aLFkVVVb/F1X1T70OxJj5rKUvRm1FmpbV4Weg8fT0E1ZFnNGATVsmPVQbCVpVhSy09saoi8yyakhkEbXJLmtZFZlN0jCmANr6YtSmu97R9AwsWp/5iQ0b4PDTkzZHIyFarRtwWi1DyqKKGU4ITimrJKq9HOvsyUtMZmYsWRlTAG39I5OTVbwfeo5A7ZrMT6xdC937IXHy0OloJGglgKbRH0swmoTKyhlOCE4Roa5MaWrrzE9gZkY8T1YicpWI7BaRvSJyc5rHPyAi29zbEyKS5kqzMXNXYswZSVYzceh6y3aoXe2M/MskEIKaldC646TNtZEQnYMjJJM2nDqTY70xGnwDSMXMh62n1Ed8HO0ayENUZqY8TVZuxfZbgKtxFmu8TkQmLtp4ALhcVc8BvgDc5mVMxhRax8AI0XAQv2/CdanmF6Bu7fQHqDvF2XecUMBHOOina8hqBGZytGeYerohUj/rYy2qCHKs3xZgLCavW1YXA3tVdb+qxoG7gGvG76CqT6hqt3v3KWCFxzEZU1AtvTHqKtJcrzq2DWqzGFRbuxZatk3aXF9ZZiPUpnCss8+ZEFxePetj1VdFaI35rSVbRF4nq+XAkXH3m9xtmXwMuN/TiIwpsGPpklWsF4Y6nOHp06lZAX3NED+5G6quImQj1KZwtOUYdaExp4r9LJVV1RGWBB2Ddp2wWLxOVumqdqb9aiIir8dJVp/P8PgNIrJVRLa2t7fnMURjJsvn+dbcMzw5WbW95Ays8GXxT9AfgOgq5znj1FWEaOm1mnWZHG7roT5fpRMj9TRKN8099uWgWLxOVk3AynH3VwDNE3cSkXOAbwHXqGraITeqepuqblLVTQ0NDZ4Ea0xKPs+3Y71phq23bHcSULaiq6Dl5EEWNeGgtaymcKRnhMbIFEuy5KK8mnrtoamtKz/HMznzOlltAdaLyFoRCQHXAveO30FEVuGsi/UhVX3F43iMKbij3cMsqpyYrHZAdHX2B6ld4yS4cRZVhjjSZessZdI8KDRU56lpJT4WhRIcaU5fWNh4z9NkpaqjwE3Ag8DLwN2qulNEbhSRG93d/hZYBHxDRF4QEas5aOaV5p7hkyemJoahr8m5FpWtmpXQcwjGTlwzqa8so6nHugHTGRkdozsRpDYazdsxG8JwqMVaVsUyyzok01PVzcDmCdtuHff7HwF/5HUcxhRLc+8w9eNbVu27oHrF1POrJgqEnPqB7a84BW6Bhsoymi1ZpdXcE2OR9OGvnP2w9ZTGCj+Pd9v7XSxWwcIYD8USY/QOJ06+ZtW6E6IrMz8pk+hq57muusoQnQNxEmPJPEQ6vxw62sJS6YZQZd6O2VAT4chAhpWejecsWRnjoaM9wzRUluEbPyG41a1ckauaVc5zXQGfj9pIyOZapXHoyCEayxLpCwTPUENtlNZ4OaP25aAoLFkZ46EjXUM0Vpef2JAchY69uQ2uSKlbAx2vgI4d39RYXUaTdU1NcuBYR/6GrbtCVQ1EpZ/m7sH8HthkxZKVMR460j3helXnPqhYBMEZfJKGKiBUBd0Hjm9qrCrjcJd9eE60vzPGkso8X5IPhFjq72P/gf35Pa7JiiUrYzx0qHOQRZXjRgK27YSaGbSqUmrXQMuJycENVWUc6LBkNdHBQT+Lo/m7XpWypGyUg0cO5/24ZnqWrIzx0P72QZbWjOsGbH4R6maRrKKrT6oTuLi6nIMdNtdqvHhijJZEBUsa8zcSMKUxAvuOdU+/o8k7S1bGeOhAxyBLa9wuP01C+8tOYdqZqlvrjgh0qpYtri7nYKe1rMY7fOQQDb4+guX5b1ktqylnT1ci78c107NkZYxHxpLK0Z5hFle73YDdB5yh1GWzWAwwHHXmXPU49aGXVJdzuGsIVasGnrJ33x6Wh4bzOhIwZXlDlH3DFXk/rpmeJStjPNLcM0xNeZCygFuf7ti27Navmk7dKce7AivKApQH/VYjcJw9h5tZHPZmePmi2joGkiF6u23V4EKzZGWMR/a09bOibtyov+bn85Osatc4x3KtqA2zt81WsU15qW2YldE064flgc/vZ0Wgl727t0+/s8krS1bGeGR3ywDLo26ySo46S3zUrZv9getOdYrauvOtltWUs8eS1XG7BsKsaoh6dvyVkQS79h307PgmPUtWxnhk17E+lqWSVftuZ3n1UB6ud5RXQVmNM7kYWBoNs+tY3+yPOw/Eelo5OlbL8vqoZ6+xsjrA9uZez45v0rNkZYxHdrX0s7I24tw5+iwsykOrKqX+VOeYwJpFFey0ZAXA7u1bWB4cIOD37qNtTUMNO3vLpt/R5JUlK2M8EEuMcbBzkFV1brJqegbqN+TvBRatd44JrKqLsL99wGrWAdv2HOCUyrinr7GqMcre0UYSXUc8fR1zMktWxnhgd0s/y6JhQgEfDHXCQNvM6gFmUrsGeptguJvyoJ/6yjL2tdt8q+eah1lTWz79jrMQCfloDMbYve1pT1/HnMySlTEe2H60l7X17vWpI09Bw2ngy+M/N3/Aaam5rat1DZW8cGSBV1aI9fH8YD3rli7y/KVOrUrw3G6rEVhIlqyM8cCWA12ckkpWB34LDWfk/0UaznCODZzSUMGWgws7WXXs+h3tRFldm8OiljO0rrGSp5sTYJOxC8aSlTEe2HKwi9OXVMNwF3TucVpW+dZ4ulO+aaSPDYur2HpwYS+5vuWFFzijcgifB5UrJjpzWQ1Pxdeh7ohM4z3Pk5WIXCUiu0Vkr4jcnOZxEZGvuo9vE5ELvI7JGC819wwzGB9jWbQcDjwGjWfktoR9tgJlUH8aHHycVbURugbjtPYt0EoWqjx6cJDTGvO8iFUGjRV+ggEfe7c+VJDXMx4nKxHxA7cAVwMbgetEZOOE3a4G1ru3G4BvehlTLnr2PMXgvZ9l5EcfoXfbfRxtbSl2SPPWtkMd3PbYPj7ynWf411/t4tkDHcUOacYefaWdc1bUOKXpXnkAlnn4/WvZ+bD7QXw+4czlNfx2z9x932ZDjz7Hb+IbOX9FTcFe8+w65X+2H5h+R5MXXresLgb2qup+VY0DdwHXTNjnGuB76ngKiIrIUo/jmlb3vq1E734nFc/dRtnun1Jzz/upOvrbYoc1L7V09nD7E0f45827eGR3O1/7n318/qc7efno3OzW+vXLrZyzIgptL0NiOD8lljJZdCrEeqBjN2cvq+Ghl1q9e60S9tKTmxF/iGVVhbuyccHKKA/0roIuG2hRCF7/n10OjJ+M0ORuy3WfgvMfedL5oBmn8ql/o719YX4YeGl/d5xfbms+advetgH2dsy95dr7Ywme2tfJeSuisOMnsPpVIB7+M/P5YOUlsOMnXLimlsf3tjMcH5v+efPJ6Ag/39HFpcsCSAGuV6Wc2RhkHys4+sTdBXvNhczrZJXuzJk4fCabfRCRG0Rkq4hsbW9vz0twUxFNM8FSx2wpBg+oph9UVcy3eqbn2/07WjhzWQ2VQ0ecWoDLNnkYpWvlRXBsG9UjxzhtcRUP7Dzm/WuWkMS2e/jp6Kt49ZpZLL0yA0G/cOkS4SdbD0NigV4rLCCvk1UTsHLc/RVA8wz2QVVvU9VNqrqpoaEh74FONLriUucC9jgDF32axsYlnr/2QrO2LsSVZy4+aduaRRFObSjMxfJ0ZnK+qSrf+d0BXrehHrbcDmsvd9ae8lqgHNZcBs98iys2NPDdJw55/5qlIjnG/Q/dz+JKPyurCz+4+Q3rqvh+4gpGnrmz4K+90Hj9f3cLsF5E1opICLgWuHfCPvcCf+iOCrwU6FXVon81LF9xDr1/8FOGz7yWsbVvoPea79C74vJihzUvLVsU5c9et4a/ePMGLlpTy8deu5Yvv+dsNi6vK3ZoOXlsTwf9sVHOT7wIg+1O91yhrHoV9B3lguRO2vpjPLF3YQy0GH3mDr7S/ybeelp1UV5/dY2PldEQ//nrp2Fobl5jnSvE624tEXkL8BXAD9yhqv8kIjcCqOqt4nQyfx24ChgCrlfVrVMdc9OmTbp165S75NXo6CiBQKBgr7eQFem9nvJCRzbnWywxxlv+/XHeeWqAi3b8A1x4PVQXeJxQbxM8932ePOvv2bw/weZPXuaUe5qvOvbyza9/iQcib+Pzr64q6PWq8Q73Jfnib3u5//T7WfLB27JZobg4gc5xnp/JqrpZVTeo6jpV/Sd3262qeqv7u6rqx93Hz54uURWDJarCmYvvdTKp/OU921lSnmDTzn+CM95W+EQFULMCTruKS3d+gbpgnL/+2fb5e4215zCPffvz3Db2Vj56fmXREhXAqmofb14X4YZXLmLwvr+BpBUU9sI8/tplTGF84efPsWvvHv6461+R098KS84pXjDLzkdOv4o/6f4KL+7ay59972k6BkaKF0++JcdIvHA3d3z1H/hk/4f45EWVNEaK/zH2zg0hFi9Zzu8/vY5D37keOvcVO6R5x/NuQC8UuhvQzHsz6wYc7oEfvoe377uGy2taOWP9BigrzrWTSWK9xA8+wd1dp/Bk8iwuqOrhokZlTV2YuqowkbIQwWCAgN+P+HwIcrx1UoxGiqrzH9UkqklIjjE6mmA0HmNosJ+e3h6a2zrZ1gkPJy9kZXiEj5zhZ0mkdD6/FPjOLh+b5GX+98hX4S+PZtrVugFnYE4mKxFpBwo55KkemItXrC3u7HSo6lWZHsx0vq2ukeDBT1ed86mRG/WB5KVSwZCnQc6ET6CHSkaZe92rU6nTnmKHkFYfFXxOvpd80/BD8XVfHdiZYbcpzzeT3pxMVoUmIltVtQATZvLL4p575vLfPldjn6txLzTF7+w1xhhjpmHJyhhjTMmzZJWd24odwAxZ3HPPXP7b52rsczXuBcWuWRljjCl51rIyxhhT8ixZjTMXVzXOIuYrRKRXRF5wb39bjDgnEpE7RKRNRHZkeLzk3ut8movnWspcPOcW+vk2L6iq3ZyuUD+wDzgFCAEvAhsn7PMW4H6cSX2XAk/PgZivAH5Z7Pc3TeyvAy4AdmR4vKTe64V+ruUYe8mdcwv5fJsvN2tZnTAXVzXOJuaSpKqPAVOVqS619zqf5uK5ljInz7kFfr7NC5asTpiLqxpnG8+rRORFEblfRM4sTGizVmrvdT7NxXMtZb6ec6X6fhvX/KrBMjt5W9W4gLKJ5zlgtaoOuMu1/AxY73VgeVBq73U+zcVzLWW+nnOl+n4bl7WsTsjbqsYFNG08qtqnqgPu75uBoIjUFy7EGSu19zqf5uK5ljJfz7lSfb+Ny5LVCXNxVeNpYxaRJe4Cl4jIxTj/zzsLHmnuSu29zqe5eK6lzNdzrlTfb+OybkCXqo6KyE3Ag5xY1XinjFvVGNiMM2poL+6qxsWK140pm5jfA/ypiIwCw8C1qlr07g0R+RHOqLF6EWkC/g4IQmm+1/k0F8+1lLl6zi3k822+sAoWxhhjSp51AxpjjCl5lqyMMcaUPEtWxhhjSp4lK2OMMSXPkpUxxpiSZ8nKGGNMybNk5SERGZjh8z4tIpF8x5PF614hIr8s9Oua/BCRT4rIyyLSnW7pjimet0ZE3u9lbFO89p0i8p5ivLaZWyxZlaZPA54nKxHxe/0apqD+DHiLqtaq6hcnPigimYoArAE8T1Z2vpnZsGRVACJSKSIPi8hzIrJdRK5xt1eIyH1udeodIvIHIvJJYBnwGxH5TYbjvU9E/s39/VMist/9fZ2I/Nb9/Y0i8rz7eneISJm7/aCI/K2733vFWUhvl3v/96f5OyrcY21xj536O8Iicpe7aN1/icjTIrIpP++eyYaI3IqzxtS9IvLnIvJ1d/udIvJv7rn0JRG5XE4sivi8iFQBXwQuc7f9eYbjbxaRc9zfnxd3QUUR+YKI/JFbpujL7nm8XUT+wH38ChH5jYj8J7Dd3e/rIvKSiNwHNE7zd10oIo+KyLMi8qC4y3a4218UkSdTr5uXN9KUrmIvqDWfb8CA+zMAVLu/1+OUdBHg3cDt4/avcX8eBOqnOO4SYIv7+49x6rUtBz4M/B+gHGe5gw3uPt8DPj3u2J9zf0/tt96N526mWDQP+Gfgg+7vUeAVoAL4XzhldwDOAUaBTcV+/xfaLXXeAB8Bvu5uuxP4JeB37/8CeI37e6V7bl4x1f93d9+bgY8D1e759qC7/TfAae65/BBOCabFwGFgqXvsQWCtu//vj9tvGdADvCfDawaBJ4AG9/4fjDvPtgGXu79/mQyLKtpt/tysZVUYAvyziGwDfo2TWBYD24E3iciXROQyVe3N5mCq2gJUut+KVwL/ibMS6mXA4zgfHgdU9RX3Kd91H0/5L/fn6e5+e9T5V/+DaV7694CbReQF4BGcZLfKPfYP3Ni24XyQmNLx36o65v7+O+Df3BZ8VFVHszzG4zj/n18L3Idz/kWANaq6293+I1UdU9VW4FHgIve5z6jqAff3143brxn4nyle8zTgLOAh95z7a2CFiNS4sT/q7vf9LP8GM4dZIdvC+ADQAFyoqgkROQiUq+orInIhTgHN/yMiv1LV/y/LYz6JU2xzN84HyUeBVwF/Aayd5rmD437PpTikAO92P5xObHQKbFuRydJ1/P+3qn7R7X57C/CUiLwpy2NsATYB+3FaRvXAHwPPuo+nWw9q0uunwsjyNQXYqaqvOmmjSDSHY5h5wlpWhVEDtLmJ6vXAagARWQYMqeoPgH8BLnD37weqpjnmY8Bn3J/PA68HRtzW2S5gjYic6u77IZxvuhPtAtaKyDr3/nXTvOaDwCdEji//cP64WD7gbjsLpyvQlCARWaeq21X1S8BWnNb1tOebOkvYHwHeBzyF8wXpM+5PcM6BPxARv4g04LSgnklzqMeAa939luKct5nsBhpE5FVu7EEROVNVe4BeEXmtu98Hpvu7zdxnyaowfghsEpGtOP+wdrnbzwaecbs4/gr4R3f7bcD9mQZYuB7H6QJ8zO3iOQL8FkBVYzitrv8Wke1AErh14gHc/W4A7nMHWBya5u/4As51hG3uBe0vuNu/idMttA34HOk/pExp+LQ7COJFnOU77sfpth11ByykHWDhehxoVdUh9/cVnEhWP3WP8yJO197n3O7qiX4K7MHpAv8m6b9EAccT5HtwBoa8CLwAvNp9+HrgFhF50v07zDxnS4SYvBORR4DPqOrWYsdi5j8RWYMzQOSsYsdivGMtK2OMMSXPWlYlTkSeBsombP6Qqm738DWvBz41YfPvVPXjXr2mKQ0iciXwpQmbD6jquzx+3Z8yeWDQ51X1QS9f18wdlqyMMcaUPOsGNMYYU/IsWRljjCl5lqyMMcaUPEtWxhhjSp4lK2OMMSXv/weOli+b3tND0gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 427.5x360 with 6 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.pairplot(new_df[['last_word_eq', 'first_word_eq', 'is_duplicate']],hue='is_duplicate')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "83d3ea6c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>398782</th>\n",
       "      <td>what is the best marketing automation tool for...</td>\n",
       "      <td>what is the best marketing automation tool for...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115086</th>\n",
       "      <td>i am poor but i want to invest  what should i do</td>\n",
       "      <td>i am quite poor and i want to be very rich  wh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>327711</th>\n",
       "      <td>i am from india and live abroad  i met a guy f...</td>\n",
       "      <td>t i e t to thapar university to thapar univers...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>367788</th>\n",
       "      <td>why do so many people in the u s  hate the sou...</td>\n",
       "      <td>my boyfriend doesnt feel guilty when he hurts ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151235</th>\n",
       "      <td>consequences of bhopal gas tragedy</td>\n",
       "      <td>what was the reason behind the bhopal gas tragedy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                question1  \\\n",
       "398782  what is the best marketing automation tool for...   \n",
       "115086   i am poor but i want to invest  what should i do   \n",
       "327711  i am from india and live abroad  i met a guy f...   \n",
       "367788  why do so many people in the u s  hate the sou...   \n",
       "151235                 consequences of bhopal gas tragedy   \n",
       "\n",
       "                                                question2  \n",
       "398782  what is the best marketing automation tool for...  \n",
       "115086  i am quite poor and i want to be very rich  wh...  \n",
       "327711  t i e t to thapar university to thapar univers...  \n",
       "367788  my boyfriend doesnt feel guilty when he hurts ...  \n",
       "151235  what was the reason behind the bhopal gas tragedy  "
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ques_df = new_df[['question1','question2']]\n",
    "ques_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "358c9867",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(30000, 23)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>q1_len</th>\n",
       "      <th>q2_len</th>\n",
       "      <th>q1_num_words</th>\n",
       "      <th>q2_num_words</th>\n",
       "      <th>word_common</th>\n",
       "      <th>word_total</th>\n",
       "      <th>word_share</th>\n",
       "      <th>cwc_min</th>\n",
       "      <th>cwc_max</th>\n",
       "      <th>...</th>\n",
       "      <th>ctc_max</th>\n",
       "      <th>last_word_eq</th>\n",
       "      <th>first_word_eq</th>\n",
       "      <th>abs_len_diff</th>\n",
       "      <th>mean_len</th>\n",
       "      <th>longest_substr_ratio</th>\n",
       "      <th>fuzz_ratio</th>\n",
       "      <th>fuzz_partial_ratio</th>\n",
       "      <th>token_sort_ratio</th>\n",
       "      <th>token_set_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>398782</th>\n",
       "      <td>1</td>\n",
       "      <td>75</td>\n",
       "      <td>76</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>12</td>\n",
       "      <td>26</td>\n",
       "      <td>0.46</td>\n",
       "      <td>0.874989</td>\n",
       "      <td>0.874989</td>\n",
       "      <td>...</td>\n",
       "      <td>0.923070</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.855263</td>\n",
       "      <td>99</td>\n",
       "      <td>99</td>\n",
       "      <td>99</td>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115086</th>\n",
       "      <td>0</td>\n",
       "      <td>48</td>\n",
       "      <td>56</td>\n",
       "      <td>13</td>\n",
       "      <td>16</td>\n",
       "      <td>8</td>\n",
       "      <td>24</td>\n",
       "      <td>0.33</td>\n",
       "      <td>0.666644</td>\n",
       "      <td>0.499988</td>\n",
       "      <td>...</td>\n",
       "      <td>0.466664</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>13.5</td>\n",
       "      <td>0.224490</td>\n",
       "      <td>69</td>\n",
       "      <td>67</td>\n",
       "      <td>65</td>\n",
       "      <td>74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>327711</th>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>119</td>\n",
       "      <td>28</td>\n",
       "      <td>21</td>\n",
       "      <td>4</td>\n",
       "      <td>38</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.115384</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>0.047619</td>\n",
       "      <td>26</td>\n",
       "      <td>29</td>\n",
       "      <td>34</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>367788</th>\n",
       "      <td>0</td>\n",
       "      <td>58</td>\n",
       "      <td>145</td>\n",
       "      <td>14</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "      <td>34</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>21.5</td>\n",
       "      <td>0.050847</td>\n",
       "      <td>29</td>\n",
       "      <td>41</td>\n",
       "      <td>23</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151235</th>\n",
       "      <td>0</td>\n",
       "      <td>34</td>\n",
       "      <td>49</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "      <td>13</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0.749981</td>\n",
       "      <td>0.599988</td>\n",
       "      <td>...</td>\n",
       "      <td>0.333330</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.542857</td>\n",
       "      <td>55</td>\n",
       "      <td>70</td>\n",
       "      <td>48</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        is_duplicate  q1_len  q2_len  q1_num_words  q2_num_words  word_common  \\\n",
       "398782             1      75      76            13            13           12   \n",
       "115086             0      48      56            13            16            8   \n",
       "327711             0     104     119            28            21            4   \n",
       "367788             0      58     145            14            32            1   \n",
       "151235             0      34      49             5             9            3   \n",
       "\n",
       "        word_total  word_share   cwc_min   cwc_max  ...   ctc_max  \\\n",
       "398782          26        0.46  0.874989  0.874989  ...  0.923070   \n",
       "115086          24        0.33  0.666644  0.499988  ...  0.466664   \n",
       "327711          38        0.11  0.000000  0.000000  ...  0.115384   \n",
       "367788          34        0.03  0.000000  0.000000  ...  0.000000   \n",
       "151235          13        0.23  0.749981  0.599988  ...  0.333330   \n",
       "\n",
       "        last_word_eq  first_word_eq  abs_len_diff  mean_len  \\\n",
       "398782           1.0            1.0           0.0      13.0   \n",
       "115086           1.0            1.0           3.0      13.5   \n",
       "327711           0.0            0.0           6.0      23.0   \n",
       "367788           0.0            0.0          17.0      21.5   \n",
       "151235           1.0            0.0           4.0       7.0   \n",
       "\n",
       "        longest_substr_ratio  fuzz_ratio  fuzz_partial_ratio  \\\n",
       "398782              0.855263          99                  99   \n",
       "115086              0.224490          69                  67   \n",
       "327711              0.047619          26                  29   \n",
       "367788              0.050847          29                  41   \n",
       "151235              0.542857          55                  70   \n",
       "\n",
       "        token_sort_ratio  token_set_ratio  \n",
       "398782                99               99  \n",
       "115086                65               74  \n",
       "327711                34               43  \n",
       "367788                23               30  \n",
       "151235                48               69  \n",
       "\n",
       "[5 rows x 23 columns]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_df = new_df.drop(columns=['id','qid1','qid2','question1','question2'])\n",
    "print(final_df.shape)\n",
    "final_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "ba9a7f21",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "# merge texts\n",
    "questions = list(ques_df['question1']) + list(ques_df['question2'])\n",
    "\n",
    "cv = CountVectorizer(max_features=3000)\n",
    "q1_arr, q2_arr = np.vsplit(cv.fit_transform(questions).toarray(),2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "f0e33e72",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(30000, 6000)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp_df1 = pd.DataFrame(q1_arr, index= ques_df.index)\n",
    "temp_df2 = pd.DataFrame(q2_arr, index= ques_df.index)\n",
    "temp_df = pd.concat([temp_df1, temp_df2], axis=1)\n",
    "temp_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "3d9b1b7e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(30000, 6023)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>q1_len</th>\n",
       "      <th>q2_len</th>\n",
       "      <th>q1_num_words</th>\n",
       "      <th>q2_num_words</th>\n",
       "      <th>word_common</th>\n",
       "      <th>word_total</th>\n",
       "      <th>word_share</th>\n",
       "      <th>cwc_min</th>\n",
       "      <th>cwc_max</th>\n",
       "      <th>...</th>\n",
       "      <th>2990</th>\n",
       "      <th>2991</th>\n",
       "      <th>2992</th>\n",
       "      <th>2993</th>\n",
       "      <th>2994</th>\n",
       "      <th>2995</th>\n",
       "      <th>2996</th>\n",
       "      <th>2997</th>\n",
       "      <th>2998</th>\n",
       "      <th>2999</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>398782</th>\n",
       "      <td>1</td>\n",
       "      <td>75</td>\n",
       "      <td>76</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>12</td>\n",
       "      <td>26</td>\n",
       "      <td>0.46</td>\n",
       "      <td>0.874989</td>\n",
       "      <td>0.874989</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115086</th>\n",
       "      <td>0</td>\n",
       "      <td>48</td>\n",
       "      <td>56</td>\n",
       "      <td>13</td>\n",
       "      <td>16</td>\n",
       "      <td>8</td>\n",
       "      <td>24</td>\n",
       "      <td>0.33</td>\n",
       "      <td>0.666644</td>\n",
       "      <td>0.499988</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>327711</th>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>119</td>\n",
       "      <td>28</td>\n",
       "      <td>21</td>\n",
       "      <td>4</td>\n",
       "      <td>38</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>367788</th>\n",
       "      <td>0</td>\n",
       "      <td>58</td>\n",
       "      <td>145</td>\n",
       "      <td>14</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "      <td>34</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151235</th>\n",
       "      <td>0</td>\n",
       "      <td>34</td>\n",
       "      <td>49</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "      <td>13</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0.749981</td>\n",
       "      <td>0.599988</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 6023 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        is_duplicate  q1_len  q2_len  q1_num_words  q2_num_words  word_common  \\\n",
       "398782             1      75      76            13            13           12   \n",
       "115086             0      48      56            13            16            8   \n",
       "327711             0     104     119            28            21            4   \n",
       "367788             0      58     145            14            32            1   \n",
       "151235             0      34      49             5             9            3   \n",
       "\n",
       "        word_total  word_share   cwc_min   cwc_max  ...  2990  2991  2992  \\\n",
       "398782          26        0.46  0.874989  0.874989  ...     0     0     0   \n",
       "115086          24        0.33  0.666644  0.499988  ...     0     0     0   \n",
       "327711          38        0.11  0.000000  0.000000  ...     0     0     0   \n",
       "367788          34        0.03  0.000000  0.000000  ...     0     0     0   \n",
       "151235          13        0.23  0.749981  0.599988  ...     0     0     0   \n",
       "\n",
       "        2993  2994  2995  2996  2997  2998  2999  \n",
       "398782     0     0     0     0     0     0     0  \n",
       "115086     0     0     0     0     0     0     0  \n",
       "327711     0     0     0     0     0     0     0  \n",
       "367788     1     0     0     0     0     0     0  \n",
       "151235     0     0     0     0     0     0     0  \n",
       "\n",
       "[5 rows x 6023 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_df = pd.concat([final_df, temp_df], axis=1)\n",
    "print(final_df.shape)\n",
    "final_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "08c91265",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train,X_test,y_train,y_test = train_test_split(final_df.iloc[:,1:].values,final_df.iloc[:,0].values,test_size=0.2,random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "23de1cb3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7876666666666666"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "rf = RandomForestClassifier()\n",
    "rf.fit(X_train,y_train)\n",
    "y_pred = rf.predict(X_test)\n",
    "accuracy_score(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "105a66ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7968333333333333"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from xgboost import XGBClassifier\n",
    "xgb = XGBClassifier()\n",
    "xgb.fit(X_train,y_train)\n",
    "y_pred1 = xgb.predict(X_test)\n",
    "accuracy_score(y_test,y_pred1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "a9274aad",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "9f055b10",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3284,  528],\n",
       "       [ 746, 1442]], dtype=int64)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "08b2c11a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3248,  564],\n",
       "       [ 655, 1533]], dtype=int64)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test,y_pred1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "0061e50b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_common_words(q1,q2):\n",
    "    w1 = set(map(lambda word: word.lower().strip(), q1.split(\" \")))\n",
    "    w2 = set(map(lambda word: word.lower().strip(), q2.split(\" \")))    \n",
    "    return len(w1 & w2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "db598b35",
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_total_words(q1,q2):\n",
    "    w1 = set(map(lambda word: word.lower().strip(), q1.split(\" \")))\n",
    "    w2 = set(map(lambda word: word.lower().strip(), q2.split(\" \")))    \n",
    "    return (len(w1) + len(w2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "72e3707b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_fetch_token_features(q1,q2):\n",
    "    \n",
    "    SAFE_DIV = 0.0001 \n",
    "\n",
    "    STOP_WORDS = stopwords.words(\"english\")\n",
    "    \n",
    "    token_features = [0.0]*8\n",
    "    \n",
    "    # Converting the Sentence into Tokens: \n",
    "    q1_tokens = q1.split()\n",
    "    q2_tokens = q2.split(\n",
    "    \n",
    "    if len(q1_tokens) == 0 or len(q2_tokens) == 0:\n",
    "        return token_features\n",
    "\n",
    "    # Get the non-stopwords in Questions\n",
    "    q1_words = set([word for word in q1_tokens if word not in STOP_WORDS])\n",
    "    q2_words = set([word for word in q2_tokens if word not in STOP_WORDS])\n",
    "    \n",
    "    #Get the stopwords in Questions\n",
    "    q1_stops = set([word for word in q1_tokens if word in STOP_WORDS])\n",
    "    q2_stops = set([word for word in q2_tokens if word in STOP_WORDS])\n",
    "    \n",
    "    # Get the common non-stopwords from Question pair\n",
    "    common_word_count = len(q1_words.intersection(q2_words))\n",
    "    \n",
    "    # Get the common stopwords from Question pair\n",
    "    common_stop_count = len(q1_stops.intersection(q2_stops))\n",
    "    \n",
    "    # Get the common Tokens from Question pair\n",
    "    common_token_count = len(set(q1_tokens).intersection(set(q2_tokens)))\n",
    "    \n",
    "    \n",
    "    token_features[0] = common_word_count / (min(len(q1_words), len(q2_words)) + SAFE_DIV)\n",
    "    token_features[1] = common_word_count / (max(len(q1_words), len(q2_words)) + SAFE_DIV)\n",
    "    token_features[2] = common_stop_count / (min(len(q1_stops), len(q2_stops)) + SAFE_DIV)\n",
    "    token_features[3] = common_stop_count / (max(len(q1_stops), len(q2_stops)) + SAFE_DIV)\n",
    "    token_features[4] = common_token_count / (min(len(q1_tokens), len(q2_tokens)) + SAFE_DIV)\n",
    "    token_features[5] = common_token_count / (max(len(q1_tokens), len(q2_tokens)) + SAFE_DIV)\n",
    "    \n",
    "    # Last word of both question is same or not\n",
    "    token_features[6] = int(q1_tokens[-1] == q2_tokens[-1])\n",
    "    \n",
    "    # First word of both question is same or not\n",
    "    token_features[7] = int(q1_tokens[0] == q2_tokens[0])\n",
    "    \n",
    "    return token_features\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "dca0d6d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_fetch_length_features(q1,q2):\n",
    "    \n",
    "    length_features = [0.0]*3\n",
    "    \n",
    "    # Converting the Sentence into Tokens: \n",
    "    q1_tokens = q1.split()\n",
    "    q2_tokens = q2.split()\n",
    "    \n",
    "    if len(q1_tokens) == 0 or len(q2_tokens) == 0:\n",
    "        return length_features\n",
    "    \n",
    "    # Absolute length features\n",
    "    length_features[0] = abs(len(q1_tokens) - len(q2_tokens))\n",
    "    \n",
    "    #Average Token Length of both Questions\n",
    "    length_features[1] = (len(q1_tokens) + len(q2_tokens))/2\n",
    "    \n",
    "    strs = list(distance.lcsubstrings(q1, q2))\n",
    "    length_features[2] = len(strs[0]) / (min(len(q1), len(q2)) + 1)\n",
    "    \n",
    "    return length_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "764b3432",
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_fetch_fuzzy_features(q1,q2):\n",
    "    \n",
    "    fuzzy_features = [0.0]*4\n",
    "    \n",
    "    # fuzz_ratio\n",
    "    fuzzy_features[0] = fuzz.QRatio(q1, q2)\n",
    "\n",
    "    # fuzz_partial_ratio\n",
    "    fuzzy_features[1] = fuzz.partial_ratio(q1, q2)\n",
    "\n",
    "    # token_sort_ratio\n",
    "    fuzzy_features[2] = fuzz.token_sort_ratio(q1, q2)\n",
    "\n",
    "    # token_set_ratio\n",
    "    fuzzy_features[3] = fuzz.token_set_ratio(q1, q2)\n",
    "\n",
    "    return fuzzy_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "b51bd2d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def query_point_creator(q1,q2):\n",
    "    \n",
    "    input_query = []\n",
    "    \n",
    "    # preprocess\n",
    "    q1 = preprocessor(q1)\n",
    "    q2 = preprocessor(q2)\n",
    "    \n",
    "    # fetch basic features\n",
    "    input_query.append(len(q1))\n",
    "    input_query.append(len(q2))\n",
    "    \n",
    "    input_query.append(len(q1.split(\" \")))\n",
    "    input_query.append(len(q2.split(\" \")))\n",
    "    \n",
    "    input_query.append(test_common_words(q1,q2))\n",
    "    input_query.append(test_total_words(q1,q2))\n",
    "    input_query.append(round(test_common_words(q1,q2)/test_total_words(q1,q2),2))\n",
    "    \n",
    "    # fetch token features\n",
    "    token_features = test_fetch_token_features(q1,q2)\n",
    "    input_query.extend(token_features)\n",
    "    \n",
    "    # fetch length based features\n",
    "    length_features = test_fetch_length_features(q1,q2)\n",
    "    input_query.extend(length_features)\n",
    "    \n",
    "    # fetch fuzzy features\n",
    "    fuzzy_features = test_fetch_fuzzy_features(q1,q2)\n",
    "    input_query.extend(fuzzy_features)\n",
    "    \n",
    "    # bow feature for q1\n",
    "    q1_bow = cv.transform([q1]).toarray()\n",
    "    \n",
    "    # bow feature for q2\n",
    "    q2_bow = cv.transform([q2]).toarray()\n",
    "    \n",
    "    \n",
    "    \n",
    "    return np.hstack((np.array(input_query).reshape(1,22),q1_bow,q2_bow))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "9cee343a",
   "metadata": {},
   "outputs": [],
   "source": [
    "q1 = 'Where is the capital of India?'\n",
    "q2 = 'What is the current capital of Pakistan?'\n",
    "q3 = 'Which city serves as the capital of India?'\n",
    "q4 = 'What is the business capital of India?'\n",
    "q5 = 'My name is Diya Goyal'\n",
    "q6 = 'My pet name is diya goyal'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "ca6dfecf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1], dtype=int64)"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf.predict(query_point_creator(q5,q6))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "0e4f71b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "pickle.dump(rf,open('model.pkl','wb'))\n",
    "pickle.dump(cv,open('cv.pkl','wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ce1ee30b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "753b3a2d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0a46017",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}