Basic - To get insight about dataframe.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "64563897",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0420d7be",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(404290, 6)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=pd.read_csv(\"train.csv\")\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f87aec90",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>What is the step by step guide to invest in sh...</td>\n",
       "      <td>What is the step by step guide to invest in sh...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>What is the story of Kohinoor (Koh-i-Noor) Dia...</td>\n",
       "      <td>What would happen if the Indian government sto...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>How can I increase the speed of my internet co...</td>\n",
       "      <td>How can Internet speed be increased by hacking...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>Why am I mentally very lonely? How can I solve...</td>\n",
       "      <td>Find the remainder when [math]23^{24}[/math] i...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>Which one dissolve in water quikly sugar, salt...</td>\n",
       "      <td>Which fish would survive in salt water?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  qid1  qid2                                          question1  \\\n",
       "0   0     1     2  What is the step by step guide to invest in sh...   \n",
       "1   1     3     4  What is the story of Kohinoor (Koh-i-Noor) Dia...   \n",
       "2   2     5     6  How can I increase the speed of my internet co...   \n",
       "3   3     7     8  Why am I mentally very lonely? How can I solve...   \n",
       "4   4     9    10  Which one dissolve in water quikly sugar, salt...   \n",
       "\n",
       "                                           question2  is_duplicate  \n",
       "0  What is the step by step guide to invest in sh...             0  \n",
       "1  What would happen if the Indian government sto...             0  \n",
       "2  How can Internet speed be increased by hacking...             0  \n",
       "3  Find the remainder when [math]23^{24}[/math] i...             0  \n",
       "4            Which fish would survive in salt water?             0  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "4f08cfbb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>69641</th>\n",
       "      <td>69641</td>\n",
       "      <td>120144</td>\n",
       "      <td>120145</td>\n",
       "      <td>How much money do you need to earn to live a c...</td>\n",
       "      <td>How much money do you need to earn to live a c...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15794</th>\n",
       "      <td>15794</td>\n",
       "      <td>30143</td>\n",
       "      <td>30144</td>\n",
       "      <td>Where is the best place to eat roast duck (烤鸭,...</td>\n",
       "      <td>Where is the best place to eat roast duck (烤鸭,...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115281</th>\n",
       "      <td>115281</td>\n",
       "      <td>15643</td>\n",
       "      <td>188012</td>\n",
       "      <td>How safe is Papua New Guinea, and how does it'...</td>\n",
       "      <td>How safe is Papua New Guinea, and how does it'...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>342004</th>\n",
       "      <td>342004</td>\n",
       "      <td>469922</td>\n",
       "      <td>55258</td>\n",
       "      <td>What are the top 10 problems in Machine Learni...</td>\n",
       "      <td>What are the Top 10 problems in Machine Learni...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>399071</th>\n",
       "      <td>399071</td>\n",
       "      <td>532317</td>\n",
       "      <td>532318</td>\n",
       "      <td>Why is Indonesia a lot friendlier to Singapore...</td>\n",
       "      <td>Is my hypothesis right/wrong that Singapore wo...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>112938</th>\n",
       "      <td>112938</td>\n",
       "      <td>7378</td>\n",
       "      <td>36052</td>\n",
       "      <td>What are some symptoms of eccentric and concen...</td>\n",
       "      <td>What are some characteristics of concentric an...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>313770</th>\n",
       "      <td>313770</td>\n",
       "      <td>438402</td>\n",
       "      <td>438403</td>\n",
       "      <td>Why was screenshot feature created?</td>\n",
       "      <td>Why was the screenshot feature was invented?</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>301535</th>\n",
       "      <td>301535</td>\n",
       "      <td>424438</td>\n",
       "      <td>424439</td>\n",
       "      <td>How does Ethereum's dilution compare to that o...</td>\n",
       "      <td>How does the emergence of ethereum compare to ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>388918</th>\n",
       "      <td>388918</td>\n",
       "      <td>521360</td>\n",
       "      <td>521361</td>\n",
       "      <td>Has our two-party system, of Democrat / Republ...</td>\n",
       "      <td>What would happen if the character Jigsaw from...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>331984</th>\n",
       "      <td>331984</td>\n",
       "      <td>458947</td>\n",
       "      <td>458948</td>\n",
       "      <td>What did you find out about your teacher that ...</td>\n",
       "      <td>What do teachers think about gooru.org?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101804</th>\n",
       "      <td>101804</td>\n",
       "      <td>168570</td>\n",
       "      <td>168571</td>\n",
       "      <td>How do I secure my Facebook account from hackers?</td>\n",
       "      <td>Someone's threatening me about hacking my FB a...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>214583</th>\n",
       "      <td>214583</td>\n",
       "      <td>247921</td>\n",
       "      <td>228430</td>\n",
       "      <td>How can one earn online?</td>\n",
       "      <td>How can you earn online?</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>341904</th>\n",
       "      <td>341904</td>\n",
       "      <td>469806</td>\n",
       "      <td>143347</td>\n",
       "      <td>Any Idea how much money politicians pay to get...</td>\n",
       "      <td>Does ADHD-Hyperfocus go away with Medication?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>271494</th>\n",
       "      <td>271494</td>\n",
       "      <td>389583</td>\n",
       "      <td>389584</td>\n",
       "      <td>What are some similar youtube channels to scho...</td>\n",
       "      <td>Are there any Youtube channels that follow a f...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>180241</th>\n",
       "      <td>180241</td>\n",
       "      <td>276385</td>\n",
       "      <td>25499</td>\n",
       "      <td>Why are black people generally more muscular/a...</td>\n",
       "      <td>Why do people of some races seem to be better ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>284385</th>\n",
       "      <td>284385</td>\n",
       "      <td>404631</td>\n",
       "      <td>404632</td>\n",
       "      <td>Which phone do I buy?</td>\n",
       "      <td>Which phone should I buy at this moment?</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>189518</th>\n",
       "      <td>189518</td>\n",
       "      <td>25768</td>\n",
       "      <td>288298</td>\n",
       "      <td>When will euthanasia be legal in India?</td>\n",
       "      <td>Should euthanasia be legalize in India? Why or...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138753</th>\n",
       "      <td>138753</td>\n",
       "      <td>208241</td>\n",
       "      <td>220882</td>\n",
       "      <td>Where can I find my verification code for my p...</td>\n",
       "      <td>Can any online services provide a phone number...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93582</th>\n",
       "      <td>93582</td>\n",
       "      <td>156455</td>\n",
       "      <td>156456</td>\n",
       "      <td>What's the best way to learn accounting online?</td>\n",
       "      <td>How can I learn accounting online?</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89113</th>\n",
       "      <td>89113</td>\n",
       "      <td>53308</td>\n",
       "      <td>149774</td>\n",
       "      <td>When and how did time begin?</td>\n",
       "      <td>According to science, how did time begin?</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>375357</th>\n",
       "      <td>375357</td>\n",
       "      <td>506376</td>\n",
       "      <td>154092</td>\n",
       "      <td>Why do trained falcons wear blinders?</td>\n",
       "      <td>How do falcons get rid from honey bees?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>308279</th>\n",
       "      <td>308279</td>\n",
       "      <td>6480</td>\n",
       "      <td>20342</td>\n",
       "      <td>What is your New Year’s Resolution(s) for 2017?</td>\n",
       "      <td>What is your resolution for 2017?</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>194188</th>\n",
       "      <td>194188</td>\n",
       "      <td>17270</td>\n",
       "      <td>68832</td>\n",
       "      <td>Is Donald Trump secretly a Democrat?</td>\n",
       "      <td>Is it possible that Donald Trump is in cahoots...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>235068</th>\n",
       "      <td>235068</td>\n",
       "      <td>345697</td>\n",
       "      <td>345698</td>\n",
       "      <td>In psychology, is there a term for the behavio...</td>\n",
       "      <td>Which college should I choose modern institute...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>326961</th>\n",
       "      <td>326961</td>\n",
       "      <td>453338</td>\n",
       "      <td>453339</td>\n",
       "      <td>What is the scope of green building technology...</td>\n",
       "      <td>I want to design and build my own home in Indi...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>272669</th>\n",
       "      <td>272669</td>\n",
       "      <td>390976</td>\n",
       "      <td>375487</td>\n",
       "      <td>Which US Founding Fathers were the smartest?</td>\n",
       "      <td>What is a Founding Father?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383556</th>\n",
       "      <td>383556</td>\n",
       "      <td>515511</td>\n",
       "      <td>37860</td>\n",
       "      <td>Why did Tiger Kill Kabali at the end of the mo...</td>\n",
       "      <td>In the end of Kabali movie, it is implied that...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2277</th>\n",
       "      <td>2277</td>\n",
       "      <td>4528</td>\n",
       "      <td>4529</td>\n",
       "      <td>Is the Korean War a hoax?</td>\n",
       "      <td>Who won the Korean War?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>379838</th>\n",
       "      <td>379838</td>\n",
       "      <td>511388</td>\n",
       "      <td>33963</td>\n",
       "      <td>I got placed in bitwise solution when will I g...</td>\n",
       "      <td>I got an intent letter from Infosys. When can ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71378</th>\n",
       "      <td>71378</td>\n",
       "      <td>122842</td>\n",
       "      <td>122843</td>\n",
       "      <td>How does Quora decide which topics to suggest ...</td>\n",
       "      <td>How does Quora decide the topics of a question?</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116584</th>\n",
       "      <td>116584</td>\n",
       "      <td>189882</td>\n",
       "      <td>189883</td>\n",
       "      <td>How do I cope with being alone on Thanksgiving?</td>\n",
       "      <td>How do you cope with being alone?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>324139</th>\n",
       "      <td>324139</td>\n",
       "      <td>345816</td>\n",
       "      <td>254886</td>\n",
       "      <td>How do you know if someone has blocked you on ...</td>\n",
       "      <td>How do you know if you are blocked on Snapchat?</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>215522</th>\n",
       "      <td>215522</td>\n",
       "      <td>173473</td>\n",
       "      <td>321435</td>\n",
       "      <td>How can you format text (make text bold, itali...</td>\n",
       "      <td>Why there is no rich text editor in Facebook m...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>320792</th>\n",
       "      <td>320792</td>\n",
       "      <td>446379</td>\n",
       "      <td>446380</td>\n",
       "      <td>Executive Certificate Programme in Digital mar...</td>\n",
       "      <td>What is the market value of a digital marketin...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107264</th>\n",
       "      <td>107264</td>\n",
       "      <td>176511</td>\n",
       "      <td>176512</td>\n",
       "      <td>How much mobile data gets used if I play battl...</td>\n",
       "      <td>How much mobile data gets used if I play games...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>260557</th>\n",
       "      <td>260557</td>\n",
       "      <td>376535</td>\n",
       "      <td>90516</td>\n",
       "      <td>Why does messenger say a person is \"active now...</td>\n",
       "      <td>What is the person who is shown as 'active now...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126443</th>\n",
       "      <td>126443</td>\n",
       "      <td>203768</td>\n",
       "      <td>203769</td>\n",
       "      <td>Is the law of fascism axiomatic?</td>\n",
       "      <td>Is Fascism axiomatic?</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>375126</th>\n",
       "      <td>375126</td>\n",
       "      <td>506115</td>\n",
       "      <td>506116</td>\n",
       "      <td>How do you think news of a Donald Trump presid...</td>\n",
       "      <td>How do you think Hillary Clinton or Donald Tru...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122715</th>\n",
       "      <td>122715</td>\n",
       "      <td>105118</td>\n",
       "      <td>198559</td>\n",
       "      <td>How do I be popular in school?</td>\n",
       "      <td>How can I be really popular in school?</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282220</th>\n",
       "      <td>282220</td>\n",
       "      <td>77724</td>\n",
       "      <td>402117</td>\n",
       "      <td>Should I learn French or German first?</td>\n",
       "      <td>Should I quit German to learn French?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>269829</th>\n",
       "      <td>269829</td>\n",
       "      <td>387596</td>\n",
       "      <td>197904</td>\n",
       "      <td>I am a 40 year old male. A beautiful 18 year o...</td>\n",
       "      <td>For a 21 year old female, is it fine to date w...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155545</th>\n",
       "      <td>155545</td>\n",
       "      <td>243665</td>\n",
       "      <td>243666</td>\n",
       "      <td>Which is the best institute for Java training ...</td>\n",
       "      <td>Which is the best institute for Java training ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30412</th>\n",
       "      <td>30412</td>\n",
       "      <td>56171</td>\n",
       "      <td>56172</td>\n",
       "      <td>How can I lose 20 kilos in two months?</td>\n",
       "      <td>How can yo lose 20 pounds in a 2 month period?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21301</th>\n",
       "      <td>21301</td>\n",
       "      <td>40124</td>\n",
       "      <td>37663</td>\n",
       "      <td>What if the South won the American Civil War?</td>\n",
       "      <td>What would have happened if the American civil...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102284</th>\n",
       "      <td>102284</td>\n",
       "      <td>169251</td>\n",
       "      <td>146436</td>\n",
       "      <td>What is the best way to deal with conflict?</td>\n",
       "      <td>What are good ways to deal with conflicts in a...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48827</th>\n",
       "      <td>48827</td>\n",
       "      <td>86978</td>\n",
       "      <td>13725</td>\n",
       "      <td>You message someone on Facebook and they see y...</td>\n",
       "      <td>Why has my Facebook message gone from seen to ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257456</th>\n",
       "      <td>257456</td>\n",
       "      <td>372771</td>\n",
       "      <td>372772</td>\n",
       "      <td>How can I become a salesforce consultant?</td>\n",
       "      <td>How can I be salesforce consultant?</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2293</th>\n",
       "      <td>2293</td>\n",
       "      <td>4559</td>\n",
       "      <td>4560</td>\n",
       "      <td>Job interview defect?</td>\n",
       "      <td>What was the purpose of the Iran nuclear deal ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16944</th>\n",
       "      <td>16944</td>\n",
       "      <td>32254</td>\n",
       "      <td>32255</td>\n",
       "      <td>What is the difference between commercial and ...</td>\n",
       "      <td>What's the difference between commercial banki...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>366160</th>\n",
       "      <td>366160</td>\n",
       "      <td>244135</td>\n",
       "      <td>213634</td>\n",
       "      <td>Why do I dream every night?</td>\n",
       "      <td>Why don't I dream at night?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            id    qid1    qid2  \\\n",
       "69641    69641  120144  120145   \n",
       "15794    15794   30143   30144   \n",
       "115281  115281   15643  188012   \n",
       "342004  342004  469922   55258   \n",
       "399071  399071  532317  532318   \n",
       "112938  112938    7378   36052   \n",
       "313770  313770  438402  438403   \n",
       "301535  301535  424438  424439   \n",
       "388918  388918  521360  521361   \n",
       "331984  331984  458947  458948   \n",
       "101804  101804  168570  168571   \n",
       "214583  214583  247921  228430   \n",
       "341904  341904  469806  143347   \n",
       "271494  271494  389583  389584   \n",
       "180241  180241  276385   25499   \n",
       "284385  284385  404631  404632   \n",
       "189518  189518   25768  288298   \n",
       "138753  138753  208241  220882   \n",
       "93582    93582  156455  156456   \n",
       "89113    89113   53308  149774   \n",
       "375357  375357  506376  154092   \n",
       "308279  308279    6480   20342   \n",
       "194188  194188   17270   68832   \n",
       "235068  235068  345697  345698   \n",
       "326961  326961  453338  453339   \n",
       "272669  272669  390976  375487   \n",
       "383556  383556  515511   37860   \n",
       "2277      2277    4528    4529   \n",
       "379838  379838  511388   33963   \n",
       "71378    71378  122842  122843   \n",
       "116584  116584  189882  189883   \n",
       "324139  324139  345816  254886   \n",
       "215522  215522  173473  321435   \n",
       "320792  320792  446379  446380   \n",
       "107264  107264  176511  176512   \n",
       "260557  260557  376535   90516   \n",
       "126443  126443  203768  203769   \n",
       "375126  375126  506115  506116   \n",
       "122715  122715  105118  198559   \n",
       "282220  282220   77724  402117   \n",
       "269829  269829  387596  197904   \n",
       "155545  155545  243665  243666   \n",
       "30412    30412   56171   56172   \n",
       "21301    21301   40124   37663   \n",
       "102284  102284  169251  146436   \n",
       "48827    48827   86978   13725   \n",
       "257456  257456  372771  372772   \n",
       "2293      2293    4559    4560   \n",
       "16944    16944   32254   32255   \n",
       "366160  366160  244135  213634   \n",
       "\n",
       "                                                question1  \\\n",
       "69641   How much money do you need to earn to live a c...   \n",
       "15794   Where is the best place to eat roast duck (烤鸭,...   \n",
       "115281  How safe is Papua New Guinea, and how does it'...   \n",
       "342004  What are the top 10 problems in Machine Learni...   \n",
       "399071  Why is Indonesia a lot friendlier to Singapore...   \n",
       "112938  What are some symptoms of eccentric and concen...   \n",
       "313770                Why was screenshot feature created?   \n",
       "301535  How does Ethereum's dilution compare to that o...   \n",
       "388918  Has our two-party system, of Democrat / Republ...   \n",
       "331984  What did you find out about your teacher that ...   \n",
       "101804  How do I secure my Facebook account from hackers?   \n",
       "214583                           How can one earn online?   \n",
       "341904  Any Idea how much money politicians pay to get...   \n",
       "271494  What are some similar youtube channels to scho...   \n",
       "180241  Why are black people generally more muscular/a...   \n",
       "284385                              Which phone do I buy?   \n",
       "189518            When will euthanasia be legal in India?   \n",
       "138753  Where can I find my verification code for my p...   \n",
       "93582     What's the best way to learn accounting online?   \n",
       "89113                        When and how did time begin?   \n",
       "375357              Why do trained falcons wear blinders?   \n",
       "308279    What is your New Year’s Resolution(s) for 2017?   \n",
       "194188               Is Donald Trump secretly a Democrat?   \n",
       "235068  In psychology, is there a term for the behavio...   \n",
       "326961  What is the scope of green building technology...   \n",
       "272669       Which US Founding Fathers were the smartest?   \n",
       "383556  Why did Tiger Kill Kabali at the end of the mo...   \n",
       "2277                            Is the Korean War a hoax?   \n",
       "379838  I got placed in bitwise solution when will I g...   \n",
       "71378   How does Quora decide which topics to suggest ...   \n",
       "116584    How do I cope with being alone on Thanksgiving?   \n",
       "324139  How do you know if someone has blocked you on ...   \n",
       "215522  How can you format text (make text bold, itali...   \n",
       "320792  Executive Certificate Programme in Digital mar...   \n",
       "107264  How much mobile data gets used if I play battl...   \n",
       "260557  Why does messenger say a person is \"active now...   \n",
       "126443                   Is the law of fascism axiomatic?   \n",
       "375126  How do you think news of a Donald Trump presid...   \n",
       "122715                     How do I be popular in school?   \n",
       "282220             Should I learn French or German first?   \n",
       "269829  I am a 40 year old male. A beautiful 18 year o...   \n",
       "155545  Which is the best institute for Java training ...   \n",
       "30412              How can I lose 20 kilos in two months?   \n",
       "21301       What if the South won the American Civil War?   \n",
       "102284        What is the best way to deal with conflict?   \n",
       "48827   You message someone on Facebook and they see y...   \n",
       "257456          How can I become a salesforce consultant?   \n",
       "2293                                Job interview defect?   \n",
       "16944   What is the difference between commercial and ...   \n",
       "366160                        Why do I dream every night?   \n",
       "\n",
       "                                                question2  is_duplicate  \n",
       "69641   How much money do you need to earn to live a c...             0  \n",
       "15794   Where is the best place to eat roast duck (烤鸭,...             0  \n",
       "115281  How safe is Papua New Guinea, and how does it'...             0  \n",
       "342004  What are the Top 10 problems in Machine Learni...             0  \n",
       "399071  Is my hypothesis right/wrong that Singapore wo...             0  \n",
       "112938  What are some characteristics of concentric an...             1  \n",
       "313770       Why was the screenshot feature was invented?             1  \n",
       "301535  How does the emergence of ethereum compare to ...             0  \n",
       "388918  What would happen if the character Jigsaw from...             0  \n",
       "331984            What do teachers think about gooru.org?             0  \n",
       "101804  Someone's threatening me about hacking my FB a...             1  \n",
       "214583                           How can you earn online?             1  \n",
       "341904      Does ADHD-Hyperfocus go away with Medication?             0  \n",
       "271494  Are there any Youtube channels that follow a f...             1  \n",
       "180241  Why do people of some races seem to be better ...             0  \n",
       "284385           Which phone should I buy at this moment?             1  \n",
       "189518  Should euthanasia be legalize in India? Why or...             0  \n",
       "138753  Can any online services provide a phone number...             0  \n",
       "93582                  How can I learn accounting online?             1  \n",
       "89113           According to science, how did time begin?             1  \n",
       "375357            How do falcons get rid from honey bees?             0  \n",
       "308279                  What is your resolution for 2017?             1  \n",
       "194188  Is it possible that Donald Trump is in cahoots...             1  \n",
       "235068  Which college should I choose modern institute...             0  \n",
       "326961  I want to design and build my own home in Indi...             0  \n",
       "272669                         What is a Founding Father?             0  \n",
       "383556  In the end of Kabali movie, it is implied that...             1  \n",
       "2277                              Who won the Korean War?             0  \n",
       "379838  I got an intent letter from Infosys. When can ...             0  \n",
       "71378     How does Quora decide the topics of a question?             1  \n",
       "116584                  How do you cope with being alone?             0  \n",
       "324139    How do you know if you are blocked on Snapchat?             1  \n",
       "215522  Why there is no rich text editor in Facebook m...             0  \n",
       "320792  What is the market value of a digital marketin...             0  \n",
       "107264  How much mobile data gets used if I play games...             0  \n",
       "260557  What is the person who is shown as 'active now...             0  \n",
       "126443                              Is Fascism axiomatic?             1  \n",
       "375126  How do you think Hillary Clinton or Donald Tru...             1  \n",
       "122715             How can I be really popular in school?             1  \n",
       "282220              Should I quit German to learn French?             0  \n",
       "269829  For a 21 year old female, is it fine to date w...             0  \n",
       "155545  Which is the best institute for Java training ...             0  \n",
       "30412      How can yo lose 20 pounds in a 2 month period?             0  \n",
       "21301   What would have happened if the American civil...             1  \n",
       "102284  What are good ways to deal with conflicts in a...             0  \n",
       "48827   Why has my Facebook message gone from seen to ...             0  \n",
       "257456                How can I be salesforce consultant?             1  \n",
       "2293    What was the purpose of the Iran nuclear deal ...             0  \n",
       "16944   What's the difference between commercial banki...             0  \n",
       "366160                        Why don't I dream at night?             0  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sample(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f8254956",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 404290 entries, 0 to 404289\n",
      "Data columns (total 6 columns):\n",
      " #   Column        Non-Null Count   Dtype \n",
      "---  ------        --------------   ----- \n",
      " 0   id            404290 non-null  int64 \n",
      " 1   qid1          404290 non-null  int64 \n",
      " 2   qid2          404290 non-null  int64 \n",
      " 3   question1     404289 non-null  object\n",
      " 4   question2     404288 non-null  object\n",
      " 5   is_duplicate  404290 non-null  int64 \n",
      "dtypes: int64(4), object(2)\n",
      "memory usage: 18.5+ MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7a746177",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id              0\n",
       "qid1            0\n",
       "qid2            0\n",
       "question1       1\n",
       "question2       2\n",
       "is_duplicate    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0500b7a0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.duplicated().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2a3abee9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    255027\n",
      "1    149263\n",
      "Name: is_duplicate, dtype: int64\n",
      "0    63.080215\n",
      "1    36.919785\n",
      "Name: is_duplicate, dtype: float64\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAD1CAYAAABOfbKwAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAQMElEQVR4nO3db6je5X3H8fdnphNZq0Q9is2fRWbKpsIshij0SUcgydoHWlB2fFDDFkgRhRb6YNonFiWgsFYQpmAxGKWrBttiWGtdph2lzKnHItXoXA7Vapqg6RKse6Bb0u8e3Ndp75zeuc7JSXJOYt4v+HH/7u/vuq5z3XDkk991/e5jqgpJko7kjxZ6ApKkk5tBIUnqMigkSV0GhSSpy6CQJHUZFJKkrkULPYHj7fzzz68VK1Ys9DQk6ZTy4osv/rqqxkZd+8gFxYoVK5iYmFjoaUjSKSXJL490zaUnSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkro+cl+4O1WsuPUHCz2Fj5Q37/r8Qk9B+sia8Y4iybIkP07yWpKdSb7c6l9P8qskL7Xjc0N9bksymeT1JOuG6lcmeblduzdJWv3MJI+1+nNJVgz12ZBkVzs2HNdPL0ma0WzuKA4CX62qnyX5BPBikh3t2j1V9Q/DjZNcCowDlwGfBP41yaeq6hBwP7AJ+A/gh8B64ElgI3Cgqi5JMg7cDfxNknOB24FVQLWfvb2qDhzbx5YkzdaMdxRVtbeqftbO3wdeA5Z0ulwDPFpVH1bVG8AksDrJRcDZVfVsDf5H3Q8D1w712drOHwfWtLuNdcCOqtrfwmEHg3CRJM2To9rMbktCnwaea6Vbkvw8yZYki1ttCfD2ULfdrbaknU+vH9anqg4C7wHndcaaPq9NSSaSTOzbt+9oPpIkaQazDookHwe+C3ylqn7DYBnpz4ArgL3AN6aajuhenfpc+/y+UPVAVa2qqlVjYyP/Sq4kaY5mFRRJPsYgJL5dVd8DqKp3qupQVf0W+BawujXfDSwb6r4U2NPqS0fUD+uTZBFwDrC/M5YkaZ7M5qmnAA8Cr1XVN4fqFw01+wLwSjvfDoy3J5kuBlYCz1fVXuD9JFe3MW8EnhjqM/VE03XAM20f4ylgbZLFbWlrbatJkubJbJ56+gzwReDlJC+12teAG5JcwWAp6E3gSwBVtTPJNuBVBk9M3dyeeAK4CXgIOIvB005PtvqDwCNJJhncSYy3sfYnuRN4obW7o6r2z+WDSpLmZsagqKqfMnqv4IedPpuBzSPqE8DlI+ofANcfYawtwJaZ5ilJOjH8Ex6SpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKlrxqBIsizJj5O8lmRnki+3+rlJdiTZ1V4XD/W5LclkkteTrBuqX5nk5Xbt3iRp9TOTPNbqzyVZMdRnQ/sZu5JsOK6fXpI0o9ncURwEvlpVfwFcDdyc5FLgVuDpqloJPN3e066NA5cB64H7kpzRxrof2ASsbMf6Vt8IHKiqS4B7gLvbWOcCtwNXAauB24cDSZJ04s0YFFW1t6p+1s7fB14DlgDXAFtbs63Ate38GuDRqvqwqt4AJoHVSS4Czq6qZ6uqgIen9Zka63FgTbvbWAfsqKr9VXUA2MHvw0WSNA+Oao+iLQl9GngOuLCq9sIgTIALWrMlwNtD3Xa32pJ2Pr1+WJ+qOgi8B5zXGUuSNE9mHRRJPg58F/hKVf2m13RErTr1ufYZntumJBNJJvbt29eZmiTpaM0qKJJ8jEFIfLuqvtfK77TlJNrru62+G1g21H0psKfVl46oH9YnySLgHGB/Z6zDVNUDVbWqqlaNjY3N5iNJkmZpNk89BXgQeK2qvjl0aTsw9RTSBuCJofp4e5LpYgab1s+35an3k1zdxrxxWp+psa4Dnmn7GE8Ba5MsbpvYa1tNkjRPFs2izWeALwIvJ3mp1b4G3AVsS7IReAu4HqCqdibZBrzK4Impm6vqUOt3E/AQcBbwZDtgEESPJJlkcCcx3sban+RO4IXW7o6q2j+3jypJmosZg6KqfsrovQKANUfosxnYPKI+AVw+ov4BLWhGXNsCbJlpnpKkE8NvZkuSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUNWNQJNmS5N0krwzVvp7kV0leasfnhq7dlmQyyetJ1g3Vr0zycrt2b5K0+plJHmv155KsGOqzIcmudmw4bp9akjRrs7mjeAhYP6J+T1Vd0Y4fAiS5FBgHLmt97ktyRmt/P7AJWNmOqTE3Ageq6hLgHuDuNta5wO3AVcBq4PYki4/6E0qSjsmMQVFVPwH2z3K8a4BHq+rDqnoDmARWJ7kIOLuqnq2qAh4Grh3qs7WdPw6saXcb64AdVbW/qg4AOxgdWJKkE+hY9ihuSfLztjQ19S/9JcDbQ212t9qSdj69flifqjoIvAec1xlLkjSPFs2x3/3AnUC1128AfwdkRNvq1Jljn8Mk2cRgWYvly5f35i1pFlbc+oOFnsJHxpt3fX6hp3DM5nRHUVXvVNWhqvot8C0Gewgw+Ff/sqGmS4E9rb50RP2wPkkWAecwWOo60lij5vNAVa2qqlVjY2Nz+UiSpCOYU1C0PYcpXwCmnojaDoy3J5kuZrBp/XxV7QXeT3J123+4EXhiqM/UE03XAc+0fYyngLVJFrelrbWtJkmaRzMuPSX5DvBZ4Pwkuxk8ifTZJFcwWAp6E/gSQFXtTLINeBU4CNxcVYfaUDcxeILqLODJdgA8CDySZJLBncR4G2t/kjuBF1q7O6pqtpvqkqTjZMagqKobRpQf7LTfDGweUZ8ALh9R/wC4/ghjbQG2zDRHSdKJ4zezJUldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklS14xBkWRLkneTvDJUOzfJjiS72uvioWu3JZlM8nqSdUP1K5O83K7dmyStfmaSx1r9uSQrhvpsaD9jV5INx+1TS5JmbTZ3FA8B66fVbgWerqqVwNPtPUkuBcaBy1qf+5Kc0frcD2wCVrZjasyNwIGqugS4B7i7jXUucDtwFbAauH04kCRJ82PGoKiqnwD7p5WvAba2863AtUP1R6vqw6p6A5gEVie5CDi7qp6tqgIentZnaqzHgTXtbmMdsKOq9lfVAWAHfxhYkqQTbK57FBdW1V6A9npBqy8B3h5qt7vVlrTz6fXD+lTVQeA94LzOWJKkeXS8N7Mzolad+lz7HP5Dk01JJpJM7Nu3b1YTlSTNzlyD4p22nER7fbfVdwPLhtotBfa0+tIR9cP6JFkEnMNgqetIY/2BqnqgqlZV1aqxsbE5fiRJ0ihzDYrtwNRTSBuAJ4bq4+1JposZbFo/35an3k9yddt/uHFan6mxrgOeafsYTwFrkyxum9hrW02SNI8WzdQgyXeAzwLnJ9nN4Emku4BtSTYCbwHXA1TVziTbgFeBg8DNVXWoDXUTgyeozgKebAfAg8AjSSYZ3EmMt7H2J7kTeKG1u6Oqpm+qS5JOsBmDoqpuOMKlNUdovxnYPKI+AVw+ov4BLWhGXNsCbJlpjpKkE8dvZkuSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUdUxBkeTNJC8neSnJRKudm2RHkl3tdfFQ+9uSTCZ5Pcm6ofqVbZzJJPcmSaufmeSxVn8uyYpjma8k6egdjzuKv6qqK6pqVXt/K/B0Va0Enm7vSXIpMA5cBqwH7ktyRutzP7AJWNmO9a2+EThQVZcA9wB3H4f5SpKOwolYeroG2NrOtwLXDtUfraoPq+oNYBJYneQi4OyqeraqCnh4Wp+psR4H1kzdbUiS5sexBkUB/5LkxSSbWu3CqtoL0F4vaPUlwNtDfXe32pJ2Pr1+WJ+qOgi8B5x3jHOWJB2FRcfY/zNVtSfJBcCOJP/ZaTvqTqA69V6fwwcehNQmgOXLl/dnLEk6Ksd0R1FVe9rru8D3gdXAO205ifb6bmu+G1g21H0psKfVl46oH9YnySLgHGD/iHk8UFWrqmrV2NjYsXwkSdI0cw6KJH+S5BNT58Ba4BVgO7ChNdsAPNHOtwPj7UmmixlsWj/flqfeT3J123+4cVqfqbGuA55p+xiSpHlyLEtPFwLfb3vLi4B/qqofJXkB2JZkI/AWcD1AVe1Msg14FTgI3FxVh9pYNwEPAWcBT7YD4EHgkSSTDO4kxo9hvpKkOZhzUFTVL4C/HFH/b2DNEfpsBjaPqE8Al4+of0ALGknSwvCb2ZKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1nRJBkWR9kteTTCa5daHnI0mnk5M+KJKcAfwj8NfApcANSS5d2FlJ0unjpA8KYDUwWVW/qKr/BR4FrlngOUnSaWPRQk9gFpYAbw+93w1cNdwgySZgU3v7P0len6e5nQ7OB3690JOYSe5e6BlogZz0v5+n0O/mnx7pwqkQFBlRq8PeVD0APDA/0zm9JJmoqlULPQ9pFH8/58epsPS0G1g29H4psGeB5iJJp51TISheAFYmuTjJHwPjwPYFnpMknTZO+qWnqjqY5BbgKeAMYEtV7VzgaZ1OXNLTyczfz3mQqpq5lSTptHUqLD1JkhaQQSFJ6jIoJEldJ/1mtuZXkj9n8M33JQy+r7IH2F5Vry3oxCQtGO8o9DtJ/p7Bn0gJ8DyDR5MDfMc/xqiTWZK/Xeg5fJT51JN+J8l/AZdV1f9Nq/8xsLOqVi7MzKS+JG9V1fKFnsdHlUtPGvZb4JPAL6fVL2rXpAWT5OdHugRcOJ9zOd0YFBr2FeDpJLv4/R9iXA5cAtyyUJOSmguBdcCBafUA/z7/0zl9GBT6nar6UZJPMfjT7ksY/Ae4G3ihqg4t6OQk+Gfg41X10vQLSf5t3mdzGnGPQpLU5VNPkqQug0KS1GVQSJK6DApJUpdBIUnq+n/InXmx1HDi4wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(df['is_duplicate'].value_counts())\n",
    "print(df['is_duplicate'].value_counts()/df['is_duplicate'].count()*100)\n",
    "df['is_duplicate'].value_counts().plot(kind='bar')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "06a98db2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of unique questons   537933\n",
      "Number of duplicates  111780\n"
     ]
    }
   ],
   "source": [
    "#finding count of duplicate questions in the whole dataset\n",
    "qid=pd.Series(df['qid1'].tolist()+df['qid2'].tolist())\n",
    "#print(qid)\n",
    "print(\"Number of unique questons  \",np.unique(qid).shape[0])\n",
    "\n",
    "x=qid.value_counts()>1\n",
    "print(\"Number of duplicates \",x[x].shape[0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "940d3103",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAQrUlEQVR4nO3dfYylZ1nH8e/PrVOkaHlp0bov7sKUxv1HgbG8+JKKiLuUaRWJdgMJ6NoNJiW+RGWbGhP+sqAxhlipq9YaqNtsaoVduqQqisWkgW4rxV2WlaUUOrSwWzE1okmpXP5xztLjdGb3zHmZc+ae7yfZzDz3Oed5fju755p7ruee50lVIUlqy7dNOoAkafQs7pLUIIu7JDXI4i5JDbK4S1KDzpt0AICLLrqotm7dOukYkrSm3H///Y9X1cVLPTbR4p5kHpifnZ3lyJEjk4wiSWtOki8u99hE2zJVdaiq9lx44YWTjCFJzZlocU8yn2TfE088MckYktQcZ+6S1CBn7pLUIGfuktQg17lLUoNsy0hSg2zLSFKD1nxbZuveu9i6965Jx5CkqWJbRpIaZFtGkhq05tsykqRnsrhLUoMs7pLUIE+oSlKDPKEqSQ2yLSNJDbK4S1KDLO6S1CCLuyQ1yNUyktQgV8tIUoNsy0hSgyzuktQgi7skNcjiLkkNsrhLUoNGXtyTXJHk40luTnLFqPcvSTq3vop7kluSnEpydNH4jiQnkpxMsrc7XMB/Ac8CFkYbV5LUj35n7rcCO3oHkmwAbgJ2AtuBXUm2Ax+vqp3AO4F3jS6qJKlffRX3qroH+Nqi4cuBk1X1UFU9CdwOXF1V3+w+/h/A+cvtM8meJEeSHDl9+vQA0SVJyxmm574ReKRnewHYmOSNSf4EeD/wR8u9uKr2VdVcVc1dfPHFQ8SQJC123hCvzRJjVVV3Anf2tYNkHpifnZ0dIoYkabFhZu4LwOae7U3AoyvZgdeWkaTxGKa43wdcmmRbkhngGuDgSnbgVSElaTz6XQq5H7gXuCzJQpLdVfUUcB1wN3AcOFBVx1ZycGfukjQeffXcq2rXMuOHgcODHtyeuySNRzPXc9+69y627r1rBKkkae3zTkyS1KBmZu6SpKd5VUhJapBtGUlqkG0ZSWqQbRlJapBtGUlqkG0ZSWqQbRlJapDFXZIaZHGXpAZ5QlWSGuQJVUlqkG0ZSWqQxV2SGmRxl6QGNVfcvWGHJLlaRpKa5GoZSWpQc20ZSZLFXZKaZHGXpAZZ3CWpQRZ3SWrQWIp7kguS3J/kDePYvyTp7Poq7kluSXIqydFF4zuSnEhyMsnenofeCRwYZVBJUv/6nbnfCuzoHUiyAbgJ2AlsB3Yl2Z7ktcBngK+OMKckaQXO6+dJVXVPkq2Lhi8HTlbVQwBJbgeuBp4DXECn4P9PksNV9c3F+0yyB9gDsGXLloH/ApKkZ+qruC9jI/BIz/YC8Iqqug4gyduAx5cq7ABVtQ/YBzA3N1dD5HiGM9eXefjGK0e5W0laM4Yp7lli7FtFuqpuPecOknlgfnZ2dogYkqTFhlktswBs7tneBDy6kh14bRlJGo9hivt9wKVJtiWZAa4BDq5kB14VUpLGo9+lkPuBe4HLkiwk2V1VTwHXAXcDx4EDVXVsJQd35i5J49Hvapldy4wfBg4PenB77pI0Hl7PXZIa5J2YJKlBztwlqUFeFVKSGtR0W2br3ru+9duqkrSe2JaRpAbZlpGkBjXdlpGk9cq2jCQ1yLaMJDXI4i5JDbK4S1KDPKEqSQ1aFydU/UUmSeuNbRlJapDFXZIaZHGXpAZZ3CWpQa6WkaQGrYvVMuDlfyWtL7ZlJKlBFndJapDFXZIaZHGXpAat2+LuyVVJLRt5cU/y/UluTnJHkl8e9f4lSefWV3FPckuSU0mOLhrfkeREkpNJ9gJU1fGqejvwc8Dc6CNLks6l35n7rcCO3oEkG4CbgJ3AdmBXku3dx64C/hn46MiSSpL61ldxr6p7gK8tGr4cOFlVD1XVk8DtwNXd5x+sqlcDbx5lWElSf84b4rUbgUd6theAVyS5AngjcD5weLkXJ9kD7AHYsmXLEDEkSYsNU9yzxFhV1ceAj53rxVW1L8ljwPzMzMzLh8gxsDMrZh6+8cpJHF6SxmaY1TILwOae7U3AoyvZwWpeW0aS1pNhivt9wKVJtiWZAa4BDq5kB14VUpLGo9+lkPuBe4HLkiwk2V1VTwHXAXcDx4EDVXVsJQd35i5J49FXz72qdi0zfpiznDQ9lyTzwPzs7OyguxgJe++SWrNurucuSeuJd2KSpAY5c+/h3ZoktWLdXhVSklpmW0aSGmRbZgm2ZiStdbZlJKlBFndJapA992W4ckbSWmbPXZIaZFtGkhpkce+TLRpJa4k9d0lqkD13SWqQbZkVcAWNpLXC4i5JDbK4S1KDLO6S1CBXywzA3rukadfXPVTHpaoOAYfm5uaunWSOQfUWeO+/Kmma2JaRpAZZ3CWpQRNty7TEFo2kaeLMXZIaZHGXpAaNpbgn+ekkf5rkQ0leN45jSJKW13dxT3JLklNJji4a35HkRJKTSfYCVNUHq+pa4G3Az480sSTpnFYyc78V2NE7kGQDcBOwE9gO7Eqyvecpv919fF3yl50kTUrfq2Wq6p4kWxcNXw6crKqHAJLcDlyd5DhwI/CRqnpgVGHXKlfSSFptw/bcNwKP9GwvdMfeAbwWeFOSty/1wiR7khxJcuT06dNDxpAk9Rp2nXuWGKuqei/w3rO9sKr2JXkMmJ+ZmXn5kDkkST2GnbkvAJt7tjcBj/b74vV4Jyb78JJWw7DF/T7g0iTbkswA1wAH+33xWr0qpCRNu5UshdwP3AtclmQhye6qegq4DrgbOA4cqKpj/e5zPc7cz3AGL2mcVrJaZtcy44eBw4McPMk8MD87OzvIyyVJy5jo5QfW88x9Kc7mJY2Kd2KSpAY5c58wZ+qSxsGrQkpSg2zLSFKDbMtMIVs1koZlW0aSGmRxl6QG2XNfg1wPL+lchr0q5FCq6hBwaG5u7tpJ5lgrLOiS+mVbZo2wsEtaCYv7lLL1ImkYE23LeOGwc7PASxqE69wlqUG2ZSSpQRZ3SWqQxV2SGmRxl6QG+RuqDRhkRY1LLaW2uVpGkhpkW6YRzsQl9bK4S1KDLO6S1CCL+zpnK0dqk8VdkhpkcZekBo38qpBJXgTcAFxYVW8a9f7Vn952y8M3XjnBJJImoa+Ze5JbkpxKcnTR+I4kJ5KcTLIXoKoeqqrd4wirc3NJpCTovy1zK7CjdyDJBuAmYCewHdiVZPtI00mSBtJXca+qe4CvLRq+HDjZnak/CdwOXN3vgZPsSXIkyZHTp0/3HVij52xfas8wJ1Q3Ao/0bC8AG5O8IMnNwEuTXL/ci6tqH/Au4IGZmZkhYmi1+c1Amn7DFPcsMVZV9e9V9faqenFV/e7ZduC1ZSRpPIZZLbMAbO7Z3gQ8upIdeA/V1Xe2GfdSj7nSRlqbhpm53wdcmmRbkhngGuDgSnbgzF2SxqPfpZD7gXuBy5IsJNldVU8B1wF3A8eBA1V1bCUH93ruq2PcPXL779L06astU1W7lhk/DBwe9OBVdQg4NDc3d+2g+5AkPZOXH5CkBnmbPUlqkLfZk6QGjfzCYSvhUsi148xJ096lkZ5IlaaXM3dJapAnVCWpQRZ3SWqQPXeNzeKevJcykFaPPXdJapBtGUlqkMVdkhrkb6hqRZZb2+4NPKTpYs9dkhpkW0aSGmRxl6QGWdwlqUEWd0lqkKtlJKlBrpaRpAbZlpGkBlncJalBFndJapDFXZIaZHGXpAZZ3CWpQSO/E1OSC4A/Bp4EPlZVt436GJKks+tr5p7kliSnkhxdNL4jyYkkJ5Ps7Q6/Ebijqq4FrhpxXklSH/pty9wK7OgdSLIBuAnYCWwHdiXZDmwCHuk+7X9HE1OStBJ9tWWq6p4kWxcNXw6crKqHAJLcDlwNLNAp8J/iLN88kuwB9gBs2bJlpbk15Za6ccdSY0vdNPvM8x6+8cr/9/mgz1tp5t59jGu/02yUedfa3301jftrM8wJ1Y08PUOHTlHfCNwJ/GyS9wGHlntxVe2rqrmqmrv44ouHiCFJWmyYE6pZYqyq6uvAL/S1g2QemJ+dnR0ihiRpsWFm7gvA5p7tTcCjw8WRJI3CMMX9PuDSJNuSzADXAAdXsgOvCilJ49HvUsj9wL3AZUkWkuyuqqeA64C7gePAgao6tpKDez13SRqPflfL7Fpm/DBweNCDV9Uh4NDc3Ny1g+5DkvRMXn5AkhrkbfYkqUHeZk+SGpSqmnQGkpwGvrjCl10EPD6GOKNgtsGYbTBmG0wL2b6vqpb8LdCpKO6DSHKkquYmnWMpZhuM2QZjtsG0ns0TqpLUIIu7JDVoLRf3fZMOcBZmG4zZBmO2wTSdbc323CVJy1vLM3dJ0jIs7pLUoDVZ3Je5d+uksmxO8o9Jjic5luRXuuPPT/J3ST7X/fi8CeXbkORfknx4mnJ1szw3yR1JPtv9+r1qWvIl+bXuv+fRJPuTPGtS2Za6h/HZsiS5vvveOJHkpyaQ7fe6/6afTvI3SZ47Ldl6HvuNJJXkomnKluQd3eMfS/KeobJV1Zr6A2wAPg+8CJgBHgS2TzDPJcDLup9/J/BvdO4p+x5gb3d8L/DuCeX7deCvgA93t6ciV/f4fwn8UvfzGeC505CPzh3FvgB8R3f7APC2SWUDfgx4GXC0Z2zJLN3/ew8C5wPbuu+VDauc7XXAed3P3z1N2brjm+lczfaLwEXTkg34ceDvgfO72y8cJtuqvmlG9EV5FXB3z/b1wPWTztWT50PATwIngEu6Y5cAJyaQZRPwUeA1PcV94rm6x/6ubgHNovGJ5+PpW0g+n86VUz/cLVgTywZsXVQIlsyy+P3QLWKvWs1six77GeC2acoG3AH8APBwT3GfeDY6k4jXLvG8gbKtxbbMcvdunbjuTcRfCnwC+O6qegyg+/GFE4j0h8BvAd/sGZuGXND5yes08BfdttGfJblgGvJV1ZeB3we+BDwGPFFVfzsN2Xosl2Xa3h+/CHyk+/nEsyW5CvhyVT246KGJZwNeAvxokk8k+ackPzRMtrVY3Je8d+uqp1gkyXOAvwZ+tar+cwryvAE4VVX3TzrLMs6j82Pp+6rqpcDX6bQXJq7bv76azo/A3wtckOQtk03Vt6l5fyS5AXgKuO3M0BJPW7VsSZ4N3AD8zlIPLzG22l+384DnAa8EfhM4kCQMmG0tFvepu3drkm+nU9hvq6o7u8NfTXJJ9/FLgFOrHOuHgauSPAzcDrwmyQemINcZC8BCVX2iu30HnWI/DfleC3yhqk5X1TeAO4FXT0m2M5bLMhXvjyRvBd4AvLm6vYQpyPZiOt+wH+y+LzYBDyT5ninIRjfDndXxSTo/cV80aLa1WNyHvnfrKHW/s/45cLyq/qDnoYPAW7ufv5VOL37VVNX1VbWpqrbS+Rr9Q1W9ZdK5evJ9BXgkyWXdoZ8APsN05PsS8Mokz+7++/4EnVtJTkO2M5bLchC4Jsn5SbYBlwKfXM1gSXYA7wSuqqr/7nlootmq6l+r6oVVtbX7vligsxjiK5PO1vVBOufHSPISOosMHh842zhPGIzxRMTr6axK+Txww4Sz/AidH5E+DXyq++f1wAvonMz8XPfj8yeY8QqePqE6Tbl+EDjS/dp9kM6PpFORD3gX8FngKPB+OisVJpIN2E+n9/8NOgVp99my0Gk9fJ7OSdedE8h2kk6P+Mz74eZpybbo8YfpnlCdhmx0ivkHuv/nHgBeM0w2Lz8gSQ1ai20ZSdI5WNwlqUEWd0lqkMVdkhpkcZekBlncJalBFndJatD/AePpeVp0BUPxAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(qid.value_counts().values,bins=180)\n",
    "plt.yscale('log')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b4593de1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}