notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "66d44557-bc0d-43b1-8e4f-1a1973bd554b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests \n",
    "\n",
    "docs_url = 'https://github.com/alexeygrigorev/llm-rag-workshop/raw/main/notebooks/documents.json'\n",
    "docs_response = requests.get(docs_url)\n",
    "documents_raw = docs_response.json()\n",
    "\n",
    "documents = []\n",
    "\n",
    "for course in documents_raw:\n",
    "    course_name = course['course']\n",
    "\n",
    "    for doc in course['documents']:\n",
    "        doc['course'] = course_name\n",
    "        documents.append(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "152dc317-118b-46f3-bc2e-5dee4f66c4e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0faa21b7-adbf-44e4-bf02-d38998a12f5c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>course</th>\n",
       "      <th>section</th>\n",
       "      <th>question</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>data-engineering-zoomcamp</td>\n",
       "      <td>General course-related questions</td>\n",
       "      <td>Course - When will the course start?</td>\n",
       "      <td>The purpose of this document is to capture fre...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>data-engineering-zoomcamp</td>\n",
       "      <td>General course-related questions</td>\n",
       "      <td>Course - What are the prerequisites for this c...</td>\n",
       "      <td>GitHub - DataTalksClub data-engineering-zoomca...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>data-engineering-zoomcamp</td>\n",
       "      <td>General course-related questions</td>\n",
       "      <td>Course - Can I still join the course after the...</td>\n",
       "      <td>Yes, even if you don't register, you're still ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>data-engineering-zoomcamp</td>\n",
       "      <td>General course-related questions</td>\n",
       "      <td>Course - I have registered for the Data Engine...</td>\n",
       "      <td>You don't need it. You're accepted. You can al...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>data-engineering-zoomcamp</td>\n",
       "      <td>General course-related questions</td>\n",
       "      <td>Course - What can I do before the course starts?</td>\n",
       "      <td>You can start by installing and setting up all...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      course                           section  \\\n",
       "0  data-engineering-zoomcamp  General course-related questions   \n",
       "1  data-engineering-zoomcamp  General course-related questions   \n",
       "2  data-engineering-zoomcamp  General course-related questions   \n",
       "3  data-engineering-zoomcamp  General course-related questions   \n",
       "4  data-engineering-zoomcamp  General course-related questions   \n",
       "\n",
       "                                            question  \\\n",
       "0               Course - When will the course start?   \n",
       "1  Course - What are the prerequisites for this c...   \n",
       "2  Course - Can I still join the course after the...   \n",
       "3  Course - I have registered for the Data Engine...   \n",
       "4   Course - What can I do before the course starts?   \n",
       "\n",
       "                                                text  \n",
       "0  The purpose of this document is to capture fre...  \n",
       "1  GitHub - DataTalksClub data-engineering-zoomca...  \n",
       "2  Yes, even if you don't register, you're still ...  \n",
       "3  You don't need it. You're accepted. You can al...  \n",
       "4  You can start by installing and setting up all...  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(documents, columns=['course', 'section', 'question', 'text'])\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2f9ecbf7-13ce-4a4f-baaf-5ad989bdeb39",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>course</th>\n",
       "      <th>section</th>\n",
       "      <th>question</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>data-engineering-zoomcamp</td>\n",
       "      <td>General course-related questions</td>\n",
       "      <td>Course - When will the course start?</td>\n",
       "      <td>The purpose of this document is to capture fre...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>data-engineering-zoomcamp</td>\n",
       "      <td>General course-related questions</td>\n",
       "      <td>Course - What are the prerequisites for this c...</td>\n",
       "      <td>GitHub - DataTalksClub data-engineering-zoomca...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>data-engineering-zoomcamp</td>\n",
       "      <td>General course-related questions</td>\n",
       "      <td>Course - Can I still join the course after the...</td>\n",
       "      <td>Yes, even if you don't register, you're still ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>data-engineering-zoomcamp</td>\n",
       "      <td>General course-related questions</td>\n",
       "      <td>Course - I have registered for the Data Engine...</td>\n",
       "      <td>You don't need it. You're accepted. You can al...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>data-engineering-zoomcamp</td>\n",
       "      <td>General course-related questions</td>\n",
       "      <td>Course - What can I do before the course starts?</td>\n",
       "      <td>You can start by installing and setting up all...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      course                           section  \\\n",
       "0  data-engineering-zoomcamp  General course-related questions   \n",
       "1  data-engineering-zoomcamp  General course-related questions   \n",
       "2  data-engineering-zoomcamp  General course-related questions   \n",
       "3  data-engineering-zoomcamp  General course-related questions   \n",
       "4  data-engineering-zoomcamp  General course-related questions   \n",
       "\n",
       "                                            question  \\\n",
       "0               Course - When will the course start?   \n",
       "1  Course - What are the prerequisites for this c...   \n",
       "2  Course - Can I still join the course after the...   \n",
       "3  Course - I have registered for the Data Engine...   \n",
       "4   Course - What can I do before the course starts?   \n",
       "\n",
       "                                                text  \n",
       "0  The purpose of this document is to capture fre...  \n",
       "1  GitHub - DataTalksClub data-engineering-zoomca...  \n",
       "2  Yes, even if you don't register, you're still ...  \n",
       "3  You don't need it. You're accepted. You can al...  \n",
       "4  You can start by installing and setting up all...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.course == 'data-engineering-zoomcamp'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "94f4da02-d93a-412f-8a99-3b5d6c23056e",
   "metadata": {},
   "outputs": [],
   "source": [
    "docs_example = [\n",
    "    \"January course details, register now\",\n",
    "    \"Course prerequisites listed in January catalog\",\n",
    "    \"Submit January course homework by end of month\",\n",
    "    \"Register for January course, no prerequisites\",\n",
    "    \"January course setup: Python and Google Cloud\"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c6f62369-fc0d-484f-abc4-ba846a3a4bb1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b51522f4-2e65-42ef-8ccc-310adb032658",
   "metadata": {},
   "outputs": [],
   "source": [
    "cv = CountVectorizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d2c5bdbc-aaaa-4da6-b19d-5f9d4fdb7eb5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {\n",
       "  /* Definition of color scheme common for light and dark mode */\n",
       "  --sklearn-color-text: black;\n",
       "  --sklearn-color-line: gray;\n",
       "  /* Definition of color scheme for unfitted estimators */\n",
       "  --sklearn-color-unfitted-level-0: #fff5e6;\n",
       "  --sklearn-color-unfitted-level-1: #f6e4d2;\n",
       "  --sklearn-color-unfitted-level-2: #ffe0b3;\n",
       "  --sklearn-color-unfitted-level-3: chocolate;\n",
       "  /* Definition of color scheme for fitted estimators */\n",
       "  --sklearn-color-fitted-level-0: #f0f8ff;\n",
       "  --sklearn-color-fitted-level-1: #d4ebff;\n",
       "  --sklearn-color-fitted-level-2: #b3dbfd;\n",
       "  --sklearn-color-fitted-level-3: cornflowerblue;\n",
       "\n",
       "  /* Specific color for light theme */\n",
       "  --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
       "  --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, white)));\n",
       "  --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
       "  --sklearn-color-icon: #696969;\n",
       "\n",
       "  @media (prefers-color-scheme: dark) {\n",
       "    /* Redefinition of color scheme for dark theme */\n",
       "    --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
       "    --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, #111)));\n",
       "    --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
       "    --sklearn-color-icon: #878787;\n",
       "  }\n",
       "}\n",
       "\n",
       "#sk-container-id-1 {\n",
       "  color: var(--sklearn-color-text);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 pre {\n",
       "  padding: 0;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 input.sk-hidden--visually {\n",
       "  border: 0;\n",
       "  clip: rect(1px 1px 1px 1px);\n",
       "  clip: rect(1px, 1px, 1px, 1px);\n",
       "  height: 1px;\n",
       "  margin: -1px;\n",
       "  overflow: hidden;\n",
       "  padding: 0;\n",
       "  position: absolute;\n",
       "  width: 1px;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-dashed-wrapped {\n",
       "  border: 1px dashed var(--sklearn-color-line);\n",
       "  margin: 0 0.4em 0.5em 0.4em;\n",
       "  box-sizing: border-box;\n",
       "  padding-bottom: 0.4em;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-container {\n",
       "  /* jupyter's `normalize.less` sets `[hidden] { display: none; }`\n",
       "     but bootstrap.min.css set `[hidden] { display: none !important; }`\n",
       "     so we also need the `!important` here to be able to override the\n",
       "     default hidden behavior on the sphinx rendered scikit-learn.org.\n",
       "     See: https://github.com/scikit-learn/scikit-learn/issues/21755 */\n",
       "  display: inline-block !important;\n",
       "  position: relative;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-text-repr-fallback {\n",
       "  display: none;\n",
       "}\n",
       "\n",
       "div.sk-parallel-item,\n",
       "div.sk-serial,\n",
       "div.sk-item {\n",
       "  /* draw centered vertical line to link estimators */\n",
       "  background-image: linear-gradient(var(--sklearn-color-text-on-default-background), var(--sklearn-color-text-on-default-background));\n",
       "  background-size: 2px 100%;\n",
       "  background-repeat: no-repeat;\n",
       "  background-position: center center;\n",
       "}\n",
       "\n",
       "/* Parallel-specific style estimator block */\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item::after {\n",
       "  content: \"\";\n",
       "  width: 100%;\n",
       "  border-bottom: 2px solid var(--sklearn-color-text-on-default-background);\n",
       "  flex-grow: 1;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel {\n",
       "  display: flex;\n",
       "  align-items: stretch;\n",
       "  justify-content: center;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  position: relative;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item:first-child::after {\n",
       "  align-self: flex-end;\n",
       "  width: 50%;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item:last-child::after {\n",
       "  align-self: flex-start;\n",
       "  width: 50%;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item:only-child::after {\n",
       "  width: 0;\n",
       "}\n",
       "\n",
       "/* Serial-specific style estimator block */\n",
       "\n",
       "#sk-container-id-1 div.sk-serial {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "  align-items: center;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  padding-right: 1em;\n",
       "  padding-left: 1em;\n",
       "}\n",
       "\n",
       "\n",
       "/* Toggleable style: style used for estimator/Pipeline/ColumnTransformer box that is\n",
       "clickable and can be expanded/collapsed.\n",
       "- Pipeline and ColumnTransformer use this feature and define the default style\n",
       "- Estimators will overwrite some part of the style using the `sk-estimator` class\n",
       "*/\n",
       "\n",
       "/* Pipeline and ColumnTransformer style (default) */\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable {\n",
       "  /* Default theme specific background. It is overwritten whether we have a\n",
       "  specific estimator or a Pipeline/ColumnTransformer */\n",
       "  background-color: var(--sklearn-color-background);\n",
       "}\n",
       "\n",
       "/* Toggleable label */\n",
       "#sk-container-id-1 label.sk-toggleable__label {\n",
       "  cursor: pointer;\n",
       "  display: block;\n",
       "  width: 100%;\n",
       "  margin-bottom: 0;\n",
       "  padding: 0.5em;\n",
       "  box-sizing: border-box;\n",
       "  text-align: center;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 label.sk-toggleable__label-arrow:before {\n",
       "  /* Arrow on the left of the label */\n",
       "  content: \"▸\";\n",
       "  float: left;\n",
       "  margin-right: 0.25em;\n",
       "  color: var(--sklearn-color-icon);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {\n",
       "  color: var(--sklearn-color-text);\n",
       "}\n",
       "\n",
       "/* Toggleable content - dropdown */\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content {\n",
       "  max-height: 0;\n",
       "  max-width: 0;\n",
       "  overflow: hidden;\n",
       "  text-align: left;\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content.fitted {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content pre {\n",
       "  margin: 0.2em;\n",
       "  border-radius: 0.25em;\n",
       "  color: var(--sklearn-color-text);\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content.fitted pre {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {\n",
       "  /* Expand drop-down */\n",
       "  max-height: 200px;\n",
       "  max-width: 100%;\n",
       "  overflow: auto;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {\n",
       "  content: \"▾\";\n",
       "}\n",
       "\n",
       "/* Pipeline/ColumnTransformer-specific style */\n",
       "\n",
       "#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-label.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Estimator-specific style */\n",
       "\n",
       "/* Colorize estimator box */\n",
       "#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-estimator.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-label label.sk-toggleable__label,\n",
       "#sk-container-id-1 div.sk-label label {\n",
       "  /* The background is the default theme color */\n",
       "  color: var(--sklearn-color-text-on-default-background);\n",
       "}\n",
       "\n",
       "/* On hover, darken the color of the background */\n",
       "#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "/* Label box, darken color on hover, fitted */\n",
       "#sk-container-id-1 div.sk-label.fitted:hover label.sk-toggleable__label.fitted {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Estimator label */\n",
       "\n",
       "#sk-container-id-1 div.sk-label label {\n",
       "  font-family: monospace;\n",
       "  font-weight: bold;\n",
       "  display: inline-block;\n",
       "  line-height: 1.2em;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-label-container {\n",
       "  text-align: center;\n",
       "}\n",
       "\n",
       "/* Estimator-specific */\n",
       "#sk-container-id-1 div.sk-estimator {\n",
       "  font-family: monospace;\n",
       "  border: 1px dotted var(--sklearn-color-border-box);\n",
       "  border-radius: 0.25em;\n",
       "  box-sizing: border-box;\n",
       "  margin-bottom: 0.5em;\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-estimator.fitted {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "/* on hover */\n",
       "#sk-container-id-1 div.sk-estimator:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-estimator.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Specification for estimator info (e.g. \"i\" and \"?\") */\n",
       "\n",
       "/* Common style for \"i\" and \"?\" */\n",
       "\n",
       ".sk-estimator-doc-link,\n",
       "a:link.sk-estimator-doc-link,\n",
       "a:visited.sk-estimator-doc-link {\n",
       "  float: right;\n",
       "  font-size: smaller;\n",
       "  line-height: 1em;\n",
       "  font-family: monospace;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  border-radius: 1em;\n",
       "  height: 1em;\n",
       "  width: 1em;\n",
       "  text-decoration: none !important;\n",
       "  margin-left: 1ex;\n",
       "  /* unfitted */\n",
       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-unfitted-level-1);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link.fitted,\n",
       "a:link.sk-estimator-doc-link.fitted,\n",
       "a:visited.sk-estimator-doc-link.fitted {\n",
       "  /* fitted */\n",
       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-fitted-level-1);\n",
       "}\n",
       "\n",
       "/* On hover */\n",
       "div.sk-estimator:hover .sk-estimator-doc-link:hover,\n",
       ".sk-estimator-doc-link:hover,\n",
       "div.sk-label-container:hover .sk-estimator-doc-link:hover,\n",
       ".sk-estimator-doc-link:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "div.sk-estimator.fitted:hover .sk-estimator-doc-link.fitted:hover,\n",
       ".sk-estimator-doc-link.fitted:hover,\n",
       "div.sk-label-container:hover .sk-estimator-doc-link.fitted:hover,\n",
       ".sk-estimator-doc-link.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "/* Span, style for the box shown on hovering the info icon */\n",
       ".sk-estimator-doc-link span {\n",
       "  display: none;\n",
       "  z-index: 9999;\n",
       "  position: relative;\n",
       "  font-weight: normal;\n",
       "  right: .2ex;\n",
       "  padding: .5ex;\n",
       "  margin: .5ex;\n",
       "  width: min-content;\n",
       "  min-width: 20ex;\n",
       "  max-width: 50ex;\n",
       "  color: var(--sklearn-color-text);\n",
       "  box-shadow: 2pt 2pt 4pt #999;\n",
       "  /* unfitted */\n",
       "  background: var(--sklearn-color-unfitted-level-0);\n",
       "  border: .5pt solid var(--sklearn-color-unfitted-level-3);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link.fitted span {\n",
       "  /* fitted */\n",
       "  background: var(--sklearn-color-fitted-level-0);\n",
       "  border: var(--sklearn-color-fitted-level-3);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link:hover span {\n",
       "  display: block;\n",
       "}\n",
       "\n",
       "/* \"?\"-specific style due to the `<a>` HTML tag */\n",
       "\n",
       "#sk-container-id-1 a.estimator_doc_link {\n",
       "  float: right;\n",
       "  font-size: 1rem;\n",
       "  line-height: 1em;\n",
       "  font-family: monospace;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  border-radius: 1rem;\n",
       "  height: 1rem;\n",
       "  width: 1rem;\n",
       "  text-decoration: none;\n",
       "  /* unfitted */\n",
       "  color: var(--sklearn-color-unfitted-level-1);\n",
       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 a.estimator_doc_link.fitted {\n",
       "  /* fitted */\n",
       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-fitted-level-1);\n",
       "}\n",
       "\n",
       "/* On hover */\n",
       "#sk-container-id-1 a.estimator_doc_link:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 a.estimator_doc_link.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-3);\n",
       "}\n",
       "</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>CountVectorizer()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator fitted sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label fitted sk-toggleable__label-arrow fitted\">&nbsp;&nbsp;CountVectorizer<a class=\"sk-estimator-doc-link fitted\" rel=\"noreferrer\" target=\"_blank\" href=\"https://scikit-learn.org/1.4/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html\">?<span>Documentation for CountVectorizer</span></a><span class=\"sk-estimator-doc-link fitted\">i<span>Fitted</span></span></label><div class=\"sk-toggleable__content fitted\"><pre>CountVectorizer()</pre></div> </div></div></div></div>"
      ],
      "text/plain": [
       "CountVectorizer()"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv.fit(docs_example)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8041bcfe-0b1b-4c06-ba64-917440f89f6b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['and', 'by', 'catalog', 'cloud', 'course', 'details', 'end', 'for',\n",
       "       'google', 'homework', 'in', 'january', 'listed', 'month', 'no',\n",
       "       'now', 'of', 'prerequisites', 'python', 'register', 'setup',\n",
       "       'submit'], dtype=object)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "names = cv.get_feature_names_out()\n",
    "names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f151c0e0-bfa1-4ea4-9cbd-e3544ebfcb9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = cv.transform(docs_example)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "061bca2a-4af8-4ccf-82a5-310952a1cab3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0],\n",
       "       [0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0],\n",
       "       [0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1],\n",
       "       [0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0],\n",
       "       [1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "11af73f9-96e1-4e3b-b9a1-34fd9c60dc57",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>and</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>by</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>catalog</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cloud</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>course</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>details</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>end</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>for</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>google</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>homework</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>in</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>january</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>listed</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>now</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>of</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>prerequisites</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>python</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>register</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>setup</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>submit</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               0  1  2  3  4\n",
       "and            0  0  0  0  1\n",
       "by             0  0  1  0  0\n",
       "catalog        0  1  0  0  0\n",
       "cloud          0  0  0  0  1\n",
       "course         1  1  1  1  1\n",
       "details        1  0  0  0  0\n",
       "end            0  0  1  0  0\n",
       "for            0  0  0  1  0\n",
       "google         0  0  0  0  1\n",
       "homework       0  0  1  0  0\n",
       "in             0  1  0  0  0\n",
       "january        1  1  1  1  1\n",
       "listed         0  1  0  0  0\n",
       "month          0  0  1  0  0\n",
       "no             0  0  0  1  0\n",
       "now            1  0  0  0  0\n",
       "of             0  0  1  0  0\n",
       "prerequisites  0  1  0  1  0\n",
       "python         0  0  0  0  1\n",
       "register       1  0  0  1  0\n",
       "setup          0  0  0  0  1\n",
       "submit         0  0  1  0  0"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_docs = pd.DataFrame(X.toarray(), columns=names).T\n",
    "df_docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a9630a4d-797d-45d8-9849-2c4b3c02c249",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>catalog</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cloud</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>course</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>details</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>end</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>google</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>homework</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>january</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>listed</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>prerequisites</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>python</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>register</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>setup</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>submit</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               0  1  2  3  4\n",
       "catalog        0  1  0  0  0\n",
       "cloud          0  0  0  0  1\n",
       "course         1  1  1  1  1\n",
       "details        1  0  0  0  0\n",
       "end            0  0  1  0  0\n",
       "google         0  0  0  0  1\n",
       "homework       0  0  1  0  0\n",
       "january        1  1  1  1  1\n",
       "listed         0  1  0  0  0\n",
       "month          0  0  1  0  0\n",
       "prerequisites  0  1  0  1  0\n",
       "python         0  0  0  0  1\n",
       "register       1  0  0  1  0\n",
       "setup          0  0  0  0  1\n",
       "submit         0  0  1  0  0"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv = CountVectorizer(stop_words='english')\n",
    "X = cv.fit_transform(docs_example)\n",
    "\n",
    "names = cv.get_feature_names_out()\n",
    "\n",
    "df_docs = pd.DataFrame(X.toarray(), columns=names).T\n",
    "df_docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "11b9f3d6-5d93-4a29-baf6-4cf8c5cf1d02",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "23a63806-bbb3-4238-ad78-11aed2c1c9ff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>catalog</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.57</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cloud</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>course</th>\n",
       "      <td>0.33</td>\n",
       "      <td>0.27</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0.36</td>\n",
       "      <td>0.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>details</th>\n",
       "      <td>0.69</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>end</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.47</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>google</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>homework</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.47</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>january</th>\n",
       "      <td>0.33</td>\n",
       "      <td>0.27</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0.36</td>\n",
       "      <td>0.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>listed</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.57</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.47</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>prerequisites</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.46</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>python</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>register</th>\n",
       "      <td>0.56</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>setup</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>submit</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.47</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  0     1     2     3     4\n",
       "catalog        0.00  0.57  0.00  0.00  0.00\n",
       "cloud          0.00  0.00  0.00  0.00  0.47\n",
       "course         0.33  0.27  0.23  0.36  0.23\n",
       "details        0.69  0.00  0.00  0.00  0.00\n",
       "end            0.00  0.00  0.47  0.00  0.00\n",
       "google         0.00  0.00  0.00  0.00  0.47\n",
       "homework       0.00  0.00  0.47  0.00  0.00\n",
       "january        0.33  0.27  0.23  0.36  0.23\n",
       "listed         0.00  0.57  0.00  0.00  0.00\n",
       "month          0.00  0.00  0.47  0.00  0.00\n",
       "prerequisites  0.00  0.46  0.00  0.61  0.00\n",
       "python         0.00  0.00  0.00  0.00  0.47\n",
       "register       0.56  0.00  0.00  0.61  0.00\n",
       "setup          0.00  0.00  0.00  0.00  0.47\n",
       "submit         0.00  0.00  0.47  0.00  0.00"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv = TfidfVectorizer(stop_words='english')\n",
    "X = cv.fit_transform(docs_example)\n",
    "\n",
    "names = cv.get_feature_names_out()\n",
    "\n",
    "df_docs = pd.DataFrame(X.toarray(), columns=names).T\n",
    "df_docs.round(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "7c39d3be-281d-4d01-a645-e5a49678f447",
   "metadata": {},
   "outputs": [],
   "source": [
    "query = \"Do I need to know python to sign up for the January course?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "2a0f5f35-b1b4-4e8a-b0af-7b83e8830b89",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.        , 0.        , 0.39515588, 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.39515588, 0.        , 0.        ,\n",
       "        0.        , 0.829279  , 0.        , 0.        , 0.        ]])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q = cv.transform([query])\n",
    "q.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "9bb650c2-5ec3-4419-a27c-8db1af022206",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'catalog': 0.0,\n",
       " 'cloud': 0.0,\n",
       " 'course': 0.39515588491314224,\n",
       " 'details': 0.0,\n",
       " 'end': 0.0,\n",
       " 'google': 0.0,\n",
       " 'homework': 0.0,\n",
       " 'january': 0.39515588491314224,\n",
       " 'listed': 0.0,\n",
       " 'month': 0.0,\n",
       " 'prerequisites': 0.0,\n",
       " 'python': 0.8292789960182417,\n",
       " 'register': 0.0,\n",
       " 'setup': 0.0,\n",
       " 'submit': 0.0}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query_dict = dict(zip(names, q.toarray()[0]))\n",
    "query_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "b099d689-1156-4f49-8cfa-4bdda6c60734",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'catalog': 0.5675015398728066,\n",
       " 'cloud': 0.0,\n",
       " 'course': 0.2704175244456293,\n",
       " 'details': 0.0,\n",
       " 'end': 0.0,\n",
       " 'google': 0.0,\n",
       " 'homework': 0.0,\n",
       " 'january': 0.2704175244456293,\n",
       " 'listed': 0.5675015398728066,\n",
       " 'month': 0.0,\n",
       " 'prerequisites': 0.45785666908911726,\n",
       " 'python': 0.0,\n",
       " 'register': 0.0,\n",
       " 'setup': 0.0,\n",
       " 'submit': 0.0}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc_dict = dict(zip(names, X.toarray()[1]))\n",
    "doc_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "8b4600a5-1924-4680-acc9-55b1f34a4547",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_qd = pd.DataFrame([query_dict, doc_dict], index=['query', 'doc']).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "8afb041c-da6f-4f20-b58e-12bc1800c223",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.21371415233666782"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(df_qd['query'] * df_qd['doc']).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "dbe38cf2-5925-48c1-be6b-4867e63b1731",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.25955955],\n",
       "       [0.21371415],\n",
       "       [0.17843726],\n",
       "       [0.28419115],\n",
       "       [0.57137158]])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.dot(q.T).toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "66c49ec8-1ddb-4c77-bbd9-87afd6aaeac6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "79d49a2a-1e7e-432d-b5fe-cb7d83a7eef0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.25955955],\n",
       "       [0.21371415],\n",
       "       [0.17843726],\n",
       "       [0.28419115],\n",
       "       [0.57137158]])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cosine_similarity(X, q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "5257f8a9-c717-4b78-b253-017e9c994a5d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['course', 'section', 'question', 'text'], dtype='object')"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "01a55dc8-240d-4c8f-a1f4-400ffddd7e80",
   "metadata": {},
   "outputs": [],
   "source": [
    "fields = ['section', 'question', 'text']\n",
    "transformers = {}\n",
    "matrices = {}\n",
    "\n",
    "for field in fields:\n",
    "    cv = TfidfVectorizer(stop_words='english', min_df=3)\n",
    "    X = cv.fit_transform(df[field])\n",
    "\n",
    "    transformers[field] = cv\n",
    "    matrices[field] = X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "bb381300-e472-4f07-ac8c-7ebcd6261d7b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['001', '01', '02', ..., 'zones', 'zoom', 'zoomcamp'], dtype=object)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transformers['text'].get_feature_names_out()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "ed3da1a0-a0fe-43db-acbb-f78eaad1d935",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<948x2118 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 26463 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matrices['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "cc72288c-58ed-47e5-93a1-586140389daf",
   "metadata": {},
   "outputs": [],
   "source": [
    "query = \"I just singned up. Is it too late to join the course?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "076ed1ee-717d-4ba2-bda6-5dadb84dee51",
   "metadata": {},
   "outputs": [],
   "source": [
    "q = transformers['text'].transform([query])\n",
    "score = cosine_similarity(matrices['text'], q).flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "f35c13cc-ae93-4b64-b6f2-8519f1b1175e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.3336047 , 0.        , 0.        , 0.1328874 , 0.        ,\n",
       "       0.        , 0.        , 0.12722114, 0.        , 0.        ])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mask = (df.course == 'data-engineering-zoomcamp').values\n",
    "score = score * mask\n",
    "score[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "7af8de09-054e-4638-af4d-27eb9335187f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "fd76b3b5-092c-4c4d-b06b-c1a2ea73e032",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  0,  15,  22,  27,  38, 287,   3,   7, 113,  11])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx = np.argsort(-score)[:10]\n",
    "idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "92c5e093-18a5-4e07-b1ba-825ae8281144",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.3336047 , 0.23530268, 0.22668   , 0.1894954 , 0.16484429,\n",
       "       0.13921764, 0.1328874 , 0.12722114, 0.1207499 , 0.10830554])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "e1629617-e109-4aeb-a758-42b621665bca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      The purpose of this document is to capture fre...\n",
       "15     No, late submissions are not allowed. But if t...\n",
       "22     It's up to you which platform and environment ...\n",
       "27     You can do most of the course without a cloud....\n",
       "38     You will have two attempts for a project. If t...\n",
       "287    This error could result if you are using some ...\n",
       "3      You don't need it. You're accepted. You can al...\n",
       "7      Yes, we will keep all the materials after the ...\n",
       "113    In the join queries, if we mention the column ...\n",
       "11     No, you can only get a certificate if you fini...\n",
       "Name: text, dtype: object"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[idx].text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "070e9bfc-7364-4746-8a13-5429c1e3793f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['section', 'question', 'text']"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fields"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "980555fc-4603-465a-a46c-84efd5464424",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "39a0f583-7c95-493d-b54e-19737c4e6807",
   "metadata": {},
   "outputs": [],
   "source": [
    "query = \"I just singned up. Is it too late to join the course?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "ecde3950-b575-4837-974f-7353431e3c66",
   "metadata": {},
   "outputs": [],
   "source": [
    "boost = {'question': 3.0}\n",
    "\n",
    "score = np.zeros(len(df))\n",
    "\n",
    "for f in fields:\n",
    "    b = boost.get(f, 1.0)\n",
    "    q = transformers[f].transform([query])\n",
    "    s = cosine_similarity(matrices[f], q).flatten()\n",
    "    score = score + b * s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb9c27bf-1d0a-4d4d-b26b-ff4f2c7e559e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "76926c4a-60c0-4547-a3e9-3ed5c01cc9b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "filters = {\n",
    "    'course': 'data-engineering-zoomcamp'\n",
    "}\n",
    "\n",
    "for field, value in filters.items():\n",
    "    mask = (df[field] == value).values\n",
    "    score = score * mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "53da7a34-8052-48a1-9041-bb0f94416448",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'course': 'data-engineering-zoomcamp',\n",
       "  'section': 'General course-related questions',\n",
       "  'question': 'Course - What are the prerequisites for this course?',\n",
       "  'text': 'GitHub - DataTalksClub data-engineering-zoomcamp#prerequisites'},\n",
       " {'course': 'data-engineering-zoomcamp',\n",
       "  'section': 'General course-related questions',\n",
       "  'question': 'How can we contribute to the course?',\n",
       "  'text': 'Star the repo! Share it with friends if you find it useful ❣️\\nCreate a PR if you see you can improve the text or the structure of the repository.'},\n",
       " {'course': 'data-engineering-zoomcamp',\n",
       "  'section': 'General course-related questions',\n",
       "  'question': 'Course - What can I do before the course starts?',\n",
       "  'text': 'You can start by installing and setting up all the dependencies and requirements:\\nGoogle cloud account\\nGoogle Cloud SDK\\nPython 3 (installed with Anaconda)\\nTerraform\\nGit\\nLook over the prerequisites and syllabus to see if you are comfortable with these subjects.'},\n",
       " {'course': 'data-engineering-zoomcamp',\n",
       "  'section': 'General course-related questions',\n",
       "  'question': 'Course - how many Zoomcamps in a year?',\n",
       "  'text': \"There are 3 Zoom Camps in a year, as of 2024. However, they are for separate courses:\\nData-Engineering (Jan - Apr)\\nMLOps (May - Aug)\\nMachine Learning (Sep - Jan)\\nThere's only one Data-Engineering Zoomcamp “live” cohort per year, for the certification. Same as for the other Zoomcamps.\\nThey follow pretty much the same schedule for each cohort per zoomcamp. For Data-Engineering it is (generally) from Jan-Apr of the year. If you’re not interested in the Certificate, you can take any zoom camps at any time, at your own pace, out of sync with any “live” cohort.\"},\n",
       " {'course': 'data-engineering-zoomcamp',\n",
       "  'section': 'General course-related questions',\n",
       "  'question': 'Course - Which playlist on YouTube should I refer to?',\n",
       "  'text': 'All the main videos are stored in the Main “DATA ENGINEERING” playlist (no year specified). The Github repository has also been updated to show each video with a thumbnail, that would bring you directly to the same playlist below.\\nBelow is the MAIN PLAYLIST’. And then you refer to the year specific playlist for additional videos for that year like for office hours videos etc. Also find this playlist pinned to the slack channel.\\nh\\nttps://youtube.com/playlist?list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&si=NspQhtZhZQs1B9F-'},\n",
       " {'course': 'data-engineering-zoomcamp',\n",
       "  'section': 'General course-related questions',\n",
       "  'question': 'Course - When will the course start?',\n",
       "  'text': \"The purpose of this document is to capture frequently asked technical questions\\nThe exact day and hour of the course will be 15th Jan 2024 at 17h00. The course will start with the first  “Office Hours'' live.1\\nSubscribe to course public Google Calendar (it works from Desktop only).\\nRegister before the course starts using this link.\\nJoin the course Telegram channel with announcements.\\nDon’t forget to register in DataTalks.Club's Slack and join the channel.\"},\n",
       " {'course': 'data-engineering-zoomcamp',\n",
       "  'section': 'General course-related questions',\n",
       "  'question': 'Course - Can I follow the course after it finishes?',\n",
       "  'text': 'Yes, we will keep all the materials after the course finishes, so you can follow the course at your own pace after it finishes.\\nYou can also continue looking at the homeworks and continue preparing for the next cohort. I guess you can also start working on your final capstone project.'},\n",
       " {'course': 'data-engineering-zoomcamp',\n",
       "  'section': 'General course-related questions',\n",
       "  'question': 'Course - Can I still join the course after the start date?',\n",
       "  'text': \"Yes, even if you don't register, you're still eligible to submit the homeworks.\\nBe aware, however, that there will be deadlines for turning in the final projects. So don't leave everything for the last minute.\"},\n",
       " {'course': 'data-engineering-zoomcamp',\n",
       "  'section': 'General course-related questions',\n",
       "  'question': 'Course - Can I get support if I take the course in the self-paced mode?',\n",
       "  'text': 'Yes, the slack channel remains open and you can ask questions there. But always sDocker containers exit code w search the channel first and second, check the FAQ (this document), most likely all your questions are already answered here.\\nYou can also tag the bot @ZoomcampQABot to help you conduct the search, but don’t rely on its answers 100%, it is pretty good though.'},\n",
       " {'course': 'data-engineering-zoomcamp',\n",
       "  'section': 'General course-related questions',\n",
       "  'question': 'Course - \\u200b\\u200bHow many hours per week am I expected to spend on this  course?',\n",
       "  'text': 'It depends on your background and previous experience with modules. It is expected to require about 5 - 15 hours per week. [source1] [source2]\\nYou can also calculate it yourself using this data and then update this answer.'}]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx = np.argsort(-score)[:10]\n",
    "results = df.iloc[idx]\n",
    "results.to_dict(orient='records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8d96950-a7e5-4b50-870d-ddf472738b92",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "74ab72b1-47fd-451b-9660-e4df629c4b14",
   "metadata": {},
   "outputs": [],
   "source": [
    "class TextSearch:\n",
    "\n",
    "    def __init__(self, text_fields):\n",
    "        self.text_fields = text_fields\n",
    "        self.matrices = {}\n",
    "        self.vectorizers = {}\n",
    "\n",
    "    def fit(self, records, vectorizer_params={}):\n",
    "        self.df = pd.DataFrame(records)\n",
    "\n",
    "        for f in self.text_fields:\n",
    "            cv = TfidfVectorizer(**vectorizer_params)\n",
    "            X = cv.fit_transform(self.df[f])\n",
    "            self.matrices[f] = X\n",
    "            self.vectorizers[f] = cv\n",
    "\n",
    "    def search(self, query, n_results=10, boost={}, filters={}):\n",
    "        score = np.zeros(len(self.df))\n",
    "\n",
    "        for f in self.text_fields:\n",
    "            b = boost.get(f, 1.0)\n",
    "            q = self.vectorizers[f].transform([query])\n",
    "            s = cosine_similarity(self.matrices[f], q).flatten()\n",
    "            score = score + b * s\n",
    "\n",
    "        for field, value in filters.items():\n",
    "            mask = (self.df[field] == value).values\n",
    "            score = score * mask\n",
    "\n",
    "        idx = np.argsort(-score)[:n_results]\n",
    "        results = self.df.iloc[idx]\n",
    "        return results.to_dict(orient='records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "8fe5e13d-02aa-4f77-862e-70f5b14c8067",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['section', 'question', 'text']"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fields"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "db06ee90-31d1-4842-829a-662a812eed17",
   "metadata": {},
   "outputs": [],
   "source": [
    "index = TextSearch(text_fields=['section', 'question', 'text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "7c3ec354-9382-4fee-94f4-458a5044f019",
   "metadata": {},
   "outputs": [],
   "source": [
    "index.fit(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "ecde6ac7-ce80-4b07-b079-55e822f399f7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'I just singned up. Is it too late to join the course?'"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "09895ca8-6a96-4e34-90fe-de6a1b117681",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'text': \"Yes, even if you don't register, you're still eligible to submit the homeworks.\\nBe aware, however, that there will be deadlines for turning in the final projects. So don't leave everything for the last minute.\",\n",
       "  'section': 'General course-related questions',\n",
       "  'question': 'Course - Can I still join the course after the start date?',\n",
       "  'course': 'data-engineering-zoomcamp'},\n",
       " {'text': \"The purpose of this document is to capture frequently asked technical questions\\nThe exact day and hour of the course will be 15th Jan 2024 at 17h00. The course will start with the first  “Office Hours'' live.1\\nSubscribe to course public Google Calendar (it works from Desktop only).\\nRegister before the course starts using this link.\\nJoin the course Telegram channel with announcements.\\nDon’t forget to register in DataTalks.Club's Slack and join the channel.\",\n",
       "  'section': 'General course-related questions',\n",
       "  'question': 'Course - When will the course start?',\n",
       "  'course': 'data-engineering-zoomcamp'},\n",
       " {'text': 'Yes, we will keep all the materials after the course finishes, so you can follow the course at your own pace after it finishes.\\nYou can also continue looking at the homeworks and continue preparing for the next cohort. I guess you can also start working on your final capstone project.',\n",
       "  'section': 'General course-related questions',\n",
       "  'question': 'Course - Can I follow the course after it finishes?',\n",
       "  'course': 'data-engineering-zoomcamp'},\n",
       " {'text': 'No, late submissions are not allowed. But if the form is still not closed and it’s after the due date, you can still submit the homework. confirm your submission by the date-timestamp on the Course page.y\\nOlder news:[source1] [source2]',\n",
       "  'section': 'General course-related questions',\n",
       "  'question': 'Homework - Are late submissions of homework allowed?',\n",
       "  'course': 'data-engineering-zoomcamp'},\n",
       " {'text': 'You can start by installing and setting up all the dependencies and requirements:\\nGoogle cloud account\\nGoogle Cloud SDK\\nPython 3 (installed with Anaconda)\\nTerraform\\nGit\\nLook over the prerequisites and syllabus to see if you are comfortable with these subjects.',\n",
       "  'section': 'General course-related questions',\n",
       "  'question': 'Course - What can I do before the course starts?',\n",
       "  'course': 'data-engineering-zoomcamp'}]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "index.search(\n",
    "    query='I just singned up. Is it too late to join the course?',\n",
    "    n_results=5,\n",
    "    boost={'question': 3.0},\n",
    "    filters={'course': 'data-engineering-zoomcamp'}\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "da26fadb-2e59-451b-86f8-09d5c0547fc4",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import TruncatedSVD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "cf7a1601-cdae-4299-b3ad-5fa73964b806",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = matrices['text']\n",
    "cv = transformers['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "22c13291-b493-480e-9e10-256eedb9c1d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "svd = TruncatedSVD(n_components=16)\n",
    "X_emb = svd.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "69b68353-1296-43e4-8501-955010f0fc87",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.08800419, -0.07512153,  0.10092401,  0.05127205,  0.05138965,\n",
       "       -0.05951887,  0.02004213,  0.0607822 , -0.20294215,  0.33437825,\n",
       "        0.03862638, -0.08756796, -0.13102394, -0.02912721,  0.03607929,\n",
       "        0.03620086])"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_emb[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "2600579f-a557-4dde-bc3e-b24afc526ad2",
   "metadata": {},
   "outputs": [],
   "source": [
    "query = 'I just singned up. Is it too late to join the course?'\n",
    "\n",
    "Q = cv.transform([query])\n",
    "Q_emb = svd.transform(Q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "bd4b9419-b36b-45a8-9152-caa31a4761d4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.0435364 , -0.03070852,  0.04387019,  0.01232817,  0.0260592 ,\n",
       "       -0.04886131,  0.01335269,  0.02430529, -0.12124854,  0.1794836 ,\n",
       "        0.01503188,  0.06727616, -0.0731015 ,  0.04771703,  0.01171046,\n",
       "        0.03114754])"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Q_emb[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "103badbb-881b-46ff-b4ab-302265e190cd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.120735142515765"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.dot(X_emb[0], Q_emb[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "c015d99a-396f-4de6-bd8d-719b30eebbf0",
   "metadata": {},
   "outputs": [],
   "source": [
    "score = cosine_similarity(X_emb, Q_emb).flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "53728975-091a-4f59-9ba8-8b3cb97e8c27",
   "metadata": {},
   "outputs": [],
   "source": [
    "idx = np.argsort(-score)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "af6d4f7c-cef3-4c15-8c79-d2041afbf6f4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['If you have submitted two projects (and peer-reviewed at least 3 course-mates’ projects for each submission), you will get the certificate for the course. According to the course coordinator, Alexey Grigorev, only two projects are needed to get the course certificate.\\n(optional) David Odimegwu',\n",
       " 'Yes, you can. You won’t be able to submit some of the homeworks, but you can still take part in the course.\\nIn order to get a certificate, you need to submit 2 out of 3 course projects and review 3 peers’ Projects by the deadline. It means that if you join the course at the end of November and manage to work on two projects, you will still be eligible for a certificate.',\n",
       " 'The course videos are pre-recorded, you can start watching the course right now.\\nWe will also occasionally have office hours - live sessions where we will answer your questions. The office hours sessions are recorded too.\\nYou can see the office hours as well as the pre-recorded course videos in the course playlist on YouTube.',\n",
       " \"The purpose of this document is to capture frequently asked technical questions\\nThe exact day and hour of the course will be 15th Jan 2024 at 17h00. The course will start with the first  “Office Hours'' live.1\\nSubscribe to course public Google Calendar (it works from Desktop only).\\nRegister before the course starts using this link.\\nJoin the course Telegram channel with announcements.\\nDon’t forget to register in DataTalks.Club's Slack and join the channel.\",\n",
       " 'No, late submissions are not allowed. But if the form is still not closed and it’s after the due date, you can still submit the homework. confirm your submission by the date-timestamp on the Course page.y\\nOlder news:[source1] [source2]',\n",
       " \"No, you can only get a certificate if you finish the course with a “live” cohort. We don't award certificates for the self-paced mode. The reason is you need to peer-review capstone(s) after submitting a project. You can only peer-review projects at the time the course is running.\",\n",
       " 'Yes, we will keep all the materials after the course finishes, so you can follow the course at your own pace after it finishes.\\nYou can also continue looking at the homeworks and continue preparing for the next cohort. I guess you can also start working on your final capstone project.',\n",
       " 'No, it’s not possible. The form is closed after the due date. But don’t worry, homework is not mandatory for finishing the course.',\n",
       " 'Please choose the closest one to your answer. Also do not post your answer in the course slack channel.',\n",
       " 'The course is available in the self-paced mode too, so you can go through the materials at any time. But if you want to do it as a cohort with other students, the next iterations will happen in September 2023, September 2024 (and potentially other Septembers as well).']"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(df.loc[idx].text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "b12b1f88-7bb4-4906-a8d0-3ff3bd5f5a0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import NMF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "66ab8528-e6e3-4191-9425-fbae117c518b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.30663126,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        ])"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nmf = NMF(n_components=16)\n",
    "X_emb = nmf.fit_transform(X)\n",
    "X_emb[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "a73571ba-6f8a-4b24-bd97-18ca6eac92ca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.        , 0.00114538, 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.1713533 ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.00066279,\n",
       "       0.        ])"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Q = cv.transform([query])\n",
    "Q_emb = nmf.transform(Q)\n",
    "Q_emb[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "77b4efea-db0d-4df5-8408-717ce953ae96",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"The purpose of this document is to capture frequently asked technical questions\\nThe exact day and hour of the course will be 15th Jan 2024 at 17h00. The course will start with the first  “Office Hours'' live.1\\nSubscribe to course public Google Calendar (it works from Desktop only).\\nRegister before the course starts using this link.\\nJoin the course Telegram channel with announcements.\\nDon’t forget to register in DataTalks.Club's Slack and join the channel.\",\n",
       " 'Please choose the closest one to your answer. Also do not post your answer in the course slack channel.',\n",
       " 'No, it’s not possible. The form is closed after the due date. But don’t worry, homework is not mandatory for finishing the course.',\n",
       " 'Yes, you can. You won’t be able to submit some of the homeworks, but you can still take part in the course.\\nIn order to get a certificate, you need to submit 2 out of 3 course projects and review 3 peers’ Projects by the deadline. It means that if you join the course at the end of November and manage to work on two projects, you will still be eligible for a certificate.',\n",
       " \"Yes, even if you don't register, you're still eligible to submit the homeworks.\\nBe aware, however, that there will be deadlines for turning in the final projects. So don't leave everything for the last minute.\",\n",
       " 'If you have submitted two projects (and peer-reviewed at least 3 course-mates’ projects for each submission), you will get the certificate for the course. According to the course coordinator, Alexey Grigorev, only two projects are needed to get the course certificate.\\n(optional) David Odimegwu',\n",
       " 'The course videos are pre-recorded, you can start watching the course right now.\\nWe will also occasionally have office hours - live sessions where we will answer your questions. The office hours sessions are recorded too.\\nYou can see the office hours as well as the pre-recorded course videos in the course playlist on YouTube.',\n",
       " 'Everything is recorded, so you won’t miss anything. You will be able to ask your questions for office hours in advance and we will cover them during the live stream. Also, you can always ask questions in Slack.',\n",
       " \"No, you can only get a certificate if you finish the course with a “live” cohort. We don't award certificates for the self-paced mode. The reason is you need to peer-review capstone(s) after submitting a project. You can only peer-review projects at the time the course is running.\",\n",
       " 'Yes, we will keep all the materials after the course finishes, so you can follow the course at your own pace after it finishes.\\nYou can also continue looking at the homeworks and continue preparing for the next cohort. I guess you can also start working on your final capstone project.']"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score = cosine_similarity(X_emb, Q_emb).flatten()\n",
    "idx = np.argsort(-score)[:10]\n",
    "list(df.loc[idx].text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "03778f26-e16a-4879-897e-7e15797fc364",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from transformers import BertModel, BertTokenizer\n",
    "\n",
    "tokenizer = BertTokenizer.from_pretrained(\"bert-base-uncased\")\n",
    "model = BertModel.from_pretrained(\"bert-base-uncased\")\n",
    "model.eval()  # Set the model to evaluation mode if not training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "c7f95191-3b91-4137-ad6c-d6d22c03afa6",
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = [\n",
    "    \"Yes, we will keep all the materials after the course finishes.\",\n",
    "    \"You can follow the course at your own pace after it finishes\"\n",
    "]\n",
    "encoded_input = tokenizer(texts, padding=True, truncation=True, return_tensors='pt')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "0c51bc5b-3908-42dc-88cc-c07b33a6ddc4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': tensor([[  101,  2748,  1010,  2057,  2097,  2562,  2035,  1996,  4475,  2044,\n",
       "          1996,  2607, 12321,  1012,   102],\n",
       "        [  101,  2017,  2064,  3582,  1996,  2607,  2012,  2115,  2219,  6393,\n",
       "          2044,  2009, 12321,   102,     0]]), 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n",
       "        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0]])}"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded_input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "37cbdc75-0916-4360-b1a7-c8d03142398f",
   "metadata": {},
   "outputs": [],
   "source": [
    "with torch.no_grad():  # Disable gradient calculation for inference\n",
    "    outputs = model(**encoded_input)\n",
    "    hidden_states = outputs.last_hidden_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "2e0c208b-d1a6-48d2-9080-130dd2f37f25",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 15, 768])"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hidden_states.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "47fc911c-5ed7-4a97-86bd-b4bc2e311214",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 768])"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_embeddings = hidden_states.mean(dim=1)\n",
    "sentence_embeddings.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "3dbe88f8-d39f-4b22-8f17-4d911c14e185",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.35999233, -0.16072303,  0.35452363, ...,  0.04289245,\n",
       "         0.03482292, -0.03822247],\n",
       "       [ 0.17849916, -0.500025  ,  0.25277564, ..., -0.11413109,\n",
       "        -0.3360847 ,  0.4109514 ]], dtype=float32)"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_embeddings.numpy()\n",
    "\n",
    "# note that if use a GPU, first you need to move your tensors to CPU\n",
    "# sentence_embeddings_cpu = sentence_embeddings.cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "032e6145-bf8d-4ccf-a592-0bf82c5822f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_batches(seq, n):\n",
    "    result = []\n",
    "    for i in range(0, len(seq), n):\n",
    "        batch = seq[i:i+n]\n",
    "        result.append(batch)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "4d013583-f000-4c1c-88e4-b05dc5f91504",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm.auto import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "57853a2c-2d45-460d-b929-6f6b564ed55b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_embeddings(texts, batch_size=8):\n",
    "    text_batches = make_batches(texts, 8)\n",
    "    \n",
    "    all_embeddings = []\n",
    "    \n",
    "    for batch in tqdm(text_batches):\n",
    "        encoded_input = tokenizer(batch, padding=True, truncation=True, return_tensors='pt')\n",
    "    \n",
    "        with torch.no_grad():\n",
    "            outputs = model(**encoded_input)\n",
    "            hidden_states = outputs.last_hidden_state\n",
    "            \n",
    "            batch_embeddings = hidden_states.mean(dim=1)\n",
    "            batch_embeddings_np = batch_embeddings.cpu().numpy()\n",
    "            all_embeddings.append(batch_embeddings_np)\n",
    "    \n",
    "    final_embeddings = np.vstack(all_embeddings)\n",
    "    return final_embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "ba794951-7869-407c-a790-ae75419ca294",
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "71b79a56-3361-401b-a734-34088dee1833",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20357dc1-3fd1-4b0e-a476-88ce433e347a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "computing embeddings for section...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b9ca8a72092a49ec87819b0b3290eb59",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/119 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "computing embeddings for question...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cebee722915a480bb9ad74495d324619",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/119 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# fields = ['section', 'question', 'text']\n",
    "\n",
    "for f in fields:\n",
    "    print(f'computing embeddings for {f}...')\n",
    "    embeddings[f] = compute_embeddings(df[f].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e30cf0c0-0522-411c-8e6f-3368aafd84e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb3fbe18-b993-4d03-890f-0a9f37ad11d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('embeddings.bin', 'wb') as f_out:\n",
    "    pickle.dump(embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8fe22344-e0f9-40c4-8d51-9e88c79417af",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}