movie-content-recommendations-h2o.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f2fd9b44",
   "metadata": {
    "papermill": {
     "duration": 0.042102,
     "end_time": "2022-04-21T12:11:10.295058",
     "exception": false,
     "start_time": "2022-04-21T12:11:10.252956",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Build your own Recommendation Systems !!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e3423514",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_kg_hide-output": true,
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "collapsed": true,
    "execution": {
     "iopub.execute_input": "2022-04-21T12:11:10.377001Z",
     "iopub.status.busy": "2022-04-21T12:11:10.376151Z",
     "iopub.status.idle": "2022-04-21T12:11:22.632236Z",
     "shell.execute_reply": "2022-04-21T12:11:22.631289Z"
    },
    "jupyter": {
     "outputs_hidden": true
    },
    "papermill": {
     "duration": 12.299289,
     "end_time": "2022-04-21T12:11:22.634824",
     "exception": false,
     "start_time": "2022-04-21T12:11:10.335535",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in links: http://h2o-release.s3.amazonaws.com/h2o/latest_stable_Py.html\r\n",
      "Requirement already satisfied: h2o in /opt/conda/lib/python3.7/site-packages (3.36.0.4)\r\n",
      "Requirement already satisfied: future in /opt/conda/lib/python3.7/site-packages (from h2o) (0.18.2)\r\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.7/site-packages (from h2o) (2.27.1)\r\n",
      "Requirement already satisfied: tabulate in /opt/conda/lib/python3.7/site-packages (from h2o) (0.8.9)\r\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /opt/conda/lib/python3.7/site-packages (from requests->h2o) (2.0.12)\r\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests->h2o) (2021.10.8)\r\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests->h2o) (1.26.8)\r\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests->h2o) (3.3)\r\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m\u001b[33mWARNING: The repository located at h2o-release.s3.amazonaws.com is not a trusted or secure host and is being ignored. If this repository is available via HTTPS we recommend you use HTTPS instead, otherwise you may silence this warning and allow it anyway with '--trusted-host h2o-release.s3.amazonaws.com'.\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "!pip install -f http://h2o-release.s3.amazonaws.com/h2o/latest_stable_Py.html h2o"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "85191b2f",
   "metadata": {
    "papermill": {
     "duration": 0.040585,
     "end_time": "2022-04-21T12:11:22.716852",
     "exception": false,
     "start_time": "2022-04-21T12:11:22.676267",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "402b6cc9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-21T12:11:22.801703Z",
     "iopub.status.busy": "2022-04-21T12:11:22.801404Z",
     "iopub.status.idle": "2022-04-21T12:11:23.095957Z",
     "shell.execute_reply": "2022-04-21T12:11:23.095146Z"
    },
    "papermill": {
     "duration": 0.338588,
     "end_time": "2022-04-21T12:11:23.097954",
     "exception": false,
     "start_time": "2022-04-21T12:11:22.759366",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((8001, 15), (1825, 14))"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('../input/ctrl-shift-intelligence-2k22/train.csv')\n",
    "test = pd.read_csv('../input/ctrl-shift-intelligence-2k22/test.csv')\n",
    "\n",
    "train.shape, test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9d7f0f12",
   "metadata": {
    "papermill": {
     "duration": 0.040386,
     "end_time": "2022-04-21T12:11:23.180051",
     "exception": false,
     "start_time": "2022-04-21T12:11:23.139665",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Understanding Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "db94dfc5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-21T12:11:23.265544Z",
     "iopub.status.busy": "2022-04-21T12:11:23.264992Z",
     "iopub.status.idle": "2022-04-21T12:11:23.303995Z",
     "shell.execute_reply": "2022-04-21T12:11:23.303317Z"
    },
    "papermill": {
     "duration": 0.083904,
     "end_time": "2022-04-21T12:11:23.306556",
     "exception": false,
     "start_time": "2022-04-21T12:11:23.222652",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 8001 entries, 0 to 8000\n",
      "Data columns (total 15 columns):\n",
      " #   Column      Non-Null Count  Dtype  \n",
      "---  ------      --------------  -----  \n",
      " 0   Unnamed: 0  8001 non-null   int64  \n",
      " 1   userID      8001 non-null   int64  \n",
      " 2   title       8000 non-null   object \n",
      " 3   year        7946 non-null   float64\n",
      " 4   kind        7971 non-null   object \n",
      " 5   genre       7916 non-null   object \n",
      " 6   vote        7228 non-null   float64\n",
      " 7   country     7415 non-null   object \n",
      " 8   language    7305 non-null   object \n",
      " 9   cast        7694 non-null   object \n",
      " 10  director    6368 non-null   object \n",
      " 11  composer    4852 non-null   object \n",
      " 12  writer      6532 non-null   object \n",
      " 13  runtime     7073 non-null   float64\n",
      " 14  rating      7228 non-null   float64\n",
      "dtypes: float64(4), int64(2), object(9)\n",
      "memory usage: 937.7+ KB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bd0f24d1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-21T12:11:23.391431Z",
     "iopub.status.busy": "2022-04-21T12:11:23.390955Z",
     "iopub.status.idle": "2022-04-21T12:11:23.431644Z",
     "shell.execute_reply": "2022-04-21T12:11:23.430936Z"
    },
    "papermill": {
     "duration": 0.085323,
     "end_time": "2022-04-21T12:11:23.433785",
     "exception": false,
     "start_time": "2022-04-21T12:11:23.348462",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>userID</th>\n",
       "      <th>year</th>\n",
       "      <th>vote</th>\n",
       "      <th>runtime</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>8001.000000</td>\n",
       "      <td>8001.000000</td>\n",
       "      <td>7946.000000</td>\n",
       "      <td>7.228000e+03</td>\n",
       "      <td>7073.000000</td>\n",
       "      <td>7228.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>4000.000000</td>\n",
       "      <td>4000.000000</td>\n",
       "      <td>1995.146111</td>\n",
       "      <td>1.902652e+04</td>\n",
       "      <td>97.644140</td>\n",
       "      <td>6.716146</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>2309.834085</td>\n",
       "      <td>2309.834085</td>\n",
       "      <td>15.829040</td>\n",
       "      <td>9.951044e+04</td>\n",
       "      <td>64.906578</td>\n",
       "      <td>1.291765</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1905.000000</td>\n",
       "      <td>5.000000e+00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.200000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2000.000000</td>\n",
       "      <td>2000.000000</td>\n",
       "      <td>1990.000000</td>\n",
       "      <td>2.427500e+02</td>\n",
       "      <td>75.000000</td>\n",
       "      <td>6.100000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>4000.000000</td>\n",
       "      <td>4000.000000</td>\n",
       "      <td>1999.000000</td>\n",
       "      <td>1.266500e+03</td>\n",
       "      <td>93.000000</td>\n",
       "      <td>6.900000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>6000.000000</td>\n",
       "      <td>6000.000000</td>\n",
       "      <td>2003.000000</td>\n",
       "      <td>6.138500e+03</td>\n",
       "      <td>109.000000</td>\n",
       "      <td>7.600000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>8000.000000</td>\n",
       "      <td>8000.000000</td>\n",
       "      <td>2023.000000</td>\n",
       "      <td>2.462087e+06</td>\n",
       "      <td>1620.000000</td>\n",
       "      <td>9.600000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Unnamed: 0       userID         year          vote      runtime  \\\n",
       "count  8001.000000  8001.000000  7946.000000  7.228000e+03  7073.000000   \n",
       "mean   4000.000000  4000.000000  1995.146111  1.902652e+04    97.644140   \n",
       "std    2309.834085  2309.834085    15.829040  9.951044e+04    64.906578   \n",
       "min       0.000000     0.000000  1905.000000  5.000000e+00     1.000000   \n",
       "25%    2000.000000  2000.000000  1990.000000  2.427500e+02    75.000000   \n",
       "50%    4000.000000  4000.000000  1999.000000  1.266500e+03    93.000000   \n",
       "75%    6000.000000  6000.000000  2003.000000  6.138500e+03   109.000000   \n",
       "max    8000.000000  8000.000000  2023.000000  2.462087e+06  1620.000000   \n",
       "\n",
       "            rating  \n",
       "count  7228.000000  \n",
       "mean      6.716146  \n",
       "std       1.291765  \n",
       "min       1.200000  \n",
       "25%       6.100000  \n",
       "50%       6.900000  \n",
       "75%       7.600000  \n",
       "max       9.600000  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6e982c98",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-21T12:11:23.521471Z",
     "iopub.status.busy": "2022-04-21T12:11:23.521009Z",
     "iopub.status.idle": "2022-04-21T12:11:23.541921Z",
     "shell.execute_reply": "2022-04-21T12:11:23.541269Z"
    },
    "papermill": {
     "duration": 0.066766,
     "end_time": "2022-04-21T12:11:23.543869",
     "exception": false,
     "start_time": "2022-04-21T12:11:23.477103",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>userID</th>\n",
       "      <th>title</th>\n",
       "      <th>year</th>\n",
       "      <th>kind</th>\n",
       "      <th>genre</th>\n",
       "      <th>vote</th>\n",
       "      <th>country</th>\n",
       "      <th>language</th>\n",
       "      <th>cast</th>\n",
       "      <th>director</th>\n",
       "      <th>composer</th>\n",
       "      <th>writer</th>\n",
       "      <th>runtime</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Dinosaur Planet</td>\n",
       "      <td>2003.0</td>\n",
       "      <td>tv mini series</td>\n",
       "      <td>['Documentary', 'Animation', 'Family']</td>\n",
       "      <td>474.0</td>\n",
       "      <td>['United States']</td>\n",
       "      <td>['English']</td>\n",
       "      <td>['Christian Slater', 'Scott Sampson']</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['Mike Carrol', 'Mike Carroll', 'Georgann Kane']</td>\n",
       "      <td>50.0</td>\n",
       "      <td>7.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Character</td>\n",
       "      <td>2021.0</td>\n",
       "      <td>movie</td>\n",
       "      <td>['Crime', 'Horror', 'Thriller']</td>\n",
       "      <td>46.0</td>\n",
       "      <td>['Japan']</td>\n",
       "      <td>['Japanese']</td>\n",
       "      <td>['Masaki Suda', 'Fukase', 'Mitsuki Takahata', ...</td>\n",
       "      <td>['Akira Nagai']</td>\n",
       "      <td>['Youki Kojima']</td>\n",
       "      <td>['Takashi Nagasaki', 'Takashi Nagasaki', 'Anna...</td>\n",
       "      <td>125.0</td>\n",
       "      <td>8.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>Get Up and Dance!</td>\n",
       "      <td>1994.0</td>\n",
       "      <td>video movie</td>\n",
       "      <td>['Family']</td>\n",
       "      <td>18.0</td>\n",
       "      <td>['United States']</td>\n",
       "      <td>['English']</td>\n",
       "      <td>['Paula Abdul', 'Aurorah Allain', 'Bill Bohl',...</td>\n",
       "      <td>['Steve Purcell']</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>54.0</td>\n",
       "      <td>8.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>The Rise and Fall of El Chapo</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>tv movie</td>\n",
       "      <td>['Documentary']</td>\n",
       "      <td>42.0</td>\n",
       "      <td>['United States']</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>85.0</td>\n",
       "      <td>6.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>Sick - IMDb</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['Thriller']</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['United States']</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['Marc Menchaca', 'Gideon Adlon', 'Dylan Spray...</td>\n",
       "      <td>['John Hyams']</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['Katelyn Crabb', 'Kevin Williamson']</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  userID                          title    year            kind  \\\n",
       "0           0       0                Dinosaur Planet  2003.0  tv mini series   \n",
       "1           1       1                      Character  2021.0           movie   \n",
       "2           2       2              Get Up and Dance!  1994.0     video movie   \n",
       "3           3       3  The Rise and Fall of El Chapo  2016.0        tv movie   \n",
       "4           4       4                    Sick - IMDb     NaN             NaN   \n",
       "\n",
       "                                    genre   vote            country  \\\n",
       "0  ['Documentary', 'Animation', 'Family']  474.0  ['United States']   \n",
       "1         ['Crime', 'Horror', 'Thriller']   46.0          ['Japan']   \n",
       "2                              ['Family']   18.0  ['United States']   \n",
       "3                         ['Documentary']   42.0  ['United States']   \n",
       "4                            ['Thriller']    NaN  ['United States']   \n",
       "\n",
       "       language                                               cast  \\\n",
       "0   ['English']              ['Christian Slater', 'Scott Sampson']   \n",
       "1  ['Japanese']  ['Masaki Suda', 'Fukase', 'Mitsuki Takahata', ...   \n",
       "2   ['English']  ['Paula Abdul', 'Aurorah Allain', 'Bill Bohl',...   \n",
       "3           NaN                                                NaN   \n",
       "4           NaN  ['Marc Menchaca', 'Gideon Adlon', 'Dylan Spray...   \n",
       "\n",
       "            director          composer  \\\n",
       "0                NaN               NaN   \n",
       "1    ['Akira Nagai']  ['Youki Kojima']   \n",
       "2  ['Steve Purcell']               NaN   \n",
       "3                NaN               NaN   \n",
       "4     ['John Hyams']               NaN   \n",
       "\n",
       "                                              writer  runtime  rating  \n",
       "0   ['Mike Carrol', 'Mike Carroll', 'Georgann Kane']     50.0     7.7  \n",
       "1  ['Takashi Nagasaki', 'Takashi Nagasaki', 'Anna...    125.0     8.3  \n",
       "2                                                NaN     54.0     8.1  \n",
       "3                                                NaN     85.0     6.9  \n",
       "4              ['Katelyn Crabb', 'Kevin Williamson']      NaN     NaN  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d95734f7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-21T12:11:23.632025Z",
     "iopub.status.busy": "2022-04-21T12:11:23.631450Z",
     "iopub.status.idle": "2022-04-21T12:11:23.641974Z",
     "shell.execute_reply": "2022-04-21T12:11:23.641326Z"
    },
    "papermill": {
     "duration": 0.057718,
     "end_time": "2022-04-21T12:11:23.644056",
     "exception": false,
     "start_time": "2022-04-21T12:11:23.586338",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Kevin Dunn']                                    33\n",
       "['Akira Kurosawa']                                16\n",
       "['Tom Clegg']                                     14\n",
       "['Werner Herzog']                                 14\n",
       "['Takashi Miike']                                 14\n",
       "                                                  ..\n",
       "['Clive Donner']                                   1\n",
       "['Margaret Selby', 'Tex Avery', 'Chuck Jones']     1\n",
       "['Elio Petri']                                     1\n",
       "['Gary Dauberman']                                 1\n",
       "['Arthur Wong', 'Brandy Yuen']                     1\n",
       "Name: director, Length: 4241, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['director'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "261f4c59",
   "metadata": {
    "papermill": {
     "duration": 0.043216,
     "end_time": "2022-04-21T12:11:23.731573",
     "exception": false,
     "start_time": "2022-04-21T12:11:23.688357",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Check Missing Values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "71ad5ef9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-21T12:11:23.821015Z",
     "iopub.status.busy": "2022-04-21T12:11:23.820404Z",
     "iopub.status.idle": "2022-04-21T12:11:23.837179Z",
     "shell.execute_reply": "2022-04-21T12:11:23.836164Z"
    },
    "papermill": {
     "duration": 0.064008,
     "end_time": "2022-04-21T12:11:23.839672",
     "exception": false,
     "start_time": "2022-04-21T12:11:23.775664",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Unnamed: 0       0\n",
       "userID           0\n",
       "title            1\n",
       "year            55\n",
       "kind            30\n",
       "genre           85\n",
       "vote           773\n",
       "country        586\n",
       "language       696\n",
       "cast           307\n",
       "director      1633\n",
       "composer      3149\n",
       "writer        1469\n",
       "runtime        928\n",
       "rating         773\n",
       "dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b118cad1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-21T12:11:23.931547Z",
     "iopub.status.busy": "2022-04-21T12:11:23.930937Z",
     "iopub.status.idle": "2022-04-21T12:11:23.940848Z",
     "shell.execute_reply": "2022-04-21T12:11:23.940014Z"
    },
    "papermill": {
     "duration": 0.058174,
     "end_time": "2022-04-21T12:11:23.942997",
     "exception": false,
     "start_time": "2022-04-21T12:11:23.884823",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Unnamed: 0      0\n",
       "userID          0\n",
       "title           0\n",
       "year           15\n",
       "kind            8\n",
       "genre           8\n",
       "vote          104\n",
       "country        79\n",
       "language       97\n",
       "cast           37\n",
       "director      227\n",
       "composer      473\n",
       "writer        187\n",
       "runtime       128\n",
       "dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "54dcb61f",
   "metadata": {
    "papermill": {
     "duration": 0.044499,
     "end_time": "2022-04-21T12:11:24.033227",
     "exception": false,
     "start_time": "2022-04-21T12:11:23.988728",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Data Clean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0ee77679",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-21T12:11:24.125931Z",
     "iopub.status.busy": "2022-04-21T12:11:24.125345Z",
     "iopub.status.idle": "2022-04-21T12:11:24.132952Z",
     "shell.execute_reply": "2022-04-21T12:11:24.132298Z"
    },
    "papermill": {
     "duration": 0.055408,
     "end_time": "2022-04-21T12:11:24.134840",
     "exception": false,
     "start_time": "2022-04-21T12:11:24.079432",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True    8001\n",
       "dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "verif = train['Unnamed: 0'] == train['userID']\n",
    "verif.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a7a06c3b",
   "metadata": {
    "papermill": {
     "duration": 0.043768,
     "end_time": "2022-04-21T12:11:24.224708",
     "exception": false,
     "start_time": "2022-04-21T12:11:24.180940",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "So, we are going to delete this feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ce3368fb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-21T12:11:24.314330Z",
     "iopub.status.busy": "2022-04-21T12:11:24.313817Z",
     "iopub.status.idle": "2022-04-21T12:11:24.318476Z",
     "shell.execute_reply": "2022-04-21T12:11:24.317610Z"
    },
    "papermill": {
     "duration": 0.05174,
     "end_time": "2022-04-21T12:11:24.320456",
     "exception": false,
     "start_time": "2022-04-21T12:11:24.268716",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "del train['Unnamed: 0']\n",
    "del test['Unnamed: 0']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32348ec2",
   "metadata": {
    "papermill": {
     "duration": 0.0435,
     "end_time": "2022-04-21T12:11:24.408192",
     "exception": false,
     "start_time": "2022-04-21T12:11:24.364692",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Object cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b8a2c099",
   "metadata": {
    "_kg_hide-output": true,
    "execution": {
     "iopub.execute_input": "2022-04-21T12:11:24.499339Z",
     "iopub.status.busy": "2022-04-21T12:11:24.499058Z",
     "iopub.status.idle": "2022-04-21T12:11:24.505515Z",
     "shell.execute_reply": "2022-04-21T12:11:24.504712Z"
    },
    "papermill": {
     "duration": 0.054672,
     "end_time": "2022-04-21T12:11:24.507486",
     "exception": false,
     "start_time": "2022-04-21T12:11:24.452814",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['title',\n",
       " 'kind',\n",
       " 'genre',\n",
       " 'country',\n",
       " 'language',\n",
       " 'cast',\n",
       " 'director',\n",
       " 'composer',\n",
       " 'writer']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "obj_cols = []\n",
    "\n",
    "for i in train.columns:\n",
    "    if train[i].dtypes == 'object':\n",
    "        obj_cols.append(i)\n",
    "        \n",
    "        \n",
    "obj_cols"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98482ec1",
   "metadata": {
    "papermill": {
     "duration": 0.044724,
     "end_time": "2022-04-21T12:11:24.596894",
     "exception": false,
     "start_time": "2022-04-21T12:11:24.552170",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Encode Label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f7c0edd7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-21T12:11:24.689811Z",
     "iopub.status.busy": "2022-04-21T12:11:24.688889Z",
     "iopub.status.idle": "2022-04-21T12:11:25.896629Z",
     "shell.execute_reply": "2022-04-21T12:11:25.895705Z"
    },
    "papermill": {
     "duration": 1.256009,
     "end_time": "2022-04-21T12:11:25.898910",
     "exception": false,
     "start_time": "2022-04-21T12:11:24.642901",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn import preprocessing\n",
    "le = preprocessing.LabelEncoder()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5e5113d8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-21T12:11:25.991449Z",
     "iopub.status.busy": "2022-04-21T12:11:25.991119Z",
     "iopub.status.idle": "2022-04-21T12:11:26.134875Z",
     "shell.execute_reply": "2022-04-21T12:11:26.134247Z"
    },
    "papermill": {
     "duration": 0.19376,
     "end_time": "2022-04-21T12:11:26.137236",
     "exception": false,
     "start_time": "2022-04-21T12:11:25.943476",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 8001 entries, 0 to 8000\n",
      "Data columns (total 14 columns):\n",
      " #   Column    Non-Null Count  Dtype  \n",
      "---  ------    --------------  -----  \n",
      " 0   userID    8001 non-null   int64  \n",
      " 1   title     8001 non-null   int64  \n",
      " 2   year      7946 non-null   float64\n",
      " 3   kind      8001 non-null   int64  \n",
      " 4   genre     8001 non-null   int64  \n",
      " 5   vote      7228 non-null   float64\n",
      " 6   country   8001 non-null   int64  \n",
      " 7   language  8001 non-null   int64  \n",
      " 8   cast      8001 non-null   int64  \n",
      " 9   director  8001 non-null   int64  \n",
      " 10  composer  8001 non-null   int64  \n",
      " 11  writer    8001 non-null   int64  \n",
      " 12  runtime   7073 non-null   float64\n",
      " 13  rating    7228 non-null   float64\n",
      "dtypes: float64(4), int64(10)\n",
      "memory usage: 875.2 KB\n"
     ]
    }
   ],
   "source": [
    "for col in obj_cols:\n",
    "    train[col] = le.fit_transform(train[col])\n",
    "    test[col] = le.fit_transform(test[col])\n",
    "    \n",
    "train.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f923eee",
   "metadata": {
    "papermill": {
     "duration": 0.044322,
     "end_time": "2022-04-21T12:11:26.226293",
     "exception": false,
     "start_time": "2022-04-21T12:11:26.181971",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Imputer Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "376b04de",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-21T12:11:26.319971Z",
     "iopub.status.busy": "2022-04-21T12:11:26.319545Z",
     "iopub.status.idle": "2022-04-21T12:11:26.530774Z",
     "shell.execute_reply": "2022-04-21T12:11:26.529842Z"
    },
    "papermill": {
     "duration": 0.260939,
     "end_time": "2022-04-21T12:11:26.533471",
     "exception": false,
     "start_time": "2022-04-21T12:11:26.272532",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.impute import KNNImputer\n",
    "\n",
    "knni = KNNImputer(n_neighbors=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "787d82fc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-21T12:11:26.624769Z",
     "iopub.status.busy": "2022-04-21T12:11:26.624254Z",
     "iopub.status.idle": "2022-04-21T12:11:26.638102Z",
     "shell.execute_reply": "2022-04-21T12:11:26.637275Z"
    },
    "papermill": {
     "duration": 0.061872,
     "end_time": "2022-04-21T12:11:26.640128",
     "exception": false,
     "start_time": "2022-04-21T12:11:26.578256",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4, 3)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "null_train_cols = []\n",
    "for col in train.columns:\n",
    "    if train[col].isnull().sum().any():\n",
    "        null_train_cols.append(col)\n",
    "        \n",
    "null_test_cols = []\n",
    "for col in test.columns:\n",
    "    if test[col].isnull().sum().any():\n",
    "        null_test_cols.append(col)\n",
    "        \n",
    "len(null_train_cols), len(null_test_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c4065872",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-21T12:11:26.731983Z",
     "iopub.status.busy": "2022-04-21T12:11:26.731381Z",
     "iopub.status.idle": "2022-04-21T12:11:26.736971Z",
     "shell.execute_reply": "2022-04-21T12:11:26.736136Z"
    },
    "papermill": {
     "duration": 0.053753,
     "end_time": "2022-04-21T12:11:26.738947",
     "exception": false,
     "start_time": "2022-04-21T12:11:26.685194",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['year', 'vote', 'runtime', 'rating']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "null_train_cols"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8db5e5c7",
   "metadata": {
    "papermill": {
     "duration": 0.046444,
     "end_time": "2022-04-21T12:11:26.830943",
     "exception": false,
     "start_time": "2022-04-21T12:11:26.784499",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "- Impute Train Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "a53f56ff",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-21T12:11:26.923961Z",
     "iopub.status.busy": "2022-04-21T12:11:26.923686Z",
     "iopub.status.idle": "2022-04-21T12:11:27.877730Z",
     "shell.execute_reply": "2022-04-21T12:11:27.876767Z"
    },
    "papermill": {
     "duration": 1.003323,
     "end_time": "2022-04-21T12:11:27.880102",
     "exception": false,
     "start_time": "2022-04-21T12:11:26.876779",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "userID      0\n",
       "title       0\n",
       "year        0\n",
       "kind        0\n",
       "genre       0\n",
       "vote        0\n",
       "country     0\n",
       "language    0\n",
       "cast        0\n",
       "director    0\n",
       "composer    0\n",
       "writer      0\n",
       "runtime     0\n",
       "rating      0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imput_train = pd.DataFrame(knni.fit_transform(train[null_train_cols]))\n",
    "\n",
    "iterate = 0\n",
    "for col in null_train_cols:\n",
    "    train[col] = imput_train[iterate]\n",
    "    iterate +=1\n",
    "    \n",
    "del imput_train\n",
    "train.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7e33906b",
   "metadata": {
    "papermill": {
     "duration": 0.045917,
     "end_time": "2022-04-21T12:11:27.972244",
     "exception": false,
     "start_time": "2022-04-21T12:11:27.926327",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "- Impute Test Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "fca6c45f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-21T12:11:28.066771Z",
     "iopub.status.busy": "2022-04-21T12:11:28.065837Z",
     "iopub.status.idle": "2022-04-21T12:11:28.113835Z",
     "shell.execute_reply": "2022-04-21T12:11:28.112800Z"
    },
    "papermill": {
     "duration": 0.09821,
     "end_time": "2022-04-21T12:11:28.116162",
     "exception": false,
     "start_time": "2022-04-21T12:11:28.017952",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "userID      0\n",
       "title       0\n",
       "year        0\n",
       "kind        0\n",
       "genre       0\n",
       "vote        0\n",
       "country     0\n",
       "language    0\n",
       "cast        0\n",
       "director    0\n",
       "composer    0\n",
       "writer      0\n",
       "runtime     0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imput_test = pd.DataFrame(knni.fit_transform(test[null_test_cols]))\n",
    "\n",
    "iterate = 0\n",
    "for col in null_test_cols:\n",
    "    test[col] = imput_test[iterate]\n",
    "    iterate +=1\n",
    "    \n",
    "test.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e54c3600",
   "metadata": {
    "papermill": {
     "duration": 0.046567,
     "end_time": "2022-04-21T12:11:28.246654",
     "exception": false,
     "start_time": "2022-04-21T12:11:28.200087",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Modeling"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d21c0651",
   "metadata": {
    "papermill": {
     "duration": 0.046002,
     "end_time": "2022-04-21T12:11:28.339251",
     "exception": false,
     "start_time": "2022-04-21T12:11:28.293249",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Split Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "29221a9a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-21T12:11:28.433568Z",
     "iopub.status.busy": "2022-04-21T12:11:28.433273Z",
     "iopub.status.idle": "2022-04-21T12:11:28.443753Z",
     "shell.execute_reply": "2022-04-21T12:11:28.442619Z"
    },
    "papermill": {
     "duration": 0.06035,
     "end_time": "2022-04-21T12:11:28.446141",
     "exception": false,
     "start_time": "2022-04-21T12:11:28.385791",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8001, 12)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del train['userID']\n",
    "del test['userID']\n",
    "\n",
    "X = train.copy()\n",
    "y = X.pop('rating')\n",
    "\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aa17c779",
   "metadata": {
    "papermill": {
     "duration": 0.046464,
     "end_time": "2022-04-21T12:11:28.539993",
     "exception": false,
     "start_time": "2022-04-21T12:11:28.493529",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Try H2O"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "07505672",
   "metadata": {
    "_kg_hide-output": true,
    "execution": {
     "iopub.execute_input": "2022-04-21T12:11:28.634517Z",
     "iopub.status.busy": "2022-04-21T12:11:28.634228Z",
     "iopub.status.idle": "2022-04-21T12:11:28.787476Z",
     "shell.execute_reply": "2022-04-21T12:11:28.786236Z"
    },
    "papermill": {
     "duration": 0.203398,
     "end_time": "2022-04-21T12:11:28.789805",
     "exception": false,
     "start_time": "2022-04-21T12:11:28.586407",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "## import packages\n",
    "import h2o\n",
    "from h2o.automl import H2OAutoML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "328350fb",
   "metadata": {
    "collapsed": true,
    "execution": {
     "iopub.execute_input": "2022-04-21T12:11:28.885060Z",
     "iopub.status.busy": "2022-04-21T12:11:28.884761Z",
     "iopub.status.idle": "2022-04-21T12:11:38.587775Z",
     "shell.execute_reply": "2022-04-21T12:11:38.586731Z"
    },
    "jupyter": {
     "outputs_hidden": true
    },
    "papermill": {
     "duration": 9.75358,
     "end_time": "2022-04-21T12:11:38.590366",
     "exception": false,
     "start_time": "2022-04-21T12:11:28.836786",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking whether there is an H2O instance running at http://localhost:54321 ..... not found.\n",
      "Attempting to start a local H2O server...\n",
      "  Java Version: openjdk version \"11.0.14.1\" 2022-02-08; OpenJDK Runtime Environment (build 11.0.14.1+1-Ubuntu-0ubuntu1.20.04); OpenJDK 64-Bit Server VM (build 11.0.14.1+1-Ubuntu-0ubuntu1.20.04, mixed mode, sharing)\n",
      "  Starting server from /opt/conda/lib/python3.7/site-packages/h2o/backend/bin/h2o.jar\n",
      "  Ice root: /tmp/tmpnrq9qcxj\n",
      "  JVM stdout: /tmp/tmpnrq9qcxj/h2o_unknownUser_started_from_python.out\n",
      "  JVM stderr: /tmp/tmpnrq9qcxj/h2o_unknownUser_started_from_python.err\n",
      "  Server is running at http://127.0.0.1:54321\n",
      "Connecting to H2O server at http://127.0.0.1:54321 ... successful.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td>H2O_cluster_uptime:</td>\n",
       "<td>02 secs</td></tr>\n",
       "<tr><td>H2O_cluster_timezone:</td>\n",
       "<td>Etc/UTC</td></tr>\n",
       "<tr><td>H2O_data_parsing_timezone:</td>\n",
       "<td>UTC</td></tr>\n",
       "<tr><td>H2O_cluster_version:</td>\n",
       "<td>3.36.0.4</td></tr>\n",
       "<tr><td>H2O_cluster_version_age:</td>\n",
       "<td>21 days, 17 hours and 36 minutes </td></tr>\n",
       "<tr><td>H2O_cluster_name:</td>\n",
       "<td>H2O_from_python_unknownUser_vnm0rl</td></tr>\n",
       "<tr><td>H2O_cluster_total_nodes:</td>\n",
       "<td>1</td></tr>\n",
       "<tr><td>H2O_cluster_free_memory:</td>\n",
       "<td>4.396 Gb</td></tr>\n",
       "<tr><td>H2O_cluster_total_cores:</td>\n",
       "<td>4</td></tr>\n",
       "<tr><td>H2O_cluster_allowed_cores:</td>\n",
       "<td>4</td></tr>\n",
       "<tr><td>H2O_cluster_status:</td>\n",
       "<td>locked, healthy</td></tr>\n",
       "<tr><td>H2O_connection_url:</td>\n",
       "<td>http://127.0.0.1:54321</td></tr>\n",
       "<tr><td>H2O_connection_proxy:</td>\n",
       "<td>{\"http\": null, \"https\": null}</td></tr>\n",
       "<tr><td>H2O_internal_security:</td>\n",
       "<td>False</td></tr>\n",
       "<tr><td>Python_version:</td>\n",
       "<td>3.7.12 final</td></tr></table></div>"
      ],
      "text/plain": [
       "--------------------------  ----------------------------------\n",
       "H2O_cluster_uptime:         02 secs\n",
       "H2O_cluster_timezone:       Etc/UTC\n",
       "H2O_data_parsing_timezone:  UTC\n",
       "H2O_cluster_version:        3.36.0.4\n",
       "H2O_cluster_version_age:    21 days, 17 hours and 36 minutes\n",
       "H2O_cluster_name:           H2O_from_python_unknownUser_vnm0rl\n",
       "H2O_cluster_total_nodes:    1\n",
       "H2O_cluster_free_memory:    4.396 Gb\n",
       "H2O_cluster_total_cores:    4\n",
       "H2O_cluster_allowed_cores:  4\n",
       "H2O_cluster_status:         locked, healthy\n",
       "H2O_connection_url:         http://127.0.0.1:54321\n",
       "H2O_connection_proxy:       {\"http\": null, \"https\": null}\n",
       "H2O_internal_security:      False\n",
       "Python_version:             3.7.12 final\n",
       "--------------------------  ----------------------------------"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parse progress: |████████████████████████████████████████████████████████████████| (done) 100%\n",
      "Parse progress: |████████████████████████████████████████████████████████████████| (done) 100%\n",
      "Parse progress: |████████████████████████████████████████████████████████████████| (done) 100%\n"
     ]
    }
   ],
   "source": [
    "## prepare data\n",
    "h2o.init()\n",
    "\n",
    "h2o_train = h2o.H2OFrame(train)\n",
    "h2o_test = h2o.H2OFrame(test)\n",
    "target = train['rating'].to_numpy().ravel()\n",
    "h2o_train['rating'] = h2o.H2OFrame(target)\n",
    "\n",
    "#from sklearn.model_selection import train_test_split\n",
    "#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bd9936df",
   "metadata": {
    "papermill": {
     "duration": 0.055982,
     "end_time": "2022-04-21T12:11:38.708021",
     "exception": false,
     "start_time": "2022-04-21T12:11:38.652039",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "86ed1e11",
   "metadata": {
    "_kg_hide-output": true,
    "execution": {
     "iopub.execute_input": "2022-04-21T12:11:38.816393Z",
     "iopub.status.busy": "2022-04-21T12:11:38.816032Z",
     "iopub.status.idle": "2022-04-21T13:11:08.521582Z",
     "shell.execute_reply": "2022-04-21T13:11:08.520526Z"
    },
    "papermill": {
     "duration": 3569.764482,
     "end_time": "2022-04-21T13:11:08.524951",
     "exception": false,
     "start_time": "2022-04-21T12:11:38.760469",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AutoML progress: |███████████████████████████████████████████████████████████████| (done) 100%\n",
      "Model Details\n",
      "=============\n",
      "H2OStackedEnsembleEstimator :  Stacked Ensemble\n",
      "Model Key:  StackedEnsemble_AllModels_7_AutoML_1_20220421_121138\n",
      "\n",
      "No model summary for this model\n",
      "\n",
      "ModelMetricsRegressionGLM: stackedensemble\n",
      "** Reported on train data. **\n",
      "\n",
      "MSE: 0.07934311918657587\n",
      "RMSE: 0.2816791067625994\n",
      "MAE: 0.19550762511021896\n",
      "RMSLE: 0.046145222114896745\n",
      "R^2: 0.9498849875472184\n",
      "Mean Residual Deviance: 0.07934311918657587\n",
      "Null degrees of freedom: 8000\n",
      "Residual degrees of freedom: 7970\n",
      "Null deviance: 12667.34787724165\n",
      "Residual deviance: 634.8242966117936\n",
      "AIC: 2495.5320308338787\n",
      "\n",
      "ModelMetricsRegressionGLM: stackedensemble\n",
      "** Reported on cross-validation data. **\n",
      "\n",
      "MSE: 0.6559348278944998\n",
      "RMSE: 0.8098980354924314\n",
      "MAE: 0.5728276928467241\n",
      "RMSLE: 0.12317397252246735\n",
      "R^2: 0.5856958687135252\n",
      "Mean Residual Deviance: 0.6559348278944998\n",
      "Null degrees of freedom: 8000\n",
      "Residual degrees of freedom: 7973\n",
      "Null deviance: 12669.885052668791\n",
      "Residual deviance: 5248.134557983893\n",
      "AIC: 19389.881972937204\n"
     ]
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## run model\n",
    "features = [x for x in h2o_train.columns if x != 'rating']\n",
    "\n",
    "model_h2o = H2OAutoML(stopping_metric='rmse', max_runtime_secs=3600)\n",
    "model_h2o.train(x=features, y='rating', training_frame=h2o_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "15c509e7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-21T13:11:08.697750Z",
     "iopub.status.busy": "2022-04-21T13:11:08.696730Z",
     "iopub.status.idle": "2022-04-21T13:11:08.717221Z",
     "shell.execute_reply": "2022-04-21T13:11:08.716293Z"
    },
    "papermill": {
     "duration": 0.101741,
     "end_time": "2022-04-21T13:11:08.719682",
     "exception": false,
     "start_time": "2022-04-21T13:11:08.617941",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th>model_id                                               </th><th style=\"text-align: right;\">  mean_residual_deviance</th><th style=\"text-align: right;\">    rmse</th><th style=\"text-align: right;\">     mse</th><th style=\"text-align: right;\">     mae</th><th style=\"text-align: right;\">   rmsle</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>StackedEnsemble_AllModels_7_AutoML_1_20220421_121138   </td><td style=\"text-align: right;\">                0.655935</td><td style=\"text-align: right;\">0.809898</td><td style=\"text-align: right;\">0.655935</td><td style=\"text-align: right;\">0.572828</td><td style=\"text-align: right;\">0.123174</td></tr>\n",
       "<tr><td>StackedEnsemble_AllModels_6_AutoML_1_20220421_121138   </td><td style=\"text-align: right;\">                0.659292</td><td style=\"text-align: right;\">0.811968</td><td style=\"text-align: right;\">0.659292</td><td style=\"text-align: right;\">0.573932</td><td style=\"text-align: right;\">0.123232</td></tr>\n",
       "<tr><td>StackedEnsemble_Best1000_1_AutoML_1_20220421_121138    </td><td style=\"text-align: right;\">                0.666547</td><td style=\"text-align: right;\">0.816423</td><td style=\"text-align: right;\">0.666547</td><td style=\"text-align: right;\">0.580979</td><td style=\"text-align: right;\">0.123758</td></tr>\n",
       "<tr><td>StackedEnsemble_BestOfFamily_7_AutoML_1_20220421_121138</td><td style=\"text-align: right;\">                0.668269</td><td style=\"text-align: right;\">0.817477</td><td style=\"text-align: right;\">0.668269</td><td style=\"text-align: right;\">0.581215</td><td style=\"text-align: right;\">0.124114</td></tr>\n",
       "<tr><td>StackedEnsemble_AllModels_4_AutoML_1_20220421_121138   </td><td style=\"text-align: right;\">                0.66912 </td><td style=\"text-align: right;\">0.817997</td><td style=\"text-align: right;\">0.66912 </td><td style=\"text-align: right;\">0.58267 </td><td style=\"text-align: right;\">0.124008</td></tr>\n",
       "<tr><td>StackedEnsemble_AllModels_3_AutoML_1_20220421_121138   </td><td style=\"text-align: right;\">                0.66912 </td><td style=\"text-align: right;\">0.817998</td><td style=\"text-align: right;\">0.66912 </td><td style=\"text-align: right;\">0.582829</td><td style=\"text-align: right;\">0.123977</td></tr>\n",
       "<tr><td>StackedEnsemble_BestOfFamily_6_AutoML_1_20220421_121138</td><td style=\"text-align: right;\">                0.67639 </td><td style=\"text-align: right;\">0.82243 </td><td style=\"text-align: right;\">0.67639 </td><td style=\"text-align: right;\">0.58405 </td><td style=\"text-align: right;\">0.124515</td></tr>\n",
       "<tr><td>StackedEnsemble_BestOfFamily_4_AutoML_1_20220421_121138</td><td style=\"text-align: right;\">                0.678166</td><td style=\"text-align: right;\">0.823508</td><td style=\"text-align: right;\">0.678166</td><td style=\"text-align: right;\">0.586466</td><td style=\"text-align: right;\">0.124791</td></tr>\n",
       "<tr><td>StackedEnsemble_AllModels_1_AutoML_1_20220421_121138   </td><td style=\"text-align: right;\">                0.690022</td><td style=\"text-align: right;\">0.830676</td><td style=\"text-align: right;\">0.690022</td><td style=\"text-align: right;\">0.594914</td><td style=\"text-align: right;\">0.125726</td></tr>\n",
       "<tr><td>StackedEnsemble_AllModels_2_AutoML_1_20220421_121138   </td><td style=\"text-align: right;\">                0.690156</td><td style=\"text-align: right;\">0.830756</td><td style=\"text-align: right;\">0.690156</td><td style=\"text-align: right;\">0.594902</td><td style=\"text-align: right;\">0.125726</td></tr>\n",
       "</tbody>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## check leaderboard\n",
    "model_h2o.leaderboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d5730e9b",
   "metadata": {
    "papermill": {
     "duration": 0.077069,
     "end_time": "2022-04-21T13:11:08.879306",
     "exception": false,
     "start_time": "2022-04-21T13:11:08.802237",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fda0d0c3",
   "metadata": {
    "papermill": {
     "duration": 0.076347,
     "end_time": "2022-04-21T13:11:09.033187",
     "exception": false,
     "start_time": "2022-04-21T13:11:08.956840",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d6c30e6",
   "metadata": {
    "papermill": {
     "duration": 0.07753,
     "end_time": "2022-04-21T13:11:09.189078",
     "exception": false,
     "start_time": "2022-04-21T13:11:09.111548",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "6e638202",
   "metadata": {
    "papermill": {
     "duration": 0.077643,
     "end_time": "2022-04-21T13:11:09.345231",
     "exception": false,
     "start_time": "2022-04-21T13:11:09.267588",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "6491e8b1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-21T13:11:09.501633Z",
     "iopub.status.busy": "2022-04-21T13:11:09.501354Z",
     "iopub.status.idle": "2022-04-21T13:11:10.056913Z",
     "shell.execute_reply": "2022-04-21T13:11:10.055775Z"
    },
    "papermill": {
     "duration": 0.638167,
     "end_time": "2022-04-21T13:11:10.059941",
     "exception": false,
     "start_time": "2022-04-21T13:11:09.421774",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "stackedensemble prediction progress: |███████████████████████████████████████████| (done) 100%\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>predict</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6.023389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6.008534</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6.572310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5.127644</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6.673857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1820</th>\n",
       "      <td>6.071470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1821</th>\n",
       "      <td>6.957226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1822</th>\n",
       "      <td>6.574042</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1823</th>\n",
       "      <td>6.438963</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1824</th>\n",
       "      <td>7.187960</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1825 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       predict\n",
       "0     6.023389\n",
       "1     6.008534\n",
       "2     6.572310\n",
       "3     5.127644\n",
       "4     6.673857\n",
       "...        ...\n",
       "1820  6.071470\n",
       "1821  6.957226\n",
       "1822  6.574042\n",
       "1823  6.438963\n",
       "1824  7.187960\n",
       "\n",
       "[1825 rows x 1 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## generate predictions\n",
    "preds_h2o = model_h2o.leader.predict(h2o_test).as_data_frame()\n",
    "preds_h2o"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e611a27f",
   "metadata": {
    "papermill": {
     "duration": 0.078937,
     "end_time": "2022-04-21T13:11:10.228484",
     "exception": false,
     "start_time": "2022-04-21T13:11:10.149547",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "5a6764fe",
   "metadata": {
    "papermill": {
     "duration": 0.077935,
     "end_time": "2022-04-21T13:11:10.386314",
     "exception": false,
     "start_time": "2022-04-21T13:11:10.308379",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "02be28d3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-21T13:11:10.545447Z",
     "iopub.status.busy": "2022-04-21T13:11:10.544871Z",
     "iopub.status.idle": "2022-04-21T13:11:10.576182Z",
     "shell.execute_reply": "2022-04-21T13:11:10.574922Z"
    },
    "papermill": {
     "duration": 0.113493,
     "end_time": "2022-04-21T13:11:10.578359",
     "exception": false,
     "start_time": "2022-04-21T13:11:10.464866",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1    1825\n",
      "Name: rating, dtype: int64 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userID</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8001</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8002</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8003</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8004</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8005</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   userID  rating\n",
       "0    8001       1\n",
       "1    8002       1\n",
       "2    8003       1\n",
       "3    8004       1\n",
       "4    8005       1"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub = pd.read_csv('../input/ctrl-shift-intelligence-2k22/sample_sub.csv')\n",
    "print(sub['rating'].value_counts(),'\\n')\n",
    "sub.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dc6e01ab",
   "metadata": {
    "papermill": {
     "duration": 0.078961,
     "end_time": "2022-04-21T13:11:10.737010",
     "exception": false,
     "start_time": "2022-04-21T13:11:10.658049",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Save Submission File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "5a05603c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-21T13:11:10.903541Z",
     "iopub.status.busy": "2022-04-21T13:11:10.902568Z",
     "iopub.status.idle": "2022-04-21T13:11:10.923984Z",
     "shell.execute_reply": "2022-04-21T13:11:10.923381Z"
    },
    "papermill": {
     "duration": 0.108167,
     "end_time": "2022-04-21T13:11:10.925797",
     "exception": false,
     "start_time": "2022-04-21T13:11:10.817630",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userID</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8001</td>\n",
       "      <td>6.023389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8002</td>\n",
       "      <td>6.008534</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8003</td>\n",
       "      <td>6.572310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8004</td>\n",
       "      <td>5.127644</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8005</td>\n",
       "      <td>6.673857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1820</th>\n",
       "      <td>9821</td>\n",
       "      <td>6.071470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1821</th>\n",
       "      <td>9822</td>\n",
       "      <td>6.957226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1822</th>\n",
       "      <td>9823</td>\n",
       "      <td>6.574042</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1823</th>\n",
       "      <td>9824</td>\n",
       "      <td>6.438963</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1824</th>\n",
       "      <td>9825</td>\n",
       "      <td>7.187960</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1825 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      userID    rating\n",
       "0       8001  6.023389\n",
       "1       8002  6.008534\n",
       "2       8003  6.572310\n",
       "3       8004  5.127644\n",
       "4       8005  6.673857\n",
       "...      ...       ...\n",
       "1820    9821  6.071470\n",
       "1821    9822  6.957226\n",
       "1822    9823  6.574042\n",
       "1823    9824  6.438963\n",
       "1824    9825  7.187960\n",
       "\n",
       "[1825 rows x 2 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub['rating'] = preds_h2o\n",
    "\n",
    "sub.to_csv('submission.csv', index=False)\n",
    "sub"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 3612.608654,
   "end_time": "2022-04-21T13:11:12.437852",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2022-04-21T12:10:59.829198",
   "version": "2.3.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}