deep_learning_29_keras_l1_l2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from scipy.stats import zscore\n",
    "\n",
    "# Read the data set\n",
    "df = pd.read_csv(\n",
    "    \"https://data.heatonresearch.com/data/t81-558/jh-simple-dataset.csv\",\n",
    "    na_values=['NA','?'])\n",
    "\n",
    "# Generate dummies for job\n",
    "df = pd.concat([df,pd.get_dummies(df['job'],prefix=\"job\",dtype=int)],axis=1)\n",
    "df.drop('job', axis=1, inplace=True)\n",
    "\n",
    "# Generate dummies for area\n",
    "df = pd.concat([df,pd.get_dummies(df['area'],prefix=\"area\",dtype=int)],axis=1)\n",
    "df.drop('area', axis=1, inplace=True)\n",
    "\n",
    "# Missing values for income\n",
    "med = df['income'].median()\n",
    "df['income'] = df['income'].fillna(med)\n",
    "\n",
    "# Standardize ranges\n",
    "df['income'] = zscore(df['income'])\n",
    "df['aspect'] = zscore(df['aspect'])\n",
    "df['save_rate'] = zscore(df['save_rate'])\n",
    "df['age'] = zscore(df['age'])\n",
    "df['subscriptions'] = zscore(df['subscriptions'])\n",
    "\n",
    "# Convert to numpy - Classification\n",
    "x_columns = df.columns.drop('product').drop('id')\n",
    "x = df[x_columns].values\n",
    "dummies = pd.get_dummies(df['product']) # Classification\n",
    "products = dummies.columns\n",
    "y = dummies.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>income</th>\n",
       "      <th>aspect</th>\n",
       "      <th>subscriptions</th>\n",
       "      <th>dist_healthy</th>\n",
       "      <th>save_rate</th>\n",
       "      <th>dist_unhealthy</th>\n",
       "      <th>age</th>\n",
       "      <th>pop_dense</th>\n",
       "      <th>retail_dense</th>\n",
       "      <th>...</th>\n",
       "      <th>job_qp</th>\n",
       "      <th>job_qw</th>\n",
       "      <th>job_rn</th>\n",
       "      <th>job_sa</th>\n",
       "      <th>job_vv</th>\n",
       "      <th>job_zz</th>\n",
       "      <th>area_a</th>\n",
       "      <th>area_b</th>\n",
       "      <th>area_c</th>\n",
       "      <th>area_d</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>-0.607550</td>\n",
       "      <td>-0.664918</td>\n",
       "      <td>-0.208449</td>\n",
       "      <td>9.017895</td>\n",
       "      <td>-0.215764</td>\n",
       "      <td>11.738935</td>\n",
       "      <td>0.854321</td>\n",
       "      <td>0.885827</td>\n",
       "      <td>0.492126</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0.338053</td>\n",
       "      <td>-0.207748</td>\n",
       "      <td>0.839031</td>\n",
       "      <td>7.766643</td>\n",
       "      <td>0.196869</td>\n",
       "      <td>6.805396</td>\n",
       "      <td>1.394432</td>\n",
       "      <td>0.874016</td>\n",
       "      <td>0.342520</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>-0.184205</td>\n",
       "      <td>1.127906</td>\n",
       "      <td>-0.208449</td>\n",
       "      <td>3.632069</td>\n",
       "      <td>-0.714362</td>\n",
       "      <td>13.671772</td>\n",
       "      <td>-0.495957</td>\n",
       "      <td>0.944882</td>\n",
       "      <td>0.724409</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>-0.526467</td>\n",
       "      <td>-0.440815</td>\n",
       "      <td>-0.208449</td>\n",
       "      <td>5.372942</td>\n",
       "      <td>-0.542432</td>\n",
       "      <td>4.333286</td>\n",
       "      <td>1.124377</td>\n",
       "      <td>0.889764</td>\n",
       "      <td>0.444882</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>-2.851675</td>\n",
       "      <td>1.638861</td>\n",
       "      <td>1.886511</td>\n",
       "      <td>3.822477</td>\n",
       "      <td>-0.473660</td>\n",
       "      <td>5.967121</td>\n",
       "      <td>-2.116291</td>\n",
       "      <td>0.744094</td>\n",
       "      <td>0.661417</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1995</th>\n",
       "      <td>1996</td>\n",
       "      <td>-0.593504</td>\n",
       "      <td>1.414758</td>\n",
       "      <td>-0.208449</td>\n",
       "      <td>5.454545</td>\n",
       "      <td>-0.232957</td>\n",
       "      <td>14.013489</td>\n",
       "      <td>-1.306124</td>\n",
       "      <td>0.881890</td>\n",
       "      <td>0.744094</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1996</th>\n",
       "      <td>1997</td>\n",
       "      <td>-3.028085</td>\n",
       "      <td>1.011372</td>\n",
       "      <td>0.839031</td>\n",
       "      <td>3.632069</td>\n",
       "      <td>-0.473660</td>\n",
       "      <td>8.380497</td>\n",
       "      <td>-2.116291</td>\n",
       "      <td>0.944882</td>\n",
       "      <td>0.877953</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1997</th>\n",
       "      <td>1998</td>\n",
       "      <td>-2.826971</td>\n",
       "      <td>1.513363</td>\n",
       "      <td>1.886511</td>\n",
       "      <td>7.168218</td>\n",
       "      <td>0.884591</td>\n",
       "      <td>4.626950</td>\n",
       "      <td>-2.656402</td>\n",
       "      <td>0.759843</td>\n",
       "      <td>0.744094</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1998</th>\n",
       "      <td>1999</td>\n",
       "      <td>1.093101</td>\n",
       "      <td>-1.274478</td>\n",
       "      <td>-1.255928</td>\n",
       "      <td>8.936292</td>\n",
       "      <td>-0.370502</td>\n",
       "      <td>3.281439</td>\n",
       "      <td>0.044154</td>\n",
       "      <td>0.909449</td>\n",
       "      <td>0.598425</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1999</th>\n",
       "      <td>2000</td>\n",
       "      <td>0.447425</td>\n",
       "      <td>-0.351174</td>\n",
       "      <td>-1.255928</td>\n",
       "      <td>4.312097</td>\n",
       "      <td>-0.679976</td>\n",
       "      <td>9.405648</td>\n",
       "      <td>0.584265</td>\n",
       "      <td>0.925197</td>\n",
       "      <td>0.539370</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2000 rows × 49 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        id    income    aspect  subscriptions  dist_healthy  save_rate  \\\n",
       "0        1 -0.607550 -0.664918      -0.208449      9.017895  -0.215764   \n",
       "1        2  0.338053 -0.207748       0.839031      7.766643   0.196869   \n",
       "2        3 -0.184205  1.127906      -0.208449      3.632069  -0.714362   \n",
       "3        4 -0.526467 -0.440815      -0.208449      5.372942  -0.542432   \n",
       "4        5 -2.851675  1.638861       1.886511      3.822477  -0.473660   \n",
       "...    ...       ...       ...            ...           ...        ...   \n",
       "1995  1996 -0.593504  1.414758      -0.208449      5.454545  -0.232957   \n",
       "1996  1997 -3.028085  1.011372       0.839031      3.632069  -0.473660   \n",
       "1997  1998 -2.826971  1.513363       1.886511      7.168218   0.884591   \n",
       "1998  1999  1.093101 -1.274478      -1.255928      8.936292  -0.370502   \n",
       "1999  2000  0.447425 -0.351174      -1.255928      4.312097  -0.679976   \n",
       "\n",
       "      dist_unhealthy       age  pop_dense  retail_dense  ...  job_qp job_qw  \\\n",
       "0          11.738935  0.854321   0.885827      0.492126  ...       0      0   \n",
       "1           6.805396  1.394432   0.874016      0.342520  ...       0      0   \n",
       "2          13.671772 -0.495957   0.944882      0.724409  ...       0      0   \n",
       "3           4.333286  1.124377   0.889764      0.444882  ...       0      0   \n",
       "4           5.967121 -2.116291   0.744094      0.661417  ...       0      0   \n",
       "...              ...       ...        ...           ...  ...     ...    ...   \n",
       "1995       14.013489 -1.306124   0.881890      0.744094  ...       0      0   \n",
       "1996        8.380497 -2.116291   0.944882      0.877953  ...       0      0   \n",
       "1997        4.626950 -2.656402   0.759843      0.744094  ...       0      0   \n",
       "1998        3.281439  0.044154   0.909449      0.598425  ...       1      0   \n",
       "1999        9.405648  0.584265   0.925197      0.539370  ...       0      0   \n",
       "\n",
       "      job_rn  job_sa  job_vv  job_zz  area_a  area_b  area_c  area_d  \n",
       "0          0       0       1       0       0       0       1       0  \n",
       "1          0       0       0       0       0       0       1       0  \n",
       "2          0       0       0       0       0       0       1       0  \n",
       "3          0       0       0       0       0       0       1       0  \n",
       "4          0       0       0       0       0       0       0       1  \n",
       "...      ...     ...     ...     ...     ...     ...     ...     ...  \n",
       "1995       0       0       1       0       0       0       1       0  \n",
       "1996       0       0       0       0       0       0       0       1  \n",
       "1997       0       0       0       0       0       0       0       1  \n",
       "1998       0       0       0       0       0       0       1       0  \n",
       "1999       0       0       0       0       0       0       1       0  \n",
       "\n",
       "[2000 rows x 49 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold #1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\HP\\AppData\\Roaming\\Python\\Python311\\site-packages\\keras\\src\\layers\\core\\dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n",
      "  super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 7ms/step\n",
      "Fold score (accuracy): 0.6425\n",
      "Fold #2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\HP\\AppData\\Roaming\\Python\\Python311\\site-packages\\keras\\src\\layers\\core\\dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n",
      "  super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 6ms/step\n",
      "Fold score (accuracy): 0.7225\n",
      "Fold #3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\HP\\AppData\\Roaming\\Python\\Python311\\site-packages\\keras\\src\\layers\\core\\dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n",
      "  super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 6ms/step\n",
      "Fold score (accuracy): 0.6925\n",
      "Fold #4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\HP\\AppData\\Roaming\\Python\\Python311\\site-packages\\keras\\src\\layers\\core\\dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n",
      "  super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 13ms/step\n",
      "Fold score (accuracy): 0.67\n",
      "Fold #5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\HP\\AppData\\Roaming\\Python\\Python311\\site-packages\\keras\\src\\layers\\core\\dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n",
      "  super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m13/13\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 6ms/step\n",
      "Fold score (accuracy): 0.62\n",
      "Final score (accuracy): 0.6695\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import numpy as np\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import KFold\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Activation\n",
    "from tensorflow.keras import regularizers\n",
    "\n",
    "# Cross-validate\n",
    "kf = KFold(5, shuffle=True, random_state=42)\n",
    "    \n",
    "oos_y = []\n",
    "oos_pred = []\n",
    "fold = 0\n",
    "\n",
    "for train, test in kf.split(x):\n",
    "    fold+=1\n",
    "    print(f\"Fold #{fold}\")\n",
    "        \n",
    "    x_train = x[train]\n",
    "    y_train = y[train]\n",
    "    x_test = x[test]\n",
    "    y_test = y[test]\n",
    "    \n",
    "    #kernel_regularizer=regularizers.l2(0.01),\n",
    "    \n",
    "    model = Sequential()\n",
    "    # Hidden 1\n",
    "    model.add(Dense(50, input_dim=x.shape[1], \n",
    "            activation='relu',\n",
    "             activity_regularizer=regularizers.l1(1e-4))) \n",
    "    # Hidden 2\n",
    "    model.add(Dense(25, activation='relu', \n",
    "                    activity_regularizer=regularizers.l1(1e-4))) \n",
    "     # Output\n",
    "    model.add(Dense(y.shape[1],activation='softmax'))\n",
    "    model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
    "\n",
    "    model.fit(x_train,y_train,validation_data=(x_test,y_test),\n",
    "              verbose=0,epochs=500)\n",
    "    \n",
    "    pred = model.predict(x_test)\n",
    "    \n",
    "    oos_y.append(y_test)\n",
    "    # raw probabilities to chosen class (highest probability)\n",
    "    pred = np.argmax(pred,axis=1) \n",
    "    oos_pred.append(pred)        \n",
    "\n",
    "    # Measure this fold's accuracy\n",
    "    y_compare = np.argmax(y_test,axis=1) # For accuracy calculation\n",
    "    score = metrics.accuracy_score(y_compare, pred)\n",
    "    print(f\"Fold score (accuracy): {score}\")\n",
    "\n",
    "\n",
    "# Build the oos prediction list and calculate the error.\n",
    "oos_y = np.concatenate(oos_y)\n",
    "oos_pred = np.concatenate(oos_pred)\n",
    "oos_y_compare = np.argmax(oos_y,axis=1) # For accuracy calculation\n",
    "\n",
    "score = metrics.accuracy_score(oos_y_compare, oos_pred)\n",
    "print(f\"Final score (accuracy): {score}\")    \n",
    "    \n",
    "# Write the cross-validated prediction\n",
    "oos_y = pd.DataFrame(oos_y)\n",
    "oos_pred = pd.DataFrame(oos_pred)\n",
    "oosDF = pd.concat( [df, oos_y, oos_pred],axis=1 )\n",
    "#oosDF.to_csv(filename_write,index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}