fraud_detection_modelling_evalml.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "fraud_detection_modelling_evalml.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "SxqxWzDSrQaR",
        "tw5RYBfZGHrm",
        "qIPat55lhxnT",
        "A0w-IANygeqz",
        "knFnySYPXfOt"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l72nk1o9qOAf"
      },
      "source": [
        "# VESTA FRAUD DETECTION WITH AUTOML - EVALML"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "17BvGXuMqYxT"
      },
      "source": [
        "**Installation & Importaion of Libraries**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WQwbsq2ufWKO"
      },
      "source": [
        "!pip install \"featuretools[complete]\"\n",
        "!pip install composeml\n",
        "!pip install utils\n",
        "%pip install evalml\n",
        "!pip install dask[complete] distributed --upgrade\n",
        "%pip install -U tornado\n",
        "!pip install woodwork\n",
        "!pip install scikit-multilearn"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WMH2feYcIPDR"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qDdQI-s0ffrp"
      },
      "source": [
        "import utils\n",
        "import os\n",
        "os.environ[\"PYARROW_IGNORE_TIMEZONE\"] = \"1\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BW-NrLSvfhxx"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GIFoa8vPfKgk"
      },
      "source": [
        "# featuretools and compose-ML\n",
        "import composeml as cp\n",
        "import featuretools as ft\n",
        "import woodwork as ww\n",
        "\n",
        "import evalml\n",
        "from evalml import AutoMLSearch\n",
        "from evalml.utils import infer_feature_types\n",
        "from evalml.preprocessing import split_data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ckQXikQ5foBh"
      },
      "source": [
        "# basic machine learning\n",
        "from sklearn.linear_model import LinearRegression\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn.naive_bayes import GaussianNB\n",
        "from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "\n",
        "# stats\n",
        "from sklearn import metrics\n",
        "from sklearn.metrics import confusion_matrix\n",
        "from sklearn.metrics import roc_curve\n",
        "from sklearn.metrics import mean_squared_error\n",
        "from sklearn.metrics import precision_score, recall_score\n",
        "from sklearn.model_selection import cross_val_score\n",
        "\n",
        "from sklearn import preprocessing\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import StandardScaler"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fPYXJ5qMfqYh"
      },
      "source": [
        "from sklearn.ensemble import RandomForestRegressor\n",
        "\n",
        "from sklearn.preprocessing import LabelEncoder, OneHotEncoder\n",
        "\n",
        "from sklearn.metrics import r2_score\n",
        "\n",
        "from sklearn.metrics import classification_report, confusion_matrix"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A1l-ulRSfs__"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "import seaborn as sns"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bKfWhWROpiUr"
      },
      "source": [
        "import evalml\n",
        "from evalml import AutoMLSearch\n",
        "from evalml.objectives import FraudCost\n",
        "from evalml.objectives import CostBenefitMatrix\n",
        "from evalml import AutoMLSearch\n",
        "evalml.problem_types.problem_types.ProblemTypes.all_problem_types\n",
        "\n",
        "from skmultilearn.model_selection import iterative_train_test_split"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tTFcVZTPfrLW"
      },
      "source": [
        "# Reading Preprocessed Files and Lable encoding\n",
        "Note: original test file does not have isFraud column.\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mf7Gp6D6i0bq"
      },
      "source": [
        "df_train = pd.read_csv('/content/drive/MyDrive/train_v3.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Abu4vHlwf2Rf"
      },
      "source": [
        "##df_test = pd.read_csv('/content/drive/MyDrive/Vesta_Fraud/test_v2.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "flWIzEcfjbw2"
      },
      "source": [
        "df_train"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sbAPO2OEvbRM"
      },
      "source": [
        "**One Hot Encode**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KPGYyPvkvguT"
      },
      "source": [
        "prodenc = LabelEncoder()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kEfhX0O0mfl3"
      },
      "source": [
        "df_train['ProductCD'] = prodenc.fit_transform(df_train['ProductCD'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0zB3t8aOODBM"
      },
      "source": [
        "df_train.head(5)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Rp3Da_IXctS"
      },
      "source": [
        "#before one hot encoding and converting to categorical values for card 4\n",
        "card4enc = LabelEncoder()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M2vAmouEDVT0"
      },
      "source": [
        "df_train['card4'] = card4enc.fit_transform(df_train['card4'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TchGR82FpiW8"
      },
      "source": [
        "emailenc = LabelEncoder()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vyndp3i2pucl"
      },
      "source": [
        "df_train['P_emaildomain'] = emailenc.fit_transform(df_train['P_emaildomain'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kx_NvoMLqAcO"
      },
      "source": [
        "df_train"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EjVw5pR4q0h7"
      },
      "source": [
        "card6enc = LabelEncoder()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j_CDlbsHq75z"
      },
      "source": [
        "df_train['card6'] = card6enc.fit_transform(df_train['card6'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AhoJSk6kORms"
      },
      "source": [
        "df_train['DeviceType'].value_counts()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l3O0nELVOFTv"
      },
      "source": [
        "devicetype_enc= LabelEncoder()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0pDETPaxOP1c"
      },
      "source": [
        "df_train['DeviceType'] =devicetype_enc.fit_transform(df_train['DeviceType'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aqRfx6AZOwTE"
      },
      "source": [
        "df_train"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2FYmyRK3Or28"
      },
      "source": [
        "deviceinfo_enc = LabelEncoder()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zr68L_kCOzPp"
      },
      "source": [
        "df_train['DeviceInfo'] =deviceinfo_enc.fit_transform(df_train['DeviceInfo'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aJpvY2MmO-X0"
      },
      "source": [
        "df_train"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gsx9BQh7uG34"
      },
      "source": [
        "df_train.columns.to_list()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fkH0cq2gDkQu"
      },
      "source": [
        "df_list = df.columns.to_list()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QYtTj6w2Doc7"
      },
      "source": [
        "df_list"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SxqxWzDSrQaR"
      },
      "source": [
        "##Splitting Data with All Columns##\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sNz6hy85u7YB"
      },
      "source": [
        "# data features - independent variables\n",
        "X = df_train.loc[:, df_train.columns != 'isFraud']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l81GDKgnqg7S"
      },
      "source": [
        "pd.set_option('max_columns', None)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2zRJ466eqJcs"
      },
      "source": [
        "X"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3RQ2YXpgvGU8"
      },
      "source": [
        "# data lables - target to predict - dependent variable\n",
        "y = df_train.isFraud"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WRoEZu69LxX4"
      },
      "source": [
        "##Splitting Data with Feature Importance Columns##"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "37ZljCqnL2Gf"
      },
      "source": [
        "##columns lables that will predict transaction is fraud\n",
        "col_features = df_train[['TransactionAmt','TransactionDT','TransactionID','addr1','ProductCD','P_emaildomain','card1',\n",
        "                          'card2','card3','card4','card5','card6','D1','D15','C1','C11','C12','C13','C14','C2','C4','C6','C7','V15',\n",
        "                          'V16','V17','V18','V33','V34','V53','V58','V73','V74','V79','V80','V81','V84','V86','V87','V92','V93','V94','V312','V317','V318','DeviceInfo']]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e3l3TLc-hq_f"
      },
      "source": [
        "df_train['V16'].value_counts()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JF0isiDZL3z4"
      },
      "source": [
        "#Y lables this is what to be predicted for X features\n",
        "col_labels = df_train['isFraud']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I8ACvpyQL9Iy"
      },
      "source": [
        "features = col_features\n",
        "labels = col_labels"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OpSa4pdQL_mL"
      },
      "source": [
        "# variable X are the features\n",
        "# variable y are the labels\n",
        "X = features\n",
        "y = labels\n",
        "\n",
        "print(features.head())\n",
        "print(X.shape)\n",
        "\n",
        "print(labels.head())\n",
        "print(y.shape)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tw5RYBfZGHrm"
      },
      "source": [
        "##Not required to run"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i8mtcf3WFVv1"
      },
      "source": [
        "import evalml\n",
        "from evalml import AutoMLSearch\n",
        "from evalml.utils import infer_feature_types\n",
        "from evalml.preprocessing import split_data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W9JzrF-OW4ll"
      },
      "source": [
        "#Fraud Objective\n",
        "#retry_percentage - what percentage of customers will retry a transaction if it is declined?\n",
        "#interchange_fee - how much of each successful transaction do you collect?\n",
        "#fraud_payout_percentage - the percentage of fraud will you be unable to collect\n",
        "#amount_col - the column in the data the represents the transaction amount\n",
        "\n",
        "from evalml.objectives import FraudCost\n",
        "fraud_objective = FraudCost(retry_percentage=.4,\n",
        "                            interchange_fee=.03,\n",
        "                            fraud_payout_percentage=.65,\n",
        "                            amount_col='TransactionAmt')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t6gEd37GFbSV"
      },
      "source": [
        "#autosearch and optimized for fraud objective\n",
        "automl = AutoMLSearch(X_train = X_train, y_train = y_train,\n",
        "                      problem_type='binary',\n",
        "                      objective= fraud_objective,\n",
        "                      additional_objectives=['auc', 'f1', 'precision','Accuracy Binary'],\n",
        "                      allowed_model_families=[\"lightgbm\",\"xgboost\",\"random_forest\", \"decision_tree\",\"linear_model\"],\n",
        "                      max_batches =1, \n",
        "                      sampler_balanced_ratio = 0.75,\n",
        "                      optimize_thresholds= True,\n",
        "                      verbose = True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Sf8kk4WVFm4a"
      },
      "source": [
        "#autosearch and optimized for AUC objective\n",
        "automl_auc = AutoMLSearch(X_train=X_train, y_train=y_train,\n",
        "                          problem_type='binary',\n",
        "                          objective='auc',\n",
        "                          additional_objectives=['f1', 'precision'],\n",
        "                          max_batches=1,\n",
        "                          sampler_balanced_ratio = 0.75,\n",
        "                          allowed_model_families=[\"random_forest\", \"decision_tree\", \"lightgbm\", \"xgboost\",\"linear_model\"],\n",
        "                          optimize_thresholds=True,\n",
        "                          verbose = True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oiDpxrSLckYS"
      },
      "source": [
        "#MODELLING#"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7YbuxVajELyW"
      },
      "source": [
        "##ADDING FRAUD OBJECTIVE##"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jWeIa4ydF2LY"
      },
      "source": [
        "Example of fraud objective\n",
        "\n",
        "https://evalml.alteryx.com/en/v0.11.0/demos/fraud.html\n",
        "\n",
        "https://evalml.alteryx.com/en/stable/demos/fraud.html\n",
        "\n",
        "To execute fraud objectives , X cannot be scaled. If not, it will throw an error."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1zIqcRnVEzD9"
      },
      "source": [
        "#retry_percentage - what percentage of customers will retry a transaction if it is declined?\n",
        "#interchange_fee - how much of each successful transaction do you collect?\n",
        "#fraud_payout_percentage - the percentage of fraud will you be unable to collect\n",
        "#amount_col - the column in the data the represents the transaction amount\n",
        "\n",
        "from evalml.objectives import FraudCost\n",
        "fraud_objective = FraudCost(retry_percentage=.4,\n",
        "                            interchange_fee=.03,\n",
        "                            fraud_payout_percentage=.65,\n",
        "                            amount_col='TransactionAmt')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dZ-FdW_zE8qP"
      },
      "source": [
        "# split data into training and testing data sets\n",
        "# holdout is also known as testing data\n",
        "X_train, X_holdout, y_train, y_holdout = evalml.preprocessing.split_data(X, y, problem_type='binary', test_size=0.2)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ghzAV26FNiD1"
      },
      "source": [
        "X_train"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_afPRNIYNlvg"
      },
      "source": [
        "X_holdout"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "33kmikBbTR7q"
      },
      "source": [
        "##Confusion Matrix function##"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vtCMYOftgMYI"
      },
      "source": [
        "from sklearn.metrics import confusion_matrix\n",
        "from sklearn.metrics import precision_score, recall_score, accuracy_score\n",
        "def plot_confusion_matrix(y_test, y_pred):\n",
        "    acc = round(accuracy_score(y_test, y_pred), 2)\n",
        "    cm = confusion_matrix(y_test, y_pred)\n",
        "    sns.heatmap(cm, annot=True, fmt=\".0f\")\n",
        "    plt.xlabel('Predicted')\n",
        "    plt.ylabel('Actual')\n",
        "    plt.title('Accuracy Score: {0}'.format(acc), size=10)\n",
        "    plt.show()\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HBznG0evoOMa"
      },
      "source": [
        "##Handling Imbalanced Data##\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ADMo98nnoTv2"
      },
      "source": [
        "# Undersampler case\n",
        "# we don't want to undersample this class, so class 1 (minority) will have a ratio of 1 to itself\n",
        "# for the majority class 0, we want to undersample it to have a minority/majority ratio of 0.5, \n",
        "#which means we want majority to have 2x the samples as the minority\n",
        "#sampler_ratio_dict = {0: 0.5, 1: 1}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cEbPUFvkois3"
      },
      "source": [
        "#pipeline_parameters = {\"Undersampler\": {\"sampler_balanced_ratio\": sampler_ratio_dict}}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xbJ9qCZEnjD3"
      },
      "source": [
        "##use autoML to search for the best model by specifying model families with fraud objective ##"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J27W-0ouGhBF"
      },
      "source": [
        "#autosearch with fraud objective\n",
        "automl = AutoMLSearch(X_train = X_train, y_train = y_train,\n",
        "                      problem_type='binary',\n",
        "                      objective= fraud_objective,\n",
        "                      additional_objectives=['auc', 'f1', 'precision','Accuracy Binary'],\n",
        "                      allowed_model_families=[\"lightgbm\",\"xgboost\",\"random_forest\", \"decision_tree\",\"linear_model\"],\n",
        "                      max_batches =1, \n",
        "                      sampler_balanced_ratio = 0.75,\n",
        "                      optimize_thresholds= True,\n",
        "                      verbose = True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cMRM1CbyYis9"
      },
      "source": [
        "automl.allowed_pipelines[-1]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F96QKoqxHPpH"
      },
      "source": [
        "automl.search()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "purLZXhpHzFM"
      },
      "source": [
        "automl.rankings"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HOScpn3VuU1O"
      },
      "source": [
        "automl.describe_pipeline(automl.rankings.iloc[2][\"id\"])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uMyPGGVdIZLU"
      },
      "source": [
        "best_pipeline = automl.best_pipeline"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_Gt5r5umcsam"
      },
      "source": [
        "best_pipeline.graph()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZpdStfZpH7gq"
      },
      "source": [
        "best_pipeline.score(X_train, y_train, objectives=[\"auc\", fraud_objective])\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JQYe58YdPilK"
      },
      "source": [
        "best_pipeline.score(X_train, y_train,['Accuracy Binary', 'F1', 'AUC', 'recall', fraud_objective])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q_gnMDEyEIUo"
      },
      "source": [
        "best_pipeline.score(X_holdout, y_holdout,['Accuracy Binary', 'F1', 'AUC', 'recall', fraud_objective])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RLUeC_pH35zP"
      },
      "source": [
        "print(best_pipeline.threshold)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YCkBye1b4pPW"
      },
      "source": [
        "y_pred_proba = best_pipeline.predict_proba(X_train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QL1i_9dOn2aw"
      },
      "source": [
        "print(y_pred_proba)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RjJrGgvDMVQn"
      },
      "source": [
        "##Avoiding Overfitting##"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kIFDGP_FMbq-"
      },
      "source": [
        "**Detecting lable leakage**\n",
        "TargetLeakageDataCheck checks for features that could potentially be “leaking”\n",
        "information by calculating the Pearson correlation coefficient between each feature and the target to warn users if there\n",
        "are features are highly correlated with the target.\n",
        "https://evalml.alteryx.com/_/downloads/en/stable/pdf/"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M2zsZydPRq94"
      },
      "source": [
        "** How does EvalML avoid overfitting?\n",
        "EvalML provides data checks to combat overfitting. Such data checks include detecting label leakage, unstable\n",
        "pipelines, hold-out datasets and cross validation. EvalML defaults to using Stratified K-Fold cross-validation for\n",
        "classification problems and K-Fold cross-validation for regression problems but allows you to utilize your own crossvalidation methods as well.**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1FMpKEmZMlkF"
      },
      "source": [
        "from evalml.data_checks import TargetLeakageDataCheck "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gKJVpz8FOOwd"
      },
      "source": [
        "Target_leakage_check = TargetLeakageDataCheck()\n",
        "results = Target_leakage_check.validate(X_train, y_train)\n",
        "\n",
        "for message in results['warnings']:print(\"Warning:\", message['message'])\n",
        "for message in results['errors']: print(\"Error:\", message['message'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ni77sF8ugDPE"
      },
      "source": [
        "##use autoML to search for the best model by specifying 4 model families (AUC objective)##"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DluY7mmCJ9Tj"
      },
      "source": [
        "automl_auc = AutoMLSearch(X_train=X_train, y_train=y_train,\n",
        "                          problem_type='binary',\n",
        "                          objective='auc',\n",
        "                          additional_objectives=['f1', 'precision'],\n",
        "                          max_batches=1,\n",
        "                          sampler_balanced_ratio = 0.75,\n",
        "                          allowed_model_families=[\"random_forest\", \"decision_tree\", \"lightgbm\", \"xgboost\",\"linear_model\"],\n",
        "                          optimize_thresholds=True,\n",
        "                          verbose = True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U8hNZKnVYoaK"
      },
      "source": [
        "automl_auc.allowed_pipelines[-1]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "--KtlMHUKGEW"
      },
      "source": [
        "automl_auc.search()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qtmmLtrfKeQ3"
      },
      "source": [
        "automl_auc.rankings"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pu_87kxVK1kd"
      },
      "source": [
        "best_pipeline_auc = automl_auc.best_pipeline"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KIM60CvoczOj"
      },
      "source": [
        "best_pipeline_auc.graph()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8WrDw-MmK775"
      },
      "source": [
        "# get the fraud score on holdout data\n",
        "best_pipeline_auc.score(X_holdout, y_holdout,  objectives=[\"auc\", fraud_objective])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KNWj_66m9EoH"
      },
      "source": [
        "# get the fraud score on holdout data\n",
        "best_pipeline_auc.score(X_train, y_train,  objectives=[\"auc\", fraud_objective])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ct0cwk-sLBG8"
      },
      "source": [
        "# fraud score on fraud optimized again. Found that fraud cost is lower compared to above.\n",
        "best_pipeline.score(X_holdout, y_holdout, objectives=[\"auc\", fraud_objective])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lfXnXR1-GGvX"
      },
      "source": [
        "best_pipeline_auc.score(X_holdout, y_holdout,['Accuracy Binary', 'F1', 'AUC', 'recall', fraud_objective])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8IyR24DMr0wd"
      },
      "source": [
        "print(best_pipeline_auc.threshold)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1MsPUu5Ib8RA"
      },
      "source": [
        "#VISUALISATION AND EVALUATION#"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Xk4iI7z2A-DB"
      },
      "source": [
        "##Predicting with best pipeline generated from fraud objective##"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GHX1-8fP4oYW"
      },
      "source": [
        "y_pred = best_pipeline.predict(X_holdout)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lG5qT6by846t"
      },
      "source": [
        "from evalml.model_understanding.graphs import graph_confusion_matrix"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GUFqSEcCn-f2"
      },
      "source": [
        "**Note: Many false positives.**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R4V978vp9A7S"
      },
      "source": [
        "#graph_confusion_matrix(y_holdout, y_pred, normalize_method= None)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YPYlf-Oehk6s"
      },
      "source": [
        "plot_confusion_matrix(y_holdout, y_pred)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kUryY1x5wchC"
      },
      "source": [
        "best_pipeline.graph_feature_importance()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Bv84sAOZAsI2"
      },
      "source": [
        "##Predicting with best pipeline generated from auc metric##\n",
        "\n",
        "**Note: many false negatives**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A8sP1aFYgpmF"
      },
      "source": [
        "from evalml.model_understanding.graphs import graph_confusion_matrix"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2uDWG6DRAZVp"
      },
      "source": [
        "y_pred_auc = best_pipeline_auc.predict(X_holdout)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4yjKh8brBEvA"
      },
      "source": [
        "#graph_confusion_matrix(y_holdout, y_pred_2, normalize_method= None)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j72oADNLgTGH"
      },
      "source": [
        "plot_confusion_matrix(y_holdout, y_pred_auc)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f09j9rv16tjt"
      },
      "source": [
        "best_pipeline_auc.graph_feature_importance()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KzUtCujyqGU8"
      },
      "source": [
        "##Training and Scoring Multiple 4 Pipelines generated with Fraud Objective##"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MPBpdNbQqMsl"
      },
      "source": [
        "trained_pipelines = automl.train_pipelines([automl.get_pipeline(i) for i in [1, 2,3, 4,5,6]])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "82yvlSQYqd1m"
      },
      "source": [
        "trained_pipelines"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lVFqxvTor_qX"
      },
      "source": [
        "pipeline_train_scores = automl.score_pipelines([trained_pipelines[name] for name in trained_pipelines.keys()],\n",
        "                                                X_train,\n",
        "                                                y_train,\n",
        "                                                ['Accuracy Binary', 'F1', 'AUC', 'recall',fraud_objective])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5jqvUuEesQQJ"
      },
      "source": [
        "pipeline_train_scores"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9EpokzaCChzg"
      },
      "source": [
        "pipeline_test_scores = automl.score_pipelines([trained_pipelines[name] for name in trained_pipelines.keys()],\n",
        "                                                X_holdout,\n",
        "                                                y_holdout,\n",
        "                                                ['Accuracy Binary', 'F1', 'AUC', 'recall',fraud_objective])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o0qzUYGKICLi"
      },
      "source": [
        "pipeline_test_scores"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WoBvU31jr-AP"
      },
      "source": [
        "**manual fitting with other pipelines to compare (with Fraud objective)**\n",
        "\n",
        "**fitting with first best pipeline - Decision Tree**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B_NwWm5cLUv0"
      },
      "source": [
        "first_best_pipeline = automl.get_pipeline(2)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Tx-Gb8_HLe21"
      },
      "source": [
        "first_best_pipeline.fit(X_train,y_train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HXlJgKVULh0W"
      },
      "source": [
        "y_pred_1 = first_best_pipeline.predict(X_holdout, fraud_objective)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3elQ8TOyMFPt"
      },
      "source": [
        "plot_confusion_matrix(y_holdout, y_pred_1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K_L1c35CMIxt"
      },
      "source": [
        "**fitting with second best pipeline - Elastic Net Classifier**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XutLoGTntBqJ"
      },
      "source": [
        "second_best_pipeline = automl.get_pipeline(1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ajjT8thutTfr"
      },
      "source": [
        "second_best_pipeline.fit(X_train,y_train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_0x8mMrTtaHB"
      },
      "source": [
        "y_pred_2 = second_best_pipeline.predict(X_holdout, fraud_objective)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YnmtJPdTTlm-"
      },
      "source": [
        "plot_confusion_matrix(y_holdout, y_pred_2)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "flt0g5rVtxuL"
      },
      "source": [
        "##graph_confusion_matrix(y_holdout, y_pred_3, normalize_method= None)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_MJZfTwg_L07"
      },
      "source": [
        "y_pred_proba = second_best_pipeline.predict_proba(X_train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4by81D5A_qzu"
      },
      "source": [
        "\n",
        "#doesnt work\n",
        "#pipeline.threshold = fraud_objective.optimize_threshold(y_pred_proba, y_train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iUbOLwPG7Znt"
      },
      "source": [
        "print(second_best_pipeline.threshold)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0lYjOIX7Dk1x"
      },
      "source": [
        "**fitting with third best pipeline - Logistic Regression**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GPOnsSnhDr-m"
      },
      "source": [
        "third_best_pipeline = automl.get_pipeline(4)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5CBEM5yGD1wt"
      },
      "source": [
        "third_best_pipeline.fit(X_train,y_train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rs_aQu93D5es"
      },
      "source": [
        "y_pred_3 = third_best_pipeline.predict(X_holdout)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CJCGlNB4TxRC"
      },
      "source": [
        "plot_confusion_matrix(y_holdout, y_pred_3)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yG6qDkQrD9-P"
      },
      "source": [
        "#graph_confusion_matrix(y_holdout, y_pred_4, normalize_method= None)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZWu4IZqoNP9P"
      },
      "source": [
        "**fitting with fourth best pipeline - Random Forest**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8kI9ItO5NWOx"
      },
      "source": [
        "fourth_best_pipeline = automl.get_pipeline(6)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cTIJ_gkNNe5x"
      },
      "source": [
        "fourth_best_pipeline.fit(X_train,y_train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oKHYGotiNkM8"
      },
      "source": [
        "y_pred_4 = fourth_best_pipeline.predict(X_holdout)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1Kf-d1FENo5W"
      },
      "source": [
        "plot_confusion_matrix(y_holdout, y_pred_4)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "--OnJ6adNsGd"
      },
      "source": [
        "**fitting with fifth best pipeline - XGBoost**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TSEYXyC6N1DF"
      },
      "source": [
        "fifth_best_pipeline = automl.get_pipeline(3)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yOnnN_GRN9B8"
      },
      "source": [
        "fifth_best_pipeline.fit(X_train,y_train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I41iDJhuOKOQ"
      },
      "source": [
        "y_pred_5 = fifth_best_pipeline.predict(X_holdout)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T3TTyzXeOO8k"
      },
      "source": [
        "plot_confusion_matrix(y_holdout, y_pred_5)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HRzXrzItOTBw"
      },
      "source": [
        "**fitting with sixth best pipeline - LightGBM**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y2K2XCSyOhQs"
      },
      "source": [
        "sixth_best_pipeline = automl.get_pipeline(5)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aBcXET1fOnkt"
      },
      "source": [
        "sixth_best_pipeline.fit(X_train,y_train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MQmwq8-fOrnf"
      },
      "source": [
        "y_pred_6 = sixth_best_pipeline.predict(X_holdout)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iRf1q8xCOvyX"
      },
      "source": [
        "plot_confusion_matrix(y_holdout, y_pred_6)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qIPat55lhxnT"
      },
      "source": [
        "\n",
        "\n",
        "##Precision Recall Curve Using EvalML for BEST PIPELINES##"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GkcWybRth83t"
      },
      "source": [
        "**Precision recall curve for best pipeline with fraud objective**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1H3ei6Xkh5sq"
      },
      "source": [
        "from evalml.model_understanding.graphs import graph_precision_recall_curve"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z5txpGBjiXZV"
      },
      "source": [
        "#Decision Tree\n",
        "y_pred_proba = best_pipeline.predict_proba(X_holdout)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h9JHT2EcjBDM"
      },
      "source": [
        "graph_precision_recall_curve(y_holdout, y_pred_proba)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ToaR6JooqHd9"
      },
      "source": [
        "**Precision recall curve for best pipeline with auc objective**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "egpnF83dqJqx"
      },
      "source": [
        "y_pred_proba = best_pipeline_auc.predict_proba(X_holdout)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4hWEmai2qMnX"
      },
      "source": [
        "graph_precision_recall_curve(y_holdout, y_pred_proba)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dVxDwBQlui1m"
      },
      "source": [
        "##Precision recall curve for other pipelines with fraud objective##"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aCZnL8qsulUG"
      },
      "source": [
        "#ElasticNet\n",
        "y_pred_proba_2 = second_best_pipeline.predict_proba(X_holdout)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NVn-w7eMus5S"
      },
      "source": [
        "graph_precision_recall_curve(y_holdout, y_pred_proba_2)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L6aMVdp5vdQC"
      },
      "source": [
        "#LogisticRegression\n",
        "y_pred_proba_3 = third_best_pipeline.predict_proba(X_holdout)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yBmGnyZvvi1d"
      },
      "source": [
        "graph_precision_recall_curve(y_holdout, y_pred_proba_3)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EDbwPJhAtVqe"
      },
      "source": [
        "#RandomForest\n",
        "y_pred_proba_4 = fourth_best_pipeline.predict_proba(X_holdout)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EEP98VoXtb2H"
      },
      "source": [
        "graph_precision_recall_curve(y_holdout, y_pred_proba_4)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_DLjHMOFthfP"
      },
      "source": [
        "#XGBoost\n",
        "y_pred_proba_5 = fifth_best_pipeline.predict_proba(X_holdout)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9zIQvatqtw7i"
      },
      "source": [
        "graph_precision_recall_curve(y_holdout, y_pred_proba_5)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n0jHS17duB4F"
      },
      "source": [
        "#LightGBM\n",
        "y_pred_proba_6 = sixth_best_pipeline.predict_proba(X_holdout)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cevoF6HYuJPB"
      },
      "source": [
        "graph_precision_recall_curve(y_holdout, y_pred_proba_6)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A0w-IANygeqz"
      },
      "source": [
        "##ROC Curve using EvalML for BEST PIPELINES##"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2TBXzYIkgxRu"
      },
      "source": [
        "**ROC curve for best pipeline with fraud objective**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QH89TcT8gvfL"
      },
      "source": [
        "from evalml.model_understanding.graphs import graph_roc_curve"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9XFkuow0g7Wf"
      },
      "source": [
        "#Decision Tree\n",
        "y_pred_proba_1 = best_pipeline.predict_proba(X_holdout)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YqQpSNnuhGrw"
      },
      "source": [
        "graph_roc_curve(y_holdout, y_pred_proba_1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vraYmrdYpcDo"
      },
      "source": [
        "**ROC curve for best pipeline with auc objective**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qfvj-bdrpiDj"
      },
      "source": [
        "#LightGBM\n",
        "y_pred_proba_auc = best_pipeline_auc.predict_proba(X_holdout)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9cAejMyqpl3_"
      },
      "source": [
        "graph_roc_curve(y_holdout, y_pred_proba_auc)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g834WLaDq_le"
      },
      "source": [
        "##ROC curve for other pipelines with fraud objective##"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DdOhZ6zdwCYv"
      },
      "source": [
        "#DecisionTree\n",
        "y_pred_proba_1_1 = first_best_pipeline.predict_proba(X_holdout)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "curwD0xMwFgl"
      },
      "source": [
        "graph_roc_curve(y_holdout, y_pred_proba_1_1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gkTxfGGirJ9t"
      },
      "source": [
        "#ElasticNet\n",
        "y_pred_proba_2 = second_best_pipeline.predict_proba(X_holdout)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6cz5TB_NrR6G"
      },
      "source": [
        "graph_roc_curve(y_holdout, y_pred_proba_2)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KKCJERdarnfk"
      },
      "source": [
        "#LogisticRegression\n",
        "y_pred_proba_3 = third_best_pipeline.predict_proba(X_holdout)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VoY5hlNNsi-j"
      },
      "source": [
        "graph_roc_curve(y_holdout, y_pred_proba_3)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-OgglEhUu2HM"
      },
      "source": [
        "#RandomForest\n",
        "y_pred_proba_4 = fourth_best_pipeline.predict_proba(X_holdout)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aYuk-gjSvFVo"
      },
      "source": [
        "graph_roc_curve(y_holdout, y_pred_proba_4)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rDHXGh-pvJy0"
      },
      "source": [
        "#XGBoost\n",
        "y_pred_proba_5 = fifth_best_pipeline.predict_proba(X_holdout)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ha_te3HsvOn6"
      },
      "source": [
        "graph_roc_curve(y_holdout, y_pred_proba_5)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9DQDNYmbvXpG"
      },
      "source": [
        "#LightGBM\n",
        "y_pred_proba_6 = sixth_best_pipeline.predict_proba(X_holdout)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fuQFwDIgvc4v"
      },
      "source": [
        "graph_roc_curve(y_holdout, y_pred_proba_6)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iuh_hru3sY1r"
      },
      "source": [
        "##FEATURE IMPORTANCE for all pipelines generated with Fraud Objective##"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PBU2AoV-si8o"
      },
      "source": [
        "**auto-best pipeline**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MnWUK2Hks7S0"
      },
      "source": [
        "best_pipeline.graph_feature_importance()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A9ZeUaoCtKkD"
      },
      "source": [
        "**manual-fit best pipeline**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H0Gd4VCjtTQC"
      },
      "source": [
        "first_best_pipeline.graph_feature_importance()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ABV9s9-JtfVG"
      },
      "source": [
        "**second best pipeline**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y0G6weLgtkdq"
      },
      "source": [
        "#ElasticNet\n",
        "second_best_pipeline.graph_feature_importance()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ver-uzzctriu"
      },
      "source": [
        "**third best pipeline**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MEn0Nkudt6MI"
      },
      "source": [
        "#LogisticRegression\n",
        "third_best_pipeline.graph_feature_importance()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WtiQ5ayBuFlM"
      },
      "source": [
        "**fourth pipleline**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7Zz6W_v9uFBP"
      },
      "source": [
        "#RandomForest\n",
        "fourth_best_pipeline.graph_feature_importance()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q_ZJ8sBbwnjN"
      },
      "source": [
        "#XGBoost\n",
        "fifth_best_pipeline.graph_feature_importance()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H39eaIIEx1tR"
      },
      "source": [
        "#LightGBM\n",
        "sixth_best_pipeline.graph_feature_importance()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "knFnySYPXfOt"
      },
      "source": [
        "##Training and Scoring Multiple Pipelines generated with AUC objective##"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hMPbDz7WXm8w"
      },
      "source": [
        "trained_pipelines_auc = automl_auc.train_pipelines([automl_auc.get_pipeline(i) for i in [0, 1, 2,3, 4]])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EQfmPtZsYL7n"
      },
      "source": [
        "pipeline_holdout_scores_auc = automl_auc.score_pipelines([trained_pipelines_auc[name] for name in trained_pipelines_auc.keys()],\n",
        "                                                X_holdout,\n",
        "                                                y_holdout,\n",
        "                                                ['Accuracy Binary', 'F1', 'AUC'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HxHxm4OKYdcc"
      },
      "source": [
        "pipeline_holdout_scores_auc"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_XvTnk7YYsF-"
      },
      "source": [
        "**manual fitting with first best pipeline instead of using best_pipeline default function to compare**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l6k4aJ6BYhTZ"
      },
      "source": [
        "first_best_pipeline_auc =  automl_auc.get_pipeline(4)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c43uEimlZ6L5"
      },
      "source": [
        "first_best_pipeline_auc"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9ZkDpK8GYuR4"
      },
      "source": [
        "first_best_pipeline_auc.fit(X_train,y_train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m9YSd-CpZm1C"
      },
      "source": [
        "y_pred_7 = first_best_pipeline_auc.predict(X_holdout)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K-TQO0OnUMt_"
      },
      "source": [
        "plot_confusion_matrix(y_holdout, y_pred_7)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W_R8qKIVZ1eU"
      },
      "source": [
        "#graph_confusion_matrix(y_holdout, y_pred_7, normalize_method= None)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w7BhW3C9IEnC"
      },
      "source": [
        "from evalml.pipelines import BinaryClassificationPipeline"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kodG0n2kIIHF"
      },
      "source": [
        "pipeline_binary = BinaryClassificationPipeline(['Simple Imputer', 'Random Forest Classifier'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mUK4d1k702K_"
      },
      "source": [
        "from evalml.model_understanding.graphs import confusion_matrix\n",
        "y_pred = pipeline_binary.predict(X_holdout)\n",
        "confusion_matrix(y_holdout, y_pred)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6xiPQrIn3DL7"
      },
      "source": [
        "from evalml.model_understanding.graphs import graph_confusion_matrix\n",
        "y_pred = pipeline_binary.predict(X_holdout)\n",
        "#graph_confusion_matrix(y_holdout, y_pred)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CMZoswKeUV50"
      },
      "source": [
        "plot_confusion_matrix(y_holdout, y_pred)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}