TrainChurnModel.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "impaired-stylus",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from pathlib import Path\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.metrics import (confusion_matrix, f1_score, make_scorer,\n",
    "                             roc_auc_score)\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.pipeline import Pipeline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "accepting-wallet",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_file_paths = [f'Churn_Modelling_{country}.csv'  for country in ['Spain', 'France']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "continued-juvenile",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CustomerId</th>\n",
       "      <th>Surname</th>\n",
       "      <th>CreditScore</th>\n",
       "      <th>Geography</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Age</th>\n",
       "      <th>Tenure</th>\n",
       "      <th>Balance</th>\n",
       "      <th>NumOfProducts</th>\n",
       "      <th>HasCrCard</th>\n",
       "      <th>IsActiveMember</th>\n",
       "      <th>EstimatedSalary</th>\n",
       "      <th>Exited</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>15647311</td>\n",
       "      <td>Hill</td>\n",
       "      <td>608</td>\n",
       "      <td>Spain</td>\n",
       "      <td>Female</td>\n",
       "      <td>41</td>\n",
       "      <td>1</td>\n",
       "      <td>83807.86</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>112542.58</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>15737888</td>\n",
       "      <td>Mitchell</td>\n",
       "      <td>850</td>\n",
       "      <td>Spain</td>\n",
       "      <td>Female</td>\n",
       "      <td>43</td>\n",
       "      <td>2</td>\n",
       "      <td>125510.82</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>79084.10</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15574012</td>\n",
       "      <td>Chu</td>\n",
       "      <td>645</td>\n",
       "      <td>Spain</td>\n",
       "      <td>Male</td>\n",
       "      <td>44</td>\n",
       "      <td>8</td>\n",
       "      <td>113755.78</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>149756.71</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15737173</td>\n",
       "      <td>Andrews</td>\n",
       "      <td>497</td>\n",
       "      <td>Spain</td>\n",
       "      <td>Male</td>\n",
       "      <td>24</td>\n",
       "      <td>3</td>\n",
       "      <td>0.00</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>76390.01</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>15600882</td>\n",
       "      <td>Scott</td>\n",
       "      <td>635</td>\n",
       "      <td>Spain</td>\n",
       "      <td>Female</td>\n",
       "      <td>35</td>\n",
       "      <td>7</td>\n",
       "      <td>0.00</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>65951.65</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   CustomerId   Surname  CreditScore Geography  Gender  Age  Tenure  \\\n",
       "0    15647311      Hill          608     Spain  Female   41       1   \n",
       "1    15737888  Mitchell          850     Spain  Female   43       2   \n",
       "2    15574012       Chu          645     Spain    Male   44       8   \n",
       "3    15737173   Andrews          497     Spain    Male   24       3   \n",
       "4    15600882     Scott          635     Spain  Female   35       7   \n",
       "\n",
       "     Balance  NumOfProducts  HasCrCard  IsActiveMember  EstimatedSalary  \\\n",
       "0   83807.86              1          0               1        112542.58   \n",
       "1  125510.82              1          1               1         79084.10   \n",
       "2  113755.78              2          1               0        149756.71   \n",
       "3       0.00              2          1               0         76390.01   \n",
       "4       0.00              2          1               1         65951.65   \n",
       "\n",
       "   Exited  \n",
       "0       0  \n",
       "1       0  \n",
       "2       1  \n",
       "3       0  \n",
       "4       0  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.concat([pd.read_csv(fpath) for fpath in data_file_paths])\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bb0d9f24",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7491, 13)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "unnecessary-roots",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CustomerId         0\n",
       "Surname            0\n",
       "CreditScore        0\n",
       "Geography          0\n",
       "Gender             0\n",
       "Age                0\n",
       "Tenure             0\n",
       "Balance            0\n",
       "NumOfProducts      0\n",
       "HasCrCard          0\n",
       "IsActiveMember     0\n",
       "EstimatedSalary    0\n",
       "Exited             0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Are there missing values?\n",
    "df.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b025ccce",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "France    5014\n",
       "Spain     2477\n",
       "Name: Geography, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Geography'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "scheduled-measure",
   "metadata": {},
   "outputs": [],
   "source": [
    "feat_cols = ['CreditScore', 'Age', 'Tenure', \n",
    "               'Balance', 'NumOfProducts', 'HasCrCard',\n",
    "               'IsActiveMember', 'EstimatedSalary']\n",
    "targ_col = 'Exited'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "junior-rating",
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = df[feat_cols], df[targ_col]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c4d476a6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.16326258176478442"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "historic-doubt",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "infinite-african",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>Pipeline(steps=[(&#x27;preprocessor&#x27;, SimpleImputer()),\n",
       "                (&#x27;clf&#x27;, RandomForestClassifier(max_depth=10, random_state=42))])</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;preprocessor&#x27;, SimpleImputer()),\n",
       "                (&#x27;clf&#x27;, RandomForestClassifier(max_depth=10, random_state=42))])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomForestClassifier</label><div class=\"sk-toggleable__content\"><pre>RandomForestClassifier(max_depth=10, random_state=42)</pre></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "Pipeline(steps=[('preprocessor', SimpleImputer()),\n",
       "                ('clf', RandomForestClassifier(max_depth=10, random_state=42))])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random_state = 42\n",
    "train_params = {'n_estimators': 100, 'max_depth': 10}\n",
    "\n",
    "clf = RandomForestClassifier(random_state=random_state, \n",
    "                            **train_params)\n",
    "model = Pipeline(\n",
    "    steps=[(\"preprocessor\", SimpleImputer()), (\"clf\", clf)]\n",
    "    )\n",
    "\n",
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "12944f94",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.5474137931034483, 0.8684512806155847)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_prob = model.predict_proba(X_test)\n",
    "y_pred = y_prob[:, 1] >= 0.5\n",
    "f1 = f1_score(y_test, y_pred)\n",
    "roc_auc = roc_auc_score(y_test, y_prob[:, 1])\n",
    "f1, roc_auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "4675491c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV0AAAD4CAYAAABPLjVeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAAsTAAALEwEAmpwYAAAT90lEQVR4nO3de5RX5X3v8fd3ZhhUVEQIo3LRUUCD5KIg2lij0SB4KcQaE9PaNNWGc3KCuZaoNbE9nGhq4jE5abCWtjb2nCYuYmOkAUUT70YRVuhSUVAE5RIZkJtc1IGZ5/zBZDKDwMyE4fltNu+Xa681e+/nt/fzW+LHL89+9jORUkKSlEdVpTsgSQcSQ1eSMjJ0JSkjQ1eSMjJ0JSmjmn19g4NPmeT0CL3L+rk/qHQXVEAH1RB7e42uZM5b83+w1/frqn0eupKUVRT7L/CGrqRyiezFa5cYupLKxUpXkjKy0pWkjKqqK92DPTJ0JZWLwwuSlJHDC5KUkZWuJGVkpStJGVnpSlJGzl6QpIysdCUpoyrHdCUpHytdScrI2QuSlJEP0iQpI4cXJCkjhxckKSMrXUnKyEpXkjKy0pWkjJy9IEkZWelKUkaO6UpSRla6kpSRla4kZWSlK0n5RJWhK0nZhMMLkpRRsTPX0JVULla6kpSRoStJGVX5IE2SMip2oWvoSioXhxckKSNDV5IyKnroFnvEWZK6KCI6vXXiWuMiYlFELI6Ia3dxfnBEPBwR8yPi2Yi4sKNrGrqSSiWqotPbHq8TUQ1MBS4AhgOfiojhOzX7OjA9pXQKcDlwW0f9M3QllUo3VrqjgcUppSUppUbgLmDCTm0ScHjLz72B33R0Ucd0JZVKN47pDgCWt9lfAZy+U5u/BR6IiKuBXsBHO7qola6kconObxExMSLmtdkmdvFunwJ+mFIaCFwI/N+IPa8taaUrqVS6UummlKYB03ZzeiUwqM3+wJZjbV0FjGu51lMRcRDQD1i9u3ta6UoqlW4c050LDI2I+oioZceDshk7tVkGnNdy3/cCBwFr9nRRK11JpdJday+klLZHxCRgNlAN3JFSWhARU4B5KaUZwFeBf4qIL7PjodpnUkppT9c1dCWVSze+G5FSmgXM2unYDW1+fgE4syvXNHQllUrR30gzdCWViqErSRkZupKUUUev91aaodtNxnzovdwy+eNUV1Xxw5/9ilv+9cF25wcf3Yfb/+YK+vU5lPVvbuXK6+9k5eoNANz4xQmMO2sEVRE8NGchX/323RX4BuoOTz7+GDf/3Y00NzVzyaWXcdVn28+1b2xs5PrrvsaLCxbQ+4gj+Pb//i4DBgxkw4b1fPVLX2DB888z/mOX8Ndf3/GsZsuWzfzFn/1p6+cbGlZx0cXj+dp112f9XvuTole6ztPtBlVVwfeu/QQTJt3GKZd+k8vGjeSk449q1+ZbX76Ef5/5DKM/+S1umnYfU64eD8AZH6jnDz54PKd94iZGXnYjI08+lrNGDq3E19Beampq4qYbp3Db7f/MPTNmcv+sn/PK4sXt2tzzHz/h8MMP5+f3P8gVn/4M37v1FgBqa3vy+au/yFcmf61d+169DmX6T+9t3Y4+ZgDnjTk/23faH3XnKmP7gqHbDU4bcRyvLH+DV1euZdv2Jn4y+9dcfM7727U56fijefSZRQA8OvclLj7nfQCkBD1re1Dbo4aetTXU1FSzet2b2b+D9t7zzz3LoEHHMnDQIHrU1jLuwot45OFftmvz8EMPMX7CJQCMOX8szzz9FCklDjnkEE4dOYqetT13e/1XX13KunVrOXXkqH36PfZ3+33oRsRJEXFNRHy/Zbum5c0LtTimf29WNKxv3V/ZsJ4B7+ndrs1zL61kwrkfBGDCuR/g8EMP5sjevZjz7FIem/cySx+8kaUP3MQvfvUii5Y25Oy+usnqhgaOOvp3f8PpX1dHQ0P7f5erVzdw1FFHA1BTU8Ohhx3Ghg3r6Yz7Z81k7LgLC//X54rrwtoLlbDH0I2Ia9ixnFkAz7RsAfx4Vwv6tvlc6yIS299Y0J393W9d9917OGvkEJ768TWcNXIIKxvW09TUzPGD+nFifR1Dxn6dE8Zezzmjh3HmKSdUursqoNn3zeKCCy+qdDcKr+iVbkcP0q4CTk4pbWt7MCJuBRYAf7erD7VdROLgUybt8ZW4MvjN6o0MrOvTuj+grg8r12xs1+b1NRu5/K/+GYBeB9fysfM+yMbNb3HlH3+IZ557lS1vNQIw+8kFnP7+ep6c/0q+L6Bu0b+ujlWvr2rdX93QQF1dXfs2/etYtep16o46iu3bt7N50yaOOKLPzpd6l0ULF7K9qYnhJ4/o9n6XTVXBZy90NLzQDByzi+NHt5wTMG/BawwZ/B6OPaYvPWqquWzsqcx85Nl2bfoe0av1/6yTrxzLnfc+DcDyVes5a+QQqqurqKmp4qxTh7Jw6ap33UPFd/KI97Fs2ausWLGcbY2N3D9rJmd/5Nx2bc75yLnMuPceAB58YDajTz+jUxXXfbN+bpXbSft7pfsl4JcR8TK/W8x3MDAEmLQP+7VfaWpq5ss3T+c/b/s81VXBnfc+zYtLVvGNz13Er19YxsxHn+PDo4Yy5erxpARP/HoxX/rWdAB++ov5nH3aMOZN/2sSiQd/9SKzHnu+wt9Iv4+amhquu/4GPjfxL2lubuJjl1zKkCFDmfr3/4eTTx7BOeeexyWXfpzrr53MxePGcHjv3nz7lu+2fv6CMeeyefNmtm3bxsMP/YLbp93BCUOGAPDA7PuY+g+7W4FQbRV9yDs6WBCHlgV5R7NjFXXYsZ7k3JRSU2ducCAML6jr1s/9QaW7oAI6qGbvH2+deM3sTmfOopvHZo/oDl+OSCk1A09n6Isk7bWiV7q+kSapVIr+IM3QlVQqhq4kZeTwgiRlVPQ39gxdSaVi6EpSRgXPXENXUrn4IE2SMnJ4QZIyKnjmGrqSysVKV5IyKnjmGrqSysVKV5IycvaCJGVU8ELX0JVULg4vSFJGBc9cQ1dSuVjpSlJGhq4kZeTsBUnKqOCFrqErqVwcXpCkjAqeuVRVugOS1J2qIjq9dSQixkXEoohYHBHX7qbNJyLihYhYEBE/6uiaVrqSSqW7HqRFRDUwFRgDrADmRsSMlNILbdoMBa4DzkwprY+I/h32r1t6J0kFURWd3zowGlicUlqSUmoE7gIm7NTms8DUlNJ6gJTS6g771/WvJEnFFRGd3jowAFjeZn9Fy7G2hgHDIuLJiHg6IsZ1dFGHFySVSlcepEXERGBim0PTUkrTunC7GmAocA4wEHgsIt6XUtqwpw9IUmkEnU/dloDdXciuBAa12R/YcqytFcCclNI2YGlEvMSOEJ67u3s6vCCpVLpxTHcuMDQi6iOiFrgcmLFTm5+xo8olIvqxY7hhyZ4uaqUrqVS6a/ZCSml7REwCZgPVwB0ppQURMQWYl1Ka0XLu/Ih4AWgCJqeU1u7puoaupFLpzPzbzkopzQJm7XTshjY/J+ArLVunGLqSSqXob6QZupJKxbUXJCmjgmeuoSupXKoLnrqGrqRScXhBkjIq+C+OMHQllYuVriRlVPDMNXQllYuVriRlVF3wQV1DV1KpFDtyDV1JJdOday/sC4aupFIpeOYaupLKxQdpkpRRwTPX0JVULs5ekKSMDvjhhfP++5/v61toP/RPc5ZWugsqoKvPrN/raxT9Fz9a6UoqlQO+0pWknAo+pGvoSioXH6RJUkYFz1xDV1K5FHxI19CVVC6uvSBJGTllTJIyKniha+hKKhdnL0hSRgXPXENXUrn4IE2SMip45hq6ksrF4QVJyigK/qspDV1JpVJT8Im6hq6kUnFpR0nKyDFdScqo4IVu4V9TlqQuqYro9NaRiBgXEYsiYnFEXLuHdpdGRIqIUR1d00pXUqlUd1MpGRHVwFRgDLACmBsRM1JKL+zU7jDgi8CczlzXSldSqVQRnd46MBpYnFJaklJqBO4CJuyi3f8Cbgbe7lz/JKlEIjq/dWAAsLzN/oqWY23uFacCg1JKMzvbP4cXJJVKV2YvRMREYGKbQ9NSStM6+dkq4FbgM13onqErqVy6suBNS8DuLmRXAoPa7A9sOfZbhwEjgEda5gYfBcyIiPEppXm7u6ehK6lUunHK2FxgaETUsyNsLwf+5LcnU0obgX6/u288AvzVngIXDF1JJdNdi5inlLZHxCRgNlAN3JFSWhARU4B5KaUZv891DV1JpdKdswNSSrOAWTsdu2E3bc/pzDUNXUml4toLkpRRsSPX0JVUMv66HknKqNiRa+hKKpmqgq/taOhKKpWir21g6EoqFWcvSFJGxY5cQ1dSyVjpSlJG1YauJOVT7Mg1dCWVTMELXUNXUrl04tfwVJShK6lUrHQlKaOw0pWkfJy9IEkZFTxzDV1J5WLoSlJGjulKUkYFX9nR0JVULv7mCEnKyOGFA8Spgw5n4ocGUxXBAwvXcPd/rWp3/rxhfbnyjEGs3bINgJ8vaOCBhW8A8JnTB3La4N4A3PXr13n8lXV5O6995rXn5vH4j/6BlJoZftY4Rl70yV22WzzvCe6/7Ztc9o3vU1c/jLc2v8n9t32T1Utf4qQzx3D2FZ/P3PP9l8MLB4CqgM+deSxfn/kSa7c08t0/Hs6cVzewfMPb7do9/so6bn9yWbtjowb35oR+h3D13QvoUV3Ft/7oJOYt28Bb25pzfgXtA83NTTz6/6Yy4as3ceiR/Zg+5QvUf/AMjhxwbLt2jW9t5dlf/Iy6409qPVbTo5bTP/Zp1q18jbUrX83c8/1b0Svdov9mi/3CsP69eP3Nd2jY9A7bmxOPLV7HGcf16dRnB/c5mAWvb6I5wTvbm3l13VZGDuq9j3usHBqWLKJ3/6Pp3f9oqmt6MPT0s1nyX0+9q92ce/6NUy+4jOoePVqP9eh5EMcMG9HumDonovNbJRi63aDvIbWs2dzYuv/Glkb69nr3fywfqu/D33/8ZK4bcwL9etUCsHTtVk4d1JueNVUcflAN7z/mMN5zaG22vmvf2bJhLYcd+Z7W/UP79GPL+rXt2qx+7WU2rV/DcR84PXf3Siu6sFXC7x26EfEXezg3MSLmRcS8ZY/f8/veolSeeW0DV/7oWa6+ewHzV7zJlz9SD8D8FW8yb9lGvjPhvUw+73gWNmyhKVW4s8oiNTfzxF3T+MNPfrbSXSmV6ohOb5WwN5Xu/9zdiZTStJTSqJTSqMFnXbIXt9g/rN3a2K467dertvWB2W9teqeJ7c070vSBhWsY0u+Q1nPT57/OF/5jAd+Y+RIAv9lpLFj7p15H9GXTujWt+5vXv0GvPn1b9xvffot1K1/jnpu/xp2TP03DKwuZ+f2/pWHpSxXobYkUvNTd44O0iHh2d6eAuu7vzv7ppdVbOKZ3T+oO2xG2Hx5yJN/55Svt2vQ5pAfrt+4I4tOPPaL1IVtVQK/aaja908RxRx5Mfd+DufXhjdm/g7pfXf2JbGz4DW+uWUWvPn15ec6jnP/frmk93/OQXvzl96e37v/05smc+YnPUlc/rBLdLY2iP0jraPZCHTAWWL/T8QB+tU96tB9qTnD7E8uYcuGJVAU8uOgNlq1/mz8ddQwvr9nKM69tYPyIOkYfewTNKbHp7e1875GlAFRXBTdPeC8AWxubuOWhJTQ7vFAKVdXVfPiK/8G9t15Pam5m+B+eT98BxzHnnn+j/3FDqT/lD/b4+Tsnf5rGt7fSvH07S+Y/xYSv3PiumQ96t4K/G0GktPv/wiPiX4B/TSk9sYtzP0op/UlHN7j4H+caIXqXsSP6VboLKqCrz6zf68icu2RjpzPntON7Z4/oPVa6KaWr9nCuw8CVpOwKXun6coSkUnHtBUnKqNiRa+hKKpuCp66hK6lUij5lzNeAJZVKd669EBHjImJRRCyOiGt3cf4rEfFCRDwbEb+MiA7n9Bm6kkqlu0I3IqqBqcAFwHDgUxExfKdm84FRKaX3A3cD3+6of4aupFKJLvzTgdHA4pTSkpRSI3AXMKFtg5TSwymlrS27TwMDO7qooSupVLpxeGEAsLzN/oqWY7tzFXBfRxf1QZqkUunKY7SImAhMbHNoWkppWpfvGXEFMAo4u6O2hq6kculC6rYE7O5CdiUwqM3+wJZj7W8X8VHgeuDslNI7Hd3T4QVJpdKNY7pzgaERUR8RtcDlwIx294o4BfhHYHxKaXVn+melK6lUuusXU6aUtkfEJGA2UA3ckVJaEBFTgHkppRnAd4BDgZ/EjkHiZSml8Xu6rqErqVy68d2IlNIsYNZOx25o8/NHu3pNQ1dSqRT9jTRDV1KpFHyRMUNXUrkUPHMNXUklU/DUNXQllYqLmEtSRsWOXENXUtkUPHUNXUml4pQxScqo4EO6hq6kcjF0JSkjhxckKSMrXUnKqOCZa+hKKhcrXUnKqtipa+hKKpXuWsR8XzF0JZWKwwuSlJFTxiQpp2JnrqErqVwKnrmGrqRycUxXkjKKgqeuoSupVIoduYaupJIpeKFr6EoqF6eMSVJGVrqSlJGhK0kZObwgSRlZ6UpSRgXPXENXUskUPHUNXUml4piuJGXkIuaSlJOhK0n5OLwgSRkVfcpYpJQq3YcDRkRMTClNq3Q/VCz+uTiwVFW6AweYiZXugArJPxcHEENXkjIydCUpI0M3L8fttCv+uTiA+CBNkjKy0pWkjAxdScrI0M0kIsZFxKKIWBwR11a6P6q8iLgjIlZHxPOV7ovyMXQziIhqYCpwATAc+FREDK9sr1QAPwTGVboTysvQzWM0sDiltCSl1AjcBUyocJ9UYSmlx4B1le6H8jJ08xgALG+zv6LlmKQDjKErSRkZunmsBAa12R/YckzSAcbQzWMuMDQi6iOiFrgcmFHhPkmqAEM3g5TSdmASMBt4EZieUlpQ2V6p0iLix8BTwIkRsSIirqp0n7Tv+RqwJGVkpStJGRm6kpSRoStJGRm6kpSRoStJGRm6kpSRoStJGf1/zEe741VYPdUAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cm = confusion_matrix(y_test, y_pred, normalize='true') \n",
    "sns.heatmap(cm, annot=True, cmap=plt.cm.Blues)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "9d8adf04",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['CreditScore', 'Age', 'Tenure', 'Balance', 'NumOfProducts',\n",
       "       'HasCrCard', 'IsActiveMember', 'EstimatedSalary'], dtype=object)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out_feat_names = model[:-1].get_feature_names_out(feat_cols)\n",
    "out_feat_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d9262a4f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <style>\n",
       "    table.eli5-weights tr:hover {\n",
       "        filter: brightness(85%);\n",
       "    }\n",
       "</style>\n",
       "\n",
       "\n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "        <table class=\"eli5-weights eli5-feature-importances\" style=\"border-collapse: collapse; border: none; margin-top: 0em; table-layout: auto;\">\n",
       "    <thead>\n",
       "    <tr style=\"border: none;\">\n",
       "        <th style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">Weight</th>\n",
       "        <th style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">Feature</th>\n",
       "    </tr>\n",
       "    </thead>\n",
       "    <tbody>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 80.00%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.2444\n",
       "                \n",
       "                    &plusmn; 0.0285\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                Age\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 83.76%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.1816\n",
       "                \n",
       "                    &plusmn; 0.0108\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                NumOfProducts\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 86.69%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.1365\n",
       "                \n",
       "                    &plusmn; 0.0315\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                IsActiveMember\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 95.12%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0326\n",
       "                \n",
       "                    &plusmn; 0.0050\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                Balance\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 96.35%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0215\n",
       "                \n",
       "                    &plusmn; 0.0136\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                Tenure\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 97.26%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0143\n",
       "                \n",
       "                    &plusmn; 0.0118\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                CreditScore\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 98.57%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0057\n",
       "                \n",
       "                    &plusmn; 0.0124\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                EstimatedSalary\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.89%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0002\n",
       "                \n",
       "                    &plusmn; 0.0083\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                HasCrCard\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "    \n",
       "    </tbody>\n",
       "</table>\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import eli5\n",
    "from sklearn.metrics import make_scorer\n",
    "from eli5.sklearn import PermutationImportance\n",
    "\n",
    "\n",
    "preprocessor = model.named_steps['preprocessor']\n",
    "clf = model.named_steps['clf']\n",
    "X_test_transformed = preprocessor.transform(X_test)\n",
    "\n",
    "perm = PermutationImportance(clf, scoring=make_scorer(f1_score), random_state=random_state).fit(X_test_transformed, y_test)\n",
    "eli5.show_weights(perm, feature_names=out_feat_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "6e44b74c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>importance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Age</td>\n",
       "      <td>0.244377</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NumOfProducts</td>\n",
       "      <td>0.181561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>IsActiveMember</td>\n",
       "      <td>0.136543</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Balance</td>\n",
       "      <td>0.032561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Tenure</td>\n",
       "      <td>0.021488</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CreditScore</td>\n",
       "      <td>0.014311</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>EstimatedSalary</td>\n",
       "      <td>0.005650</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>HasCrCard</td>\n",
       "      <td>0.000152</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           feature  importance\n",
       "1              Age    0.244377\n",
       "4    NumOfProducts    0.181561\n",
       "6   IsActiveMember    0.136543\n",
       "3          Balance    0.032561\n",
       "2           Tenure    0.021488\n",
       "0      CreditScore    0.014311\n",
       "7  EstimatedSalary    0.005650\n",
       "5        HasCrCard    0.000152"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feat_imp = zip(X_test.columns.tolist(), perm.feature_importances_)\n",
    "df_feat_imp = pd.DataFrame(feat_imp, \n",
    "                    columns=['feature', 'importance'])\n",
    "df_feat_imp = df_feat_imp.sort_values(by='importance', ascending=False)\n",
    "df_feat_imp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "6d00dcf7",
   "metadata": {},
   "outputs": [],
   "source": [
    "feat_importance_fpath = 'feat_imp.csv'\n",
    "df_feat_imp.to_csv(feat_importance_fpath, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "dac3462a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from joblib import dump\n",
    "\n",
    "dump(model, 'clf-model.joblib');"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.10 ('.venv': venv)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  },
  "vscode": {
   "interpreter": {
    "hash": "060614c890ed22051a9be2360999a13d2882d827ad8c9dd21319e1709b800224"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}