1. Data Collection and Concatenation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.simplefilter('ignore')\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The raw data are stored in separate files, ranging from $2007$ until $2018$. Possibly some years have been separated into $4$ quarters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read raw data\n",
    "data_2011 = pd.read_csv('./data/2011.csv')\n",
    "data_2013 = pd.read_csv('./data/2013.csv')\n",
    "data_2014 = pd.read_csv('./data/2014.csv')\n",
    "data_2015 = pd.read_csv('./data/2015.csv')\n",
    "data_2016_1 = pd.read_csv('./data/2016(1).csv')\n",
    "data_2016_2 = pd.read_csv('./data/2016(2).csv')\n",
    "data_2016_3 = pd.read_csv('./data/2016(3).csv')\n",
    "data_2016_4 = pd.read_csv('./data/2016(4).csv')\n",
    "data_2017_1 = pd.read_csv('./data/2017(1).csv')\n",
    "data_2017_2 = pd.read_csv('./data/2017(2).csv')\n",
    "data_2017_3 = pd.read_csv('./data/2017(3).csv')\n",
    "data_2017_4 = pd.read_csv('./data/2017(4).csv')\n",
    "data_2018 = pd.read_csv('./data/2018(1).csv')\n",
    "\n",
    "# merge data from 2016 and 2017\n",
    "data_2016 = pd.concat(objs=[data_2016_1, data_2016_2, data_2016_3, data_2016_4], axis=0)\n",
    "data_2017 = pd.concat(objs=[data_2017_1, data_2017_2, data_2017_3, data_2017_4], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Year 2011:\t (42538, 143)\n",
      "Year 2013:\t (188181, 143)\n",
      "Year 2014:\t (235629, 143)\n",
      "Year 2015:\t (421095, 143)\n",
      "Year 2016:\t (434407, 143)\n",
      "Year 2017:\t (443579, 143)\n",
      "Year 2018:\t (107864, 143)\n"
     ]
    }
   ],
   "source": [
    "# basic information\n",
    "print('Year 2011:\\t', data_2011.shape)\n",
    "print('Year 2013:\\t', data_2013.shape)\n",
    "print('Year 2014:\\t', data_2014.shape)\n",
    "print('Year 2015:\\t', data_2015.shape)\n",
    "print('Year 2016:\\t', data_2016.shape)\n",
    "print('Year 2017:\\t', data_2017.shape)\n",
    "print('Year 2018:\\t', data_2018.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 42538 entries, 0 to 42537\n",
      "Columns: 143 entries, loan_amnt to settlement_term\n",
      "dtypes: float64(114), object(29)\n",
      "memory usage: 46.4+ MB\n"
     ]
    }
   ],
   "source": [
    "data_2011.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 188181 entries, 0 to 188180\n",
      "Columns: 143 entries, loan_amnt to settlement_term\n",
      "dtypes: float64(92), int64(15), object(36)\n",
      "memory usage: 205.3+ MB\n"
     ]
    }
   ],
   "source": [
    "data_2013.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 235629 entries, 0 to 235628\n",
      "Columns: 143 entries, loan_amnt to settlement_term\n",
      "dtypes: float64(66), int64(41), object(36)\n",
      "memory usage: 257.1+ MB\n"
     ]
    }
   ],
   "source": [
    "data_2014.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 421095 entries, 0 to 421094\n",
      "Columns: 143 entries, loan_amnt to settlement_term\n",
      "dtypes: float64(66), int64(40), object(37)\n",
      "memory usage: 459.4+ MB\n"
     ]
    }
   ],
   "source": [
    "data_2015.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 434407 entries, 0 to 103545\n",
      "Columns: 143 entries, loan_amnt to settlement_term\n",
      "dtypes: float64(66), int64(40), object(37)\n",
      "memory usage: 477.3+ MB\n"
     ]
    }
   ],
   "source": [
    "data_2016.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 443579 entries, 0 to 118647\n",
      "Columns: 143 entries, loan_amnt to settlement_term\n",
      "dtypes: float64(55), int64(51), object(37)\n",
      "memory usage: 487.3+ MB\n"
     ]
    }
   ],
   "source": [
    "data_2017.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 107864 entries, 0 to 107863\n",
      "Columns: 143 entries, loan_amnt to settlement_term\n",
      "dtypes: float64(59), int64(54), object(30)\n",
      "memory usage: 117.7+ MB\n"
     ]
    }
   ],
   "source": [
    "data_2018.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Concatenation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After observation in depth, we notice that early date, especially the data before $2011$, have some difference with the most recent data, such as the data in $2017$ and $2018$. Since the data before $2014$ is only a small part of all data (~$12\\%$), in this project, we will only use the data from $2014$ until $2018$."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# add year information \n",
    "data_2014['year'] = 2014\n",
    "data_2015['year'] = 2015\n",
    "data_2016_1['year'] = 2016\n",
    "data_2016_2['year'] = 2016\n",
    "data_2016_3['year'] = 2016\n",
    "data_2016_4['year'] = 2016\n",
    "data_2017_1['year'] = 2017\n",
    "data_2017_2['year'] = 2017\n",
    "data_2017_3['year'] = 2017\n",
    "data_2017_4['year'] = 2017\n",
    "data_2018['year'] = 2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loan_amnt</th>\n",
       "      <th>funded_amnt</th>\n",
       "      <th>funded_amnt_inv</th>\n",
       "      <th>term</th>\n",
       "      <th>int_rate</th>\n",
       "      <th>installment</th>\n",
       "      <th>grade</th>\n",
       "      <th>sub_grade</th>\n",
       "      <th>emp_title</th>\n",
       "      <th>emp_length</th>\n",
       "      <th>...</th>\n",
       "      <th>hardship_last_payment_amount</th>\n",
       "      <th>disbursement_method</th>\n",
       "      <th>debt_settlement_flag</th>\n",
       "      <th>debt_settlement_flag_date</th>\n",
       "      <th>settlement_status</th>\n",
       "      <th>settlement_date</th>\n",
       "      <th>settlement_amount</th>\n",
       "      <th>settlement_percentage</th>\n",
       "      <th>settlement_term</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10400</td>\n",
       "      <td>10400</td>\n",
       "      <td>10400.0</td>\n",
       "      <td>36 months</td>\n",
       "      <td>6.99%</td>\n",
       "      <td>321.08</td>\n",
       "      <td>A</td>\n",
       "      <td>A3</td>\n",
       "      <td>Truck Driver Delivery Personel</td>\n",
       "      <td>8 years</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>15000</td>\n",
       "      <td>15000</td>\n",
       "      <td>15000.0</td>\n",
       "      <td>60 months</td>\n",
       "      <td>12.39%</td>\n",
       "      <td>336.64</td>\n",
       "      <td>C</td>\n",
       "      <td>C1</td>\n",
       "      <td>MANAGEMENT</td>\n",
       "      <td>10+ years</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9600</td>\n",
       "      <td>9600</td>\n",
       "      <td>9600.0</td>\n",
       "      <td>36 months</td>\n",
       "      <td>13.66%</td>\n",
       "      <td>326.53</td>\n",
       "      <td>C</td>\n",
       "      <td>C3</td>\n",
       "      <td>Admin Specialist</td>\n",
       "      <td>10+ years</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7650</td>\n",
       "      <td>7650</td>\n",
       "      <td>7650.0</td>\n",
       "      <td>36 months</td>\n",
       "      <td>13.66%</td>\n",
       "      <td>260.20</td>\n",
       "      <td>C</td>\n",
       "      <td>C3</td>\n",
       "      <td>Technical Specialist</td>\n",
       "      <td>&lt; 1 year</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>21425</td>\n",
       "      <td>21425</td>\n",
       "      <td>21425.0</td>\n",
       "      <td>60 months</td>\n",
       "      <td>15.59%</td>\n",
       "      <td>516.36</td>\n",
       "      <td>D</td>\n",
       "      <td>D1</td>\n",
       "      <td>Programming Analysis Supervisor</td>\n",
       "      <td>6 years</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2014</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 144 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   loan_amnt  funded_amnt  funded_amnt_inv        term int_rate  installment  \\\n",
       "0      10400        10400          10400.0   36 months    6.99%       321.08   \n",
       "1      15000        15000          15000.0   60 months   12.39%       336.64   \n",
       "2       9600         9600           9600.0   36 months   13.66%       326.53   \n",
       "3       7650         7650           7650.0   36 months   13.66%       260.20   \n",
       "4      21425        21425          21425.0   60 months   15.59%       516.36   \n",
       "\n",
       "  grade sub_grade                        emp_title emp_length  ...   \\\n",
       "0     A        A3   Truck Driver Delivery Personel    8 years  ...    \n",
       "1     C        C1                       MANAGEMENT  10+ years  ...    \n",
       "2     C        C3                 Admin Specialist  10+ years  ...    \n",
       "3     C        C3             Technical Specialist   < 1 year  ...    \n",
       "4     D        D1  Programming Analysis Supervisor    6 years  ...    \n",
       "\n",
       "  hardship_last_payment_amount  disbursement_method debt_settlement_flag  \\\n",
       "0                          NaN                 Cash                    N   \n",
       "1                          NaN                 Cash                    N   \n",
       "2                          NaN                 Cash                    N   \n",
       "3                          NaN                 Cash                    N   \n",
       "4                          NaN                 Cash                    N   \n",
       "\n",
       "  debt_settlement_flag_date settlement_status settlement_date  \\\n",
       "0                       NaN               NaN             NaN   \n",
       "1                       NaN               NaN             NaN   \n",
       "2                       NaN               NaN             NaN   \n",
       "3                       NaN               NaN             NaN   \n",
       "4                       NaN               NaN             NaN   \n",
       "\n",
       "   settlement_amount settlement_percentage settlement_term  year  \n",
       "0                NaN                   NaN             NaN  2014  \n",
       "1                NaN                   NaN             NaN  2014  \n",
       "2                NaN                   NaN             NaN  2014  \n",
       "3                NaN                   NaN             NaN  2014  \n",
       "4                NaN                   NaN             NaN  2014  \n",
       "\n",
       "[5 rows x 144 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# concate data from 2014 until 2018\n",
    "objs = [data_2014, data_2015, data_2016_1, data_2016_2, data_2016_3, data_2016_4, \n",
    "        data_2017_1, data_2017_2, data_2017_3, data_2017_4, data_2018]\n",
    "data = pd.concat(objs=objs, axis=0)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 1642574 entries, 0 to 107863\n",
      "Columns: 144 entries, loan_amnt to year\n",
      "dtypes: float64(67), int64(39), object(38)\n",
      "memory usage: 1.8+ GB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Cleaning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook, we only do the very basic data cleaning steps: to remove the features with large amount of missing values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>missing ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>url</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>orig_projected_additional_accrued_interest</td>\n",
       "      <td>0.996483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>133</th>\n",
       "      <td>hardship_payoff_balance_amount</td>\n",
       "      <td>0.995612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124</th>\n",
       "      <td>deferral_term</td>\n",
       "      <td>0.995612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125</th>\n",
       "      <td>hardship_amount</td>\n",
       "      <td>0.995612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>hardship_start_date</td>\n",
       "      <td>0.995612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>127</th>\n",
       "      <td>hardship_end_date</td>\n",
       "      <td>0.995612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128</th>\n",
       "      <td>payment_plan_start_date</td>\n",
       "      <td>0.995612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129</th>\n",
       "      <td>hardship_length</td>\n",
       "      <td>0.995612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130</th>\n",
       "      <td>hardship_dpd</td>\n",
       "      <td>0.995612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123</th>\n",
       "      <td>hardship_status</td>\n",
       "      <td>0.995612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>131</th>\n",
       "      <td>hardship_loan_status</td>\n",
       "      <td>0.995612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121</th>\n",
       "      <td>hardship_type</td>\n",
       "      <td>0.995612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134</th>\n",
       "      <td>hardship_last_payment_amount</td>\n",
       "      <td>0.995612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122</th>\n",
       "      <td>hardship_reason</td>\n",
       "      <td>0.995612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>desc</td>\n",
       "      <td>0.990657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119</th>\n",
       "      <td>sec_app_mths_since_last_major_derog</td>\n",
       "      <td>0.988438</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138</th>\n",
       "      <td>settlement_status</td>\n",
       "      <td>0.988106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>142</th>\n",
       "      <td>settlement_term</td>\n",
       "      <td>0.988106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137</th>\n",
       "      <td>debt_settlement_flag_date</td>\n",
       "      <td>0.988106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141</th>\n",
       "      <td>settlement_percentage</td>\n",
       "      <td>0.988106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140</th>\n",
       "      <td>settlement_amount</td>\n",
       "      <td>0.988106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139</th>\n",
       "      <td>settlement_date</td>\n",
       "      <td>0.988106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114</th>\n",
       "      <td>sec_app_revol_util</td>\n",
       "      <td>0.966871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>revol_bal_joint</td>\n",
       "      <td>0.966293</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118</th>\n",
       "      <td>sec_app_collections_12_mths_ex_med</td>\n",
       "      <td>0.966293</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>sec_app_inq_last_6mths</td>\n",
       "      <td>0.966293</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>sec_app_chargeoff_within_12_mths</td>\n",
       "      <td>0.966293</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>110</th>\n",
       "      <td>sec_app_earliest_cr_line</td>\n",
       "      <td>0.966293</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>112</th>\n",
       "      <td>sec_app_mort_acc</td>\n",
       "      <td>0.966293</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113</th>\n",
       "      <td>sec_app_open_acc</td>\n",
       "      <td>0.966293</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116</th>\n",
       "      <td>sec_app_num_rev_accts</td>\n",
       "      <td>0.966293</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115</th>\n",
       "      <td>sec_app_open_act_il</td>\n",
       "      <td>0.966293</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>verification_status_joint</td>\n",
       "      <td>0.958762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>dti_joint</td>\n",
       "      <td>0.958570</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>annual_inc_joint</td>\n",
       "      <td>0.958567</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>mths_since_last_record</td>\n",
       "      <td>0.823255</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>mths_since_recent_bc_dlq</td>\n",
       "      <td>0.752743</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>mths_since_last_major_derog</td>\n",
       "      <td>0.721055</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>mths_since_recent_revol_delinq</td>\n",
       "      <td>0.650519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>mths_since_last_delinq</td>\n",
       "      <td>0.491939</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>next_pymnt_d</td>\n",
       "      <td>0.485112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>il_util</td>\n",
       "      <td>0.471999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>mths_since_rcnt_il</td>\n",
       "      <td>0.404905</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>all_util</td>\n",
       "      <td>0.386912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>total_cu_tl</td>\n",
       "      <td>0.386840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>inq_last_12m</td>\n",
       "      <td>0.386840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>open_acc_6m</td>\n",
       "      <td>0.386840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>total_bal_il</td>\n",
       "      <td>0.386840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>open_il_24m</td>\n",
       "      <td>0.386840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>open_il_12m</td>\n",
       "      <td>0.386840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>open_act_il</td>\n",
       "      <td>0.386840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>open_rv_12m</td>\n",
       "      <td>0.386840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>inq_fi</td>\n",
       "      <td>0.386840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>open_rv_24m</td>\n",
       "      <td>0.386840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>max_bal_bc</td>\n",
       "      <td>0.386840</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        feature  missing ratio\n",
       "16                                          url       1.000000\n",
       "132  orig_projected_additional_accrued_interest       0.996483\n",
       "133              hardship_payoff_balance_amount       0.995612\n",
       "124                               deferral_term       0.995612\n",
       "125                             hardship_amount       0.995612\n",
       "126                         hardship_start_date       0.995612\n",
       "127                           hardship_end_date       0.995612\n",
       "128                     payment_plan_start_date       0.995612\n",
       "129                             hardship_length       0.995612\n",
       "130                                hardship_dpd       0.995612\n",
       "123                             hardship_status       0.995612\n",
       "131                        hardship_loan_status       0.995612\n",
       "121                               hardship_type       0.995612\n",
       "134                hardship_last_payment_amount       0.995612\n",
       "122                             hardship_reason       0.995612\n",
       "17                                         desc       0.990657\n",
       "119         sec_app_mths_since_last_major_derog       0.988438\n",
       "138                           settlement_status       0.988106\n",
       "142                             settlement_term       0.988106\n",
       "137                   debt_settlement_flag_date       0.988106\n",
       "141                       settlement_percentage       0.988106\n",
       "140                           settlement_amount       0.988106\n",
       "139                             settlement_date       0.988106\n",
       "114                          sec_app_revol_util       0.966871\n",
       "109                             revol_bal_joint       0.966293\n",
       "118          sec_app_collections_12_mths_ex_med       0.966293\n",
       "111                      sec_app_inq_last_6mths       0.966293\n",
       "117            sec_app_chargeoff_within_12_mths       0.966293\n",
       "110                    sec_app_earliest_cr_line       0.966293\n",
       "112                            sec_app_mort_acc       0.966293\n",
       "113                            sec_app_open_acc       0.966293\n",
       "116                       sec_app_num_rev_accts       0.966293\n",
       "115                         sec_app_open_act_il       0.966293\n",
       "53                    verification_status_joint       0.958762\n",
       "52                                    dti_joint       0.958570\n",
       "51                             annual_inc_joint       0.958567\n",
       "27                       mths_since_last_record       0.823255\n",
       "84                     mths_since_recent_bc_dlq       0.752743\n",
       "48                  mths_since_last_major_derog       0.721055\n",
       "86               mths_since_recent_revol_delinq       0.650519\n",
       "26                       mths_since_last_delinq       0.491939\n",
       "45                                 next_pymnt_d       0.485112\n",
       "63                                      il_util       0.471999\n",
       "61                           mths_since_rcnt_il       0.404905\n",
       "67                                     all_util       0.386912\n",
       "70                                  total_cu_tl       0.386840\n",
       "71                                 inq_last_12m       0.386840\n",
       "57                                  open_acc_6m       0.386840\n",
       "62                                 total_bal_il       0.386840\n",
       "60                                  open_il_24m       0.386840\n",
       "59                                  open_il_12m       0.386840\n",
       "58                                  open_act_il       0.386840\n",
       "64                                  open_rv_12m       0.386840\n",
       "69                                       inq_fi       0.386840\n",
       "65                                  open_rv_24m       0.386840\n",
       "66                                   max_bal_bc       0.386840"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# missing values\n",
    "pd.options.display.max_rows = 150\n",
    "missing = data.isnull().sum()\n",
    "missing_ratio = missing / len(data)\n",
    "missing_ratio = missing_ratio.reset_index()\n",
    "\n",
    "# check the feature that has missing ration >= 0.2\n",
    "missing_ratio = missing_ratio.rename(columns={'index': 'feature', 0: 'missing ratio'})\n",
    "missing_ratio = missing_ratio.sort_values(by='missing ratio', ascending=False)\n",
    "missing_ratio[missing_ratio['missing ratio'] >= 0.2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Based on the missing ratio and business sense, all the above features are removed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# it's safe to remove the features with missing ratio >= 0.3\n",
    "columns = ['url', 'orig_projected_additional_accrued_interest', 'hardship_start_date', \n",
    "           'hardship_end_date', 'payment_plan_start_date', 'hardship_length', \n",
    "           'hardship_dpd', 'hardship_loan_status', 'hardship_payoff_balance_amount', \n",
    "           'hardship_last_payment_amount', 'hardship_amount', 'deferral_term', \n",
    "           'hardship_status', 'hardship_reason', 'hardship_type', 'desc', \n",
    "           'sec_app_mths_since_last_major_derog', 'settlement_term', \n",
    "           'debt_settlement_flag_date', 'settlement_date', 'settlement_amount', \n",
    "           'settlement_percentage', 'settlement_status', 'sec_app_revol_util', \n",
    "           'revol_bal_joint', 'sec_app_open_act_il', 'sec_app_earliest_cr_line', \n",
    "           'sec_app_inq_last_6mths', 'sec_app_open_acc', 'sec_app_mort_acc', \n",
    "           'sec_app_num_rev_accts', 'sec_app_chargeoff_within_12_mths', \n",
    "           'sec_app_collections_12_mths_ex_med', 'verification_status_joint',\n",
    "           'dti_joint', 'annual_inc_joint', 'mths_since_last_record', \n",
    "           'mths_since_recent_bc_dlq', 'mths_since_last_major_derog', \n",
    "           'mths_since_recent_revol_delinq', 'mths_since_last_delinq', 'next_pymnt_d', \n",
    "           'il_util', 'mths_since_rcnt_il', 'all_util', 'open_acc_6m', 'total_cu_tl', \n",
    "           'inq_last_12m', 'open_act_il', 'open_il_12m', 'max_bal_bc', 'open_rv_24m', \n",
    "           'open_rv_12m', 'total_bal_il', 'open_il_24m', 'inq_fi']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loan_amnt</th>\n",
       "      <th>funded_amnt</th>\n",
       "      <th>funded_amnt_inv</th>\n",
       "      <th>term</th>\n",
       "      <th>int_rate</th>\n",
       "      <th>installment</th>\n",
       "      <th>grade</th>\n",
       "      <th>sub_grade</th>\n",
       "      <th>emp_title</th>\n",
       "      <th>emp_length</th>\n",
       "      <th>...</th>\n",
       "      <th>pub_rec_bankruptcies</th>\n",
       "      <th>tax_liens</th>\n",
       "      <th>tot_hi_cred_lim</th>\n",
       "      <th>total_bal_ex_mort</th>\n",
       "      <th>total_bc_limit</th>\n",
       "      <th>total_il_high_credit_limit</th>\n",
       "      <th>hardship_flag</th>\n",
       "      <th>disbursement_method</th>\n",
       "      <th>debt_settlement_flag</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10400</td>\n",
       "      <td>10400</td>\n",
       "      <td>10400.0</td>\n",
       "      <td>36 months</td>\n",
       "      <td>6.99%</td>\n",
       "      <td>321.08</td>\n",
       "      <td>A</td>\n",
       "      <td>A3</td>\n",
       "      <td>Truck Driver Delivery Personel</td>\n",
       "      <td>8 years</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>179407</td>\n",
       "      <td>15030</td>\n",
       "      <td>13000</td>\n",
       "      <td>11325</td>\n",
       "      <td>N</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "      <td>2014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>15000</td>\n",
       "      <td>15000</td>\n",
       "      <td>15000.0</td>\n",
       "      <td>60 months</td>\n",
       "      <td>12.39%</td>\n",
       "      <td>336.64</td>\n",
       "      <td>C</td>\n",
       "      <td>C1</td>\n",
       "      <td>MANAGEMENT</td>\n",
       "      <td>10+ years</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>196500</td>\n",
       "      <td>149140</td>\n",
       "      <td>10000</td>\n",
       "      <td>12000</td>\n",
       "      <td>N</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "      <td>2014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9600</td>\n",
       "      <td>9600</td>\n",
       "      <td>9600.0</td>\n",
       "      <td>36 months</td>\n",
       "      <td>13.66%</td>\n",
       "      <td>326.53</td>\n",
       "      <td>C</td>\n",
       "      <td>C3</td>\n",
       "      <td>Admin Specialist</td>\n",
       "      <td>10+ years</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>52490</td>\n",
       "      <td>38566</td>\n",
       "      <td>21100</td>\n",
       "      <td>24890</td>\n",
       "      <td>N</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "      <td>2014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7650</td>\n",
       "      <td>7650</td>\n",
       "      <td>7650.0</td>\n",
       "      <td>36 months</td>\n",
       "      <td>13.66%</td>\n",
       "      <td>260.20</td>\n",
       "      <td>C</td>\n",
       "      <td>C3</td>\n",
       "      <td>Technical Specialist</td>\n",
       "      <td>&lt; 1 year</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>82331</td>\n",
       "      <td>64426</td>\n",
       "      <td>4900</td>\n",
       "      <td>64031</td>\n",
       "      <td>N</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "      <td>2014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>21425</td>\n",
       "      <td>21425</td>\n",
       "      <td>21425.0</td>\n",
       "      <td>60 months</td>\n",
       "      <td>15.59%</td>\n",
       "      <td>516.36</td>\n",
       "      <td>D</td>\n",
       "      <td>D1</td>\n",
       "      <td>Programming Analysis Supervisor</td>\n",
       "      <td>6 years</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>57073</td>\n",
       "      <td>42315</td>\n",
       "      <td>15000</td>\n",
       "      <td>35573</td>\n",
       "      <td>N</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "      <td>2014</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 88 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   loan_amnt  funded_amnt  funded_amnt_inv        term int_rate  installment  \\\n",
       "0      10400        10400          10400.0   36 months    6.99%       321.08   \n",
       "1      15000        15000          15000.0   60 months   12.39%       336.64   \n",
       "2       9600         9600           9600.0   36 months   13.66%       326.53   \n",
       "3       7650         7650           7650.0   36 months   13.66%       260.20   \n",
       "4      21425        21425          21425.0   60 months   15.59%       516.36   \n",
       "\n",
       "  grade sub_grade                        emp_title emp_length  ...   \\\n",
       "0     A        A3   Truck Driver Delivery Personel    8 years  ...    \n",
       "1     C        C1                       MANAGEMENT  10+ years  ...    \n",
       "2     C        C3                 Admin Specialist  10+ years  ...    \n",
       "3     C        C3             Technical Specialist   < 1 year  ...    \n",
       "4     D        D1  Programming Analysis Supervisor    6 years  ...    \n",
       "\n",
       "  pub_rec_bankruptcies  tax_liens tot_hi_cred_lim total_bal_ex_mort  \\\n",
       "0                    0          0          179407             15030   \n",
       "1                    0          0          196500            149140   \n",
       "2                    0          0           52490             38566   \n",
       "3                    0          0           82331             64426   \n",
       "4                    0          0           57073             42315   \n",
       "\n",
       "  total_bc_limit total_il_high_credit_limit hardship_flag disbursement_method  \\\n",
       "0          13000                      11325             N                Cash   \n",
       "1          10000                      12000             N                Cash   \n",
       "2          21100                      24890             N                Cash   \n",
       "3           4900                      64031             N                Cash   \n",
       "4          15000                      35573             N                Cash   \n",
       "\n",
       "  debt_settlement_flag  year  \n",
       "0                    N  2014  \n",
       "1                    N  2014  \n",
       "2                    N  2014  \n",
       "3                    N  2014  \n",
       "4                    N  2014  \n",
       "\n",
       "[5 rows x 88 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# drop features with too many missing values\n",
    "data = data.drop(labels=columns, axis=1)\n",
    "\n",
    "# save to local disk\n",
    "data.to_csv('./data/data_2014_2018.csv', index=False)\n",
    "\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 1642574 entries, 0 to 107863\n",
      "Data columns (total 88 columns):\n",
      "loan_amnt                     1642574 non-null int64\n",
      "funded_amnt                   1642574 non-null int64\n",
      "funded_amnt_inv               1642574 non-null float64\n",
      "term                          1642574 non-null object\n",
      "int_rate                      1642574 non-null object\n",
      "installment                   1642574 non-null float64\n",
      "grade                         1642574 non-null object\n",
      "sub_grade                     1642574 non-null object\n",
      "emp_title                     1535014 non-null object\n",
      "emp_length                    1537225 non-null object\n",
      "home_ownership                1642574 non-null object\n",
      "annual_inc                    1642574 non-null float64\n",
      "verification_status           1642574 non-null object\n",
      "issue_d                       1642574 non-null object\n",
      "loan_status                   1642574 non-null object\n",
      "pymnt_plan                    1642574 non-null object\n",
      "purpose                       1642574 non-null object\n",
      "title                         1619269 non-null object\n",
      "zip_code                      1642573 non-null object\n",
      "addr_state                    1642574 non-null object\n",
      "dti                           1641733 non-null float64\n",
      "delinq_2yrs                   1642574 non-null int64\n",
      "earliest_cr_line              1642574 non-null object\n",
      "inq_last_6mths                1642573 non-null float64\n",
      "open_acc                      1642574 non-null int64\n",
      "pub_rec                       1642574 non-null int64\n",
      "revol_bal                     1642574 non-null int64\n",
      "revol_util                    1641430 non-null object\n",
      "total_acc                     1642574 non-null int64\n",
      "initial_list_status           1642574 non-null object\n",
      "out_prncp                     1642574 non-null float64\n",
      "out_prncp_inv                 1642574 non-null float64\n",
      "total_pymnt                   1642574 non-null float64\n",
      "total_pymnt_inv               1642574 non-null float64\n",
      "total_rec_prncp               1642574 non-null float64\n",
      "total_rec_int                 1642574 non-null float64\n",
      "total_rec_late_fee            1642574 non-null float64\n",
      "recoveries                    1642574 non-null float64\n",
      "collection_recovery_fee       1642574 non-null float64\n",
      "last_pymnt_d                  1640863 non-null object\n",
      "last_pymnt_amnt               1642574 non-null float64\n",
      "last_credit_pull_d            1642514 non-null object\n",
      "collections_12_mths_ex_med    1642574 non-null int64\n",
      "policy_code                   1642574 non-null int64\n",
      "application_type              1642574 non-null object\n",
      "acc_now_delinq                1642574 non-null int64\n",
      "tot_coll_amt                  1642574 non-null int64\n",
      "tot_cur_bal                   1642574 non-null int64\n",
      "total_rev_hi_lim              1642574 non-null int64\n",
      "acc_open_past_24mths          1642574 non-null int64\n",
      "avg_cur_bal                   1642540 non-null float64\n",
      "bc_open_to_buy                1624277 non-null float64\n",
      "bc_util                       1623392 non-null float64\n",
      "chargeoff_within_12_mths      1642574 non-null int64\n",
      "delinq_amnt                   1642574 non-null int64\n",
      "mo_sin_old_il_acct            1594032 non-null float64\n",
      "mo_sin_old_rev_tl_op          1642574 non-null int64\n",
      "mo_sin_rcnt_rev_tl_op         1642574 non-null int64\n",
      "mo_sin_rcnt_tl                1642574 non-null int64\n",
      "mort_acc                      1642574 non-null int64\n",
      "mths_since_recent_bc          1625302 non-null float64\n",
      "mths_since_recent_inq         1465199 non-null float64\n",
      "num_accts_ever_120_pd         1642574 non-null int64\n",
      "num_actv_bc_tl                1642574 non-null int64\n",
      "num_actv_rev_tl               1642574 non-null int64\n",
      "num_bc_sats                   1642574 non-null int64\n",
      "num_bc_tl                     1642574 non-null int64\n",
      "num_il_tl                     1642574 non-null int64\n",
      "num_op_rev_tl                 1642574 non-null int64\n",
      "num_rev_accts                 1642573 non-null float64\n",
      "num_rev_tl_bal_gt_0           1642574 non-null int64\n",
      "num_sats                      1642574 non-null int64\n",
      "num_tl_120dpd_2m              1568345 non-null float64\n",
      "num_tl_30dpd                  1642574 non-null int64\n",
      "num_tl_90g_dpd_24m            1642574 non-null int64\n",
      "num_tl_op_past_12m            1642574 non-null int64\n",
      "pct_tl_nvr_dlq                1642573 non-null float64\n",
      "percent_bc_gt_75              1623840 non-null float64\n",
      "pub_rec_bankruptcies          1642574 non-null int64\n",
      "tax_liens                     1642574 non-null int64\n",
      "tot_hi_cred_lim               1642574 non-null int64\n",
      "total_bal_ex_mort             1642574 non-null int64\n",
      "total_bc_limit                1642574 non-null int64\n",
      "total_il_high_credit_limit    1642574 non-null int64\n",
      "hardship_flag                 1642574 non-null object\n",
      "disbursement_method           1642574 non-null object\n",
      "debt_settlement_flag          1642574 non-null object\n",
      "year                          1642574 non-null int64\n",
      "dtypes: float64(25), int64(39), object(24)\n",
      "memory usage: 1.1+ GB\n"
     ]
    }
   ],
   "source": [
    "# basic information\n",
    "data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally we got the data used for modeling after some preliminary processing and cleaning. The cleaned dataset contains $1,642,574$ records with $88$ features. \n",
    "\n",
    "Further cleaning and processing are presented in following notebooks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}