07-02-Ensemble Methods 2 - Bagging Machine Learning Algorithm.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bagging Machine Learning Algorithm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### **B**ootstrap **Agg**regat**ing** or [Bagging](https://en.wikipedia.org/wiki/Bootstrap_aggregating)\n",
    "* [Scikit- Learn Reference](http://scikit-learn.org/stable/modules/ensemble.html#bagging)\n",
    "* Bootstrap sampling: Sampling with replacement\n",
    "* Combine by averaging the output (regression)\n",
    "* Combine by voting (classification)\n",
    "* Can be applied to many classifiers which includes ANN, CART, etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "df = sns.load_dataset('titanic')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(891, 15)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>survived</th>\n",
       "      <th>pclass</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "      <th>sibsp</th>\n",
       "      <th>parch</th>\n",
       "      <th>fare</th>\n",
       "      <th>embarked</th>\n",
       "      <th>class</th>\n",
       "      <th>who</th>\n",
       "      <th>adult_male</th>\n",
       "      <th>deck</th>\n",
       "      <th>embark_town</th>\n",
       "      <th>alive</th>\n",
       "      <th>alone</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>S</td>\n",
       "      <td>Third</td>\n",
       "      <td>man</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Southampton</td>\n",
       "      <td>no</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C</td>\n",
       "      <td>First</td>\n",
       "      <td>woman</td>\n",
       "      <td>False</td>\n",
       "      <td>C</td>\n",
       "      <td>Cherbourg</td>\n",
       "      <td>yes</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>S</td>\n",
       "      <td>Third</td>\n",
       "      <td>woman</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Southampton</td>\n",
       "      <td>yes</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>S</td>\n",
       "      <td>First</td>\n",
       "      <td>woman</td>\n",
       "      <td>False</td>\n",
       "      <td>C</td>\n",
       "      <td>Southampton</td>\n",
       "      <td>yes</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>S</td>\n",
       "      <td>Third</td>\n",
       "      <td>man</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Southampton</td>\n",
       "      <td>no</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   survived  pclass     sex   age  sibsp  parch     fare embarked  class  \\\n",
       "0         0       3    male  22.0      1      0   7.2500        S  Third   \n",
       "1         1       1  female  38.0      1      0  71.2833        C  First   \n",
       "2         1       3  female  26.0      0      0   7.9250        S  Third   \n",
       "3         1       1  female  35.0      1      0  53.1000        S  First   \n",
       "4         0       3    male  35.0      0      0   8.0500        S  Third   \n",
       "\n",
       "     who  adult_male deck  embark_town alive  alone  \n",
       "0    man        True  NaN  Southampton    no  False  \n",
       "1  woman       False    C    Cherbourg   yes  False  \n",
       "2  woman       False  NaN  Southampton   yes   True  \n",
       "3  woman       False    C  Southampton   yes  False  \n",
       "4    man        True  NaN  Southampton    no   True  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "df.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 3, 2], dtype=int64)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['pclass'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    157\n",
       "2     15\n",
       "3     10\n",
       "Name: pclass, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['pclass'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['female', 'male'], dtype=object)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['sex'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "male      94\n",
       "female    88\n",
       "Name: sex, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['sex'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAElxJREFUeJzt3X+M5HV9x/Hnu6AVbu0BRaf0IF2aGBpzq9ibIP6o3QUl\nJxBpGpNC0EBLs3+0KppryBHTGv8wJWm1mrZpQxQxlbCJiNWAVRHZkjZquwfoHRyI1QtyAoelHl0k\n1avv/rFD3K67t/P9MTNfPvd8JJub+c53vt/XfGfmdd/9znw/G5mJJOn57xcmHUCS1A4LXZIKYaFL\nUiEsdEkqhIUuSYWw0CWpEBa6JBXCQpekQljoklSI48e5slNPPTWnp6eHnv+ZZ55hy5YtowtUU1dz\nQXezdTUXdDdbV3NBd7N1NRc0y7Znz54fZOZLNp0xM8f2s2PHjqzirrvuqjT/uHQ1V2Z3s3U1V2Z3\ns3U1V2Z3s3U1V2azbMBSDtGxHnKRpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhbDQJakQFrokFcJC\nl6RCjPXUf6lt07tvX3f6gesuGnMSafLcQ5ekQljoklQIC12SCmGhS1IhLHRJKoSFLkmFsNAlqRAW\nuiQVwkKXpEJsWugRcUNEHIqIfevctisiMiJOHU08SdKwhtlDvxHYuXZiRJwBXAA80nImSVINmxZ6\nZt4NPLXOTX8FXANk26EkSdXVOoYeEZcABzPzGy3nkSTVFJmb72BHxDRwW2Zuj4gTgbuACzLzcEQc\nAPqZ+YMN7jsPzAP0er0dCwsLQ4dbXl5mampq6PnHpau5oLvZRpVr78HD606f2bZ16GUca9usDV3N\n1tVc0Czb3NzcnszsbzZfnUKfAe4EfjS4+XTg+8A5mfn40ZbT7/dzaWlp0/U9Z3FxkdnZ2aHnH5eu\n5oLuZhtVrjaGzz3Wtlkbupqtq7mgWbaIGKrQK4+Hnpl7gZeuWtEBjrKHLkkaj2G+tngz8FXgrIh4\nNCKuGn0sSVJVm+6hZ+Zlm9w+3VoaSVJtnikqSYWw0CWpEBa6JBXCQpekQljoklQIC12SCmGhS1Ih\nLHRJKoSFLkmFsNAlqRAWuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhbDQJakQw/xN0Rsi\n4lBE7Fs17S8i4sGI+GZEfCYiThptTEnSZobZQ78R2Llm2h3A9sx8BfAt4NqWc0mSKtq00DPzbuCp\nNdO+lJlHBle/Bpw+gmySpAraOIb+B8A/tbAcSVIDkZmbzxQxDdyWmdvXTH8v0Ad+NzdYUETMA/MA\nvV5vx8LCwtDhlpeXmZqaGnr+celqLuhutlHl2nvw8LrTZ7ZtHXoZx9o2a0NXs3U1FzTLNjc3tycz\n+5vNd3ytpQMRcSVwMXD+RmUOkJnXA9cD9Pv9nJ2dHXodi4uLVJl/XLqaC7qbbVS5rtx9+7rTD1w+\n/LqOtW3Whq5m62ouGE+2WoUeETuBa4DfzswftRtJklTHMF9bvBn4KnBWRDwaEVcBfwO8GLgjIu6L\niL8fcU5J0iY23UPPzMvWmfyxEWSRJDXgmaKSVAgLXZIKYaFLUiEsdEkqhIUuSYWw0CWpEBa6JBXC\nQpekQljoklQIC12SCmGhS1IhLHRJKoSFLkmFsNAlqRAWuiQVwkKXpEJY6JJUCAtdkgphoUtSIYb5\nI9E3RMShiNi3atopEXFHRDw8+Pfk0caUJG1mmD30G4Gda6btBu7MzJcBdw6uS5ImaNNCz8y7gafW\nTL4E+MTg8ieA32k5lySporrH0HuZ+djg8uNAr6U8kqSaIjM3nyliGrgtM7cPrv8wM09adft/Zea6\nx9EjYh6YB+j1ejsWFhaGDre8vMzU1NTQ849LV3NBd7ONKtfeg4fXnT6zbevQyxjXNquatavPJXQ3\nW1dzQbNsc3NzezKzv9l8x9daOjwREadl5mMRcRpwaKMZM/N64HqAfr+fs7OzQ69kcXGRKvOPS1dz\nQXezjSrXlbtvX3f6gcuHX9e4tlnVrF19LqG72bqaC8aTre4hl88BVwwuXwF8tp04kqS6hvna4s3A\nV4GzIuLRiLgKuA54U0Q8DLxxcF2SNEGbHnLJzMs2uOn8lrNIkhrwTFFJKoSFLkmFsNAlqRAWuiQV\nwkKXpEJY6JJUCAtdkgphoUtSISx0SSpE3cG5VIDpjQaLuu6iYte93np3zRxhdqRrlcbDPXRJKoSF\nLkmFsNAlqRAWuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSpEo0KPiPdExP0RsS8ibo6IF7UVTJJU\nTe1Cj4htwLuAfmZuB44DLm0rmCSpmqaHXI4HToiI44ETge83jyRJqiMys/6dI64GPgA8C3wpMy9f\nZ555YB6g1+vtWFhYGHr5y8vLTE1N1c43Kl3NBdWy7T14eN3pM9u2thkJ+Plcba276nLWm793Arz0\nlPYf8zDrho2zlvI6G6eu5oJm2ebm5vZkZn+z+WoXekScDHwa+D3gh8CngFsy85Mb3aff7+fS0tLQ\n61hcXGR2drZWvlHqai6olm2cIx6uzdXWuqsuZ6PRFt95+SWV1ltH1aylvM7Gqau5oFm2iBiq0Jsc\ncnkj8N3MfDIzfwLcCry2wfIkSQ00KfRHgHMj4sSICOB8YH87sSRJVdUu9Mz8OnALcA+wd7Cs61vK\nJUmqqNFfLMrM9wHvaymLJKkBzxSVpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhbDQJakQFrokFaLR\niUUarVEPYNWmcQ70JWl97qFLUiEsdEkqhIUuSYWw0CWpEBa6JBXCQpekQljoklQIC12SCmGhS1Ih\nGhV6RJwUEbdExIMRsT8iXtNWMElSNU1P/f8I8IXMfGtEvBA4sYVMkqQaahd6RGwF3gBcCZCZPwZ+\n3E4sSVJVTQ65nAk8CXw8Iu6NiI9GxJaWckmSKorMrHfHiD7wNeB1mfn1iPgI8HRm/uma+eaBeYBe\nr7djYWFh6HUsLy8zNTVVK98ojSvX3oOHK80/s23rutnqLKeqjdbx3LLW5qqaqaqNHsN66+2dAC89\npfpjrmqzbbTWoacO88Szw88/Tsf6e7OOJtnm5ub2ZGZ/s/maFPqvAF/LzOnB9d8CdmfmhuOl9vv9\nXFpaGnodi4uLzM7O1so3SuPKVXXY2wPXXbRutjrLqWqz4XPX5hr1kL4bPYb11rtr5gjvvPySkebZ\naN2wcda/vumzfHDvzx8V7cKQxMf6e7OOJtkiYqhCr33IJTMfB74XEWcNJp0PPFB3eZKkZpp+y+Wd\nwE2Db7h8B/j95pEkSXU0KvTMvA/Y9NcASdLoeaaoJBXCQpekQljoklQIC12SCmGhS1IhLHRJKoSF\nLkmFsNAlqRAWuiQVoump/2NTdWAj1TfqgbNK0dZrcqPl7JoZ7Xqr5vG91n3uoUtSISx0SSqEhS5J\nhbDQJakQFrokFcJCl6RCWOiSVAgLXZIKYaFLUiEaF3pEHBcR90bEbW0EkiTV08Ye+tXA/haWI0lq\noFGhR8TpwEXAR9uJI0mqq+ke+oeBa4CftpBFktRAZGa9O0ZcDFyYmX8UEbPAn2TmxevMNw/MA/R6\nvR0LCwtDr2N5eZmpqSkA9h48XCnfzLatleavYnWuUarzmNfLVnU5o9A7AZ54dtIp1ne0bEd7HW20\nXTe6T9Xnoeo2q7reJjlXZxvle62qcb0362iSbW5ubk9m9jebr0mh/znwduAI8CLgl4BbM/NtG92n\n3+/n0tLS0OtYXFxkdnYWqD6k6yiH+lyda5TqPOb1snVhONxdM0f44N5ujtZ8tGxHex1VHWa26vNQ\ndZtVXW+TnKuzdWlY3XG9N+toki0ihir02odcMvPazDw9M6eBS4GvHK3MJUmj5ffQJakQrfwOnJmL\nwGIby5Ik1eMeuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhbDQJakQ3Rxc4xjT1lgr07tv\nZ9fMEa7swNgtx7JJjZ3ThTF7NFnuoUtSISx0SSqEhS5JhbDQJakQFrokFcJCl6RCWOiSVAgLXZIK\nYaFLUiEsdEkqRO1Cj4gzIuKuiHggIu6PiKvbDCZJqqbJWC5HgF2ZeU9EvBjYExF3ZOYDLWWTJFVQ\new89Mx/LzHsGl/8b2A9sayuYJKmayMzmC4mYBu4Gtmfm02tumwfmAXq93o6FhYWhl7u8vMzU1BQA\new8erpRpZtvWSvNXsTpXFVUfQx29E+CJZ0e+msq6mguOnu1or6NRP5+j3mYbPbZhHtfqbG1uo6bv\n27rvzXFokm1ubm5PZvY3m69xoUfEFPDPwAcy89ajzdvv93NpaWnoZS8uLjI7OwtUHxr0wHUXVZq/\nitW5qhjH8Ka7Zo7wwb3dGxW5q7ng6NmO9joa9fM56m220WMb5nGtztbmNmr6vq373hyHJtkiYqhC\nb/Qtl4h4AfBp4KbNylySNFpNvuUSwMeA/Zn5ofYiSZLqaLKH/jrg7cB5EXHf4OfClnJJkiqqfYAu\nM/8FiBazSJIa8ExRSSqEhS5JhbDQJakQFrokFcJCl6RCWOiSVAgLXZIKYaFLUiG6OVrSCB1tsKCq\nAwNttKxRDgym8RrHgGqT0tZjm+Q2WrvuXTNHuLLF93jTPKvduHPLSNcN7qFLUjEsdEkqhIUuSYWw\n0CWpEBa6JBXCQpekQljoklQIC12SCmGhS1IhGhV6ROyMiIci4tsRsbutUJKk6moXekQcB/wt8Gbg\n5cBlEfHytoJJkqppsod+DvDtzPxOZv4YWAAuaSeWJKmqJoW+DfjequuPDqZJkiYgMrPeHSPeCuzM\nzD8cXH878OrMfMea+eaB+cHVs4CHKqzmVOAHtQKOVldzQXezdTUXdDdbV3NBd7N1NRc0y/ZrmfmS\nzWZqMnzuQeCMVddPH0z7fzLzeuD6OiuIiKXM7NeLNzpdzQXdzdbVXNDdbF3NBd3N1tVcMJ5sTQ65\n/Dvwsog4MyJeCFwKfK6dWJKkqmrvoWfmkYh4B/BF4Djghsy8v7VkkqRKGv3Fosz8PPD5lrKsp9ah\nmjHoai7obrau5oLuZutqLuhutq7mgjFkq/2hqCSpWzz1X5IK0clC79KQAhFxQ0Qcioh9q6adEhF3\nRMTDg39PnkCuMyLiroh4ICLuj4iru5AtIl4UEf8WEd8Y5Hp/F3KtyXhcRNwbEbd1KVtEHIiIvRFx\nX0QsdSVbRJwUEbdExIMRsT8iXjPpXBFx1mA7PffzdES8e9K5VuV7z+D1vy8ibh68L0aerXOF3sEh\nBW4Edq6Zthu4MzNfBtw5uD5uR4Bdmfly4FzgjwfbadLZ/gc4LzNfCZwN7IyIczuQa7Wrgf2rrncp\n21xmnr3q621dyPYR4AuZ+RvAK1nZdhPNlZkPDbbT2cAO4EfAZyadCyAitgHvAvqZuZ2VL41cOpZs\nmdmpH+A1wBdXXb8WuHbCmaaBfauuPwScNrh8GvBQB7bbZ4E3dSkbcCJwD/DqruRi5XyJO4HzgNu6\n9HwCB4BT10ybaDZgK/BdBp+3dSXXmiwXAP/alVz87Cz6U1j54sltg4wjz9a5PXSeH0MK9DLzscHl\nx4HeJMNExDTwKuDrdCDb4JDGfcAh4I7M7ESugQ8D1wA/XTWtK9kS+HJE7BmcYQ2Tz3Ym8CTw8cFh\nqo9GxJYO5FrtUuDmweWJ58rMg8BfAo8AjwGHM/NL48jWxUJ/XsmV/24n9lWhiJgCPg28OzOfXn3b\npLJl5v/myq/CpwPnRMT2LuSKiIuBQ5m5Z6N5Jvx8vn6w3d7MyiG0N6y+cULZjgd+E/i7zHwV8Axr\nDhVMcpsNTmp8C/CptbdN8HV2MisDFZ4J/CqwJSLeNo5sXSz0oYYUmLAnIuI0gMG/hyYRIiJewEqZ\n35SZt3YpG0Bm/hC4i5XPILqQ63XAWyLiACujg54XEZ/sSLbn9uzIzEOsHA8+pwPZHgUeHfyWBXAL\nKwU/6VzPeTNwT2Y+MbjehVxvBL6bmU9m5k+AW4HXjiNbFwv9+TCkwOeAKwaXr2Dl+PVYRUQAHwP2\nZ+aHupItIl4SEScNLp/AynH9ByedCyAzr83M0zNzmpXX1Vcy821dyBYRWyLixc9dZuWY675JZ8vM\nx4HvRcRZg0nnAw9MOtcql/Gzwy3QjVyPAOdGxImD9+n5rHyQPPpsk/gQY4gPFS4EvgX8B/DeCWe5\nmZXjYD9hZW/lKuCXWflg7WHgy8ApE8j1elZ+ZfsmcN/g58JJZwNeAdw7yLUP+LPB9IlvszU5Z/nZ\nh6ITzwb8OvCNwc/9z73uO5LtbGBp8Jz+I3ByR3JtAf4T2Lpq2sRzDXK8n5UdmX3APwC/OI5snikq\nSYXo4iEXSVINFrokFcJCl6RCWOiSVAgLXZIKYaFLUiEsdEkqhIUuSYX4PwhrPK+nNve4AAAAAElF\nTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1e0c161d2b0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df['age'].hist(bins=50);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Pre-processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "X = df[['pclass', 'sex', 'age']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "from sklearn import preprocessing\n",
    "lb = preprocessing.LabelBinarizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  if __name__ == '__main__':\n"
     ]
    }
   ],
   "source": [
    "X['sex'] = lb.fit_transform(X['sex'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pclass</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>38.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>35.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>54.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>58.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    pclass  sex   age\n",
       "1        1    0  38.0\n",
       "3        1    0  35.0\n",
       "6        1    1  54.0\n",
       "10       3    0   4.0\n",
       "11       1    0  58.0"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(182, 3)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pclass</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>182.000000</td>\n",
       "      <td>182.000000</td>\n",
       "      <td>182.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.192308</td>\n",
       "      <td>0.516484</td>\n",
       "      <td>35.623187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.516411</td>\n",
       "      <td>0.501107</td>\n",
       "      <td>15.671615</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.920000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>24.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>36.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>47.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>80.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           pclass         sex         age\n",
       "count  182.000000  182.000000  182.000000\n",
       "mean     1.192308    0.516484   35.623187\n",
       "std      0.516411    0.501107   15.671615\n",
       "min      1.000000    0.000000    0.920000\n",
       "25%      1.000000    0.000000   24.000000\n",
       "50%      1.000000    1.000000   36.000000\n",
       "75%      1.000000    1.000000   47.750000\n",
       "max      3.000000    1.000000   80.000000"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 182 entries, 1 to 889\n",
      "Data columns (total 3 columns):\n",
      "pclass    182 non-null int64\n",
      "sex       182 non-null int32\n",
      "age       182 non-null float64\n",
      "dtypes: float64(1), int32(1), int64(1)\n",
      "memory usage: 10.0 KB\n"
     ]
    }
   ],
   "source": [
    "X.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "y = df['survived']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    123\n",
       "0     59\n",
       "Name: survived, dtype: int64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fit Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import BaggingClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "from sklearn.model_selection import cross_val_score, cross_val_predict\n",
    "from sklearn.metrics import accuracy_score, classification_report, confusion_matrix"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "def print_score(clf, X_train, y_train, X_test, y_test, train=True):\n",
    "    '''\n",
    "    print the accuracy score, classification report and confusion matrix of classifier\n",
    "    '''\n",
    "    if train:\n",
    "        '''\n",
    "        training performance\n",
    "        '''\n",
    "        print(\"Train Result:\\n\")\n",
    "        print(\"accuracy score: {0:.4f}\\n\".format(accuracy_score(y_train, clf.predict(X_train))))\n",
    "        print(\"Classification Report: \\n {}\\n\".format(classification_report(y_train, clf.predict(X_train))))\n",
    "        print(\"Confusion Matrix: \\n {}\\n\".format(confusion_matrix(y_train, clf.predict(X_train))))\n",
    "\n",
    "        res = cross_val_score(clf, X_train, y_train, cv=10, scoring='accuracy')\n",
    "        print(\"Average Accuracy: \\t {0:.4f}\".format(np.mean(res)))\n",
    "        print(\"Accuracy SD: \\t\\t {0:.4f}\".format(np.std(res)))\n",
    "        \n",
    "    elif train==False:\n",
    "        '''\n",
    "        test performance\n",
    "        '''\n",
    "        print(\"Test Result:\\n\")        \n",
    "        print(\"accuracy score: {0:.4f}\\n\".format(accuracy_score(y_test, clf.predict(X_test))))\n",
    "        print(\"Classification Report: \\n {}\\n\".format(classification_report(y_test, clf.predict(X_test))))\n",
    "        print(\"Confusion Matrix: \\n {}\\n\".format(confusion_matrix(y_test, clf.predict(X_test))))    \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import cross_val_score, cross_val_predict\n",
    "from sklearn.metrics import accuracy_score, classification_report\n",
    "from sklearn.metrics import confusion_matrix, roc_auc_score\n",
    "def print_score(clf, X_train, X_test, y_train, y_test, train=True):\n",
    "    '''\n",
    "    v0.1 Follow the scikit learn library format in terms of input\n",
    "    print the accuracy score, classification report and confusion matrix of classifier\n",
    "    '''\n",
    "    lb = preprocessing.LabelBinarizer()\n",
    "    lb.fit(y_train)\n",
    "    if train:\n",
    "        '''\n",
    "        training performance\n",
    "        '''\n",
    "        res = clf.predict(X_train)\n",
    "        print(\"Train Result:\\n\")\n",
    "        print(\"accuracy score: {0:.4f}\\n\".format(accuracy_score(y_train, \n",
    "                                                                res)))\n",
    "        print(\"Classification Report: \\n {}\\n\".format(classification_report(y_train, \n",
    "                                                                            res)))\n",
    "        print(\"Confusion Matrix: \\n {}\\n\".format(confusion_matrix(y_train, \n",
    "                                                                  res)))\n",
    "        print(\"ROC AUC: {0:.4f}\\n\".format(roc_auc_score(lb.transform(y_train), \n",
    "                                                      lb.transform(res))))\n",
    "\n",
    "        res = cross_val_score(clf, X_train, y_train, cv=10, scoring='accuracy')\n",
    "        print(\"Average Accuracy: \\t {0:.4f}\".format(np.mean(res)))\n",
    "        print(\"Accuracy SD: \\t\\t {0:.4f}\".format(np.std(res)))\n",
    "        \n",
    "    elif train==False:\n",
    "        '''\n",
    "        test performance\n",
    "        '''\n",
    "        res_test = clf.predict(X_test)\n",
    "        print(\"Test Result:\\n\")        \n",
    "        print(\"accuracy score: {0:.4f}\\n\".format(accuracy_score(y_test, \n",
    "                                                                res_test)))\n",
    "        print(\"Classification Report: \\n {}\\n\".format(classification_report(y_test, \n",
    "                                                                            res_test)))\n",
    "        print(\"Confusion Matrix: \\n {}\\n\".format(confusion_matrix(y_test, \n",
    "                                                                  res_test)))   \n",
    "        print(\"ROC AUC: {0:.4f}\\n\".format(roc_auc_score(lb.transform(y_test), \n",
    "                                                      lb.transform(res_test))))\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "clf = DecisionTreeClassifier(random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n",
       "            max_features=None, max_leaf_nodes=None,\n",
       "            min_impurity_split=1e-07, min_samples_leaf=1,\n",
       "            min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "            presort=False, random_state=42, splitter='best')"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Result:\n",
      "\n",
      "accuracy score: 0.9449\n",
      "\n",
      "Classification Report: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          0       0.88      0.95      0.91        38\n",
      "          1       0.98      0.94      0.96        89\n",
      "\n",
      "avg / total       0.95      0.94      0.95       127\n",
      "\n",
      "\n",
      "Confusion Matrix: \n",
      " [[36  2]\n",
      " [ 5 84]]\n",
      "\n",
      "ROC AUC: 0.9456\n",
      "\n",
      "Average Accuracy: \t 0.7457\n",
      "Accuracy SD: \t\t 0.0904\n",
      "Test Result:\n",
      "\n",
      "accuracy score: 0.8000\n",
      "\n",
      "Classification Report: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          0       0.78      0.67      0.72        21\n",
      "          1       0.81      0.88      0.85        34\n",
      "\n",
      "avg / total       0.80      0.80      0.80        55\n",
      "\n",
      "\n",
      "Confusion Matrix: \n",
      " [[14  7]\n",
      " [ 4 30]]\n",
      "\n",
      "ROC AUC: 0.7745\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print_score(clf, X_train, X_test, y_train, y_test, train=True)\n",
    "print_score(clf, X_train, X_test, y_train, y_test, train=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bagging (oob_score=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "bag_clf = BaggingClassifier(base_estimator=clf, n_estimators=1000,\n",
    "                            bootstrap=True, n_jobs=-1,\n",
    "                            random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BaggingClassifier(base_estimator=DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n",
       "            max_features=None, max_leaf_nodes=None,\n",
       "            min_impurity_split=1e-07, min_samples_leaf=1,\n",
       "            min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "            presort=False, random_state=42, splitter='best'),\n",
       "         bootstrap=True, bootstrap_features=False, max_features=1.0,\n",
       "         max_samples=1.0, n_estimators=1000, n_jobs=-1, oob_score=False,\n",
       "         random_state=42, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bag_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Result:\n",
      "\n",
      "accuracy score: 0.9449\n",
      "\n",
      "Classification Report: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          0       0.88      0.95      0.91        38\n",
      "          1       0.98      0.94      0.96        89\n",
      "\n",
      "avg / total       0.95      0.94      0.95       127\n",
      "\n",
      "\n",
      "Confusion Matrix: \n",
      " [[36  2]\n",
      " [ 5 84]]\n",
      "\n",
      "ROC AUC: 0.9456\n",
      "\n",
      "Average Accuracy: \t 0.7457\n",
      "Accuracy SD: \t\t 0.0904\n",
      "Test Result:\n",
      "\n",
      "accuracy score: 0.8000\n",
      "\n",
      "Classification Report: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          0       0.78      0.67      0.72        21\n",
      "          1       0.81      0.88      0.85        34\n",
      "\n",
      "avg / total       0.80      0.80      0.80        55\n",
      "\n",
      "\n",
      "Confusion Matrix: \n",
      " [[14  7]\n",
      " [ 4 30]]\n",
      "\n",
      "ROC AUC: 0.7745\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print_score(clf, X_train, X_test, y_train, y_test, train=True)\n",
    "print_score(clf, X_train, X_test, y_train, y_test, train=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bagging (oob_score=True)\n",
    "\n",
    "Use out-of-bag samples to estimate the generalization accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "bag_clf = BaggingClassifier(base_estimator=clf, n_estimators=1000,\n",
    "                            bootstrap=True, oob_score=True,\n",
    "                            n_jobs=-1, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BaggingClassifier(base_estimator=DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n",
       "            max_features=None, max_leaf_nodes=None,\n",
       "            min_impurity_split=1e-07, min_samples_leaf=1,\n",
       "            min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "            presort=False, random_state=42, splitter='best'),\n",
       "         bootstrap=True, bootstrap_features=False, max_features=1.0,\n",
       "         max_samples=1.0, n_estimators=1000, n_jobs=-1, oob_score=True,\n",
       "         random_state=42, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bag_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.75590551181102361"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bag_clf.oob_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Result:\n",
      "\n",
      "accuracy score: 0.9449\n",
      "\n",
      "Classification Report: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          0       0.88      0.95      0.91        38\n",
      "          1       0.98      0.94      0.96        89\n",
      "\n",
      "avg / total       0.95      0.94      0.95       127\n",
      "\n",
      "\n",
      "Confusion Matrix: \n",
      " [[36  2]\n",
      " [ 5 84]]\n",
      "\n",
      "ROC AUC: 0.9456\n",
      "\n",
      "Average Accuracy: \t 0.7457\n",
      "Accuracy SD: \t\t 0.0904\n",
      "Test Result:\n",
      "\n",
      "accuracy score: 0.8000\n",
      "\n",
      "Classification Report: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          0       0.78      0.67      0.72        21\n",
      "          1       0.81      0.88      0.85        34\n",
      "\n",
      "avg / total       0.80      0.80      0.80        55\n",
      "\n",
      "\n",
      "Confusion Matrix: \n",
      " [[14  7]\n",
      " [ 4 30]]\n",
      "\n",
      "ROC AUC: 0.7745\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print_score(clf, X_train, X_test, y_train, y_test, train=True)\n",
    "print_score(clf, X_train, X_test, y_train, y_test, train=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}