feature-selection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  Feature selection with KNN and RandomForest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.base import clone\n",
    "from itertools import combinations\n",
    "from sklearn.metrics import accuracy_score\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "      <th>D</th>\n",
       "      <th>E</th>\n",
       "      <th>F</th>\n",
       "      <th>G</th>\n",
       "      <th>H</th>\n",
       "      <th>I</th>\n",
       "      <th>J</th>\n",
       "      <th>K</th>\n",
       "      <th>L</th>\n",
       "      <th>M</th>\n",
       "      <th>N</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>14.23</td>\n",
       "      <td>1.71</td>\n",
       "      <td>2.43</td>\n",
       "      <td>15.6</td>\n",
       "      <td>127</td>\n",
       "      <td>2.80</td>\n",
       "      <td>3.06</td>\n",
       "      <td>0.28</td>\n",
       "      <td>2.29</td>\n",
       "      <td>5.64</td>\n",
       "      <td>1.04</td>\n",
       "      <td>3.92</td>\n",
       "      <td>1065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>13.20</td>\n",
       "      <td>1.78</td>\n",
       "      <td>2.14</td>\n",
       "      <td>11.2</td>\n",
       "      <td>100</td>\n",
       "      <td>2.65</td>\n",
       "      <td>2.76</td>\n",
       "      <td>0.26</td>\n",
       "      <td>1.28</td>\n",
       "      <td>4.38</td>\n",
       "      <td>1.05</td>\n",
       "      <td>3.40</td>\n",
       "      <td>1050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>13.16</td>\n",
       "      <td>2.36</td>\n",
       "      <td>2.67</td>\n",
       "      <td>18.6</td>\n",
       "      <td>101</td>\n",
       "      <td>2.80</td>\n",
       "      <td>3.24</td>\n",
       "      <td>0.30</td>\n",
       "      <td>2.81</td>\n",
       "      <td>5.68</td>\n",
       "      <td>1.03</td>\n",
       "      <td>3.17</td>\n",
       "      <td>1185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>14.37</td>\n",
       "      <td>1.95</td>\n",
       "      <td>2.50</td>\n",
       "      <td>16.8</td>\n",
       "      <td>113</td>\n",
       "      <td>3.85</td>\n",
       "      <td>3.49</td>\n",
       "      <td>0.24</td>\n",
       "      <td>2.18</td>\n",
       "      <td>7.80</td>\n",
       "      <td>0.86</td>\n",
       "      <td>3.45</td>\n",
       "      <td>1480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>13.24</td>\n",
       "      <td>2.59</td>\n",
       "      <td>2.87</td>\n",
       "      <td>21.0</td>\n",
       "      <td>118</td>\n",
       "      <td>2.80</td>\n",
       "      <td>2.69</td>\n",
       "      <td>0.39</td>\n",
       "      <td>1.82</td>\n",
       "      <td>4.32</td>\n",
       "      <td>1.04</td>\n",
       "      <td>2.93</td>\n",
       "      <td>735</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   A      B     C     D     E    F     G     H     I     J     K     L     M  \\\n",
       "0  1  14.23  1.71  2.43  15.6  127  2.80  3.06  0.28  2.29  5.64  1.04  3.92   \n",
       "1  1  13.20  1.78  2.14  11.2  100  2.65  2.76  0.26  1.28  4.38  1.05  3.40   \n",
       "2  1  13.16  2.36  2.67  18.6  101  2.80  3.24  0.30  2.81  5.68  1.03  3.17   \n",
       "3  1  14.37  1.95  2.50  16.8  113  3.85  3.49  0.24  2.18  7.80  0.86  3.45   \n",
       "4  1  13.24  2.59  2.87  21.0  118  2.80  2.69  0.39  1.82  4.32  1.04  2.93   \n",
       "\n",
       "      N  \n",
       "0  1065  \n",
       "1  1050  \n",
       "2  1185  \n",
       "3  1480  \n",
       "4   735  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data= pd.read_csv('data.csv')\n",
    "\n",
    "data.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "X=data.values[:, 1:]\n",
    "y=data.values[:, 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SBS():\n",
    "    \n",
    "    def __init__(self, estimator, k_features, scoring=accuracy_score,               \n",
    "                 test_size=0.25, random_state=1):     \n",
    "        self.scoring = scoring\n",
    "        self.estimator = clone(estimator)\n",
    "        self.k_features = k_features\n",
    "        self.test_size = test_size\n",
    "        self.random_state = random_state\n",
    "        \n",
    "        \n",
    "    def fit(self, X, y):\n",
    "            \n",
    "        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=self.test_size,\n",
    "                                                            random_state=self.random_state)\n",
    "\n",
    "        dim = X_train.shape[1]\n",
    "        self.indices_ = tuple(range(dim))\n",
    "        self.subsets_ = [self.indices_]\n",
    "        score = self._calc_score(X_train, y_train,\n",
    "                                 X_test, y_test, self.indices_)\n",
    "        self.scores_ = [score]\n",
    "        while dim > self.k_features:                 \n",
    "            scores = []\n",
    "            subsets = []\n",
    "            for p in combinations(self.indices_, r=dim-1):\n",
    "                score = self._calc_score(X_train, y_train, X_test, y_test, p)\n",
    "                scores.append(score)\n",
    "                subsets.append(p)\n",
    "                \n",
    "            best = np.argmax(scores)\n",
    "            self.indices_ = subsets[best]\n",
    "            self.subsets_.append(self.indices_)\n",
    "            dim -= 1\n",
    "            \n",
    "            self.scores_.append(scores[best])\n",
    "        self.k_score_ = self.scores_[-1]\n",
    "        return self\n",
    "    def transform(self, X):\n",
    "        return X[:, self.indices_]\n",
    "    \n",
    "    def _calc_score(self, X_train, y_train, X_test, y_test, indices):\n",
    "        self.estimator.fit(X_train[:, indices], y_train)\n",
    "        y_pred = self.estimator.predict(X_test[:, indices])\n",
    "        score = self.scoring(y_test, y_pred)\n",
    "        return score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<__main__.SBS at 0x1a625a43908>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "import matplotlib.pyplot as plt\n",
    "knn = KNeighborsClassifier(n_neighbors=2)\n",
    "sbs = SBS(knn, k_features=1)\n",
    "sbs.fit(X_train_std, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl4VdW5x/HvmzAPCgjGyowigwMgEVGqhjqA1hacWuitQ7WX2kIdqrZwa9ViB1u11qrXikq1XitFpJYqFa2SQlWUiAwCBsOghCCIyBANQ5L3/nF29BCS7EPIzjk5+X2eZz/Ze+3pXZCT9+xhrWXujoiISE0ykh2AiIikPiULEREJpWQhIiKhlCxERCSUkoWIiIRSshARkVCRJQszm2pmm83snWrW9zWz181st5ndWGndSDPLN7MCM5sYVYwiIpKYKK8sHgNG1rB+K3ANcFd8oZllAg8A5wL9gbFm1j+iGEVEJAGRJQt3n0csIVS3frO7LwT2Vlo1BChw9zXuvgeYBoyKKk4REQnXJNkBVKEzsD5uuRA4uaoNzWwcMA6gZcuWg7t27Rp9dAehvLycjIz0eEyULnVJl3qA6pKqUr0uq1at2uLuncK2S8VkYVWUVdknibtPAaYAZGdne15eXpRxHbTc3FxycnKSHUadSJe6pEs9QHVJValeFzN7P5HtUjHdFQLxlwhdgKIkxSIiIqRmslgI9DaznmbWDBgDzEpyTCIijVpkt6HM7CkgB+hoZoXArUBTAHf/o5kdAeQBhwDlZnYd0N/dd5jZBGAOkAlMdfflUcUpIiLhIksW7j42ZP2HxG4xVbVuNjA7irhEROTApeJtKBERSTFKFiIiEkrJQkREQilZiIhIKCULEREJpWQhIiKhlCxERCSUkoWIiIRSshARkVBKFiIiEkrJQkREQilZiIhIKCULEREJpWQhIiKhlCxERCSUkoWIiIRSshARkVBKFiIiEiqyZGFmU81ss5m9U816M7M/mFmBmS01sxPj1pWZ2eJgmhVVjCIikpgoryweA0bWsP5coHcwjQMejFtX4u4Dg+nr0YUoIiKJiCxZuPs8YGsNm4wC/uwxC4B2ZvalqOIREZHaS+Yzi87A+rjlwqAMoIWZ5ZnZAjMbXf+hiYhIvCZJPLdVUebBz27uXmRmvYBXzGyZu6/e7wBm44jdwiIrK4vc3NzIgq0LxcXFKR9jotKlLulSD1BdUlW61CWZyaIQ6Bq33AUoAnD3ip9rzCwXGATslyzcfQowBSA7O9tzcnKijfgg5ebmkuoxJipd6pIu9QDVJVWlS12SeRtqFnBZ8FbUUGC7u280s/Zm1hzAzDoCw4AVSYxTRKTRi+zKwsyeAnKAjmZWCNwKNAVw9z8Cs4HzgALgM+A7wa79gIfMrJxYMrvD3ZUsRESSKLJk4e5jQ9Y7ML6K8teA46OKS0REDpxacIuISCglCxERCaVkISIioZQsREQklJKFiIiEUrIQEZFQShYiIhJKyUJEREIpWYiISCglCxERCaVkISIioZQsREQklJKFiIiEUrIQEZFQShYiIhJKyUJEREIpWYiISCglCxERCaVkISIioSIbg9vMpgLnA5vd/bgq1htwL3Ae8BlwhbsvCtZdDtwcbPoLd388qjgl9Tz79gbunJNP0bYSjmzXkptG9GH0oM6RnGPDthI6L3glknPEn0d1ObBzRFmX+qhHOoosWQCPAfcDf65m/blA72A6GXgQONnMOgC3AtmAA2+Z2Sx3/yTCWCVFPPv2BibNXEbJ3jIANmwrYeLMpewtK+drA46sk3P8Y0kRP/v7O+zaWx7ZOerrPKrLwZ9j0sxlAEoYIczdozu4WQ/guWquLB4Cct39qWA5H8ipmNz9e1VtV53s7GzPy8ury/DrXG5uLjk5OckOo05EVZdhd7zChm0ldX5ckZp0bteSVyd+JZJjp/rn3szecvfssO2ivLII0xlYH7dcGJRVV74fMxsHjAPIysoiNzc3kkDrSnFxccrHmKio6lJTorj4mKZ1co4Zq/ZGfo76Oo/qUjfn2LCtJLLPZrp87pOZLKyKMq+hfP9C9ynAFIhdWaRy9obU/4ZxIKKqS9Zr/2LTjt37lXdu15K7rqybb36vV3P1UpfnqK/zqC51c44Mg7KsfpzZL6tOzhMvXT73yXwbqhDoGrfcBSiqoVzS3J7Scppm7P8r2bJpJjeN6FNn57lpRB9aNs2M9Bz1dR7V5eDP0bxJBlltm3PV43lMfGYpxbtL6+x86SSZyWIWcJnFDAW2u/tGYA5wjpm1N7P2wDlBmaS5u1/Mp3BbCd8Z1oPO7VpixL5V/vrC4+v04ePoQZ359YXH07ldS4joHJXPo7oc2Dmoh3NU1OM3F51A7o+Hc/UZR/HXvPWcd+988tZtrbNzpg13j2QCngI2AnuJXS1cBVwNXB2sN+ABYDWwDMiO2/dKoCCYvpPI+QYPHuypbu7cuckOoc7UdV3mrdrs3X/ynE+aubROjxtG/yepKVl1eXPtx/7l37zsPSc+53f8c6Xv2lt60MdM9f8XIM8T+Bsb2TMLdx8bst6B8dWsmwpMjSIuST0fF+/mR9OX0PvwNvzsq/2THY40Yif16MA/rz2dXzy3ggdzVzP33c38fsxA+h5xSLJDSzq14JakcndufHoJ20v2ct+3BtGyWWb4TiIRatO8CXdcdAKPXJbNluLdfP2+V5kybzVl5dE1M2gIlCwkqf706jrm5n/ET8/rp29vklLO6p/FnOtOZ3jfTvxq9ruMfXgB67d+luywkkbJQpJmedF27vjnu5zV73AuO6V7ssMR2c9hbZrzx28P5q5LBrCiaAfn3juf6XnrK56tNipKFpIUn+0p5Zqn3qZdq6b89uIBxLoKE0k9ZsbFg7vwwnWncVznQ/jxjKWMe+ItthTv3x4onSlZSFLc/twK1mz5lHu+OZAOrZslOxyRUF3at+Iv3x3KzV/tx79XfcSIe+bx4vIPkx1WvVGykHo3e9lGnnpzPd87/SiGHd0x2eGIJCwjw/juab34x4Qvk3VIC8Y98RY/nrGEnbuq76okXShZSL3asK2Eic8sZUDXdtxwzjHJDkekVvoc0ZZnxw9j/PCjmPFWIefeO5831nyc7LAipWQh9aa0rJzrpy2m3OEPYwbSNFO/ftJwNWuSwU0j+vL01aeQmWGMeXgBv5q9kt2lZckOLRLJ7EhQ6lBDGNDl/rkFvLluK/d8cwDdD2ud7HBE6sTg7h2Yfc1p/HL2SqbMW8O8VR/xu28MZNWmnfUyKFV9UbJIA1UNGJRqA7osXLeVP7z8HhcM6swFg7okOxyROtW6eRN+dcHxnN0vix8/s5Tz75tPhhmlQUO+VPxMHijdB0gDd87J/zxRVCjZW8adc/KTFNG+tn+2l+umLaZL+1ZMHnVsssMRiczwvocz57rTadYk4/NEUSGVPpO1oSuLNFBUzYBB1ZXXJ3fnf/62jE07djHj+6fStkXdDcgjkoo6tG7G7mDY1spS4TNZW7qySANHBl06V9ameRP2llX9S1tfpuet5/llG/nROccwsGu7pMYiUl+q+0xmHdKiniOpO0oWaeCmEX1okrFvC+hMM3buLuWC/32V9zbtTEpcBZuLuW3WCoYdfRhXn35UUmIQSYaqBlkC2LlrLy+8szEJER08JYs0MLzP4WQatGya8fmALnd/YwB//PZgirbt4qv3/YdH/7OW8nrsNXN3aRnXPPU2LZpm8LtvDCQjQ915SONR1UBOk87tQ69Obbj6/xbxo+mL2dHAGvLpmUUaePz1dewuc54fP4xjjzx0n3WDu7dn0syl3P7cCl5euYk7Lxnw+S9wlH7zz3xWbNzBI5dlN+hLb5HaGj2oM6MHdd5nDO4rv9yL+15+jwdyV/PGmq3cdckATjnqsOQGmiBdWTRwn+4uZeqrazmz7+H7JQqATm2b8/Bl2fzmouNZsn4bI++Zx8xFhZH2mjk3fzNTX13L5ad056z+WZGdR6ShaZqZwY/O6cOMq0+hWZMMxj68gF88t4Jde1O/IZ+SRQP35Bvvs+2zvYz/ytHVbmNmfPOkbvzz2tPp+6W2/Gj6En7w5CK2frqnzuPZvHMXN05fQt8j2jLpvH51fnyRdDCoW3uev+bLXDq0O4/8Zy1fu+8/vLNhe7LDqlGkycLMRppZvpkVmNnEKtZ3N7OXzWypmeWaWZe4dWVmtjiYZkUZZ0O1a28ZD89fy7CjD+PEbu1Dt+92WCumjTuFief25V8rN3HOPfN45d1NdRZPeblzw/QlFO8u5b6xg2hRxQM+EYlp1awJt48+jsevHML2kr2MfuBV7n/lPUqT/AZjdSJLFmaWCTwAnAv0B8aaWeUBlu8C/uzuJwCTgV/HrStx94HB9PWo4mzIpuet56Oduxk/vPqrisoyM4yrzziKv4//Mh3bNOPKx/KYNHMZn+4uPeh4Hv3PWua/t4Vbvtaf3lltD/p4Io3BGcd04sXrT2fkcUdw14ur+MZDr7Nuy6fJDms/UV5ZDAEK3H2Nu+8BpgGjKm3TH3g5mJ9bxXqpxt6ych769xoGd2/PKb0O/AFZ/yMP4e8ThvG9M3oxbeEHnPeH+bz1/tZax7OscDu/nfMuI489gm8N6Vbr44g0Ru1aNeP+b53IvWMGUrC5mHPvnc+Tb7yfUiPyWVTBmNnFwEh3/26wfClwsrtPiNvmL8Ab7n6vmV0IPAN0dPePzawUWAyUAne4+7NVnGMcMA4gKytr8LRp0yKpS10pLi6mTZs2dXKs+YV7efSdPVw/uDkDOh3cS235W8t4eNluPi5xvtqrKaOPbrpfu43K4uuyq9S59bUS9pbD5FNb0qZZw3lNti7/T5JNdUlNB1qXrbvKeXTZbpZ/XM4JnTK58thmtGsR3ff64cOHv+Xu2aEbunskE3AJ8Ejc8qXAfZW2ORKYCbwN3AsUAodWrAt+9gLWAUfVdL7Bgwd7qps7d26dHKe0rNxz7pzr5907z8vLy+vkmDt37fUfP73Eu//kOT/39/P83Y07atw+vi43Tl/sPSY+56+v3lInsdSnuvo/SQWqS2qqTV3Kysr9sVfXep+bZ/vAn8/x55cW1X1gASDPE/ibHuVtqEKga9xyF6AofgN3L3L3C919EPDToGx7xbrg5xogFxgUYawNyvPLNrJ2y6dMGH50nY1d3aZ5E35z8Qk8fFk2m3fu4mv3/YeH560Jbcg3a0kRT79VyIThRzO0FrfDRGR/GRnG5af24PlrTqNbh1b84MlFXDftbbaXJK8hX5TJYiHQ28x6mlkzYAywz1tNZtbRzCpimARMDcrbm1nzim2AYcCKCGNtMMrLnQdeKeDow9sw4tgj6vz4Z/fPYs51p5PTpxO/nL2SsQ8voPCTz6rcdv3Wz/jpzGWc2K0d157Zu85jEWnsjurUhhnfP5XrzurNP5ZuZOTv5/FqwZakxBJZsnD3UmACMAdYCUx39+VmNtnMKt5uygHyzWwVkAX8MijvB+SZ2RJiD77vcHclC+BfKzeRv2kn44cfFVkXGoe1ac5Dlw7mzotPYHnRDkb+fj5P563f52FbWblz7bS3Abh3zCCaaNQ7kUg0zczgurOOYeb3T6Vls0z+65E3+Pk/ltd7Q75Iu/tw99nA7Eplt8TNzwBmVLHfa8DxUcbWELk7988toFuHVnzthCMjPZeZcUl2V4b2Oowbnl7CTTOW8tKKTZx+TEcezF3Dhm0lwGdcdkp3unZoFWksIgIDurbj+R+exm9eeJc/vbqO+e9tYdSAI5m2cH29jJCpr4MNyPz3trC0cDvfzzmq3r7Jd+3Qimn/PZSfntePl1du4uZnlweJIubpvEKefXtDvcQi0ti1bJbJbV8/lieuGsLmHbu4+6VVbNhWgvPFaHxRfR5D/+KY2QQzC28eLJG7f24BRxzSggtPrN9hGTMyjP8+vReHtWm+37qGPvqXSEN0Wu9OtG6+/42hKD+PiXw9PQJYaGbTg+47Gs5L9GnkzbVbeXPtVr53Ri+aN0lONxof7dxdZXlDHv1LpKH6cPuuKsuj+jyGJgt3vxnoDTwKXAG8Z2a/MjONZlOP7p9bwGGtmzHmpOS1jq5u9K/qykUkOvX9eUzoxnfQcOPDYCoF2gMzzOy3kUQl+1iyfhvzVn3EVaf1pGWz5HXOV9XoXy2bZnLTiD5Jikik8arvz2Po21Bmdg1wObAFeAS4yd33Bu0j3gN+HElk8rkH5hZwSIsmXDq0e1LjqHjL4s45+WzYVkLniN++EJHqxX8e6+NtqERene0IXOju78cXunu5mZ0fSVTyufwPd/Liik1cc2Zv2rZomuxwqhz9S0SSo+LzWB8SuQ01G/i8O1Iza2tmJwO4+8qoApOYB+YW0LpZJt85tUeyQxGRRiyRZPEgUBy3/GlQJhFbu+VTnltaxLeHdqd962bJDkdEGrFEkoV5XD8P7l5OxC2/JebB3AKaZmZw1Wk9kx2KiDRyiSSLNWZ2jZk1DaZrgTVRB9bYbdhWwsxFGxhzUlcOb9si2eGISCOXSLK4GjgV2ECs2/GTCQYckug89O/VAIw7Q81ZRCT5Qm8nuftmYt2LSz3ZvHMX0xau56ITu9BZDd5EJAUk0s6iBXAVcCzw+f0Qd78ywrgatUfnr6W0rJzv5+iqQkRSQyK3oZ4g1j/UCODfxEa82xllUI3ZJ5/u4YkF7/O1AUfSo2PrZIcjIgIkliyOdvefAZ+6++PAV9FYE5H502vr+GxPGT/IOTrZoYiIfC6RZFEx6Os2MzsOOBToEVlEjdjOXXt57NW1jDg2iz5HtE12OCIin0ukvcSUYDyLm4mNod0G+FmkUTVSTyx4nx27SpkwXONZi0hqqfHKIugscIe7f+Lu89y9l7sf7u4PJXLwYPyLfDMrMLOJVazvbmYvm9lSM8s1sy5x6y43s/eC6fIDrlkDU7KnjEfnr+WMYzpxfJdDkx2OiMg+akwWQWvtCbU5sJllAg8A5wL9gbFm1r/SZncBf3b3E4DJwK+DfTsAtxJr0zEEuDXdR+t76s0P+PjTPUz4ip5ViEjqSeSZxUtmdqOZdTWzDhVTAvsNAQrcfY277wGmAaMqbdMfeDmYnxu3fgTwkrtvdfdPgJeAkQmcs0HaXVrGlHlrGNKzAyf1SOSfVkSkfiXyzKKiPcX4uDIHeoXs1xlYH7dc0fo73hLgIuBe4AKgrZkdVs2++/XDa2bjCFqTZ2VlkZubGxJSchUXF1cZY+76vXy4Yw/fPoaUr0OF6urS0KRLPUB1SVXpUpdEWnDXthe7qsbq9krLNwL3m9kVwDxiXYqUJrgv7j4FmAKQnZ3tqT6+QlVjQJSWlfOzu3MZ0OVQxl80jIYyxHm6jGeRLvUA1SVVpUtdEmnBfVlV5e7+55BdC4GucctdgKJKxygCLgzO0wa4yN23m1khkFNp39ywWBuiWUuKWL+1hFvOP7bBJAoRaXwSuQ11Utx8C+BMYBEQliwWAr3NrCexK4YxwLfiNzCzjsDW4EH6JGBqsGoO8Ku4h9rnBOvTSnm587+5q+l7RFvO7Ht4ssMREalWIrehfhi/bGaHEusCJGy/UjObQOwPfyYw1d2Xm9lkIM/dZxG7evi1mTmx21Djg323mtntxBIOwGR337rfSRq4Ocs/pGBzMfeNHURGhq4qRCR11WYQo8+AhFqNuftsYsOyxpfdEjc/A5hRzb5T+eJKI+24O/fPLaBXx9acd/yXkh2OiEiNEnlm8Q++eLicQex11+lRBtUY5OZ/xPKiHfz24hPI1FWFiKS4RK4s7oqbLwXed/fCiOJpFNyd+155j87tWnLBoP3eCBYRSTmJJIsPgI3uvgvAzFqaWQ93XxdpZGns9TUfs+iDbdw+6liaZibSLlJEJLkS+Uv1NFAet1wWlEkt3f9KAZ3aNueS7K7hG4uIpIBEkkWToLsOAIL5ZtGFlN4WffAJr63+mHGn9aJF08xkhyMikpBEksVHZvb1igUzGwVsiS6k9PbAKwW0b9WUb53cLdmhiIgkLJFnFlcDT5rZ/cFyIVBlq26p2fs7ynj53c3ccPYxtG5em7eWRUSSI5FGeauBoUF3HObuGn/7AD379gbunJPPhm27MOCwtrqLJyINS+htKDP7lZm1c/did99pZu3N7Bf1EVw6ePbtDUyauYwN20qAWIOV2/+xkmff3pDcwEREDkAizyzOdfdtFQvB+BLnRRdSerlzTj4le8v2KSvZW8adc/KTFJGIyIFLJFlkmlnzigUzawk0r2F7iVNxRVFZUTXlIiKpKJGnrP8HvGxmfwqWvwM8Hl1I6aF4dymT/7G82vVHtmtZj9GIiBycRB5w/9bMlgJnERuU6AWge9SBNWRvrt3Kj6YvpmhbCWf1O5z/FGxh194v2jW2bJrJTSP6JDFCEZEDk+j7mx8Sa8X9DWAt8ExkETVgu0vL+N2Lq5gyfw1d27di+vdOIbtHh7i3oUro3K4lN43ow2j1CSUiDUi1ycLMjiE2YNFY4GPgr8RenR1eT7E1KCs37uD6vy7m3Q93MnZIN27+ar/P21KMHtSZ0YM6p83wiiLS+NR0ZfEuMB/4mrsXAJjZ9fUSVQNSVu5MmbeG372Uz6EtmzH1imy+0jcr2WGJiNSpmpLFRcSuLOaa2QvANGLPLCTwwcefccPTi1m47hPOPe4IfnnB8XRorQZ3IpJ+qk0W7v434G9m1hoYDVwPZJnZg8Df3P3Feoox5bg7f124ntufW0GGGb/7xgAuGNQZM+VSEUlPoe0s3P1Td3/S3c8HugCLgYmJHNzMRppZvpkVmNl++5hZNzOba2Zvm9lSMzsvKO9hZiVmtjiY/niA9YrMRzt3899/zmPizGWc0KUdL1x/Ohee2EWJQkTS2gH1ZufuW4GHgqlGZpYJPACcTazzwYVmNsvdV8RtdjMw3d0fNLP+xMbr7hGsW+3uAw8kvqi98M6H/M/fllG8u5Sfnd+f75zagwwNiSoijUCUXZ8OAQrcfQ2AmU0DRgHxycKBQ4L5Q4GiCOOptR279vLzWSt4ZlEhx3U+hHu+MZDeWW2THZaISL0xd4/mwGYXAyPd/bvB8qXAye4+IW6bLwEvAu2B1sBZ7v6WmfUAlgOrgB3Aze4+v4pzjAPGAWRlZQ2eNm1anddj5cdlPLJsN1t3Oecf1ZRRRzWlSS2vJoqLi2nTpk0dR5gc6VKXdKkHqC6pKtXrMnz48LfcPTtsuyivLKr6i1o5M40FHnP3u83sFOAJMzsO2Ah0c/ePzWww8KyZHevuO/Y5mPsUYApAdna212Ubhl17y7hrTj6P5q2lx2GtefjKAZzYrf1BHTOd2lmkS13SpR6guqSqdKlLlMmiEIgfZLoL+99mugoYCeDur5tZC6Cju28Gdgflb5nZauAYIC/CeD/3zobt/Gj6YlZtKubSod2ZdF5fWjXTYEUi0nhF+RdwIdDbzHoCG4i12fhWpW0+AM4EHjOzfkALYsO4dgK2unuZmfUCegNrogiyoiuOom0lfKldCwZ1bceLKzbRvlUzHvvOSeT0OTyK04qINCiRJQt3LzWzCcAcIBOY6u7LzWwykOfus4AbgIeDluEOXOHubmanA5PNrBQoA64O3sSqUxUDE1WMN1G0bRdF2z5kYNdD+dMVQ2ivBnYiIkC0Vxa4+2xir8PGl90SN78CGFbFfs9QD50VVjUwEcTaUihRiIh8IZHBj9JWdQMQFW3bVc+RiIiktkadLKobgEgDE4mI7KtRJ4ubRvShZdPMfco0MJGIyP4a9fugFQMQVbwNdaQGJhIRqVKjThbwxcBEIiJSvUZ9G0pERBKjZCEiIqGULEREJJSShYiIhFKyEBGRUEoWIiISSslCRERCKVmIiEgoJQsREQmlZCEiIqGULEREJJSShYiIhFKyEBGRUJEmCzMbaWb5ZlZgZhOrWN/NzOaa2dtmttTMzotbNynYL9/MRkQZp4iI1CyyLsrNLBN4ADgbKAQWmtmsYNztCjcD0939QTPrT2y87h7B/BjgWOBI4F9mdoy77z9gtoiIRC7KK4shQIG7r3H3PcA0YFSlbRw4JJg/FCgK5kcB09x9t7uvBQqC44mISBJEOfhRZ2B93HIhcHKlbW4DXjSzHwKtgbPi9l1Qad/9Rigys3HAOICsrCxyc3PrIu7IFBcXp3yMiUqXuqRLPUB1SVXpUpcok4VVUeaVlscCj7n73WZ2CvCEmR2X4L64+xRgCkB2drbn5OQcXMQRy83NJdVjTFS61CVd6gGqS6pKl7pEmSwKga5xy1344jZThauAkQDu/rqZtQA6JriviIjUkyifWSwEeptZTzNrRuyB9axK23wAnAlgZv2AFsBHwXZjzKy5mfUEegNvRhiriIjUILIrC3cvNbMJwBwgE5jq7svNbDKQ5+6zgBuAh83semK3ma5wdweWm9l0YAVQCozXm1AiIskT5W0o3H02sddh48tuiZtfAQyrZt9fAr+MMj4REUmMWnCLiEgoJQsREQmlZCEiIqGULEREJJSShYiIhFKyEBGRUEoWIiISSslCRERCKVmIiEgoJQsREQmlZCEiIqGULEREJJSShYiIhFKyEBGRUEoWIiISSslCRERCKVmIiEgoJQsREQkVabIws5Fmlm9mBWY2sYr195jZ4mBaZWbb4taVxa2bFWWcIiJSs8jG4DazTOAB4GygEFhoZrOCcbcBcPfr47b/ITAo7hAl7j4wqvhERCRxUV5ZDAEK3H2Nu+8BpgGjath+LPBUhPGIiEgtRZksOgPr45YLg7L9mFl3oCfwSlxxCzPLM7MFZjY6ujBFRCRMZLehAKuizKvZdgwww93L4sq6uXuRmfUCXjGzZe6+ep8TmI0DxgFkZWWRm5tbB2FHp7i4OOVjTFS61CVd6gGqS6pKl7pEmSwKga5xy12Aomq2HQOMjy9w96Lg5xozyyX2PGN1pW2mAFMAsrOzPScnpy7ijkxubi6pHmOi0qUu6VIPUF1SVbrUJcrbUAuB3mbW08yaEUsI+73VZGZ9gPbA63Fl7c2seTDfERgGrKi8r4iI1I/IrizcvdTMJgBzgExgqrsvN7PJQJ67VySOscA0d4+/RdUPeMjMyokltDvi36ISEZH6FeVtKNx9NjBLwIwOAAAI3UlEQVS7UtktlZZvq2K/14Djo4xNREQSpxbcIiISSslCRERCKVmIiEgoJQsREQmlZCEiIqGULEREJJSShYiIhFKyEBGRUEoWIiISSslCRERCKVmIiEgoJQsREQmlZCEiIqGULEREJJSShYiIhFKyEBGRUEoWIiISSslCRERCKVmIiEioSJOFmY00s3wzKzCziVWsv8fMFgfTKjPbFrfucjN7L5gujzJOERGpWZOoDmxmmcADwNlAIbDQzGa5+4qKbdz9+rjtfwgMCuY7ALcC2YADbwX7fhJVvCIiUr0oryyGAAXuvsbd9wDTgFE1bD8WeCqYHwG85O5bgwTxEjAywlhFRKQGkV1ZAJ2B9XHLhcDJVW1oZt2BnsArNezbuYr9xgHjgsViM8s/yJij1hHYkuwg6ki61CVd6gGqS6pK9bp0T2SjKJOFVVHm1Ww7Bpjh7mUHsq+7TwGm1C68+mdmee6enew46kK61CVd6gGqS6pKl7pEeRuqEOgat9wFKKpm2zF8cQvqQPcVEZGIRZksFgK9zaynmTUjlhBmVd7IzPoA7YHX44rnAOeYWXszaw+cE5SJiEgSRHYbyt1LzWwCsT/ymcBUd19uZpOBPHevSBxjgWnu7nH7bjWz24klHIDJ7r41qljrUYO5ZZaAdKlLutQDVJdUlRZ1sbi/0SIiIlVSC24REQmlZCEiIqGULOqBmXU1s7lmttLMlpvZtcmO6WCYWaaZvW1mzyU7loNhZu3MbIaZvRv835yS7Jhqy8yuD3633jGzp8ysRbJjSpSZTTWzzWb2TlxZBzN7Keju56XgRZeUVk097gx+v5aa2d/MrF0yYzwYShb1oxS4wd37AUOB8WbWP8kxHYxrgZXJDqIO3Au84O59gQE00DqZWWfgGiDb3Y8j9kLJmORGdUAeY/8eGiYCL7t7b+DlYDnVPcb+9XgJOM7dTwBWAZPqO6i6omRRD9x9o7svCuZ3EvujtF+L9IbAzLoAXwUeSXYsB8PMDgFOBx4FcPc97r6t5r1SWhOgpZk1AVrRgNolufs8oPLbjqOAx4P5x4HR9RpULVRVD3d/0d1Lg8UFxNqMNUhKFvXMzHoQ6zDxjeRGUmu/B34MlCc7kIPUC/gI+FNwS+0RM2ud7KBqw903AHcBHwAbge3u/mJyozpoWe6+EWJftoDDkxxPXbgS+Geyg6gtJYt6ZGZtgGeA69x9R7LjOVBmdj6w2d3fSnYsdaAJcCLwoLsPAj6lYdzq2E9wP38Usf7VjgRam9m3kxuVxDOznxK7Hf1ksmOpLSWLemJmTYkliifdfWay46mlYcDXzWwdsV6Ev2Jm/5fckGqtECh094orvBnEkkdDdBaw1t0/cve9wEzg1CTHdLA2mdmXAIKfm5McT60F4/GcD/yXN+CGbUoW9cDMjNi98ZXu/rtkx1Nb7j7J3bu4ew9iD1BfcfcG+Q3W3T8E1gfdzQCcCayoYZdU9gEw1MxaBb9rZ9JAH9bHmQVUDHp2OfD3JMZSa2Y2EvgJ8HV3/yzZ8RwMJYv6MQy4lNg38YqRAc9LdlDCD4EnzWwpMBD4VZLjqZXg6mgGsAhYRuxz3WC6mDCzp4j1DdfHzArN7CrgDuBsM3uP2ABqdyQzxkRUU4/7gbbAS8Hn/o9JDfIgqLsPEREJpSsLEREJpWQhIiKhlCxERCSUkoWIiIRSshARkVBKFtIgmJmb2d1xyzea2W11dOzHzOziujhWyHkuCXq3nVvFujuDXmPvrMVxB+pVbImakoU0FLuBC82sY7IDiWdmmQew+VXAD9x9eBXrvgec6O431SKMgcABJQuL0edfEqZfFmkoSok1NLu+8orKVwZmVhz8zDGzf5vZdDNbZWZ3mNl/mdmbZrbMzI6KO8xZZjY/2O78YP/M4Bv/wmA8gu/FHXeumf2FWCO4yvGMDY7/jpn9Jii7Bfgy8MfKVw9mNgtoDbxhZt80s05m9kxw3oVmNizYboiZvRZ0fPiamfUxs2bAZOCbQaOvb5rZbWZ2Y9zx3zGzHsG00sz+l1gDvq5mdo6ZvW5mi8zs6aD/MoJ/qxVBve860P8sSUPurklTyk9AMXAIsA44FLgRuC1Y9xhwcfy2wc8cYBvwJaA5sAH4ebDuWuD3cfu/QOzLU29i/Ua1AMYBNwfbNAfyiHXWl0Os48GeVcR5JLHuNzoR66zwFWB0sC6X2JgTVdYvbv4vwJeD+W7EuokhqH+TYP4s4Jlg/grg/rj9bwNujFt+B+gRTOXA0KC8IzAPaB0s/wS4BegA5PNFo912yf7/15T8qUl4OhFJDe6+w8z+TGygn5IEd1voQVfXZrYaqOi6exkQfztouruXA++Z2RqgL3AOcELcVcuhxJLJHuBNd19bxflOAnLd/aPgnE8SGzfj2QTjhVgi6B/r5gmAQ8ysbXD+x82sN+BA0wM4ZoX33X1BMD8U6A+8GpyrGbHuKnYAu4BHzOx5oEGPiCh1Q8lCGprfE7uF8qe4slKCW6pBR3rN4tbtjpsvj1suZ9/f/8r93jhgwA/dfU78CjPLIXZlURWrpvxAZACnuPs+CdHM7gPmuvsFFhsXJbea/T//9wjED7EaH7cBL7n72MoHMLMhxDokHANMAL5yYFWQdKNnFtKguPtWYDqxh8UV1gGDg/lR1O4b9yVmlhE8x+hF7DbMHOD7QffymNkxFj5A0hvAGWbWMXj4PRb49wHG8iKxP9AE5x0YzB5K7FYaxG49VdhJrLO6CusIuls3sxOJ3TqrygJgmJkdHWzbKqhjG+BQd58NXEfsAbo0ckoW0hDdTex+e4WHif2BfhM4meq/9dckn9gf9X8CV7v7LmJDx64AFpnZO8BDhFyNB7e8JgFzgSXAInc/0O61rwGyg4fLK4Crg/LfAr82s1eJjbNdYS6x21aLzeybxMZN6WBmi4HvExv7uapYPyKWdJ6yWM+7C4jdfmsLPBeU/ZsqXiqQxke9zoqISChdWYiISCglCxERCaVkISIioZQsREQklJKFiIiEUrIQEZFQShYiIhLq/wFQzFlILUOOFgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "k_feat = [len(k) for k in sbs.subsets_]\n",
    "plt.plot(k_feat, sbs.scores_, marker='o')\n",
    "plt.ylim([0.7, 1.1])\n",
    "plt.ylabel('Accuracy')\n",
    "plt.xlabel('Number of features')\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['B', 'C', 'E', 'L', 'N'], dtype='object')\n"
     ]
    }
   ],
   "source": [
    "k5 = list(sbs.subsets_[8])\n",
    "print(data.columns[1:][k5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training accuracy: 0.9838709677419355\n",
      "Test accuracy: 0.9444444444444444\n"
     ]
    }
   ],
   "source": [
    "# Accuracy without feature selection\n",
    "knn.fit(X_train_std, y_train)\n",
    "print('Training accuracy:', knn.score(X_train_std, y_train))\n",
    "print('Test accuracy:', knn.score(X_test_std, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training accuracy: 0.9596774193548387\n",
      "Test accuracy: 0.9629629629629629\n"
     ]
    }
   ],
   "source": [
    "# دAccuracy with feature selection\n",
    "knn.fit(X_train_std[:, k5], y_train)\n",
    "print('Training accuracy:', knn.score(X_train_std[:, k5], y_train))\n",
    "print('Test accuracy:', knn.score(X_test_std[:, k5], y_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 1) B                              0.182483\n",
      " 2) C                              0.158610\n",
      " 3) D                              0.150948\n",
      " 4) E                              0.131987\n",
      " 5) F                              0.106589\n",
      " 6) G                              0.078243\n",
      " 7) H                              0.060718\n",
      " 8) I                              0.032033\n",
      " 9) J                              0.025400\n",
      "10) K                              0.022351\n",
      "11) L                              0.022078\n",
      "12) M                              0.014645\n",
      "13) N                              0.013916\n"
     ]
    }
   ],
   "source": [
    "# feature selection with randomForest\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "feat_labels = data.columns[1:]\n",
    "forest = RandomForestClassifier(n_estimators=10000, random_state=0, n_jobs=-1) \n",
    "forest.fit(X_train, y_train)\n",
    "importances = forest.feature_importances_\n",
    "indices = np.argsort(importances)[::-1]\n",
    "for f in range(X_train.shape[1]):\n",
    "    print(\"%2d) %-*s %f\" % (f + 1, 30, feat_labels[f], importances[indices[f]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGu5JREFUeJzt3X+cXXV95/HX22Aibf2FGWtIsMGSxVLcxRrQbiu1smholbAs1FCq0NJSu8vD2p+gu0UfKe7W3e2ya8u6oqIWf4ACanYNRluKj20VmgERiJp2iNQMSWUgoIiIDXz2j3ume7nOZM4kN8mZmdfz8biPOef7Pef7/Z47ybzv99xzz01VIUlS1zzpYA9AkqSpGFCSpE4yoCRJnWRASZI6yYCSJHWSASVJ6iQDSpLUSQaUOivJ3UkeSfLtvsfh+9jmy5KMD2uMLft8f5JLDmSf00ny1iQfPNjjkNowoNR1r66qH+p77DiYg0lyyMHsf1/M5bFrYTKgNCcleUmSzyd5MMmXkrysr+6Xk3wlyUNJtiX59ab8B4HrgcP7Z2SDM5zBWVYzk7swye3Aw0kOafa7NslEkq8leUPLca9MUs0Ytyd5IMnrkxyf5PbmeP60b/tzk/x1kj9J8s0kX01yUl/94Uk2JNmVZCzJr/XVvTXJNUk+mORbwOuBNwOvaY79S3t6vvqfiyS/k+TeJDuT/HJf/aFJ/jjJ3zfj+6skh7b4HZ3b9PVQ8/yd3eb508LiKyrNOUmWA58CXgt8GjgJuDbJ86tqArgXeBWwDTgRuD7J5qq6NckpwAerakVfe226PQv4eeA+4HHgfwOfbMpXAH+eZGtVbWp5GC8GVjXj29Acx78Cngx8McnHqupzfdteAywFTgeuS3JkVe0CPgJsAQ4Hng98Nsm2qvqLZt+1wJnA64AlTRtHVdUv9Y1l2uerqX8O8HRgOXAycE2ST1TVA8B/BX4c+JfAPzRjfXxPvyPgO8A7gOOramuSZcBhLZ83LSDOoNR1n2hegT+Y5BNN2S8BG6tqY1U9XlWfBUaBnwOoqk9V1V3V8zngM8BL93Ec76iq7VX1CHA8MFJV66vqe1W1DXg3sG4W7f1hVX23qj4DPAx8pKrurap7gP8LvLBv23uB/15V/1hVVwNbgZ9PcgTw08CFTVu3Ae+hFwqTvlBVn2iep0emGkiL5+sfgfVN/xuBbwNHJ3kS8CvAb1bVPVX1WFV9vqoeZYbfEb2QPzbJoVW1s6q2zOK50wJhQKnrTquqZzSP05qyHwHO7AuuB+n9oV4GkOSUJDc1p70epPdHcek+jmN73/KP0DtN2N//m4EfnkV73+hbfmSK9R/qW7+nnnhX57+nN2M6HNhVVQ8N1C2fZtxTavF83V9Vu/vWv9OMbynwFOCuKZqd9ndUVQ8Dr6F3ynFnkk81MyvpCQwozUXbgSv7gusZVfWDVfVHSZYA19I79fTDVfUMYCMweR5vqtv3Pwz8QN/6c6bYpn+/7cDXBvp/alX93BT7DcPyPPE85HOBHc3jsCRPHai7Z5pxf996i+drT+4Dvgv86BR10/6OAKpqU1WdTO9FxVfpzUClJzCgNBd9EHh1klcmWZTkKc2b+SuAxfTea5kAdjfvOb2ib99vAM9K8vS+stuAn0tyWJLnAG+cof+/Ab7VXDhxaDOGY5McP7QjfKJnA29I8uQkZwI/Ru/02Xbg88B/ap6Dfw6cB3xoD219A1jZnJ6DmZ+vaVXV48AVwH9rLtZYlOQnm9Cb9neU5IeTnJreRSuP0jtl+NgsnxMtAAaU5pzmD/NaeqfVJui9Wv894EnN6a43AB8FHgB+kd5FCJP7fpXehQXbmlNPhwNXAl8C7qb3/svVM/T/GPBq4Djga/RmEu+hdyHB/nAzvQsq7gPeBpxRVfc3dWcBK+nNpj4OvKV5v2c6H2t+3p/k1pmerxZ+F7gD2AzsAt5O7/cw7e+oefxOM+ZdwM8A/3YWfWqBiF9YKHVXknOBX62qnz7YY5EONGdQkqROMqAkSZ3kKT5JUic5g5IkddKcutXR0qVLa+XKlQd7GJKkfXDLLbfcV1UjM203pwJq5cqVjI6OHuxhSJL2QZK/b7Odp/gkSZ1kQEmSOsmAkiR1kgElSeokA0qS1EkGlCSpkwwoSVInGVCSpE4yoCRJnWRASZI6aU7d6mhfXbd159DbPP3oZUNvU5LkDEqS1FEGlCSpkwwoSVInGVCSpE4yoCRJndQqoJKsSbI1yViSi6aoPzHJrUl2Jzmjr/xnk9zW9/huktOauvcn+Vpf3XHDOyxJ0lw342XmSRYBlwEnA+PA5iQbqurLfZt9HTgX+N3+favqL4HjmnYOA8aAz/Rt8ntVdc2+HIAkaX5q8zmoE4CxqtoGkOQqYC3wTwFVVXc3dY/voZ0zgOur6jt7PVpJ0oLR5hTfcmB73/p4UzZb64CPDJS9LcntSS5NsmSqnZKcn2Q0yejExMRedCtJmovaBFSmKKvZdJJkGfACYFNf8ZuA5wPHA4cBF061b1VdXlWrq2r1yMjIbLqVJM1hbQJqHDiib30FsGOW/fwC8PGq+sfJgqraWT2PAu+jdypRkiSgXUBtBlYlOTLJYnqn6jbMsp+zGDi918yqSBLgNODOWbYpSZrHZgyoqtoNXEDv9NxXgI9W1ZYk65OcCpDk+CTjwJnAu5Jsmdw/yUp6M7DPDTT9oSR3AHcAS4FL9v1wJEnzRau7mVfVRmDjQNnFfcub6Z36m2rfu5niooqqevlsBipJWli8k4QkqZMMKElSJxlQkqROMqAkSZ1kQEmSOsmAkiR1kgElSeokA0qS1EkGlCSpkwwoSVInGVCSpE4yoCRJnWRASZI6yYCSJHVSq6/b0Oxct3Xn0Ns8/ehlQ29TkrrMGZQkqZMMKElSJxlQkqROMqAkSZ1kQEmSOsmAkiR1UquASrImydYkY0kumqL+xCS3Jtmd5IyBuseS3NY8NvSVH5nk5iR/l+TqJIv3/XAkSfPFjAGVZBFwGXAKcAxwVpJjBjb7OnAu8OEpmnikqo5rHqf2lb8duLSqVgEPAOftxfglSfNUmw/qngCMVdU2gCRXAWuBL09uUFV3N3WPt+k0SYCXA7/YFH0AeCvwzpbjVsMPBUuar9qc4lsObO9bH2/K2npKktEkNyU5rSl7FvBgVe2eqc0k5zf7j05MTMyiW0nSXNZmBpUpymoWfTy3qnYkeR5wQ5I7gG+1bbOqLgcuB1i9evVs+pUkzWFtZlDjwBF96yuAHW07qKodzc9twI3AC4H7gGckmQzIWbUpSZr/2gTUZmBVc9XdYmAdsGGGfQBI8swkS5rlpcBPAV+uqgL+Epi84u8c4JOzHbwkaf6aMaCa94kuADYBXwE+WlVbkqxPcipAkuOTjANnAu9KsqXZ/ceA0SRfohdIf1RVkxdXXAj8dpIxeu9JvXeYByZJmttafd1GVW0ENg6UXdy3vJneabrB/T4PvGCaNrfRu0JQkqTv450kJEmdZEBJkjrJgJIkdZIBJUnqJANKktRJBpQkqZMMKElSJxlQkqROMqAkSZ1kQEmSOsmAkiR1kgElSeokA0qS1EkGlCSpkwwoSVInGVCSpE5q9YWF0nVbd+6Xdk8/etl+aVfS3OcMSpLUSQaUJKmTDChJUicZUJKkTjKgJEmd1CqgkqxJsjXJWJKLpqg/McmtSXYnOaOv/LgkX0iyJcntSV7TV/f+JF9LclvzOG44hyRJmg9mvMw8ySLgMuBkYBzYnGRDVX25b7OvA+cCvzuw+3eA11XV3yU5HLglyaaqerCp/72qumZfD0KSNP+0+RzUCcBYVW0DSHIVsBb4p4Cqqrubusf7d6yqv+1b3pHkXmAEeBBJkvagzSm+5cD2vvXxpmxWkpwALAbu6it+W3Pq79IkS6bZ7/wko0lGJyYmZtutJGmOahNQmaKsZtNJkmXAlcAvV9XkLOtNwPOB44HDgAun2reqLq+q1VW1emRkZDbdSpLmsDYBNQ4c0be+AtjRtoMkTwM+BfyHqrppsryqdlbPo8D76J1KlCQJaBdQm4FVSY5MshhYB2xo03iz/ceBP6uqjw3ULWt+BjgNuHM2A5ckzW8zBlRV7QYuADYBXwE+WlVbkqxPcipAkuOTjANnAu9KsqXZ/ReAE4Fzp7ic/ENJ7gDuAJYClwz1yCRJc1qru5lX1UZg40DZxX3Lm+md+hvc74PAB6dp8+WzGqkkaUHxThKSpE4yoCRJnWRASZI6yYCSJHWSASVJ6iQDSpLUSQaUJKmTDChJUicZUJKkTjKgJEmdZEBJkjrJgJIkdZIBJUnqJANKktRJBpQkqZMMKElSJxlQkqROavWNutKBdN3WnUNv8/Sjlw29TUn7lzMoSVInGVCSpE4yoCRJndQqoJKsSbI1yViSi6aoPzHJrUl2JzljoO6cJH/XPM7pK39RkjuaNt+RJPt+OJKk+WLGgEqyCLgMOAU4BjgryTEDm30dOBf48MC+hwFvAV4MnAC8Jckzm+p3AucDq5rHmr0+CknSvNNmBnUCMFZV26rqe8BVwNr+Darq7qq6HXh8YN9XAp+tql1V9QDwWWBNkmXA06rqC1VVwJ8Bp+3rwUiS5o82AbUc2N63Pt6UtTHdvsub5b1pU5K0ALQJqKneG6qW7U+3b+s2k5yfZDTJ6MTERMtuJUlzXZuAGgeO6FtfAexo2f50+443yzO2WVWXV9Xqqlo9MjLSsltJ0lzXJqA2A6uSHJlkMbAO2NCy/U3AK5I8s7k44hXApqraCTyU5CXN1XuvAz65F+OXJM1TMwZUVe0GLqAXNl8BPlpVW5KsT3IqQJLjk4wDZwLvSrKl2XcX8If0Qm4zsL4pA/gN4D3AGHAXcP1Qj0ySNKe1uhdfVW0ENg6UXdy3vJknnrLr3+4K4IopykeBY2czWEnSwuGdJCRJnWRASZI6yYCSJHWSASVJ6iQDSpLUSQaUJKmTDChJUicZUJKkTjKgJEmdZEBJkjrJgJIkdZIBJUnqJANKktRJBpQkqZMMKElSJxlQkqROMqAkSZ1kQEmSOsmAkiR1kgElSeokA0qS1EmHHOwBSAfLdVt37pd2Tz962X5pV1poWs2gkqxJsjXJWJKLpqhfkuTqpv7mJCub8rOT3Nb3eDzJcU3djU2bk3XPHuaBSZLmthkDKski4DLgFOAY4Kwkxwxsdh7wQFUdBVwKvB2gqj5UVcdV1XHAa4G7q+q2vv3OnqyvqnuHcDySpHmizQzqBGCsqrZV1feAq4C1A9usBT7QLF8DnJQkA9ucBXxkXwYrSVo42gTUcmB73/p4UzblNlW1G/gm8KyBbV7D9wfU+5rTe38wRaABkOT8JKNJRicmJloMV5I0H7QJqKmCo2azTZIXA9+pqjv76s+uqhcAL20er52q86q6vKpWV9XqkZGRFsOVJM0HbQJqHDiib30FsGO6bZIcAjwd2NVXv46B2VNV3dP8fAj4ML1TiZIkAe0CajOwKsmRSRbTC5sNA9tsAM5pls8AbqiqAkjyJOBMeu9d0ZQdkmRps/xk4FXAnUiS1Jjxc1BVtTvJBcAmYBFwRVVtSbIeGK2qDcB7gSuTjNGbOa3ra+JEYLyqtvWVLQE2NeG0CPhz4N1DOSJJ0rzQ6oO6VbUR2DhQdnHf8nfpzZKm2vdG4CUDZQ8DL5rlWCVJC4i3OpIkdZIBJUnqJANKktRJBpQkqZMMKElSJxlQkqROMqAkSZ1kQEmSOsmAkiR1kgElSeokA0qS1EkGlCSpk1rdLFbSvrlu686ht3n60cuG3qbUJc6gJEmdZEBJkjrJgJIkdZIBJUnqJANKktRJBpQkqZMMKElSJxlQkqROMqAkSZ3UKqCSrEmyNclYkoumqF+S5Oqm/uYkK5vylUkeSXJb8/hfffu8KMkdzT7vSJJhHZQkae6bMaCSLAIuA04BjgHOSnLMwGbnAQ9U1VHApcDb++ruqqrjmsfr+8rfCZwPrGoea/b+MCRJ802bGdQJwFhVbauq7wFXAWsHtlkLfKBZvgY4aU8zoiTLgKdV1ReqqoA/A06b9eglSfNWm4BaDmzvWx9vyqbcpqp2A98EntXUHZnki0k+l+SlfduPz9AmAEnOTzKaZHRiYqLFcCVJ80GbgJpqJlQtt9kJPLeqXgj8NvDhJE9r2WavsOryqlpdVatHRkZaDFeSNB+0Cahx4Ii+9RXAjum2SXII8HRgV1U9WlX3A1TVLcBdwD9rtl8xQ5uSpAWsTUBtBlYlOTLJYmAdsGFgmw3AOc3yGcANVVVJRpqLLEjyPHoXQ2yrqp3AQ0le0rxX9Trgk0M4HknSPDHjFxZW1e4kFwCbgEXAFVW1Jcl6YLSqNgDvBa5MMgbsohdiACcC65PsBh4DXl9Vu5q63wDeDxwKXN88JEkCWn6jblVtBDYOlF3ct/xd4Mwp9rsWuHaaNkeBY2czWEnSwuGdJCRJnWRASZI6yYCSJHWSASVJ6iQDSpLUSQaUJKmTDChJUicZUJKkTjKgJEmdZEBJkjrJgJIkdZIBJUnqJANKktRJBpQkqZMMKElSJxlQkqROavWFhZLmhuu27twv7Z5+9LL90q60J86gJEmdZEBJkjrJgJIkdZIBJUnqpFYBlWRNkq1JxpJcNEX9kiRXN/U3J1nZlJ+c5JYkdzQ/X963z41Nm7c1j2cP66AkSXPfjFfxJVkEXAacDIwDm5NsqKov9212HvBAVR2VZB3wduA1wH3Aq6tqR5JjgU3A8r79zq6q0SEdiyRpHmkzgzoBGKuqbVX1PeAqYO3ANmuBDzTL1wAnJUlVfbGqdjTlW4CnJFkyjIFLkua3Np+DWg5s71sfB1483TZVtTvJN4Fn0ZtBTfo3wBer6tG+svcleQy4FrikqmqW45d0kOyPz1z5eSv1azODyhRlg0Gyx22S/Di9036/3ld/dlW9AHhp83jtlJ0n5ycZTTI6MTHRYriSpPmgTUCNA0f0ra8Adky3TZJDgKcDu5r1FcDHgddV1V2TO1TVPc3Ph4AP0zuV+H2q6vKqWl1Vq0dGRtockyRpHmgTUJuBVUmOTLIYWAdsGNhmA3BOs3wGcENVVZJnAJ8C3lRVfz25cZJDkixtlp8MvAq4c98ORZI0n8z4HlTzntIF9K7AWwRcUVVbkqwHRqtqA/Be4MokY/RmTuua3S8AjgL+IMkfNGWvAB4GNjXhtAj4c+DdQzwuSfPEgby/oO+rdUurm8VW1UZg40DZxX3L3wXOnGK/S4BLpmn2Re2HKUlaaLyThCSpkwwoSVInGVCSpE4yoCRJnWRASZI6yYCSJHWSASVJ6qRWn4OSJA3Pgfzw8VxmQEnSPDaX747hKT5JUicZUJKkTjKgJEmdZEBJkjrJgJIkdZIBJUnqJANKktRJBpQkqZMMKElSJxlQkqROMqAkSZ1kQEmSOsmAkiR1UquASrImydYkY0kumqJ+SZKrm/qbk6zsq3tTU741ySvbtilJWthmDKgki4DLgFOAY4CzkhwzsNl5wANVdRRwKfD2Zt9jgHXAjwNrgP+ZZFHLNiVJC1ibGdQJwFhVbauq7wFXAWsHtlkLfKBZvgY4KUma8quq6tGq+how1rTXpk1J0gLW5gsLlwPb+9bHgRdPt01V7U7yTeBZTflNA/sub5ZnahOAJOcD5zer306ytcWY99VS4L4D0M+B7Mtjmht9zcdjOpB9eUxzo68fabNRm4DKFGXVcpvpyqeauQ222Susuhy4fE8DHLYko1W1ej715THNjb7m4zEdyL48prnTVxttTvGNA0f0ra8Adky3TZJDgKcDu/awb5s2JUkLWJuA2gysSnJkksX0LnrYMLDNBuCcZvkM4IaqqqZ8XXOV35HAKuBvWrYpSVrAZjzF17yndAGwCVgEXFFVW5KsB0aragPwXuDKJGP0Zk7rmn23JPko8GVgN/DvquoxgKnaHP7h7bUDeUrxQPXlMc2NvubjMR3IvjymudPXjNKb6EiS1C3eSUKS1EkGlCSpkwwoSVInGVAHSJKjkvzUFOUvTfKjB2NM0jAleSjJt6Z5TCS5KclJQ+xvxR7qXj2sflqM440Hqq+FxoskppBkKXB/DfHJSfJ/gDdX1e0D5auBt1TVfvkPlWQEoKom9lP7v19V/7lZPrOqPtZX9x+r6s1D6ue5VfX1YbQ1Qz9rgRVVdVmzfjMw0lT/flVdM8S+/oRpPqAOUFVvGFZfB1tz/81jgQ9V1bFDanMr8Mqqunug/FeAf19VB+SFX5KvV9VzD0Rfw5Tk4j1UV1X94QEbzDQW/AwqyUuS3JjkuiQvTHIncCfwjSRrhtjVysFwAqiqUWDlEPshPW9Nch/wVeBvm1ewe/oHubfW9S2/aaBumM/fJyYXklw7xHYH/T5P/EzeEuB44GXAbwy5r1HgluZxat/y5GNo9jC7eSjJt4bZ11Sq6rGq+hLwJ0Ns9reAzyZZNVmQ5E1N+c8MsZ+ZTHXHnL1rKNmwp8ew+mk8PMWj6N38+8Ih97VX2tzqaL77U+DN9O5+cQNwSlXdlOT5wEeATw+pn6fsoe7QIfUx6Y3ATwHHNzfpJcnzgHcm+a2qunSIfWWa5anWh9XP84bY7qDFVdV/n8i/qqr7gfuT/OAwO6qqyRssk+SN/evDVlVP3V9tz0ZVvWuIbW1M8ihwfZLTgF+l92LixKp6YFj9tBnKENv6SXr3Kf0IcDPD/T/0BFX1x5PLSZ4K/CbwK/Ru3v3H0+13IC34GRRwSFV9pjk19Q9VdRNAVX11yP1sTvJrg4VJzmPIr5aB1wFnTYYTQFVtA36pqRummmZ5qvX91c8wPfMJnVZd0Lc6wv7jufa9UFV/AZwL3EjvhctJ+yOc9jQDBQ4fYlfPofeC+VjgfwAnA/dV1eeq6nND7AeAJIcluQS4nd6E5Seq6sKqunfYfe0NZ1DweN/yIwN1w/yj8Ubg40nO5v8H0mpgMfCvh9gPwJOr6vvuSFxVE0mePOS+/kVziijAoX2ni8KeZ43D7Keq6mlD6ufmJL9WVe/uL0zy6/Ru06WOaMJh8qbUS4CTgHubr/oZ5r+JAzYDbe6082ng00mWAGcBNyZZX1XDPD1Kkv8CnE7v7hEvqKpvD7P9YVjwF0kkeYzeudfQO9X2nckq4ClVNdQ/6El+lt6rI4AtVXXDMNtv+ri1qn5itnWCJM+m937Xo8CtTfGL6P0BPK2qvjHEvib/wAL8AE/8tzfUP7CaO5pg+nl64bSS3nuiV1TVPUPu53F6/85388QX453597fgA2o+6gvd76tiP4TufJTk5fS+CRr20wsJaVCSD9B7AXs9vS97vfMgD+mgMqAkqSOaWc3ki8tOzmoOJANKktRJXsUnSeokA0qS1EkGlCSpkwwoSVIn/T8iLAipWxAJCQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.title('Feature Importances')\n",
    "plt.bar(range(X_train.shape[1]), importances[indices], color='lightblue', align='center')\n",
    "plt.xticks(range(X_train.shape[1]),feat_labels, rotation=90)                                    \n",
    "plt.xlim([-1, X_train.shape[1]])\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}