extratrees_nn_lf.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from subprocess import check_output\n",
    "from sklearn import ensemble, metrics, linear_model\n",
    "import random\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "rnd=12\n",
    "random.seed(rnd)\n",
    "n_ft=15 #Number of features to add\n",
    "max_elts=5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class addNearestNeighbourLinearFeatures:\n",
    "    \n",
    "    def __init__(self, n_neighbours=1, max_elts=None, verbose=True, random_state=None):\n",
    "        self.rnd = random_state\n",
    "        self.n = n_neighbours\n",
    "        self.max_elts = max_elts\n",
    "        self.verbose = verbose\n",
    "        self.neighbours = []\n",
    "        self.clfs = []\n",
    "        \n",
    "    def fit(self, train, y):\n",
    "        if self.rnd != None:\n",
    "            random.seed(rnd)\n",
    "        if self.max_elts == None:\n",
    "            self.max_elts = len(train.columns)\n",
    "        list_vars = list(train.columns)\n",
    "        random.shuffle(list_vars)\n",
    "        \n",
    "        lastscores = np.zeros(self.n) + 1e15\n",
    "\n",
    "        for elt in list_vars[:self.n]:\n",
    "            self.neighbours.append([elt])\n",
    "        list_vars = list_vars[self.n:]\n",
    "        \n",
    "        for elt in list_vars:\n",
    "            indice = 0\n",
    "            scores = []\n",
    "            for elt2 in self.neighbours:\n",
    "                if len(elt2) < self.max_elts:\n",
    "                    clf = linear_model.LinearRegression(fit_intercept=False, \n",
    "                                                        normalize=True, copy_X=True) \n",
    "                    clf.fit(train[elt2+[elt]], y)\n",
    "                    scores.append(metrics.log_loss(y,clf.predict(train[elt2 + [elt]])))\n",
    "                    indice=indice+1\n",
    "                else:\n",
    "                    scores.append(lastscores[indice])\n",
    "                    indice=indice+1\n",
    "            gains = lastscores - scores\n",
    "            if gains.max() > 0:\n",
    "                temp = gains.argmax()\n",
    "                lastscores[temp] = scores[temp]\n",
    "                self.neighbours[temp].append(elt)\n",
    "\n",
    "        indice = 0\n",
    "        for elt in self.neighbours:\n",
    "            clf = linear_model.LinearRegression(fit_intercept=False, normalize=True, copy_X=True) \n",
    "            clf.fit(train[elt], y)\n",
    "            self.clfs.append(clf)\n",
    "            if self.verbose:\n",
    "                print(indice, lastscores[indice], elt)\n",
    "            indice = indice + 1\n",
    "                    \n",
    "    def transform(self, train):\n",
    "        indice = 0\n",
    "        for elt in self.neighbours:\n",
    "            train['_'.join(pd.Series(elt).sort_values().values)] = self.clfs[indice].predict(train[elt])\n",
    "            indice = indice + 1\n",
    "        return train\n",
    "    \n",
    "    def fit_transform(self, train, y):\n",
    "        self.fit(train, y)\n",
    "        return self.transform(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv(\"data/train.csv\")\n",
    "target = train['target'].values\n",
    "test = pd.read_csv(\"data/test.csv\")\n",
    "id_test = test['ID'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#train['v22-1']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[0]))\n",
    "#test['v22-1']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[0]))\n",
    "#train['v22-2']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[1]))\n",
    "#test['v22-2']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[1]))\n",
    "#train['v22-3']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[2]))\n",
    "#test['v22-3']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[2]))\n",
    "#train['v22-4']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[3]))\n",
    "#test['v22-4']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[3]))\n",
    "\n",
    "drop_list=['v8','v23','v25','v31','v36','v37','v46',\n",
    "                        'v51','v53','v54','v63','v73','v75','v79','v81','v82',\n",
    "                        'v89','v92','v95','v105','v107','v108','v109','v110',\n",
    "                        'v116','v117','v118','v119','v123','v124','v128']\n",
    "train = train.drop(['ID','target'] + drop_list,axis=1).fillna(-999)\n",
    "test = test.drop(['ID'] + drop_list,axis=1).fillna(-999)\n",
    "\n",
    "refcols=list(train.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for elt in refcols:\n",
    "    if train[elt].dtype=='O':\n",
    "        train[elt], temp = pd.factorize(train[elt])\n",
    "        test[elt]=temp.get_indexer(test[elt])\n",
    "    else:\n",
    "        train[train[elt] == 20] = -999\n",
    "        test[test[elt] == 20] = -999\n",
    "        train[elt]=train[elt].round(5)\n",
    "        test[elt]=test[elt].round(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 0.602739434722 ['v5', 'v98', 'v18', 'v14', 'v50']\n",
      "1 0.595501697576 ['v17', 'v71', 'v65', 'v94', 'v90']\n",
      "2 0.810994034632 ['v83', 'v103', 'v87', 'v32', 'v93']\n",
      "3 0.56572540597 ['v61', 'v19', 'v4', 'v91', 'v52']\n",
      "4 1.49456076359 ['v121', 'v24', 'v120', 'v48', 'v85']\n",
      "5 0.72309429408 ['v66', 'v129', 'v56', 'v35', 'v86']\n",
      "6 0.592261091924 ['v44', 'v101', 'v64', 'v76', 'v29']\n",
      "7 1.10813159468 ['v111', 'v40', 'v45', 'v130', 'v10']\n",
      "8 0.59014496359 ['v27', 'v60', 'v28', 'v49', 'v126']\n",
      "9 0.598807749711 ['v100', 'v38', 'v26', 'v67', 'v13']\n",
      "10 0.559437953509 ['v114', 'v43', 'v97', 'v80', 'v11']\n",
      "11 1.35177306603 ['v16', 'v69', 'v12', 'v113', 'v47']\n",
      "12 0.666550798928 ['v15', 'v77', 'v78', 'v131', 'v33']\n",
      "13 0.889217967428 ['v7', 'v57', 'v127', 'v3', 'v62']\n",
      "14 1.59216441132 ['v112', 'v21', 'v6', 'v39', 'v125']\n"
     ]
    }
   ],
   "source": [
    "a=addNearestNeighbourLinearFeatures(n_neighbours=n_ft, max_elts=max_elts, verbose=True, random_state=rnd)\n",
    "a.fit(train, target)\n",
    "\n",
    "train = a.transform(train)\n",
    "test = a.transform(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 35 55\n"
     ]
    }
   ],
   "source": [
    "preds = list()\n",
    "mf = 55\n",
    "md = 35\n",
    "for rnd in range(1):\n",
    "    print(rnd, md, mf, flush=True)\n",
    "    clf = ensemble.ExtraTreesClassifier(n_estimators=3750, max_features=mf ,criterion='entropy',\n",
    "                                        min_samples_split=4, max_depth=md, min_samples_leaf=2, \n",
    "                                        n_jobs = -1, random_state=rnd)\n",
    "    clf.fit(train,target)\n",
    "    pred_et = clf.predict_proba(test)[:, 1]\n",
    "    preds.append(pred_et)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.40670835,  0.89109065,  0.85985541, ...,  0.89307127,\n",
       "        0.92975251,  0.55945507])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(preds)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f7e83c3f8d0>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEACAYAAABGYoqtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFidJREFUeJzt3X2MZXV5wPHvgytFLbKgYSddkMHqwmLVLbZoo43jS0Vs\nAsTYDdqqI9Q0aivapnW3SUP/aawmTdFYSKhWwGgQJalEEZHC1JhIwTewLsKq3ZVdu+Mb0GijXeTp\nH/eM57LOYe7LOfeec+/3k0xyz+8+584zT3buM+f33Hs3MhNJktZz1LQTkCS1l01CklTJJiFJqmST\nkCRVsklIkirZJCRJlTZsEhHxgYhYjYi7+taOj4ibIuKeiPhMRBzXd9/uiNgbEXdHxMv61s+MiLsi\n4t6IuLRv/eiIuKY45wsR8ZQ6f0BJ0ugGuZL4IHD2EWu7gJsz8zTgFmA3QEScAewEtgPnAJdFRBTn\nXA5clJnbgG0RsfaYFwE/ysynA5cC7x7j55Ek1WjDJpGZnwfuP2L5POCq4vZVwPnF7XOBazLzoczc\nB+wFzoqIBeDYzLyjiLu675z+x/o48JIRfg5JUgNGnUmcmJmrAJl5CDixWN8K3NcXd7BY2woc6Fs/\nUKw94pzM/DnwQEScMGJekqQa1TW4rvOzPWLjEEnSJGwa8bzViNiSmavFVtL3ivWDwMl9cScVa1Xr\n/ed8NyIeAzwxM3+03jeNCD9oSpJGkJkj/QE+6JVE8Mi/8K8Hlovbrwc+0bd+QfGKpVOBpwG3F1tS\nD0bEWcUg+3VHnPP64vYf0BuEV8pMvzK55JJLpp5DW76shbWwFo/+NY4NryQi4iPAEvCkiPgOcAnw\n98DHIuJCYD+9VzSRmXsi4lpgD3AYeHOWGb4FuBI4BrghM28s1j8AfCgi9gI/BC4Y6yeaE/v27Zt2\nCq1hLUrWomQt6rFhk8jM11Tc9dKK+HcC71xn/UvAM9dZ/xlFk5EktYvvuO6o5eXlaafQGtaiZC1K\nXazFwsIiETHQ18LC4kRyinH3qyYpIrJL+UrSMHoj20Gf42LgeUNEkA0PrtUyKysr006hNaxFyVqU\nrEU9bBKSpEpuN0lSS7jdJEnqFJtER7nfWrIWJWtRshb1sElIkio5k5CkhiwsLLK6un/Is9o1k7BJ\nSFJDhhtEQ+8j8trVJNxu6ij3W0vWomQtStaiHjYJSVIlt5skqSFuN0mSZppNoqPcby1Zi5K1KFmL\netgkJGkIw3yc9yxwJiFJQxj285WcSUiSZpZNoqPcby1Zi5K1KFmLetgkJEmVnElI0hCcSUiSVLBJ\ndJT7rSVrUbIWpWFqMW8vax3GpmknIEnT1vs472G2kOaHMwlJc6+5OYMzCUnSDLNJdJR7zyVrUbIW\nJWtRD5uEJKmSMwlJc8+ZRDWvJCRJlWwSHeV+a8lalKxFyVrUwyYhSarkTELS3HMmUc0rCUlSJZtE\nR7nfWrIWJWtROuGEBT+PqQZ+dpOkmXT//av4eUzjG2smERG7gT8Cfg58DXgD8ATgo8ApwD5gZ2Y+\n2Bd/IfAQcHFm3lSsnwlcCRwD3JCZb6v4fs4kJA2kHXOGOZ5JRMQpwBuB38zMZ9G7Knk1sAu4OTNP\nA24BdhfxZwA7ge3AOcBlUV7nXQ5clJnbgG0RcfaoeUmS6jPOTOJ/gP8DnhARm4DHAQeB84Cripir\ngPOL2+cC12TmQ5m5D9gLnBURC8CxmXlHEXd13zmq4N5zyVqUrIXqNnKTyMz7gX8AvkOvOTyYmTcD\nWzJztYg5BJxYnLIVuK/vIQ4Wa1uBA33rB4o1SdKUjTy4joinAm+nN3t4EPhYRPwhv7yhVusQYXl5\nmcXFRQA2b97Mjh07WFpaAsq/oubheGlpqVX5eNye4zVtyafO41e+8oJiID2oFWCp7zY1HLPB/ZOK\n79VkvXqtrKxw5ZVXAvzi+XJUIw+uI2In8HuZ+cbi+LXA84AXA0uZuVpsJd2amdsjYheQmfmuIv5G\n4BJg/1pMsX4B8MLMfNM639PBtTTHujeMnuPBNXAP8LyIOKYYQL8E2ANcDywXMa8HPlHcvh64ICKO\njohTgacBtxdbUg9GxFnF47yu7xxVOPKvxnlmLUpdrIX/v3S7jbzdlJl3RsTVwJfovQT2K8AVwLHA\ntRFxIb2rhJ1F/J6IuJZeIzkMvLnvsuAtPPIlsDeOmpekbvH/l243P7tJ0lTN9hbSfG83SZJmnE2i\no7q499wUa1GyFqqbn90kSZ30KxMZ5juTkDRVziQmk4czCUlS7WwSHeXec8lalKyF6maTkCRVciYh\naaqcSUwmD2cSkqTa2SQ6yr3nkrUoWQvVzSYhSarkTEJS7RYWFosP7hvUtOcBbYhtNo9RZxI2CUm1\ncxg9SmyzeTi4njPuPZesRclaqG42CUlSJbebJNXO7aZRYpvNw+0mSVLtbBId5d5zyVqUrIXqZpOQ\nJFVyJiGpds4kRoltNg9nEpKk2tkkOsq955K1KFkL1c0mIWkgCwuLRMRAX5odziQkDcQ5Q9Oxzebh\nTEKSVDubREe591yyFiVrobrZJCRJlZxJSBqIM4mmY5vNw5mEJKl2NomOcu+5ZC1K1kJ1s0lIkio5\nk5A0EGcSTcc2m4czCUlS7WwSHeXec8lalKyF6maTkCRVGmsmERHHAe8HfgN4GLgQuBf4KHAKsA/Y\nmZkPFvG7i5iHgIsz86Zi/UzgSuAY4IbMfFvF93MmIdVkYWGR1dX9Q5417X37WY5tNo9pzSTeQ+9J\nfTvwbOAbwC7g5sw8DbgF2A0QEWcAO4HtwDnAZVF+XOTlwEWZuQ3YFhFnj5mXpA30GkQO8aV5NHKT\niIgnAr+bmR8EyMyHiiuG84CrirCrgPOL2+cC1xRx+4C9wFkRsQAcm5l3FHFX952jCu49l6xFyVqo\nbuNcSZwK/CAiPhgRX46IKyLi8cCWzFwFyMxDwIlF/Fbgvr7zDxZrW4EDfesHijVJ0pSN0yQ2AWcC\n/5SZZwI/obfVdOR1qdepDVhaWpp2Cq1hLUrWQnXbNMa5B4D7MvOLxfF19JrEakRsyczVYivpe8X9\nB4GT+84/qVirWl/X8vIyi4uLAGzevJkdO3b84hdj7VLbY4893vi4ZwVY6rvNoxwPGz/oMRvcP2r8\n2tq4+Y36/ZuOX1tb7/4Veq8FAlhkHOO+uunfgTdm5r0RcQnw+OKuH2XmuyLiHcDxmbmrGFx/GHgu\nve2kzwJPz8yMiNuAtwJ3AJ8C3puZN67z/Xx1U2FlZcW/GgvWojRMLYZ7BzW04xVAsxzbbB6jvrpp\nnCsJ6D2xfzgiHgt8G3gD8Bjg2oi4ENhP7xVNZOaeiLgW2AMcBt7c94z/Fh75EthfahCSpMnzs5uk\nOeWVRNtim83Dz26SJNXOJtFRvh6+ZC1K1kJ1s0lIkio5k5DmlDOJtsU2m4czCUlS7WwSHeXec8la\nlKyF6maTkCRVciYhzSlnEm2LbTYPZxKSpNrZJDrKveeStShZC9XNJiFJquRMQppTziTaFttsHs4k\nJEm1s0l0lHvPJWtRshaqm01CklTJmYQ0QxYWFlld3T/EGe3YLze2+TxGnUnYJKQZMtwwuj1PYMY2\nn4eD6znj3nPJWkjNsUlIkiq53STNELebuhzbbB5uN0mSameT6Cj34UvWQmqOTUKSVMmZhDRDnEl0\nObbZPJxJSJJqZ5PoKPfhS9ZCao5NQpJUyZmENEOcSXQ5ttk8nElIkmpnk+go9+FL1kJqjk1CklTJ\nmYQ0Q5xJdDm22TycSUiSameT6Cj34UvWQmqOTUJquYWFRSJioC+pbs4kpJZrbs7Qnv1yY5vPY2oz\niYg4KiK+HBHXF8fHR8RNEXFPRHwmIo7ri90dEXsj4u6IeFnf+pkRcVdE3BsRl46bkySpHnVsN10M\n7Ok73gXcnJmnAbcAuwEi4gxgJ7AdOAe4LMrr48uBizJzG7AtIs6uIa+Z5j58yVpIzRmrSUTEScAr\ngPf3LZ8HXFXcvgo4v7h9LnBNZj6UmfuAvcBZEbEAHJuZdxRxV/edI0maonGvJP4R+EseuTG2JTNX\nATLzEHBisb4VuK8v7mCxthU40Ld+oFjTo1haWpp2Cq1hLaTmjNwkIuL3gdXM/Cq9CUoVJ82S1FGb\nxjj3+cC5EfEK4HHAsRHxIeBQRGzJzNViK+l7RfxB4OS+808q1qrW17W8vMzi4iIAmzdvZseOHb/4\nS3Jtb3oejvv34duQzzSP19bakk8TP1/P2vHSBseDxq+tbfR4o8YPeswG948av7Y2bn6jfv+m49fW\n1rt/BbiyOF5kHLW8BDYiXgj8RWaeGxHvBn6Yme+KiHcAx2fmrmJw/WHgufS2kz4LPD0zMyJuA94K\n3AF8CnhvZt64zvfxJbCFlZUVt1kKs14LXwI7L7HN5jHqS2CbaBInANfSuzrYD+zMzAeKuN3ARcBh\n4OLMvKlYfw69tncMcENmXlzxfWwSmjs2iXmJbTaPqTaJSbFJaB7ZJOYlttk8/IC/OeN7A0rWQmqO\nTUKSVMntJqnl3G6al9hm83C7SZJUO5tER7kPX7IWUnNsEpKkSs4kpJZzJjEvsc3m4UxC6hD/tzl1\nhU2io9yHL3WxFqur++n9FTjIlzQ9NglJUiVnEtIUtGPO0J79cmObz8OZhCSpdjaJjuriPnxTrIXU\nHJuEJKmSMwlpCpxJGDvpPJxJSJJqZ5PoKPfhS9ZCao5NQpJUyZmENAXOJIyddB7OJCRJtbNJdJT7\n8CVrITXHJiFJquRMQpoCZxLGTjoPZxLSlPl/RGgW2SQ6yn34Ultq4f8RoVlkk5AkVXImIdWke3OG\n9uyXG9t8Hs4kJEm1s0l0VFv24dvAWkjNsUlIkio5k5Bq4kzC2PFim83DmYQkqXY2iY5yH75kLaTm\n2CQkSZWcSUg1cSZh7HixzebhTEKSVLuRm0REnBQRt0TE1yPiaxHx1mL9+Ii4KSLuiYjPRMRxfefs\njoi9EXF3RLysb/3MiLgrIu6NiEvH+5Hmg/vwJWshNWecK4mHgD/PzGcAvwO8JSJOB3YBN2fmacAt\nwG6AiDgD2AlsB84BLovy4zAvBy7KzG3Atog4e4y8pNr4ya6ad7XNJCLiX4H3FV8vzMzViFgAVjLz\n9IjYBWRmvquI/zTwt8B+4JbMPKNYv6A4/03rfA9nEpqo2Z4ztGe/3Njm85jqTCIiFoEdwG3Alsxc\nBcjMQ8CJRdhW4L6+0w4Wa1uBA33rB4o1SdKUbRr3ASLiV4GPAxdn5o8j4sjWVuuf/svLyywuLgKw\nefNmduzYwdLSElDuTc/Dcf8+fBvymebx2lqTj9+zdrxUcby2VnX/qMeDfv9h49fWBs1n2PhBj9ng\n/lHj19bGzW/U7990/NraevevAFcWx4uMY6ztpojYBHwS+HRmvqdYuxtY6ttuujUzt6+z3XQjcAm9\n7aZbM3N7se520wBWVlZ+8WQ275qshdtNbctjlmObzWNa203/AuxZaxCF64Hl4vbrgU/0rV8QEUdH\nxKnA04Dbiy2pByPirGKQ/bq+c1TBBlGyFlJzRr6SiIjnA58Dvkb5fzL+NXA7cC1wMr2rhJ2Z+UBx\nzm7gIuAwve2pm4r159C7NjoGuCEzL674nl5JaKK8kmhbHrMc22weo15J+I7rjnK7qeR20yRi25LH\nLMc2m4fvuJYk1c4rCelReCXRtjxmObbZPLySkCTVzibRUX5eUclaSM2xSWju+HlM0uCcSWjuOGcY\nJbYtecxybLN5OJOQJNXOJtFR7sOXrIXUHJuEJKmSMwnNHWcSo8S2JY9Zjm02D2cSkqTa2SQ6yn34\nkrWQmmOT0EzwvQ9SM5xJaCY4Z2g6ti15zHJss3k4k5Ak1c4m0VHuw5eshdQcm4QkqZIzCc0EZxJN\nx7Ylj1mObTYPZxKSpNrZJDrKffiStZCaY5OQJFVyJqFWWlhYZHV1/5BnTXtPeZZj25LHLMc2m8eo\nMwmbhFppuEE0tOOXfJZj25LHLMc2m4eD6znjPny/lWknIM0sm4Qmxs9XkrrH7SZNTHPvZRg23tjh\nY9uSxyzHNpuH202SpNrZJDrKmUS/lWknIM0sm4TG4pxBmm3OJDSWdnxmUpOPbWy78pjl2GbzcCah\n2nh1IGmNTaKjmpxJ9N7pnAN+tcHKtBOQZpZNQpJUyZnEBAzzOURbtpzCoUP7an/co456PA8//L8D\nxfZMe3+2PXu5xrYpj1mObTaPUWcSm0Y5ad41+eFzq6vHDLnXP9jjPvzwsP/4JKlF200R8fKI+EZE\n3BsR75h2Po9muD37Ya98fjbgY9467o8xQ1amnYA0s1rRJCLiKOB9wNnAM4BXR8Tpk8yhe6/o+eq0\nE2gRayE1pRVNAjgL2JuZ+zPzMHANcN4kE+jeK3oemHYCLWItpKa0ZSaxFbiv7/gAvcbxS6677rqB\nH/RNb3o73//+fRsHSpLW1ZYmMbBXvepVQ54xq8PafdNOoEX2TTsBaWa1pUkcBJ7Sd3xSsVaDYZ78\nm4ptSx6zHNuWPGY5ti15zHJs0489vFa8TyIiHgPcA7wE+G/gduDVmXn3VBOTpDnXiiuJzPx5RPwp\ncBO9YfoHbBCSNH2tuJKQJLVTW14C+wiDvLEuIt4bEXsj4qsRsWPSOU7KRrWIiNdExJ3F1+cj4pnT\nyHMSBn3DZUT8dkQcjohXTjK/SRrwd2QpIr4SEf8ZETP77ssBfkeeFBGfLp4rvhYRy1NIs3ER8YGI\nWI2Iux4lZvjnzcxs1Re9xvVN4BTgsfTeKXX6ETHnAJ8qbj8XuG3aeU+xFs8Djituv3yea9EX92/A\nJ4FXTjvvKf67OA74OrC1OH7ytPOeYi0uAd65Vgfgh8CmaefeQC1eAOwA7qq4f6TnzTZeSQzyxrrz\ngKsBMvM/gOMiYstk05yIDWuRmbdl5oPF4W303nMyiwZ9w+WfAR8HvjfJ5CZskFq8BrguMw8CZOYP\nJpzjpAxSi0PAscXtY4EfZuZDE8xxIjLz88D9jxIy0vNmG5vEem+sO/KJ78iYg+vEzIJBatHvj4FP\nN5rR9GxYi4j4NeD8zLyc7r3xZRiD/LvYBpwQEbdGxB0R8dqJZTdZg9Tin4FnRMR3gTuBiyeUW9uM\n9LzZilc3aXwR8SLgDfQuOefVpUD/nvQsN4qNbALOBF4MPAH4QkR8ITO/Od20pmI3cGdmvigifh34\nbEQ8KzN/PO3EuqCNTWKQN9YdBE7eIGYWDPQmw4h4FnAF8PLMfLTLzS4bpBa/BVwTvU9hfDJwTkQc\nzszrJ5TjpAxSiwPADzLzp8BPI+JzwLPp7d/PkkFq8Xzg7wAy81sR8V/A6cAXJ5Jhe4z0vNnG7aY7\ngKdFxCkRcTRwAXDkL/n1wOsAIuJ5wAOZuTrZNCdiw1pExFOA64DXZua3ppDjpGxYi8x8avF1Kr25\nxJtnsEHAYL8jnwBeEBGPiYjH0xtUzuJ7jwapxd3ASwGKPfhtwLcnmuXkBNVX0CM9b7buSiIr3lgX\nEX/SuzuvyMwbIuIVEfFN4Cf0tllmziC1AP4GOAG4rPgL+nBmrvvhiF02YC0eccrEk5yQAX9HvhER\nnwHuAn4OXJGZe6aYdiMG/HfxTuCDEXEnvSfQv8rMH00v62ZExEeAJeBJEfEdeq/qOpoxnzd9M50k\nqVIbt5skSS1hk5AkVbJJSJIq2SQkSZVsEpKkSjYJSVIlm4QkqZJNQpJU6f8B8pNIFLFh92kAAAAA\nSUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f7e8396cf98>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "submission=pd.read_csv('data/sample_submission.csv')\n",
    "submission.index=submission.ID\n",
    "submission.PredictedProb=np.array(preds)[0]#.mean(axis=0)\n",
    "submission.to_csv('submission/et_ln_nbag.csv', index=False)\n",
    "submission.PredictedProb.hist(bins=30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load in \n",
    "\n",
    " # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
    "\n",
    "from subprocess import check_output\n",
    "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n",
    "\n",
    "# Any results you write to the current directory are saved as output.\n",
    "\n",
    "from sklearn import ensemble, metrics, linear_model\n",
    "import random\n",
    "\n",
    "#Some parameters to play with\n",
    "rnd=12\n",
    "random.seed(rnd)\n",
    "n_ft=20 #Number of features to add\n",
    "max_elts=3 #Maximum size of a group of linear features\n",
    "\n",
    "class addNearestNeighbourLinearFeatures:\n",
    "    \n",
    "    def __init__(self, n_neighbours=1, max_elts=None, verbose=True, random_state=None):\n",
    "        self.rnd=random_state\n",
    "        self.n=n_neighbours\n",
    "        self.max_elts=max_elts\n",
    "        self.verbose=verbose\n",
    "        self.neighbours=[]\n",
    "        self.clfs=[]\n",
    "        \n",
    "    def fit(self,train,y):\n",
    "        if self.rnd!=None:\n",
    "            random.seed(rnd)\n",
    "        if self.max_elts==None:\n",
    "            self.max_elts=len(train.columns)\n",
    "        list_vars=list(train.columns)\n",
    "        random.shuffle(list_vars)\n",
    "        \n",
    "        lastscores=np.zeros(self.n)+1e15\n",
    "\n",
    "        for elt in list_vars[:self.n]:\n",
    "            self.neighbours.append([elt])\n",
    "        list_vars=list_vars[self.n:]\n",
    "        \n",
    "        for elt in list_vars:\n",
    "            indice=0\n",
    "            scores=[]\n",
    "            for elt2 in self.neighbours:\n",
    "                if len(elt2)<self.max_elts:\n",
    "                    clf=linear_model.LinearRegression(fit_intercept=False, normalize=True, copy_X=True, n_jobs=-1) \n",
    "                    clf.fit(train[elt2+[elt]], y)\n",
    "                    scores.append(metrics.log_loss(y,clf.predict(train[elt2 + [elt]])))\n",
    "                    indice=indice+1\n",
    "                else:\n",
    "                    scores.append(lastscores[indice])\n",
    "                    indice=indice+1\n",
    "            gains=lastscores-scores\n",
    "            if gains.max()>0:\n",
    "                temp=gains.argmax()\n",
    "                lastscores[temp]=scores[temp]\n",
    "                self.neighbours[temp].append(elt)\n",
    "\n",
    "        indice=0\n",
    "        for elt in self.neighbours:\n",
    "            clf=linear_model.LinearRegression(fit_intercept=False, normalize=True, copy_X=True, n_jobs=-1) \n",
    "            clf.fit(train[elt], y)\n",
    "            self.clfs.append(clf)\n",
    "            if self.verbose:\n",
    "                print(indice, lastscores[indice], elt)\n",
    "            indice=indice+1\n",
    "                    \n",
    "    def transform(self, train):\n",
    "        indice=0\n",
    "        for elt in self.neighbours:\n",
    "            train['_'.join(pd.Series(elt).sort_values().values)]=self.clfs[indice].predict(train[elt])\n",
    "            indice=indice+1\n",
    "        return train\n",
    "    \n",
    "    def fit_transform(self, train, y):\n",
    "        self.fit(train, y)\n",
    "        return self.transform(train)\n",
    "    \n",
    "    \n",
    "train = pd.read_csv(\"../input/train.csv\")\n",
    "target = train['target'].values\n",
    "test = pd.read_csv(\"../input/test.csv\")\n",
    "id_test = test['ID'].values\n",
    "\n",
    "train['v22-1']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[0]))\n",
    "test['v22-1']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[0]))\n",
    "train['v22-2']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[1]))\n",
    "test['v22-2']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[1]))\n",
    "train['v22-3']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[2]))\n",
    "test['v22-3']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[2]))\n",
    "train['v22-4']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[3]))\n",
    "test['v22-4']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[3]))\n",
    "\n",
    "drop_list=['v91','v1', 'v8', 'v10', 'v15', 'v17', 'v25', 'v29', 'v34', 'v41', 'v46', 'v54', 'v64', 'v67', 'v97', 'v105', 'v111', 'v122']\n",
    "train = train.drop(['ID','target'] + drop_list,axis=1).fillna(-999)\n",
    "test = test.drop(['ID'] + drop_list,axis=1).fillna(-999)\n",
    "\n",
    "refcols=list(train.columns)\n",
    "\n",
    "for elt in refcols:\n",
    "    if train[elt].dtype=='O':\n",
    "        train[elt], temp = pd.factorize(train[elt])\n",
    "        test[elt]=temp.get_indexer(test[elt])\n",
    "    else:\n",
    "        train[elt]=train[elt].round(5)\n",
    "        test[elt]=test[elt].round(5)\n",
    "        \n",
    "a=addNearestNeighbourLinearFeatures(n_neighbours=n_ft, max_elts=max_elts, verbose=True, random_state=rnd)\n",
    "a.fit(train, target)\n",
    "\n",
    "train = a.transform(train)\n",
    "test = a.transform(test)\n",
    "\n",
    "clf = ensemble.ExtraTreesClassifier(n_estimators=750,max_features=50,criterion= 'entropy',min_samples_split= 4,\n",
    "                        max_depth= 35, min_samples_leaf= 2, n_jobs = -1, random_state=rnd)\n",
    "\n",
    "clf.fit(train,target)\n",
    "pred_et=clf.predict_proba(test)\n",
    "\n",
    "submission=pd.read_csv('../input/sample_submission.csv')\n",
    "submission.index=submission.ID\n",
    "submission.PredictedProb=pred_et[:,1]\n",
    "submission.to_csv('./addNNLinearFt.csv', index=False)\n",
    "submission.PredictedProb.hist(bins=30)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}