extratrees_nn_lf_seed_bag.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from subprocess import check_output\n",
    "from sklearn import ensemble, metrics, linear_model\n",
    "import random\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "rnd=191989\n",
    "random.seed(rnd)\n",
    "n_ft=15 #Number of features to add\n",
    "max_elts=5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class addNearestNeighbourLinearFeatures:\n",
    "    \n",
    "    def __init__(self, n_neighbours=1, max_elts=None, verbose=True, random_state=None):\n",
    "        self.rnd = random_state\n",
    "        self.n = n_neighbours\n",
    "        self.max_elts = max_elts\n",
    "        self.verbose = verbose\n",
    "        self.neighbours = []\n",
    "        self.clfs = []\n",
    "        \n",
    "    def fit(self, train, y):\n",
    "        if self.rnd != None:\n",
    "            random.seed(rnd)\n",
    "        if self.max_elts == None:\n",
    "            self.max_elts = len(train.columns)\n",
    "        list_vars = list(train.columns)\n",
    "        random.shuffle(list_vars)\n",
    "        \n",
    "        lastscores = np.zeros(self.n) + 1e15\n",
    "\n",
    "        for elt in list_vars[:self.n]:\n",
    "            self.neighbours.append([elt])\n",
    "        list_vars = list_vars[self.n:]\n",
    "        \n",
    "        for elt in list_vars:\n",
    "            indice = 0\n",
    "            scores = []\n",
    "            for elt2 in self.neighbours:\n",
    "                if len(elt2) < self.max_elts:\n",
    "                    clf = linear_model.LinearRegression(fit_intercept=False, \n",
    "                                                        normalize=True, copy_X=True) \n",
    "                    clf.fit(train[elt2+[elt]], y)\n",
    "                    scores.append(metrics.log_loss(y,clf.predict(train[elt2 + [elt]])))\n",
    "                    indice=indice+1\n",
    "                else:\n",
    "                    scores.append(lastscores[indice])\n",
    "                    indice=indice+1\n",
    "            gains = lastscores - scores\n",
    "            if gains.max() > 0:\n",
    "                temp = gains.argmax()\n",
    "                lastscores[temp] = scores[temp]\n",
    "                self.neighbours[temp].append(elt)\n",
    "\n",
    "        indice = 0\n",
    "        for elt in self.neighbours:\n",
    "            clf = linear_model.LinearRegression(fit_intercept=False, normalize=True, copy_X=True) \n",
    "            clf.fit(train[elt], y)\n",
    "            self.clfs.append(clf)\n",
    "            if self.verbose:\n",
    "                print(indice, lastscores[indice], elt)\n",
    "            indice = indice + 1\n",
    "                    \n",
    "    def transform(self, train):\n",
    "        indice = 0\n",
    "        for elt in self.neighbours:\n",
    "            train['_'.join(pd.Series(elt).sort_values().values)] = self.clfs[indice].predict(train[elt])\n",
    "            indice = indice + 1\n",
    "        return train\n",
    "    \n",
    "    def fit_transform(self, train, y):\n",
    "        self.fit(train, y)\n",
    "        return self.transform(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv(\"data/train.csv\")\n",
    "target = train['target'].values\n",
    "test = pd.read_csv(\"data/test.csv\")\n",
    "id_test = test['ID'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train['v22-1']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[0]))\n",
    "test['v22-1']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[0]))\n",
    "train['v22-2']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[1]))\n",
    "test['v22-2']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[1]))\n",
    "train['v22-3']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[2]))\n",
    "test['v22-3']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[2]))\n",
    "train['v22-4']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[3]))\n",
    "test['v22-4']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[3]))\n",
    "\n",
    "drop_list=['v91','v1', 'v8', 'v10', 'v15', 'v17', 'v25', 'v29', \n",
    "           'v34', 'v41', 'v46', 'v54', 'v64', 'v67', 'v97', 'v105', 'v111', 'v122']\n",
    "train = train.drop(['ID','target'] + drop_list,axis=1).fillna(-999)\n",
    "test = test.drop(['ID'] + drop_list,axis=1).fillna(-999)\n",
    "\n",
    "refcols=list(train.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for elt in refcols:\n",
    "    if train[elt].dtype=='O':\n",
    "        train[elt], temp = pd.factorize(train[elt])\n",
    "        test[elt]=temp.get_indexer(test[elt])\n",
    "    else:\n",
    "        #train[train[elt] == 20] = -999\n",
    "        #test[test[elt] == 20] = -999\n",
    "        train[elt]=train[elt].round(5)\n",
    "        test[elt]=test[elt].round(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 0.857936296658 ['v9', 'v126', 'v45', 'v55', 'v21']\n",
      "1 0.560635902021 ['v53', 'v57', 'v43', 'v89', 'v2']\n",
      "2 0.60592220067 ['v114', 'v84', 'v101', 'v61', 'v93']\n",
      "3 1.19645086749 ['v112', 'v123', 'v51', 'v102', 'v62']\n",
      "4 0.722759106994 ['v13', 'v86', 'v27', 'v18', 'v85']\n",
      "5 0.564896139836 ['v130', 'v19', 'v69', 'v127', 'v22-3']\n",
      "6 0.731445183697 ['v81', 'v124', 'v36', 'v82', 'v48']\n",
      "7 1.10498487505 ['v24', 'v107', 'v28', 'v52', 'v20']\n",
      "8 0.594309329043 ['v37', 'v73', 'v65', 'v76', 'v90']\n",
      "9 0.601229963483 ['v88', 'v96', 'v3', 'v38', 'v42']\n",
      "10 1.40705040933 ['v22', 'v50', 'v92', 'v59', 'v58']\n",
      "11 0.561642727643 ['v117', 'v12', 'v109', 'v113', 'v22-4']\n",
      "12 0.660709937507 ['v33', 'v106', 'v77', 'v7', 'v32']\n",
      "13 2.03335775613 ['v71', 'v80', 'v31', 'v60', 'v6']\n",
      "14 1.74028555247 ['v119', 'v74', 'v125', 'v98', 'v16']\n"
     ]
    }
   ],
   "source": [
    "a=addNearestNeighbourLinearFeatures(n_neighbours=n_ft, max_elts=max_elts, verbose=True, random_state=rnd)\n",
    "a.fit(train, target)\n",
    "\n",
    "train = a.transform(train)\n",
    "test = a.transform(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from libtelepot import sendMessage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12 35 55\n",
      "13 35 55\n",
      "14 35 55\n",
      "15 35 55\n",
      "16 35 55\n",
      "17 35 55\n",
      "18 35 55\n",
      "19 35 55\n",
      "20 35 55\n",
      "21 35 55\n",
      "22 35 55\n",
      "23 35 55\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-9-41efca6721d7>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      8\u001b[0m                                         \u001b[0mmin_samples_split\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m4\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmax_depth\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmd\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmin_samples_leaf\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      9\u001b[0m                                         n_jobs = -1, random_state=rnd)\n\u001b[1;32m---> 10\u001b[1;33m     \u001b[0mclf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mtarget\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     11\u001b[0m     \u001b[0mpred_et\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mclf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict_proba\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtest\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     12\u001b[0m     \u001b[0mpreds\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpred_et\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/libfun/vrt/lib/python3.4/site-packages/sklearn/ensemble/forest.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[0;32m    288\u001b[0m                     \u001b[0mt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrees\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    289\u001b[0m                     verbose=self.verbose, class_weight=self.class_weight)\n\u001b[1;32m--> 290\u001b[1;33m                 for i, t in enumerate(trees))\n\u001b[0m\u001b[0;32m    291\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    292\u001b[0m             \u001b[1;31m# Collect newly grown trees\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/libfun/vrt/lib/python3.4/site-packages/sklearn/externals/joblib/parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, iterable)\u001b[0m\n\u001b[0;32m    808\u001b[0m                 \u001b[1;31m# consumption.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    809\u001b[0m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 810\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mretrieve\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    811\u001b[0m             \u001b[1;31m# Make sure that we get a last message telling us we are done\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    812\u001b[0m             \u001b[0melapsed_time\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_start_time\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/libfun/vrt/lib/python3.4/site-packages/sklearn/externals/joblib/parallel.py\u001b[0m in \u001b[0;36mretrieve\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    725\u001b[0m                 \u001b[0mjob\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_jobs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpop\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    726\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 727\u001b[1;33m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_output\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    728\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mtuple\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexceptions\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mexception\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    729\u001b[0m                 \u001b[1;31m# Stop dispatching any new job in the async callback thread\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/lib/python3.4/multiprocessing/pool.py\u001b[0m in \u001b[0;36mget\u001b[1;34m(self, timeout)\u001b[0m\n\u001b[0;32m    591\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    592\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 593\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    594\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mready\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    595\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mTimeoutError\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/lib/python3.4/multiprocessing/pool.py\u001b[0m in \u001b[0;36mwait\u001b[1;34m(self, timeout)\u001b[0m\n\u001b[0;32m    588\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    589\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mwait\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 590\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_event\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    591\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    592\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/lib/python3.4/threading.py\u001b[0m in \u001b[0;36mwait\u001b[1;34m(self, timeout)\u001b[0m\n\u001b[0;32m    551\u001b[0m             \u001b[0msignaled\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_flag\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    552\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0msignaled\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 553\u001b[1;33m                 \u001b[0msignaled\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_cond\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    554\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0msignaled\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    555\u001b[0m         \u001b[1;32mfinally\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/lib/python3.4/threading.py\u001b[0m in \u001b[0;36mwait\u001b[1;34m(self, timeout)\u001b[0m\n\u001b[0;32m    288\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m    \u001b[1;31m# restore state no matter what (e.g., KeyboardInterrupt)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    289\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 290\u001b[1;33m                 \u001b[0mwaiter\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    291\u001b[0m                 \u001b[0mgotit\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    292\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "preds = list()\n",
    "mf = 55\n",
    "md = 35\n",
    "for rnd in range(12, 40):\n",
    "    print(rnd, md, mf, flush=True)\n",
    "    sendMessage('Hey, I\\'m running your shitty script with seed = {:}, you lazy fucker'.format(rnd))\n",
    "    clf = ensemble.ExtraTreesClassifier(n_estimators=3750, max_features=mf ,criterion='entropy',\n",
    "                                        min_samples_split=4, max_depth=md, min_samples_leaf=2, \n",
    "                                        n_jobs = -1, random_state=rnd)\n",
    "    clf.fit(train,target)\n",
    "    pred_et = clf.predict_proba(test)[:, 1]\n",
    "    preds.append(pred_et)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(114393,)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(preds).mean(axis=0).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('keksec.pkl', 'wb') as f:\n",
    "    pickle.dump(np.array(preds), f, protocol=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('kek.pkl', 'rb') as f:\n",
    "    myold = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "my = np.vstack((myold, np.array(preds)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sv = pd.read_csv('woe_mean_pred.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sv = sv.PredictedProb.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((114393,), (114393,))"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my.mean(axis=0).shape, sv.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "finz = .68*my.mean(axis=0) + .32*sv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 0.38134411,  0.83253723,  0.83865838, ...,  0.92172533,\n",
       "         0.9408101 ,  0.5580156 ]),\n",
       " array([ 0.36067409,  0.83439408,  0.81989932, ...,  0.91925303,\n",
       "         0.93622129,  0.55447438]),\n",
       " array([ 0.42526789,  0.82859142,  0.87852137, ...,  0.92697898,\n",
       "         0.95056131,  0.56554069]))"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "finz, my.mean(axis=0), sv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f3b18cbfba8>"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEACAYAAABGYoqtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFmtJREFUeJzt3W+MZXV5wPHvgytS7MKClZ10QYYWF8FWt7RdbbRxWq2I\nTYA06QZt1RFqGrWV/knrbpOGV43VpCmaFhJaK4uxQdQXEEVEitPGRMpqK9Auwtp2kV27oyjSaKPd\n1acv7hnO3XUOc/+cc+85934/yST3/Oa5d559snOfOb/n3HsjM5EkaT0nTTsBSVJ72SQkSZVsEpKk\nSjYJSVIlm4QkqZJNQpJUacMmERHvj4jViHigb+2MiLgrIh6OiE9FxOl939sTEQci4qGIeHXf+sUR\n8UBEPBIR1/WtnxwRtxT3+VxEPK/Of6AkaXSDnEl8ALjkhLXdwN2ZeQFwD7AHICIuAnYBFwKXAtdH\nRBT3uQG4OjO3A9sjYu0xrwa+mZnPB64D3jPGv0eSVKMNm0RmfhZ44oTly4G9xe29wBXF7cuAWzLz\nWGYeBA4AOyNiAdicmfuKuJv77tP/WB8FXjnCv0OS1IBRZxJnZeYqQGYeAc4q1rcBj/XFHS7WtgGH\n+tYPFWvH3Sczvw98KyLOHDEvSVKN6hpc1/neHrFxiCRpEjaNeL/ViNiamavFVtLXivXDwDl9cWcX\na1Xr/ff5akQ8AzgtM7+53g+NCN9oSpJGkJkj/QE+6JlEcPxf+LcDy8XtNwG39a1fWVyxdB5wPnBf\nsSX1ZETsLAbZbzzhPm8qbv86vUF4pcz0K5Nrr7126jm05ctaWAtr8fRf49jwTCIi/h5YAp4TEV8B\nrgX+HPhIRFwFPErviiYyc39E3ArsB44Cb8syw7cDNwGnAHdk5p3F+vuBD0bEAeAbwJVj/YvmxMGD\nB6edQmtYi5K1KFmLemzYJDLz9RXfelVF/LuAd62z/gXgp9dZ/x5Fk5EktYuvuO6o5eXlaafQGtai\nZC1K1qIeMe5+1SRFRHYpX0lqg4ggGx5cq2VWVlamnUJrWIuStShZi3rYJCRJldxukqQZ53aTJKkR\nNomOcr+1ZC1K1qJkLephk5CkllhYWCQiBvpaWFicSE7OJCRpCAsLi6yuPjpQ7Nat53LkyMGBH7v3\nrkWDPsfFwG+5Mc5MwiYhSUNo6om8ycd2cD2H3G8tWYuStSi1oxbPGnj7qPwQz3YZ9a3CJUkb+h7D\nfdxO+xqF202SNIRht4SGbxJuN0mSOsIm0VHt2G9tB2tRshYla1EPm4QkqZIzCUlzb5jXPvTMz0zC\nJiFp7jU3jO5+k3C7qaPcby1Zi5K1KFmLetgkJEmV3G6SNPfcbqrmmYQkqZJNoqPcby1Zi5K1KFmL\netgkJEmVnElImnvOJKp5JiFJqmST6Cj3W0vWomQtStaiHjYJSVIlZxKS5p4ziWqeSUiaSQsLi53+\n2NC2sEl0lPutJWtRshal3ru65oBfqmKTkCRVciYhaSa1Y87gTEKSNMNsEh3l3nPJWpRmvRYOoydv\n07QTkKRBlcPoQdgo6jDWTCIi9gC/CXwfeBB4M/Bs4MPAucBBYFdmPtkXfxVwDLgmM+8q1i8GbgJO\nAe7IzN+r+HnOJKQ51r05wxzPJCLiXOAtwM9k5ovonZW8DtgN3J2ZFwD3AHuK+IuAXcCFwKXA9VGe\nE94AXJ2Z24HtEXHJqHlJkuozzkzif4D/A54dEZuAHwEOA5cDe4uYvcAVxe3LgFsy81hmHgQOADsj\nYgHYnJn7irib++6jCrO+9zwMa1GyFqrbyE0iM58A/gL4Cr3m8GRm3g1szczVIuYIcFZxl23AY30P\ncbhY2wYc6ls/VKxJkqZs5MF1RPwE8Pv0Zg9PAh+JiN/ghzfUah0iLC8vs7i4CMCWLVvYsWMHS0tL\nQPlX1DwcLy0ttSofj9tzvKYt+TT174O146WK47W1qu+Pejzoz286vleT9eq1srLCTTfdBPDU8+Wo\nRh5cR8Qu4Fcy8y3F8RuAlwK/DCxl5mqxlfSZzLwwInYDmZnvLuLvBK4FHl2LKdavBF6RmW9d52c6\nuJbmmIPr42NbPbgGHgZeGhGnFAPoVwL7gduB5SLmTcBtxe3bgSsj4uSIOA84H7iv2JJ6MiJ2Fo/z\nxr77qMIP/1U1v6xFyVqobiNvN2Xm/RFxM/AFepfA/itwI7AZuDUirqJ3lrCriN8fEbfSayRHgbf1\nnRa8neMvgb1z1LwkSfXxvZskdYbbTcfHtn27SZI042wSHeXec8lalLpYC9+Pqd187yZJU+X7MbWb\nMwlJUzXbcwZnEpKkGWaT6Kgu7j03xVqUrIXqZpOQJFVyJiGpdgsLi8VAelDTnge0IXb4x57ETMIm\nIal2DqNHiR3+sR1cq5J7zyVrUbIWqptNQpJUye0mSbVzu2mU2OEfexLbTb7iWpI66VkTeasSt5s6\nyr3nkrUoWYt58j16Zx2DfI3OJiFJquRMQlLtnEmMEttsHl4CK0mqnU2io9x7LlmLUlO1GOYzH/zc\nh9ni1U2SNjTcZz6An/swO5xJSNrQcDMGaMcef9dim83DmYQkqXY2iY5yH75kLUrWQnWzSUiSKjmT\nkLQhZxKTiG02D2cSkqTa2SQ6yr3nkrUoWQvVzSYhSarkTELShpxJTCK22TycSUiSameT6Cj3nkvW\nomQtVDebhCSpkjMJSRtyJjGJ2GbzcCYhSaqdTaKj3HsuWYuStVDdbBKSpEpjzSQi4nTgb4GfAn4A\nXAU8AnwYOBc4COzKzCeL+D1FzDHgmsy8q1i/GLgJOAW4IzN/r+LnOZOQpsCZxCRim81jWjOJ99J7\nUr8QeDHwJWA3cHdmXgDcA+wBiIiLgF3AhcClwPVRfs7hDcDVmbkd2B4Rl4yZlySpBiM3iYg4DfjF\nzPwAQGYeK84YLgf2FmF7gSuK25cBtxRxB4EDwM6IWAA2Z+a+Iu7mvvuognvPJWtRGqYWw3xutebX\nOGcS5wGPR8QHIuJfIuLGiDgV2JqZqwCZeQQ4q4jfBjzWd//Dxdo24FDf+qFiTVKDys+tHuRL82qc\nJrEJuBj468y8GPgOva2mE/9H+T+sAUtLS9NOoTWsRclaqG6bxrjvIeCxzPx8cfwxek1iNSK2ZuZq\nsZX0teL7h4Fz+u5/drFWtb6u5eVlFhcXAdiyZQs7dux46hdj7VTbY4893vi4ZwVY6rvNFI7Z4Puj\nxq+tjZvfqD+/6fi1tfW+v0LvWiCARcYx7tVN/wi8JTMfiYhrgVOLb30zM98dEe8EzsjM3cXg+kPA\nS+htJ30aeH5mZkTcC7wD2Ad8AnhfZt65zs/z6qbCysqKfzUWrEVpmFoMd8VSe67Smd3YZvMY9eqm\ncc4koPfE/qGIeCbwn8CbgWcAt0bEVcCj9K5oIjP3R8StwH7gKPC2vmf8t3P8JbA/1CAkSZPnezdJ\nc8ozibbFNpuH790kSaqdTaKjfG1AyVqUrIXqZpOQJFVyJiHNKWcSbYttNg9nEpKk2tkkOsq955K1\nKFkL1c0mIUmq5ExCmlPOJNoW22weziQkSbWzSXSUe88la1GyFqqbTUKSVMmZhDSnnEm0LbbZPJxJ\nSJJqZ5PoKPeeS9aidOaZC35utWo17udJSGqRJ55YZbjtCunpOZOQZkhzc4b27K3PbmyzeTiTkCTV\nzibRUe7Dl6yF1BybhCSpkjMJaYY4k+hybLN5OJOQJNXOJtFR7sOXrIXUHJuEJKmSMwlphjiT6HJs\ns3k4k5Ak1c4m0VHuw5eshdQcm4QkqZIzCWmGOJPocmyzeTiTkCTVzibRUe7Dl6yF1BybhCSpkjMJ\naYY4k+hybLN5OJOQJNXOJtFR7sOXrIXUHJuEJKmSMwmp5RYWFlldfXSIe8z23vrsxjabx9RmEhFx\nUkT8S0TcXhyfERF3RcTDEfGpiDi9L3ZPRByIiIci4tV96xdHxAMR8UhEXDduTtIs6TWIHPBLqlcd\n203XAPv7jncDd2fmBcA9wB6AiLgI2AVcCFwKXB+9SzEAbgCuzsztwPaIuKSGvGaa+/AlayE1Z6wm\nERFnA68F/rZv+XJgb3F7L3BFcfsy4JbMPJaZB4EDwM6IWAA2Z+a+Iu7mvvtIkqZo3DOJvwT+iOPP\nc7dm5ipAZh4BzirWtwGP9cUdLta2AYf61g8Va3oaS0tL006hNayF1JyRm0RE/CqwmplfpDdBqeJG\nqSR11KYx7vsy4LKIeC3wI8DmiPggcCQitmbmarGV9LUi/jBwTt/9zy7WqtbXtby8zOLiIgBbtmxh\nx44dT/0lubY3PQ/H/fvwbchnmsdra23Jp4l/X8/a8dIGx4PGr61t9HhNH7PB90eNX1sbN79Rf37T\n8Wtr631/BbipOF5kHLVcAhsRrwD+MDMvi4j3AN/IzHdHxDuBMzJzdzG4/hDwEnrbSZ8Gnp+ZGRH3\nAu8A9gGfAN6XmXeu83O8BLawsrLiNkth1mvRjrfaaM+lnLMb22weo14C20STOBO4ld7ZwaPArsz8\nVhG3B7gaOApck5l3Fes/S6/tnQLckZnXVPwcm4Tmjk1iXmKbzWOqTWJSbBKaRzaJeYltNg/f4G/O\n+NqAkrWQmmOTkCRVcrtJajm3m+Ylttk83G6SJNXOJtFR7sOXrIXUHJuEJKmSMwmp5ZxJzEtss3k4\nk5Ak1c4m0VHuw5e6WIuFhUUiYqAvaZrGeYM/SSMqP21uEDYKTY8zCWkKujdnaM/e+uzGNpuHMwlJ\nUu1sEh3VxX34plgLqTk2CUlSJWcS0hQ4kzB20nk4k5Ak1c4m0VHuw5eshdQcm4QkqZIzCWkKnEkY\nO+k8nElIkmpnk+go9+FL1kJqjk1CklTJmYQ0Bc4kjJ10Hs4kpCnz7b81i2wSHeU+fKkttSjf/nuQ\nL6kbbBKSpErOJKSazPacoT1767Mb22weziQkSbWzSXRUW/bh28BaSM2xSUiSKjmTkGriTKJteXQt\nttk8nElIkmpnk+go9+FL1kJqjk1CklTJmYRUE2cSbcuja7HN5uFMQpJUu5GbREScHRH3RMS/R8SD\nEfGOYv2MiLgrIh6OiE9FxOl999kTEQci4qGIeHXf+sUR8UBEPBIR1433T5oP7sOXrIXUnHHOJI4B\nf5CZLwR+AXh7RLwA2A3cnZkXAPcAewAi4iJgF3AhcClwfZRvh3kDcHVmbge2R8QlY+QlSarJyE0i\nM49k5heL298GHgLOBi4H9hZhe4ErituXAbdk5rHMPAgcAHZGxAKwOTP3FXE3991HFZaWlqadQms0\nWQvf/lvzrpaZREQsAjuAe4GtmbkKvUYCnFWEbQMe67vb4WJtG3Cob/1QsSZNnW//rXm3adwHiIgf\nBT4KXJOZ346IE39bav3tWV5eZnFxEYAtW7awY8eOp/6SXNubnofj/n34NuQzzeO1tSYfv2fteKni\neG2t6vujHg/684eNX1sbN79xj9ng+6PGr62Nm9+oP7/p+LW19b6/AtxUHC8yjrEugY2ITcDHgU9m\n5nuLtYeApcxcLbaSPpOZF0bEbiAz891F3J3AtcCjazHF+pXAKzLzrev8PC+BLaysrLjlVGiyFl7W\nOkpsW/LoWmyzeUzrEti/A/avNYjC7cBycftNwG1961dGxMkRcR5wPnBfsSX1ZETsLAbZb+y7jyrY\nIErWQmrOyGcSEfEy4J+AByk3Zf8EuA+4FTiH3lnCrsz8VnGfPcDVwFF621N3Fes/S+/c6BTgjsy8\npuJneiahifJMYpTYtuTRtdhm8xj1TMJXXHeU200lt5vaFtuWPLoW22wevuJaklQ7zySkp+GZxCix\nbcmja7HN5uGZhCSpdjaJjvL9ikrWQmqOTUKSVMmZhObKwsJi8VYbw5j2XnXXYtuSR9dim83DS2Cl\nAQw3iIZ2PHl0LbYteXQtttk8HFzPGffhS9ZCao5NQpJUye0mzRW3myYR25Y8uhbbbB5uN0mSameT\n6Cj34UvWQmqOTUKSVMmZhOaKM4lJxLYlj67FNpuHMwnNrYWFRSJioC9Jw7FJdJT78KXeK6hzwC9J\nw7BJSJIqOZNQ5zX3mQ/Dxhvbrjy6FttsHs4kJEm1s0l0lDMJSZNgk5AkVXImoVYa/nMfurdHPLux\nbcmja7HN5uHnSWimNDeMbs8v7ezGtiWPrsU2m4eD6znjTELSJNgkJEmV3G7SxLRjztCe0//ZjW1L\nHl2LbTYPZxJqvXbMGdrzSzu7sW3Jo2uxzebhTGLOOJOQNAk2CUlSJbebNJbuzRnac/o/u7FtyaNr\nsc3mMep206ZR7qTZ1uwTv6Qucbupo5qcSfj5DJLW2CTmhJ/eJmkUbjc1bJitm5NOOpUf/OB/a48t\nuS0kaTgOrhvWjtcGNPnYsxzbljy6FtuWPLoW22wenX+dRES8JiK+FBGPRMQ7p52PJKklTSIiTgL+\nCrgEeCHwuoh4wXSzqub+vqR50YomAewEDmTmo5l5FLgFuHzKOVXy6h9J86Itg+ttwGN9x4foNY4f\n8vjjjw/0gCeffDKnnXbawAkM/9oASZp9bWkSA3vuc587ROw5fP3rj20c+BSv/pGkfm1pEoeB5/Ud\nn12sjWW4BgHDPfl3LbYteXQtti15dC22LXl0Lbbpxx5eKy6BjYhnAA8DrwT+G7gPeF1mPjTVxCRp\nzrXiTCIzvx8RvwPcRW+Y/n4bhCRNXyvOJCRJ7dSWS2CPM8gL6yLifRFxICK+GBE7Jp3jpGxUi4h4\nfUTcX3x9NiJ+ehp5TsKgL7iMiJ+PiKMR8WuTzG+SBvwdWYqIf42If4uIz0w6x0kZ4HfkORHxyeK5\n4sGIWJ5Cmo2LiPdHxGpEPPA0McM/b2Zmq77oNa4vA+cCzwS+CLzghJhLgU8Ut18C3DvtvKdYi5cC\npxe3XzPPteiL+wfg48CvTTvvKf6/OB34d2Bbcfxj0857irW4FnjXWh2AbwCbpp17A7V4ObADeKDi\n+yM9b7bxTGKQF9ZdDtwMkJn/DJweEVsnm+ZEbFiLzLw3M58sDu+l95qTWTToCy5/F/go8LVJJjdh\ng9Ti9cDHMvMwQGYO9gKj7hmkFkeAzcXtzcA3MvPYBHOciMz8LPDE04SM9LzZxiax3gvrTnziOzHm\n8Doxs2CQWvT7LeCTjWY0PRvWIiJ+HLgiM29gtl/MMsj/i+3AmRHxmYjYFxFvmFh2kzVILf4GeGFE\nfBW4H7hmQrm1zUjPm624uknji4hfAt5M75RzXl0H9O9Jz3Kj2Mgm4GLgl4FnA5+LiM9l5penm9ZU\n7AHuz8xfioifBD4dES/KzG9PO7EuaGOTGOSFdYeBczaImQUDvcgwIl4E3Ai8JjOf7nSzywapxc8B\nt0TvnRV/DLg0Io5m5u0TynFSBqnFIeDxzPwu8N2I+CfgxfT272fJILV4GfBnAJn5HxHxX8ALgM9P\nJMP2GOl5s43bTfuA8yPi3Ig4GbgSOPGX/HbgjQAR8VLgW5m5Otk0J2LDWkTE84CPAW/IzP+YQo6T\nsmEtMvMniq/z6M0l3jaDDQIG+x25DXh5RDwjIk6lN6icxdceDVKLh4BXARR78NuB/5xolpMTVJ9B\nj/S82boziax4YV1E/Hbv23ljZt4REa+NiC8D36G3zTJzBqkF8KfAmcD1xV/QRzNz3TdH7LIBa3Hc\nXSae5IQM+DvypYj4FPAA8H3gxszcP8W0GzHg/4t3AR+IiPvpPYH+cWZ+c3pZNyMi/h5YAp4TEV+h\nd1XXyYz5vOmL6SRJldq43SRJagmbhCSpkk1CklTJJiFJqmSTkCRVsklIkirZJCRJlWwSkqRK/w/M\n8IXHxBDFpwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f3b197ebbe0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "submission=pd.read_csv('data/sample_submission.csv')\n",
    "submission.index=submission.ID\n",
    "submission.PredictedProb=finz\n",
    "submission.to_csv('submission/finz.csv', index=False)\n",
    "submission.PredictedProb.hist(bins=30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load in \n",
    "\n",
    " # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
    "\n",
    "from subprocess import check_output\n",
    "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n",
    "\n",
    "# Any results you write to the current directory are saved as output.\n",
    "\n",
    "from sklearn import ensemble, metrics, linear_model\n",
    "import random\n",
    "\n",
    "#Some parameters to play with\n",
    "rnd=12\n",
    "random.seed(rnd)\n",
    "n_ft=20 #Number of features to add\n",
    "max_elts=3 #Maximum size of a group of linear features\n",
    "\n",
    "class addNearestNeighbourLinearFeatures:\n",
    "    \n",
    "    def __init__(self, n_neighbours=1, max_elts=None, verbose=True, random_state=None):\n",
    "        self.rnd=random_state\n",
    "        self.n=n_neighbours\n",
    "        self.max_elts=max_elts\n",
    "        self.verbose=verbose\n",
    "        self.neighbours=[]\n",
    "        self.clfs=[]\n",
    "        \n",
    "    def fit(self,train,y):\n",
    "        if self.rnd!=None:\n",
    "            random.seed(rnd)\n",
    "        if self.max_elts==None:\n",
    "            self.max_elts=len(train.columns)\n",
    "        list_vars=list(train.columns)\n",
    "        random.shuffle(list_vars)\n",
    "        \n",
    "        lastscores=np.zeros(self.n)+1e15\n",
    "\n",
    "        for elt in list_vars[:self.n]:\n",
    "            self.neighbours.append([elt])\n",
    "        list_vars=list_vars[self.n:]\n",
    "        \n",
    "        for elt in list_vars:\n",
    "            indice=0\n",
    "            scores=[]\n",
    "            for elt2 in self.neighbours:\n",
    "                if len(elt2)<self.max_elts:\n",
    "                    clf=linear_model.LinearRegression(fit_intercept=False, normalize=True, copy_X=True, n_jobs=-1) \n",
    "                    clf.fit(train[elt2+[elt]], y)\n",
    "                    scores.append(metrics.log_loss(y,clf.predict(train[elt2 + [elt]])))\n",
    "                    indice=indice+1\n",
    "                else:\n",
    "                    scores.append(lastscores[indice])\n",
    "                    indice=indice+1\n",
    "            gains=lastscores-scores\n",
    "            if gains.max()>0:\n",
    "                temp=gains.argmax()\n",
    "                lastscores[temp]=scores[temp]\n",
    "                self.neighbours[temp].append(elt)\n",
    "\n",
    "        indice=0\n",
    "        for elt in self.neighbours:\n",
    "            clf=linear_model.LinearRegression(fit_intercept=False, normalize=True, copy_X=True, n_jobs=-1) \n",
    "            clf.fit(train[elt], y)\n",
    "            self.clfs.append(clf)\n",
    "            if self.verbose:\n",
    "                print(indice, lastscores[indice], elt)\n",
    "            indice=indice+1\n",
    "                    \n",
    "    def transform(self, train):\n",
    "        indice=0\n",
    "        for elt in self.neighbours:\n",
    "            train['_'.join(pd.Series(elt).sort_values().values)]=self.clfs[indice].predict(train[elt])\n",
    "            indice=indice+1\n",
    "        return train\n",
    "    \n",
    "    def fit_transform(self, train, y):\n",
    "        self.fit(train, y)\n",
    "        return self.transform(train)\n",
    "    \n",
    "    \n",
    "train = pd.read_csv(\"../input/train.csv\")\n",
    "target = train['target'].values\n",
    "test = pd.read_csv(\"../input/test.csv\")\n",
    "id_test = test['ID'].values\n",
    "\n",
    "train['v22-1']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[0]))\n",
    "test['v22-1']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[0]))\n",
    "train['v22-2']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[1]))\n",
    "test['v22-2']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[1]))\n",
    "train['v22-3']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[2]))\n",
    "test['v22-3']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[2]))\n",
    "train['v22-4']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[3]))\n",
    "test['v22-4']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[3]))\n",
    "\n",
    "drop_list=['v91','v1', 'v8', 'v10', 'v15', 'v17', 'v25', 'v29', 'v34', 'v41', 'v46', 'v54', 'v64', 'v67', 'v97', 'v105', 'v111', 'v122']\n",
    "train = train.drop(['ID','target'] + drop_list,axis=1).fillna(-999)\n",
    "test = test.drop(['ID'] + drop_list,axis=1).fillna(-999)\n",
    "\n",
    "refcols=list(train.columns)\n",
    "\n",
    "for elt in refcols:\n",
    "    if train[elt].dtype=='O':\n",
    "        train[elt], temp = pd.factorize(train[elt])\n",
    "        test[elt]=temp.get_indexer(test[elt])\n",
    "    else:\n",
    "        train[elt]=train[elt].round(5)\n",
    "        test[elt]=test[elt].round(5)\n",
    "        \n",
    "a=addNearestNeighbourLinearFeatures(n_neighbours=n_ft, max_elts=max_elts, verbose=True, random_state=rnd)\n",
    "a.fit(train, target)\n",
    "\n",
    "train = a.transform(train)\n",
    "test = a.transform(test)\n",
    "\n",
    "clf = ensemble.ExtraTreesClassifier(n_estimators=750,max_features=50,criterion= 'entropy',min_samples_split= 4,\n",
    "                        max_depth= 35, min_samples_leaf= 2, n_jobs = -1, random_state=rnd)\n",
    "\n",
    "clf.fit(train,target)\n",
    "pred_et=clf.predict_proba(test)\n",
    "\n",
    "submission=pd.read_csv('../input/sample_submission.csv')\n",
    "submission.index=submission.ID\n",
    "submission.PredictedProb=pred_et[:,1]\n",
    "submission.to_csv('./addNNLinearFt.csv', index=False)\n",
    "submission.PredictedProb.hist(bins=30)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}