extratrees_nn_linearfeats.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from subprocess import check_output\n",
    "from sklearn import ensemble, metrics, linear_model\n",
    "import random\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "rnd=12\n",
    "random.seed(rnd)\n",
    "n_ft=40 #Number of features to add\n",
    "max_elts=5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class addNearestNeighbourLinearFeatures:\n",
    "    \n",
    "    def __init__(self, n_neighbours=1, max_elts=None, verbose=True, random_state=None):\n",
    "        self.rnd = random_state\n",
    "        self.n = n_neighbours\n",
    "        self.max_elts = max_elts\n",
    "        self.verbose = verbose\n",
    "        self.neighbours = []\n",
    "        self.clfs = []\n",
    "        \n",
    "    def fit(self, train, y):\n",
    "        if self.rnd != None:\n",
    "            random.seed(rnd)\n",
    "        if self.max_elts == None:\n",
    "            self.max_elts = len(train.columns)\n",
    "        list_vars = list(train.columns)\n",
    "        random.shuffle(list_vars)\n",
    "        \n",
    "        lastscores = np.zeros(self.n) + 1e15\n",
    "\n",
    "        for elt in list_vars[:self.n]:\n",
    "            self.neighbours.append([elt])\n",
    "        list_vars = list_vars[self.n:]\n",
    "        \n",
    "        for elt in list_vars:\n",
    "            indice = 0\n",
    "            scores = []\n",
    "            for elt2 in self.neighbours:\n",
    "                if len(elt2) < self.max_elts:\n",
    "                    clf = linear_model.LinearRegression(fit_intercept=False, \n",
    "                                                        normalize=True, copy_X=True, n_jobs=-1) \n",
    "                    clf.fit(train[elt2+[elt]], y)\n",
    "                    scores.append(metrics.log_loss(y,clf.predict(train[elt2 + [elt]])))\n",
    "                    indice=indice+1\n",
    "                else:\n",
    "                    scores.append(lastscores[indice])\n",
    "                    indice=indice+1\n",
    "            gains = lastscores - scores\n",
    "            if gains.max() > 0:\n",
    "                temp = gains.argmax()\n",
    "                lastscores[temp] = scores[temp]\n",
    "                self.neighbours[temp].append(elt)\n",
    "\n",
    "        indice = 0\n",
    "        for elt in self.neighbours:\n",
    "            clf = linear_model.LinearRegression(fit_intercept=False, normalize=True, copy_X=True, n_jobs=-1) \n",
    "            clf.fit(train[elt], y)\n",
    "            self.clfs.append(clf)\n",
    "            if self.verbose:\n",
    "                print(indice, lastscores[indice], elt)\n",
    "            indice = indice + 1\n",
    "                    \n",
    "    def transform(self, train):\n",
    "        indice = 0\n",
    "        for elt in self.neighbours:\n",
    "            train['_'.join(pd.Series(elt).sort_values().values)] = self.clfs[indice].predict(train[elt])\n",
    "            indice = indice + 1\n",
    "        return train\n",
    "    \n",
    "    def fit_transform(self, train, y):\n",
    "        self.fit(train, y)\n",
    "        return self.transform(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv(\"data/train.csv\")\n",
    "target = train['target'].values\n",
    "test = pd.read_csv(\"data/test.csv\")\n",
    "id_test = test['ID'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#train['v22-1']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[0]))\n",
    "#test['v22-1']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[0]))\n",
    "#train['v22-2']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[1]))\n",
    "#test['v22-2']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[1]))\n",
    "#train['v22-3']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[2]))\n",
    "#test['v22-3']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[2]))\n",
    "#train['v22-4']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[3]))\n",
    "#test['v22-4']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[3]))\n",
    "\n",
    "drop_list=['v91','v1', 'v8', 'v10', 'v15', 'v17', 'v25', 'v29', 'v34', \n",
    "           'v41', 'v46', 'v54', 'v64', 'v67', 'v97', 'v105', 'v111', 'v122']\n",
    "train = train.drop(['ID','target'] + drop_list,axis=1)\n",
    "test = test.drop(['ID'] + drop_list,axis=1)\n",
    "\n",
    "refcols=list(train.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train['v22-1']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[0]))\n",
    "test['v22-1']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[0]))\n",
    "train['v22-2']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[1]))\n",
    "test['v22-2']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[1]))\n",
    "train['v22-3']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[2]))\n",
    "test['v22-3']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[2]))\n",
    "train['v22-4']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[3]))\n",
    "test['v22-4']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[3]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = train.fillna(-999)\n",
    "test = test.fillna(-999)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for elt in refcols:\n",
    "    if train[elt].dtype=='O':\n",
    "        train[elt], temp = pd.factorize(train[elt])\n",
    "        test[elt]=temp.get_indexer(test[elt])\n",
    "    else:\n",
    "        train[elt]=train[elt].round(5)\n",
    "        test[elt]=test[elt].round(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ztr = (train.v22.value_counts()).index[train.v22.value_counts() > 30].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train.loc[train.v22.map(lambda x: x not in ztr), 'v22'] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "means = train.groupby(by='v22').mean()\n",
    "maxs = train.groupby(by='v22').max()\n",
    "mins = train.groupby(by='v22').min()\n",
    "medians = train.groupby(by='v22').median()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "mens = means.loc[train.v22, :]\n",
    "mens.index = train.index\n",
    "mans = maxs.loc[train.v22, :]\n",
    "mans.index = train.index\n",
    "mns = mins.loc[train.v22, :]\n",
    "mns.index = train.index\n",
    "mdns = medians.loc[train.v22, :]\n",
    "mdns.index = train.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ntrain = pd.concat((train, mens, mans, mns, mdns), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((114321, 113), (114321, 509))"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape, ntrain.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ntrain.columns = ['v'+str(i) for i in range(509)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "refcols=list(ntrain.columns)\n",
    "for elt in refcols:\n",
    "    if ntrain[elt].dtype=='O':\n",
    "        ntrain[elt], temp = pd.factorize(ntrain[elt])\n",
    "        #test[elt]=temp.get_indexer(test[elt])\n",
    "    else:\n",
    "        ntrain[elt]=ntrain[elt].round(5)\n",
    "        #test[elt]=test[elt].round(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import xgboost as xgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dtrain = xgb.DMatrix(ntrain, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "params = {\n",
    "    'eta': .3,\n",
    "    'silent': 1,\n",
    "    'nthread': 12,\n",
    "    'max_depth': 6,\n",
    "    'min_child_weight': 1,\n",
    "    'subsample': .8,\n",
    "    'colsample_bytree': .8,\n",
    "    'objective': 'binary:logistic',\n",
    "    'eval_metric': 'logloss',\n",
    "    'seed': 0\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[0]\tcv-test-logloss:0.5978327499999999+0.0006750319899826836\tcv-train-logloss:0.5966130000000001+0.0006767787673974394\n",
      "[1]\tcv-test-logloss:0.5475265+0.0013140392878449151\tcv-train-logloss:0.5450475+0.000741485502218351\n",
      "[2]\tcv-test-logloss:0.51830175+0.0006695160098907166\tcv-train-logloss:0.5147042500000001+0.001105487533851003\n",
      "[3]\tcv-test-logloss:0.501724+0.0010684481737547938\tcv-train-logloss:0.49688825000000003+0.0020205573952501404\n",
      "[4]\tcv-test-logloss:0.49201275+0.0008954876813781488\tcv-train-logloss:0.4857785+0.0017408063505169006\n",
      "[5]\tcv-test-logloss:0.48478849999999996+0.001070837405958533\tcv-train-logloss:0.4771265+0.0009040366419564985\n",
      "[6]\tcv-test-logloss:0.48095175+0.0010338163219353947\tcv-train-logloss:0.4719545+0.0007506092525408892\n",
      "[7]\tcv-test-logloss:0.47789499999999996+0.001024887310878613\tcv-train-logloss:0.46750800000000003+0.0005765604911889093\n",
      "[8]\tcv-test-logloss:0.47620225000000005+0.0010471662654516723\tcv-train-logloss:0.46436849999999996+0.0004225958471163658\n",
      "[9]\tcv-test-logloss:0.47489149999999997+0.001072919964396246\tcv-train-logloss:0.461646+0.0005143743772778694\n",
      "[10]\tcv-test-logloss:0.47400999999999993+0.001272910248210757\tcv-train-logloss:0.4593935+0.00036019751526072404\n",
      "[11]\tcv-test-logloss:0.47337725+0.00118798555862435\tcv-train-logloss:0.45737174999999997+0.00039577163554251575\n",
      "[12]\tcv-test-logloss:0.473013+0.0013048917579630839\tcv-train-logloss:0.4554165+0.00017200072674265773\n",
      "[13]\tcv-test-logloss:0.47266400000000003+0.001164815650650344\tcv-train-logloss:0.4536795+9.863189139421134e-05\n",
      "[14]\tcv-test-logloss:0.47241475+0.0012964590959609796\tcv-train-logloss:0.45215150000000004+0.00013947490813762323\n",
      "[15]\tcv-test-logloss:0.47222700000000006+0.0011012604142526833\tcv-train-logloss:0.45077+0.00011324972406147537\n",
      "[16]\tcv-test-logloss:0.47201325+0.0011449690770933528\tcv-train-logloss:0.44931899999999997+0.00027271596946273587\n",
      "[17]\tcv-test-logloss:0.47203649999999997+0.0010210703942432197\tcv-train-logloss:0.44805400000000006+0.0004980045180517964\n",
      "[18]\tcv-test-logloss:0.47192425+0.0010236350362800145\tcv-train-logloss:0.44704900000000003+0.0006150821083400282\n",
      "[19]\tcv-test-logloss:0.47192325+0.0010384785445544736\tcv-train-logloss:0.44588350000000004+0.0008268378015064366\n",
      "[20]\tcv-test-logloss:0.47177875+0.0010520566465262015\tcv-train-logloss:0.44477675+0.001012338721723115\n",
      "[21]\tcv-test-logloss:0.47183200000000003+0.0010974310912307823\tcv-train-logloss:0.44359600000000005+0.001110589258006771\n",
      "[22]\tcv-test-logloss:0.471962+0.0012432805395404666\tcv-train-logloss:0.44272750000000005+0.001196980471854075\n",
      "[23]\tcv-test-logloss:0.47202999999999995+0.0011863517606511302\tcv-train-logloss:0.4417105+0.0011463347024320493\n",
      "[24]\tcv-test-logloss:0.472147+0.0012698549129723264\tcv-train-logloss:0.44076975+0.0011529374169919164\n",
      "[25]\tcv-test-logloss:0.47207124999999994+0.0014673367669011791\tcv-train-logloss:0.439541+0.0012098749935427167\n",
      "[26]\tcv-test-logloss:0.47206349999999997+0.001380237932387029\tcv-train-logloss:0.438447+0.001112447976311702\n",
      "[27]\tcv-test-logloss:0.471999+0.001492509463956587\tcv-train-logloss:0.437399+0.0011174781877065806\n",
      "[28]\tcv-test-logloss:0.47210799999999997+0.001540811961272353\tcv-train-logloss:0.43631549999999997+0.0011879150853491137\n",
      "[29]\tcv-test-logloss:0.4721685+0.0015279689951042754\tcv-train-logloss:0.43544825+0.0010609487204856019\n",
      "[30]\tcv-test-logloss:0.47227675+0.0014991011265088122\tcv-train-logloss:0.43443925+0.0012704501908772153\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-18-17764848f4e6>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mxgb\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mparams\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtrain\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnum_boost_round\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m500\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnfold\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m4\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstratified\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmetrics\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'logloss'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mverbose_eval\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m/home/libfun/vrt/lib/python3.4/site-packages/xgboost-0.4-py3.4.egg/xgboost/training.py\u001b[0m in \u001b[0;36mcv\u001b[1;34m(params, dtrain, num_boost_round, nfold, stratified, folds, metrics, obj, feval, maximize, early_stopping_rounds, fpreproc, as_pandas, verbose_eval, show_stdv, seed)\u001b[0m\n\u001b[0;32m    455\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mfold\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mcvfolds\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    456\u001b[0m             \u001b[0mfold\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 457\u001b[1;33m         res = aggcv([f.eval(i, feval) for f in cvfolds],\n\u001b[0m\u001b[0;32m    458\u001b[0m                     \u001b[0mshow_stdv\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mshow_stdv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mverbose_eval\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mverbose_eval\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    459\u001b[0m                     as_pandas=as_pandas, trial=i)\n",
      "\u001b[1;32m/home/libfun/vrt/lib/python3.4/site-packages/xgboost-0.4-py3.4.egg/xgboost/training.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m    455\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mfold\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mcvfolds\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    456\u001b[0m             \u001b[0mfold\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 457\u001b[1;33m         res = aggcv([f.eval(i, feval) for f in cvfolds],\n\u001b[0m\u001b[0;32m    458\u001b[0m                     \u001b[0mshow_stdv\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mshow_stdv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mverbose_eval\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mverbose_eval\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    459\u001b[0m                     as_pandas=as_pandas, trial=i)\n",
      "\u001b[1;32m/home/libfun/vrt/lib/python3.4/site-packages/xgboost-0.4-py3.4.egg/xgboost/training.py\u001b[0m in \u001b[0;36meval\u001b[1;34m(self, iteration, feval)\u001b[0m\n\u001b[0;32m    255\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0meval\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0miteration\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeval\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    256\u001b[0m         \u001b[1;34m\"\"\"\"Evaluate the CVPack for one iteration.\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 257\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbst\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0meval_set\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwatchlist\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0miteration\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeval\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    258\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    259\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/libfun/vrt/lib/python3.4/site-packages/xgboost-0.4-py3.4.egg/xgboost/core.py\u001b[0m in \u001b[0;36meval_set\u001b[1;34m(self, evals, iteration, feval)\u001b[0m\n\u001b[0;32m    809\u001b[0m             _check_call(_LIB.XGBoosterEvalOneIter(self.handle, iteration,\n\u001b[0;32m    810\u001b[0m                                                   \u001b[0mdmats\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mevnames\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mevals\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 811\u001b[1;33m                                                   ctypes.byref(msg)))\n\u001b[0m\u001b[0;32m    812\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mmsg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    813\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "xgb.cv(params, dtrain, num_boost_round=500, nfold=4, stratified=True, metrics='logloss', verbose_eval=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import cross_val_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fntrain = ntrain.fillna(-999)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.48433754, -0.48323283, -0.48327725, -0.48193341])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_val_score(ensemble.ExtraTreesClassifier(n_estimators=750, max_features=50 ,criterion='entropy',\n",
    "                                                min_samples_split=4, max_depth=35, min_samples_leaf=2, \n",
    "                                                n_jobs=-1, random_state=12),\n",
    "                fntrain, target, scoring='log_loss', cv=4, n_jobs=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 0.564640677692 ['v95', 'v22-3']\n",
      "1 0.78492204104 ['v93', 'v39', 'v55', 'v45']\n",
      "2 1.0150421579 ['v103', 'v32']\n",
      "3 1.2237846517 ['v112', 'v30', 'v38', 'v90', 'v100']\n",
      "4 0.695421552089 ['v77', 'v18']\n",
      "5 1.51026055801 ['v12', 'v52', 'v24']\n",
      "6 0.912545995271 ['v131', 'v69']\n",
      "7 0.590485761075 ['v50', 'v14']\n",
      "8 1.27423497001 ['v71', 'v51', 'v107', 'v129', 'v85']\n",
      "9 1.34909645163 ['v82', 'v89', 'v70']\n",
      "10 0.953489101882 ['v128', 'v114']\n",
      "11 0.597755677821 ['v102', 'v110', 'v11', 'v35', 'v121']\n",
      "12 0.67363493342 ['v6', 'v49']\n",
      "13 0.560716786825 ['v43', 'v53']\n",
      "14 0.55909244045 ['v47', 'v22-4']\n",
      "15 1.02753943928 ['v62', 'v31']\n",
      "16 1.71151478424 ['v40', 'v58']\n",
      "17 0.586971031973 ['v116', 'v20']\n",
      "18 0.672166410312 ['v5', 'v108', 'v81']\n",
      "19 0.549512531659 ['v22-1', 'v109']\n",
      "20 0.74920712197 ['v61', 'v127', 'v101']\n",
      "21 0.777851399127 ['v117', 'v23', 'v27', 'v126', 'v94']\n",
      "22 0.933721028047 ['v92', 'v28', 'v72', 'v3']\n",
      "23 0.63493438842 ['v88', 'v68']\n",
      "24 0.661909492268 ['v65', 'v26']\n",
      "25 0.731832815015 ['v115', 'v76', 'v33', 'v59']\n",
      "26 1.15896428988 ['v84', 'v83', 'v13']\n",
      "27 0.928826550551 ['v86', 'v106', 'v104']\n",
      "28 0.979200665933 ['v19', 'v78', 'v44']\n",
      "29 0.569156601724 ['v36', 'v123', 'v79', 'v22-2']\n",
      "30 1.78018473991 ['v125', 'v113', 'v98']\n",
      "31 0.651382893884 ['v99', 'v96']\n",
      "32 1.70085601077 ['v66', 'v22', 'v56', 'v74']\n",
      "33 1.08546777218 ['v118', 'v73']\n",
      "34 1.69479229131 ['v124', 'v119', 'v16', 'v63', 'v87']\n",
      "35 0.575071564801 ['v42', 'v130', 'v60']\n",
      "36 1.16144510508 ['v48', 'v120']\n",
      "37 0.978400519026 ['v57', 'v7']\n",
      "38 0.84544290493 ['v80', 'v9']\n",
      "39 1.66754527869 ['v21', 'v37', 'v4', 'v2', 'v75']\n"
     ]
    }
   ],
   "source": [
    "a=addNearestNeighbourLinearFeatures(n_neighbours=n_ft, max_elts=max_elts, verbose=True, random_state=rnd)\n",
    "a.fit(train, target)\n",
    "\n",
    "train = a.transform(train)\n",
    "test = a.transform(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "params = [{'max_features': 50, 'max_depth': 35, 'random'}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 35 50\n",
      "12 35 50\n",
      "42 35 50\n",
      "0 40 50\n",
      "12 40 50\n",
      "42 40 50\n",
      "0 45 50\n",
      "12 45 50\n",
      "42 45 50\n",
      "0 35 60\n",
      "12 35 60\n",
      "42 35 60\n",
      "0 40 60\n",
      "12 40 60\n",
      "42 40 60\n",
      "0 45 60\n",
      "12 45 60\n",
      "42 45 60\n",
      "0 35 70\n",
      "12 35 70\n",
      "42 35 70\n",
      "0 40 70\n",
      "12 40 70\n",
      "42 40 70\n",
      "0 45 70\n",
      "12 45 70\n",
      "42 45 70\n"
     ]
    }
   ],
   "source": [
    "preds = list()\n",
    "for mf in [50, 60, 70]:\n",
    "    for md in [35, 40, 45]:\n",
    "        for rnd in [0, 12, 42]:\n",
    "            print(rnd, md, mf, flush=True)\n",
    "            clf = ensemble.ExtraTreesClassifier(n_estimators=3750, max_features=mf ,criterion='entropy',\n",
    "                                                min_samples_split=4, max_depth=md, min_samples_leaf=2, \n",
    "                                                n_jobs = -1, random_state=rnd)\n",
    "            clf.fit(train,target)\n",
    "            pred_et = clf.predict_proba(test)[:, 1]\n",
    "            preds.append(pred_et)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(114393,)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(preds).mean(axis=0).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f7ed49fdb70>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEACAYAAABGYoqtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFglJREFUeJzt3X2sZHV5wPHvAwtBLLKgYTddkIvVhcWqW2zRRhuvLxWx\nCRCTblZb9Qo1jdqKbdO626Sh/zRWk6ZoLCRUK2A0iJpUoohI4daYSME3sC7Cqt2VXbuXqLiNNtJF\nnv4x53qG9R7uvJwzc87M95NMmPO7z8w8+4Q7zz2/Z14iM5EkaS3HTDsBSVJ72SQkSZVsEpKkSjYJ\nSVIlm4QkqZJNQpJUad0mEREfjIiViLi3b+2UiLg1Iu6PiM9FxMl9P9sdEXsj4r6IeGXf+nkRcW9E\nPBARV/atHx8RNxS3+VJEPL3Of6AkaXSDnEl8CLjgqLVdwG2ZeTZwO7AbICLOBXYA24ALgasiIorb\nXA1clplbga0RsXqflwE/ysxnAVcC7xnj3yNJqtG6TSIzvwg8fNTyxcB1xfXrgEuK6xcBN2Tmo5m5\nD9gLnB8Rm4GTMvPuIu76vtv039cngJeP8O+QJDVg1JnEaZm5ApCZh4DTivUtwIN9cQeLtS3Agb71\nA8Xa426TmT8HfhwRp46YlySpRnUNruv8bI9YP0SSNAkbRrzdSkRsysyVYivpoWL9IHBGX9zpxVrV\nev9tvh8RxwJPycwfrfWgEeEHTUnSCDJzpD/ABz2TCB7/F/5NwFJx/Y3Ap/rWdxavWDoLeCZwV7El\ndTgizi8G2W846jZvLK7/Pr1BeKXM9JLJFVdcMfUc2nKxFtbCWjzxZRzrnklExEeBReCpEfE94Arg\n74GPR8SlwH56r2giM/dExI3AHuAI8NYsM3wbcC1wAnBzZt5SrH8Q+HBE7AV+COwc6180J/bt2zft\nFFrDWpSsRcla1GPdJpGZr6v40Ssq4t8FvGuN9a8Az1lj/RGKJiNJahffcd1RS0tL006hNaxFyVqU\nrEU9Ytz9qkmKiOxSvpLUBhFBNjy4VsssLy9PO4XWsBYla1GyFvWwSUiSKrndJEkzzu0mSVIjbBId\n5X5ryVqUrEXJWtTDJiFJquRMQpKGsHnzAisr+weK3bTpTA4d2jf1+x5nJmGTkKQh9D5+btDnoROA\nR4Z8hEHvOwb+XCYH13PI/daStShZi1I7avEIvSf9QS/tY5OQNPc2b14gIga6zBu3myTNveG2kJqK\nHf6+3W6SJE2VTaKj2rHf2g7WomQtStaiHjYJSVIlZxKS5p4ziWqeSUiSKtkkOsr91pK1KFmLkrWo\nh01CklTJmYSkuedMoppnEpKkSjaJjnK/tWQtStaiZC3qYZOQJFVyJiFp7jmTqOaZhCSpkk2io9xv\nLVmLkrUoWYt62CQkSZWcSUiaScN8X3SPM4k1b9ulJ12bhKRBtWMY3f0m4XZTR7nfWrIWJWuhutkk\nJEmV3G6SNJPcbuqLdLtJ0jzYvHmBiBjoonrYJDrKveeStSjNei16r1bKAS+qg01CklRprJlEROwG\n/hD4OfAN4E3Ak4GPAWcC+4AdmXm4L/5S4FHg8sy8tVg/D7gWOAG4OTPfUfF4ziSkOda9OcMczyQi\n4kzgzcBvZOZzgQ3Aa4FdwG2ZeTZwO7C7iD8X2AFsAy4Eropy4/Bq4LLM3ApsjYgLRs1LklSfcbab\n/gf4P+DJEbEBeBJwELgYuK6IuQ64pLh+EXBDZj6amfuAvcD5EbEZOCkz7y7iru+7jSrM+t7zMKxF\nyVqobiM3icx8GPgH4Hv0msPhzLwN2JSZK0XMIeC04iZbgAf77uJgsbYFONC3fqBYkyRN2YZRbxgR\nzwD+jN7s4TDw8Yj4A355Q63WIcLS0hILCwsAbNy4ke3bt7O4uAiUf0XNw/Hi4mKr8vG4Pcer2pLP\nesc7dy4N+RlLq5aL/y5WHK+uVf181ONBH7/p+F4N16rv8vIy1157LcAvni9HNfLgOiJ2AL+bmW8u\njl8PvBB4GbCYmSvFVtIdmbktInYBmZnvLuJvAa4A9q/GFOs7gZdk5lvWeEwH19KMme1h9BwProH7\ngRdGxAnFAPrlwB7gJmCpiHkj8Kni+k3Azog4PiLOAp4J3FVsSR2OiPOL+3lD321Uwb3nkrUoWQvV\nbeTtpsy8JyKuB75C7yWwXwOuAU4CboyIS+mdJewo4vdExI30GskR4K19pwVv4/Evgb1l1LwkSfXx\ns5skTZXbTaPfd9u3myRJM84m0VHuPZesRclaqG42CUlSJWcSkqbKmcTo9+1MQpI0VTaJjnLvuWQt\nStZCdbNJSJIqOZOQNFXOJEa/b2cSkjrJ76KeHTaJjnLvuWQtSm2phd9FPTtsEpKkSs4kJNXOOcMo\nscPftzMJSdJU2SQ6qi17z21gLUrWQnWzSUiSKjmTkFQ7ZxKjxA5/384kJLXCMO978L0Ps8Um0VHu\nPZesRampWgz3vgfP9meJTUKSVMmZhKR1DTdjgHbs8Xctdvj7diYhSZoqm0RHuQ9fshYla6G62SQk\nSZWcSUhalzOJScQOG38C8MjA9zzqTGLDKDeSJE3bIwzXrEbjdlNHufdcshYla6G62SQkSZWcSUha\nlzOJScQ2m4fvk5Ak1c4m0VHuPZesRclaqG42CUlSJWcSktblTGISsc3m4UxCklQ7m0RHufdcshYl\na6G62SQkSZXGmklExMnAB4BfBx4DLgUeAD4GnAnsA3Zk5uEifncR8yhweWbeWqyfB1xL78NIbs7M\nd1Q8njMJaQqcSUwittk8pjWTeC+9J/VtwPOAbwG7gNsy82zgdmA3QEScC+wAtgEXAldF+WW4VwOX\nZeZWYGtEXDBmXpLWMcz3Vmt+jdwkIuIpwO9k5ocAMvPR4ozhYuC6Iuw64JLi+kXADUXcPmAvcH5E\nbAZOysy7i7jr+26jCu49l6xFaZhaDPe91ZpX45xJnAX8ICI+FBFfjYhrIuJEYFNmrgBk5iHgtCJ+\nC/Bg3+0PFmtbgAN96weKNUnSlI3TJDYA5wH/lJnnAT+lt9V09J8d/hnSgMXFxWmn0BrWomQtVLdx\nvk/iAPBgZn65OP4kvSaxEhGbMnOl2Ep6qPj5QeCMvtufXqxVra9paWmJhYUFADZu3Mj27dt/8Yux\neqrtsccer3/cswws9l1nCses8/NR41fXxs1v1MdvOn51ba2fL9N7LRDAAuMY99VN/w68OTMfiIgr\ngBOLH/0oM98dEe8ETsnMXcXg+iPAC+htJ30eeFZmZkTcCbwduBv4DPC+zLxljcfz1U2F5eVl/2os\nWIvSMLUY7hVL7XmVzuzGNpvHtL6Z7u3ARyLiOOC7wJuAY4EbI+JSYD+9VzSRmXsi4kZgD3AEeGvf\nM/7bePxLYH+pQUiSJs/PbpLmlGcSbYttNg8/u0mSVDubREf53oCStShZC9XNJiFJquRMQppTziTa\nFttsHs4kJEm1s0l0lHvPJWtRshaqm01CklTJmYQ0p5xJtC222TycSUiSameT6Cj3nkvWomQtVDeb\nhDRDXvOanX7bnGrlTEKaIc3NGdqztz67sc3m4UxCklQ7m0RHufdcshZSc2wSkqRKziSkGeJMosux\nzebhTEKSVDubREe5D1+yFlJzbBKSpErOJKQZ4kyiy7HN5uFMQpJUO5tER7kPX7IWUnNsEpKkSs4k\npBniTKLLsc3m4UxCklQ7m0RHuQ9fshZSc2wSkqRKziSkGeJMosuxzebhTEKSVDubREe5D1+yFlJz\nbBKSpErOJKQZ4kyiy7HN5uFMQpJUO5tER7kPX7IWUnNsElLLbd68QEQMdJHq5kxCarl2zBnas7c+\nu7HN5jG1mUREHBMRX42Im4rjUyLi1oi4PyI+FxEn98Xujoi9EXFfRLyyb/28iLg3Ih6IiCvHzUmS\nVI86tpsuB/b0He8CbsvMs4Hbgd0AEXEusAPYBlwIXBXl+fHVwGWZuRXYGhEX1JDXTHMfvmQtpOaM\n1SQi4nTg1cAH+pYvBq4rrl8HXFJcvwi4ITMfzcx9wF7g/IjYDJyUmXcXcdf33UaSNEXjnkn8I/CX\nPH5jbFNmrgBk5iHgtGJ9C/BgX9zBYm0LcKBv/UCxpiewuLg47RRaw1pIzRm5SUTE7wErmfl1ehOU\nKk6aJamjNoxx2xcBF0XEq4EnASdFxIeBQxGxKTNXiq2kh4r4g8AZfbc/vVirWl/T0tISCwsLAGzc\nuJHt27f/4i/J1b3peTju34dvQz7TPF5da0s+Tfz7elaPF9c5HjR+dW29+2v6mHV+Pmr86tq4+Y36\n+E3Hr66t9fNl4NrieIFx1PIS2Ih4CfAXmXlRRLwH+GFmvjsi3gmckpm7isH1R4AX0NtO+jzwrMzM\niLgTeDtwN/AZ4H2Zecsaj+NLYAvLy8tusxRmvRa+BHZeYpvNY9SXwDbRJE4FbqR3drAf2JGZPy7i\ndgOXAUeAyzPz1mL9+fTa3gnAzZl5ecXj2CQ0d2wS8xLbbB5TbRKTYpPQPLJJzEtss3n4AX9zxvcG\nlKyF1BybhCSpkttNUsu53TQvsc3m4XaTJKl2NomOch++ZC2k5tgkJEmVnElILedMYl5im83DmYTU\nIX7bnLrCJtFR7sOXuliLlZX99P4KHOQiTY9NQpJUyZmENAXdmzO0Z299dmObzcOZhCSpdjaJjuri\nPnxTrIXUHJuEJKmSMwlpCpxJGDvpPJxJSJJqZ5PoKPfhS9ZCao5NQpJUyZmENAXOJIyddB7OJCRJ\ntbNJdJT78CVrITXHJiFJquRMQpoCZxLGTjoPZxKSpNrZJDrKffiStZCaY5OQauK3zWkWOZOQajLb\nc4b27K3PbmyzeTiTkCTVzibRUe7Dl6yF1BybhCSpkjMJqSbOJNqWR9dim83DmYQkqXY2iY5yH75k\nLaTm2CQkSZWcSUg1cSbRtjy6FttsHs4kJEm1G7lJRMTpEXF7RHwzIr4REW8v1k+JiFsj4v6I+FxE\nnNx3m90RsTci7ouIV/atnxcR90bEAxFx5Xj/pPngPnzJWkjNGedM4lHgzzPz2cBvA2+LiHOAXcBt\nmXk2cDuwGyAizgV2ANuAC4GrovwQm6uByzJzK7A1Ii4YIy+pNn4ek+ZdbTOJiPhX4P3F5SWZuRIR\nm4HlzDwnInYBmZnvLuI/C/wtsB+4PTPPLdZ3Frd/yxqP4UxCE+WcYZTYtuTRtdhm85jqTCIiFoDt\nwJ3ApsxcAcjMQ8BpRdgW4MG+mx0s1rYAB/rWDxRrkqQp2zDuHUTErwCfAC7PzJ9ExNGtrdY//ZeW\nllhYWABg48aNbN++ncXFRaDcm56H4/59+DbkM83j1bUm779n9Xix4nh1rernox4P+vjDxq+ujZvf\nuMes8/NR41fXxs1v1MdvOn51ba2fLwPXFscLjGOs7aaI2AB8GvhsZr63WLsPWOzbbrojM7etsd10\nC3AFve2mOzJzW7HudtMAlpeXf/FkNu+arIXbTaPEtiWPrsU2m8e0tpv+Bdiz2iAKNwFLxfU3Ap/q\nW98ZEcdHxFnAM4G7ii2pwxFxfjHIfkPfbVTBBlGyFlJzRj6TiIgXAV8AvkGvnSXw18BdwI3AGfTO\nEnZk5o+L2+wGLgOO0NueurVYfz69c6MTgJsz8/KKx/RMQhPlmcQosW3Jo2uxzeYx6pmE77juKLeb\nSm43tS22LXl0LbbZPHzHtSSpdp5JSE/AM4lRYtuSR9dim83DMwlJUu1sEh3l5xWVrIXUHJuEJKmS\nMwnpCTiTGCW2LXl0LbbZPJxJSJJqZ5PoKPfhS8PUYpiP/vbjv6UaPuBP6pKVlf0Mf/ovzS9nEpor\nw80YoB171V2LbUseXYttNg9nEpKk2tkkOsqZRMlaSM2xSUiSKjmT0FxxJjGJ2Lbk0bXYZvNwJiFJ\nqp1NoqPchy+deupm3/cgNcT3SajzHn54heFO0SUNypmEOq+5z1caNt7YduXRtdhm83AmIUmqnU2i\no5xJSJoEm4QkqZIzCbXS5s0LxYfxDap7e8SzG9uWPLoW22weo84kbBJqpXZ82U+T9z3LsW3Jo2ux\nzebh4HrOOJOQNAk2CUlSJbeb1EpuN3U5ti15dC222TzcblLrDfPVoZLawSbRUV2cSZRfHTrIRVIb\n2CQ0Fs8OpNnmTEJjacfsoJt7xLMb25Y8uhbbbB7OJCRJtbNJdNQwM4lhtoQigmOPfbJbSJIAv09i\nLpQD48E89tiwp7ySZpVNokWG+byiY445kcce+9+GM5I072wSDWvqg+r8a1/SJLRmJhERr4qIb0XE\nAxHxzmnnUxffGyCpy1rRJCLiGOD9wAXAs4HXRsQ5081KktSKJgGcD+zNzP2ZeQS4Abh4yjlV8g1k\nkuZFW2YSW4AH+44P0Gscv+Tw4cMD3eFxxx3HiSeeOHACzX7JjSR1U1uaxMA2btw4cOxppz2dhx76\n3hD37hO/JPVrS5M4CDy97/j0Ym0swzUIGO7Jf5Zj25JHG2LbkkfXYtuSR9dim77v4bXis5si4ljg\nfuDlwH8DdwGvzcz7ppqYJM25VpxJZObPI+JPgFvpDdM/aIOQpOlrxZmEJKmd2vIS2McZ5I11EfG+\niNgbEV+PiO2TznFS1qtFRLwuIu4pLl+MiOdMI89JGPQNlxHxWxFxJCJeM8n8JmnA35HFiPhaRPxn\nRNwx6RwnZYDfkadGxGeL54pvRMTSFNJsXER8MCJWIuLeJ4gZ/nkzM1t1ode4vg2cCRwHfB0456iY\nC4HPFNdfANw57bynWIsXAicX1181z7Xoi/s34NPAa6ad9xT/vzgZ+CawpTh+2rTznmItrgDetVoH\n4IfAhmnn3kAtXgxsB+6t+PlIz5ttPJMY5I11FwPXA2TmfwAnR8SmyaY5EevWIjPvzMzVN4/cSe89\nJ7No0Ddc/inwCeChSSY3YYPU4nXAJzPzIEBm/mDCOU7KILU4BJxUXD8J+GFmPjrBHCciM78IPPwE\nISM9b7axSaz1xrqjn/iOjjm4RswsGKQW/f4I+GyjGU3PurWIiF8FLsnMq5ntN7MM8v/FVuDUiLgj\nIu6OiNdPLLvJGqQW/ww8OyK+D9wDXD6h3NpmpOfNVry6SeOLiJcCb6J3yjmvrgT696RnuVGsZwNw\nHvAy4MnAlyLiS5n57emmNRW7gXsy86UR8WvA5yPiuZn5k2kn1gVtbBKDvLHuIHDGOjGzYKA3GUbE\nc4FrgFdl5hOdbnbZILX4TeCG6H1o1tOACyPiSGbeNKEcJ2WQWhwAfpCZPwN+FhFfAJ5Hb/9+lgxS\nixcBfweQmd+JiP8CzgG+PJEM22Ok5802bjfdDTwzIs6MiOOBncDRv+Q3AW8AiIgXAj/OzJXJpjkR\n69YiIp4OfBJ4fWZ+Zwo5Tsq6tcjMZxSXs+jNJd46gw0CBvsd+RTw4og4NiJOpDeonMX3Hg1Si/uA\nVwAUe/Bbge9ONMvJCarPoEd63mzdmURWvLEuIv649+O8JjNvjohXR8S3gZ/S22aZOYPUAvgb4FTg\nquIv6COZueaHI3bZgLV43E0mnuSEDPg78q2I+BxwL/Bz4JrM3DPFtBsx4P8X7wI+FBH30HsC/avM\n/NH0sm5GRHwUWASeGhHfo/eqruMZ83nTN9NJkiq1cbtJktQSNglJUiWbhCSpkk1CklTJJiFJqmST\nkCRVsklIkirZJCRJlf4fwcMSWFC+d9gAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f7ed49fd390>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "submission=pd.read_csv('data/sample_submission.csv')\n",
    "submission.index=submission.ID\n",
    "submission.PredictedProb=np.array(preds).mean(axis=0)\n",
    "submission.to_csv('submission/et_ln_bag1.csv', index=False)\n",
    "submission.PredictedProb.hist(bins=30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load in \n",
    "\n",
    " # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
    "\n",
    "from subprocess import check_output\n",
    "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n",
    "\n",
    "# Any results you write to the current directory are saved as output.\n",
    "\n",
    "from sklearn import ensemble, metrics, linear_model\n",
    "import random\n",
    "\n",
    "#Some parameters to play with\n",
    "rnd=12\n",
    "random.seed(rnd)\n",
    "n_ft=20 #Number of features to add\n",
    "max_elts=3 #Maximum size of a group of linear features\n",
    "\n",
    "class addNearestNeighbourLinearFeatures:\n",
    "    \n",
    "    def __init__(self, n_neighbours=1, max_elts=None, verbose=True, random_state=None):\n",
    "        self.rnd=random_state\n",
    "        self.n=n_neighbours\n",
    "        self.max_elts=max_elts\n",
    "        self.verbose=verbose\n",
    "        self.neighbours=[]\n",
    "        self.clfs=[]\n",
    "        \n",
    "    def fit(self,train,y):\n",
    "        if self.rnd!=None:\n",
    "            random.seed(rnd)\n",
    "        if self.max_elts==None:\n",
    "            self.max_elts=len(train.columns)\n",
    "        list_vars=list(train.columns)\n",
    "        random.shuffle(list_vars)\n",
    "        \n",
    "        lastscores=np.zeros(self.n)+1e15\n",
    "\n",
    "        for elt in list_vars[:self.n]:\n",
    "            self.neighbours.append([elt])\n",
    "        list_vars=list_vars[self.n:]\n",
    "        \n",
    "        for elt in list_vars:\n",
    "            indice=0\n",
    "            scores=[]\n",
    "            for elt2 in self.neighbours:\n",
    "                if len(elt2)<self.max_elts:\n",
    "                    clf=linear_model.LinearRegression(fit_intercept=False, normalize=True, copy_X=True, n_jobs=-1) \n",
    "                    clf.fit(train[elt2+[elt]], y)\n",
    "                    scores.append(metrics.log_loss(y,clf.predict(train[elt2 + [elt]])))\n",
    "                    indice=indice+1\n",
    "                else:\n",
    "                    scores.append(lastscores[indice])\n",
    "                    indice=indice+1\n",
    "            gains=lastscores-scores\n",
    "            if gains.max()>0:\n",
    "                temp=gains.argmax()\n",
    "                lastscores[temp]=scores[temp]\n",
    "                self.neighbours[temp].append(elt)\n",
    "\n",
    "        indice=0\n",
    "        for elt in self.neighbours:\n",
    "            clf=linear_model.LinearRegression(fit_intercept=False, normalize=True, copy_X=True, n_jobs=-1) \n",
    "            clf.fit(train[elt], y)\n",
    "            self.clfs.append(clf)\n",
    "            if self.verbose:\n",
    "                print(indice, lastscores[indice], elt)\n",
    "            indice=indice+1\n",
    "                    \n",
    "    def transform(self, train):\n",
    "        indice=0\n",
    "        for elt in self.neighbours:\n",
    "            train['_'.join(pd.Series(elt).sort_values().values)]=self.clfs[indice].predict(train[elt])\n",
    "            indice=indice+1\n",
    "        return train\n",
    "    \n",
    "    def fit_transform(self, train, y):\n",
    "        self.fit(train, y)\n",
    "        return self.transform(train)\n",
    "    \n",
    "    \n",
    "train = pd.read_csv(\"../input/train.csv\")\n",
    "target = train['target'].values\n",
    "test = pd.read_csv(\"../input/test.csv\")\n",
    "id_test = test['ID'].values\n",
    "\n",
    "train['v22-1']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[0]))\n",
    "test['v22-1']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[0]))\n",
    "train['v22-2']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[1]))\n",
    "test['v22-2']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[1]))\n",
    "train['v22-3']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[2]))\n",
    "test['v22-3']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[2]))\n",
    "train['v22-4']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[3]))\n",
    "test['v22-4']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[3]))\n",
    "\n",
    "drop_list=['v91','v1', 'v8', 'v10', 'v15', 'v17', 'v25', 'v29', 'v34', 'v41', 'v46', 'v54', 'v64', 'v67', 'v97', 'v105', 'v111', 'v122']\n",
    "train = train.drop(['ID','target'] + drop_list,axis=1).fillna(-999)\n",
    "test = test.drop(['ID'] + drop_list,axis=1).fillna(-999)\n",
    "\n",
    "refcols=list(train.columns)\n",
    "\n",
    "for elt in refcols:\n",
    "    if train[elt].dtype=='O':\n",
    "        train[elt], temp = pd.factorize(train[elt])\n",
    "        test[elt]=temp.get_indexer(test[elt])\n",
    "    else:\n",
    "        train[elt]=train[elt].round(5)\n",
    "        test[elt]=test[elt].round(5)\n",
    "        \n",
    "a=addNearestNeighbourLinearFeatures(n_neighbours=n_ft, max_elts=max_elts, verbose=True, random_state=rnd)\n",
    "a.fit(train, target)\n",
    "\n",
    "train = a.transform(train)\n",
    "test = a.transform(test)\n",
    "\n",
    "clf = ensemble.ExtraTreesClassifier(n_estimators=750,max_features=50,criterion= 'entropy',min_samples_split= 4,\n",
    "                        max_depth= 35, min_samples_leaf= 2, n_jobs = -1, random_state=rnd)\n",
    "\n",
    "clf.fit(train,target)\n",
    "pred_et=clf.predict_proba(test)\n",
    "\n",
    "submission=pd.read_csv('../input/sample_submission.csv')\n",
    "submission.index=submission.ID\n",
    "submission.PredictedProb=pred_et[:,1]\n",
    "submission.to_csv('./addNNLinearFt.csv', index=False)\n",
    "submission.PredictedProb.hist(bins=30)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}