extratrees_nn_lf_param_bag.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from subprocess import check_output\n",
    "from sklearn import ensemble, metrics, linear_model\n",
    "import random\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class addNearestNeighbourLinearFeatures:\n",
    "    \n",
    "    def __init__(self, n_neighbours=1, max_elts=None, verbose=True, random_state=None):\n",
    "        self.rnd = random_state\n",
    "        self.n = n_neighbours\n",
    "        self.max_elts = max_elts\n",
    "        self.verbose = verbose\n",
    "        self.neighbours = []\n",
    "        self.clfs = []\n",
    "        \n",
    "    def fit(self, train, y):\n",
    "        if self.rnd != None:\n",
    "            random.seed(rnd)\n",
    "        if self.max_elts == None:\n",
    "            self.max_elts = len(train.columns)\n",
    "        list_vars = list(train.columns)\n",
    "        random.shuffle(list_vars)\n",
    "        \n",
    "        lastscores = np.zeros(self.n) + 1e15\n",
    "\n",
    "        for elt in list_vars[:self.n]:\n",
    "            self.neighbours.append([elt])\n",
    "        list_vars = list_vars[self.n:]\n",
    "        \n",
    "        for elt in list_vars:\n",
    "            indice = 0\n",
    "            scores = []\n",
    "            for elt2 in self.neighbours:\n",
    "                if len(elt2) < self.max_elts:\n",
    "                    clf = linear_model.LinearRegression(fit_intercept=False, \n",
    "                                                        normalize=True, copy_X=True) \n",
    "                    clf.fit(train[elt2+[elt]], y)\n",
    "                    scores.append(metrics.log_loss(y,clf.predict(train[elt2 + [elt]])))\n",
    "                    indice=indice+1\n",
    "                else:\n",
    "                    scores.append(lastscores[indice])\n",
    "                    indice=indice+1\n",
    "            gains = lastscores - scores\n",
    "            if gains.max() > 0:\n",
    "                temp = gains.argmax()\n",
    "                lastscores[temp] = scores[temp]\n",
    "                self.neighbours[temp].append(elt)\n",
    "\n",
    "        indice = 0\n",
    "        for elt in self.neighbours:\n",
    "            clf = linear_model.LinearRegression(fit_intercept=False, normalize=True, copy_X=True) \n",
    "            clf.fit(train[elt], y)\n",
    "            self.clfs.append(clf)\n",
    "            if self.verbose:\n",
    "                print(indice, lastscores[indice], elt)\n",
    "            indice = indice + 1\n",
    "                    \n",
    "    def transform(self, train):\n",
    "        indice = 0\n",
    "        for elt in self.neighbours:\n",
    "            train['_'.join(pd.Series(elt).sort_values().values)] = self.clfs[indice].predict(train[elt])\n",
    "            indice = indice + 1\n",
    "        return train\n",
    "    \n",
    "    def fit_transform(self, train, y):\n",
    "        self.fit(train, y)\n",
    "        return self.transform(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv(\"data/train.csv\")\n",
    "target = train['target'].values\n",
    "test = pd.read_csv(\"data/test.csv\")\n",
    "id_test = test['ID'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train['v22-1']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[0]))\n",
    "test['v22-1']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[0]))\n",
    "train['v22-2']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[1]))\n",
    "test['v22-2']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[1]))\n",
    "train['v22-3']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[2]))\n",
    "test['v22-3']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[2]))\n",
    "train['v22-4']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[3]))\n",
    "test['v22-4']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[3]))\n",
    "\n",
    "drop_list=['v91','v1', 'v8', 'v10', 'v15', 'v17', 'v25', 'v29', 'v34', 'v41', 'v46', 'v54', 'v64', 'v67', 'v97', 'v105', 'v111', 'v122']\n",
    "train = train.drop(['ID','target'] + drop_list,axis=1).fillna(-999)\n",
    "test = test.drop(['ID'] + drop_list,axis=1).fillna(-999)\n",
    "\n",
    "refcols=list(train.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for elt in refcols:\n",
    "    if train[elt].dtype=='O':\n",
    "        train[elt], temp = pd.factorize(train[elt])\n",
    "        test[elt]=temp.get_indexer(test[elt])\n",
    "    else:\n",
    "        train[elt]=train[elt].round(5)\n",
    "        test[elt]=test[elt].round(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "rnd=12\n",
    "random.seed(rnd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import StratifiedKFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "skf = StratifiedKFold(target, n_folds=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import log_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from libtelepot import sendMessage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from hyperopt import fmin, hp, tpe, STATUS_OK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def objective(space):\n",
    "    n_ft = int(space['n_ft'])\n",
    "    max_elts = int(space['max_elts'])\n",
    "    #n_ft = max_elts * 3\n",
    "    \n",
    "    scores = list()\n",
    "    \n",
    "    for train_index, test_index in skf:\n",
    "        X_train, X_test = train.iloc[train_index].copy(), train.iloc[test_index].copy()\n",
    "        y_train, y_test = target[train_index], target[test_index]\n",
    "        a = addNearestNeighbourLinearFeatures(n_neighbours=n_ft, max_elts=max_elts,\n",
    "                                              verbose=False, random_state=rnd)\n",
    "        a.fit(X_train, y_train)\n",
    "        X_train = a.transform(X_train)\n",
    "        X_test = a.transform(X_test)\n",
    "        \n",
    "        clf = ensemble.ExtraTreesClassifier(n_estimators=500, max_features=int(space['max_features']),\n",
    "                                            criterion='entropy', min_samples_split=4, \n",
    "                                            max_depth=int(space['max_depth']), min_samples_leaf=2, \n",
    "                                            n_jobs=-1, random_state=rnd)\n",
    "        clf.fit(X_train, y_train)\n",
    "        pred_et = clf.predict_proba(X_test)[:, 1]\n",
    "        scores.append(log_loss(y_test, pred_et))\n",
    "    \n",
    "    sendMessage('Scores for space {:} are {:.5f} +/- {:.5f}'.format(space, np.mean(scores), np.std(scores)))\n",
    "    return {'loss': np.mean(scores), 'loss_variance': np.std(scores), 'status': STATUS_OK}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "space = {'max_depth': hp.quniform('max_depth', 25, 70, 5),\n",
    "         'max_features': hp.quniform('max_features', 30, 100, 5),\n",
    "         'n_ft': hp.quniform('n_ft', 10, 60, 5),\n",
    "         'max_elts': hp.quniform('max_elts', 4, 20, 2)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "best = fmin(objective, space, algo=tpe.suggest, max_evals=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "n_ft=40 #Number of features to add\n",
    "max_elts=5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 0.564640677692 ['v95', 'v22-3']\n",
      "1 0.78492204104 ['v93', 'v39', 'v55', 'v45']\n",
      "2 1.0150421579 ['v103', 'v32']\n",
      "3 1.2237846517 ['v112', 'v30', 'v38', 'v90', 'v100']\n",
      "4 0.695421552089 ['v77', 'v18']\n",
      "5 1.51026055801 ['v12', 'v52', 'v24']\n",
      "6 0.912545995271 ['v131', 'v69']\n",
      "7 0.590485761075 ['v50', 'v14']\n",
      "8 1.27423497001 ['v71', 'v51', 'v107', 'v129', 'v85']\n",
      "9 1.34909645163 ['v82', 'v89', 'v70']\n",
      "10 0.953489101882 ['v128', 'v114']\n",
      "11 0.597755677821 ['v102', 'v110', 'v11', 'v35', 'v121']\n",
      "12 0.67363493342 ['v6', 'v49']\n",
      "13 0.560716786825 ['v43', 'v53']\n",
      "14 0.55909244045 ['v47', 'v22-4']\n",
      "15 1.02753943928 ['v62', 'v31']\n",
      "16 1.71151478424 ['v40', 'v58']\n",
      "17 0.586971031973 ['v116', 'v20']\n",
      "18 0.672166410312 ['v5', 'v108', 'v81']\n",
      "19 0.549512531659 ['v22-1', 'v109']\n",
      "20 0.74920712197 ['v61', 'v127', 'v101']\n",
      "21 0.777851399127 ['v117', 'v23', 'v27', 'v126', 'v94']\n",
      "22 0.933721028047 ['v92', 'v28', 'v72', 'v3']\n",
      "23 0.63493438842 ['v88', 'v68']\n",
      "24 0.661909492268 ['v65', 'v26']\n",
      "25 0.731832815015 ['v115', 'v76', 'v33', 'v59']\n",
      "26 1.15896428988 ['v84', 'v83', 'v13']\n",
      "27 0.928826550551 ['v86', 'v106', 'v104']\n",
      "28 0.979200665933 ['v19', 'v78', 'v44']\n",
      "29 0.569156601724 ['v36', 'v123', 'v79', 'v22-2']\n",
      "30 1.78018473991 ['v125', 'v113', 'v98']\n",
      "31 0.651382893884 ['v99', 'v96']\n",
      "32 1.70085601077 ['v66', 'v22', 'v56', 'v74']\n",
      "33 1.08546777218 ['v118', 'v73']\n",
      "34 1.69479229131 ['v124', 'v119', 'v16', 'v63', 'v87']\n",
      "35 0.575071564801 ['v42', 'v130', 'v60']\n",
      "36 1.16144510508 ['v48', 'v120']\n",
      "37 0.978400519026 ['v57', 'v7']\n",
      "38 0.84544290493 ['v80', 'v9']\n",
      "39 1.66754527869 ['v21', 'v37', 'v4', 'v2', 'v75']\n"
     ]
    }
   ],
   "source": [
    "a=addNearestNeighbourLinearFeatures(n_neighbours=n_ft, max_elts=max_elts, verbose=True, random_state=rnd)\n",
    "a.fit(train, target)\n",
    "\n",
    "train = a.transform(train)\n",
    "test = a.transform(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "params = [{'max_features': 50, 'max_depth': 35, 'random'}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 35 50\n",
      "12 35 50\n",
      "42 35 50\n",
      "0 40 50\n",
      "12 40 50\n",
      "42 40 50\n",
      "0 45 50\n",
      "12 45 50\n",
      "42 45 50\n",
      "0 35 60\n",
      "12 35 60\n",
      "42 35 60\n",
      "0 40 60\n",
      "12 40 60\n",
      "42 40 60\n",
      "0 45 60\n",
      "12 45 60\n",
      "42 45 60\n",
      "0 35 70\n",
      "12 35 70\n",
      "42 35 70\n",
      "0 40 70\n",
      "12 40 70\n",
      "42 40 70\n",
      "0 45 70\n",
      "12 45 70\n",
      "42 45 70\n"
     ]
    }
   ],
   "source": [
    "preds = list()\n",
    "for mf in [50, 60, 70]:\n",
    "    for md in [35, 40, 45]:\n",
    "        for rnd in [0, 12, 42]:\n",
    "            print(rnd, md, mf, flush=True)\n",
    "            clf = ensemble.ExtraTreesClassifier(n_estimators=3750, max_features=mf ,criterion='entropy',\n",
    "                                                min_samples_split=4, max_depth=md, min_samples_leaf=2, \n",
    "                                                n_jobs = -1, random_state=rnd)\n",
    "            clf.fit(train,target)\n",
    "            pred_et = clf.predict_proba(test)[:, 1]\n",
    "            preds.append(pred_et)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(114393,)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(preds).mean(axis=0).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f7ed49fdb70>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEACAYAAABGYoqtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFglJREFUeJzt3X2sZHV5wPHvAwtBLLKgYTddkIvVhcWqW2zRRhuvLxWx\nCRCTblZb9Qo1jdqKbdO626Sh/zRWk6ZoLCRUK2A0iJpUoohI4daYSME3sC7Cqt2VXbuXqLiNNtJF\nnv4x53qG9R7uvJwzc87M95NMmPO7z8w8+4Q7zz2/Z14iM5EkaS3HTDsBSVJ72SQkSZVsEpKkSjYJ\nSVIlm4QkqZJNQpJUad0mEREfjIiViLi3b+2UiLg1Iu6PiM9FxMl9P9sdEXsj4r6IeGXf+nkRcW9E\nPBARV/atHx8RNxS3+VJEPL3Of6AkaXSDnEl8CLjgqLVdwG2ZeTZwO7AbICLOBXYA24ALgasiIorb\nXA1clplbga0RsXqflwE/ysxnAVcC7xnj3yNJqtG6TSIzvwg8fNTyxcB1xfXrgEuK6xcBN2Tmo5m5\nD9gLnB8Rm4GTMvPuIu76vtv039cngJeP8O+QJDVg1JnEaZm5ApCZh4DTivUtwIN9cQeLtS3Agb71\nA8Xa426TmT8HfhwRp46YlySpRnUNruv8bI9YP0SSNAkbRrzdSkRsysyVYivpoWL9IHBGX9zpxVrV\nev9tvh8RxwJPycwfrfWgEeEHTUnSCDJzpD/ABz2TCB7/F/5NwFJx/Y3Ap/rWdxavWDoLeCZwV7El\ndTgizi8G2W846jZvLK7/Pr1BeKXM9JLJFVdcMfUc2nKxFtbCWjzxZRzrnklExEeBReCpEfE94Arg\n74GPR8SlwH56r2giM/dExI3AHuAI8NYsM3wbcC1wAnBzZt5SrH8Q+HBE7AV+COwc6180J/bt2zft\nFFrDWpSsRcla1GPdJpGZr6v40Ssq4t8FvGuN9a8Az1lj/RGKJiNJahffcd1RS0tL006hNaxFyVqU\nrEU9Ytz9qkmKiOxSvpLUBhFBNjy4VsssLy9PO4XWsBYla1GyFvWwSUiSKrndJEkzzu0mSVIjbBId\n5X5ryVqUrEXJWtTDJiFJquRMQpKGsHnzAisr+weK3bTpTA4d2jf1+x5nJmGTkKQh9D5+btDnoROA\nR4Z8hEHvOwb+XCYH13PI/daStShZi1I7avEIvSf9QS/tY5OQNPc2b14gIga6zBu3myTNveG2kJqK\nHf6+3W6SJE2VTaKj2rHf2g7WomQtStaiHjYJSVIlZxKS5p4ziWqeSUiSKtkkOsr91pK1KFmLkrWo\nh01CklTJmYSkuedMoppnEpKkSjaJjnK/tWQtStaiZC3qYZOQJFVyJiFp7jmTqOaZhCSpkk2io9xv\nLVmLkrUoWYt62CQkSZWcSUiaScN8X3SPM4k1b9ulJ12bhKRBtWMY3f0m4XZTR7nfWrIWJWuhutkk\nJEmV3G6SNJPcbuqLdLtJ0jzYvHmBiBjoonrYJDrKveeStSjNei16r1bKAS+qg01CklRprJlEROwG\n/hD4OfAN4E3Ak4GPAWcC+4AdmXm4L/5S4FHg8sy8tVg/D7gWOAG4OTPfUfF4ziSkOda9OcMczyQi\n4kzgzcBvZOZzgQ3Aa4FdwG2ZeTZwO7C7iD8X2AFsAy4Eropy4/Bq4LLM3ApsjYgLRs1LklSfcbab\n/gf4P+DJEbEBeBJwELgYuK6IuQ64pLh+EXBDZj6amfuAvcD5EbEZOCkz7y7iru+7jSrM+t7zMKxF\nyVqobiM3icx8GPgH4Hv0msPhzLwN2JSZK0XMIeC04iZbgAf77uJgsbYFONC3fqBYkyRN2YZRbxgR\nzwD+jN7s4TDw8Yj4A355Q63WIcLS0hILCwsAbNy4ke3bt7O4uAiUf0XNw/Hi4mKr8vG4Pcer2pLP\nesc7dy4N+RlLq5aL/y5WHK+uVf181ONBH7/p+F4N16rv8vIy1157LcAvni9HNfLgOiJ2AL+bmW8u\njl8PvBB4GbCYmSvFVtIdmbktInYBmZnvLuJvAa4A9q/GFOs7gZdk5lvWeEwH19KMme1h9BwProH7\ngRdGxAnFAPrlwB7gJmCpiHkj8Kni+k3Azog4PiLOAp4J3FVsSR2OiPOL+3lD321Uwb3nkrUoWQvV\nbeTtpsy8JyKuB75C7yWwXwOuAU4CboyIS+mdJewo4vdExI30GskR4K19pwVv4/Evgb1l1LwkSfXx\ns5skTZXbTaPfd9u3myRJM84m0VHuPZesRclaqG42CUlSJWcSkqbKmcTo9+1MQpI0VTaJjnLvuWQt\nStZCdbNJSJIqOZOQNFXOJEa/b2cSkjrJ76KeHTaJjnLvuWQtSm2phd9FPTtsEpKkSs4kJNXOOcMo\nscPftzMJSdJU2SQ6qi17z21gLUrWQnWzSUiSKjmTkFQ7ZxKjxA5/384kJLXCMO978L0Ps8Um0VHu\nPZesRampWgz3vgfP9meJTUKSVMmZhKR1DTdjgHbs8Xctdvj7diYhSZoqm0RHuQ9fshYla6G62SQk\nSZWcSUhalzOJScQOG38C8MjA9zzqTGLDKDeSJE3bIwzXrEbjdlNHufdcshYla6G62SQkSZWcSUha\nlzOJScQ2m4fvk5Ak1c4m0VHuPZesRclaqG42CUlSJWcSktblTGISsc3m4UxCklQ7m0RHufdcshYl\na6G62SQkSZXGmklExMnAB4BfBx4DLgUeAD4GnAnsA3Zk5uEifncR8yhweWbeWqyfB1xL78NIbs7M\nd1Q8njMJaQqcSUwittk8pjWTeC+9J/VtwPOAbwG7gNsy82zgdmA3QEScC+wAtgEXAldF+WW4VwOX\nZeZWYGtEXDBmXpLWMcz3Vmt+jdwkIuIpwO9k5ocAMvPR4ozhYuC6Iuw64JLi+kXADUXcPmAvcH5E\nbAZOysy7i7jr+26jCu49l6xFaZhaDPe91ZpX45xJnAX8ICI+FBFfjYhrIuJEYFNmrgBk5iHgtCJ+\nC/Bg3+0PFmtbgAN96weKNUnSlI3TJDYA5wH/lJnnAT+lt9V09J8d/hnSgMXFxWmn0BrWomQtVLdx\nvk/iAPBgZn65OP4kvSaxEhGbMnOl2Ep6qPj5QeCMvtufXqxVra9paWmJhYUFADZu3Mj27dt/8Yux\neqrtsccer3/cswws9l1nCses8/NR41fXxs1v1MdvOn51ba2fL9N7LRDAAuMY99VN/w68OTMfiIgr\ngBOLH/0oM98dEe8ETsnMXcXg+iPAC+htJ30eeFZmZkTcCbwduBv4DPC+zLxljcfz1U2F5eVl/2os\nWIvSMLUY7hVL7XmVzuzGNpvHtL6Z7u3ARyLiOOC7wJuAY4EbI+JSYD+9VzSRmXsi4kZgD3AEeGvf\nM/7bePxLYH+pQUiSJs/PbpLmlGcSbYttNg8/u0mSVDubREf53oCStShZC9XNJiFJquRMQppTziTa\nFttsHs4kJEm1s0l0lHvPJWtRshaqm01CklTJmYQ0p5xJtC222TycSUiSameT6Cj3nkvWomQtVDeb\nhDRDXvOanX7bnGrlTEKaIc3NGdqztz67sc3m4UxCklQ7m0RHufdcshZSc2wSkqRKziSkGeJMosux\nzebhTEKSVDubREe5D1+yFlJzbBKSpErOJKQZ4kyiy7HN5uFMQpJUO5tER7kPX7IWUnNsEpKkSs4k\npBniTKLLsc3m4UxCklQ7m0RHuQ9fshZSc2wSkqRKziSkGeJMosuxzebhTEKSVDubREe5D1+yFlJz\nbBKSpErOJKQZ4kyiy7HN5uFMQpJUO5tER7kPX7IWUnNsElLLbd68QEQMdJHq5kxCarl2zBnas7c+\nu7HN5jG1mUREHBMRX42Im4rjUyLi1oi4PyI+FxEn98Xujoi9EXFfRLyyb/28iLg3Ih6IiCvHzUmS\nVI86tpsuB/b0He8CbsvMs4Hbgd0AEXEusAPYBlwIXBXl+fHVwGWZuRXYGhEX1JDXTHMfvmQtpOaM\n1SQi4nTg1cAH+pYvBq4rrl8HXFJcvwi4ITMfzcx9wF7g/IjYDJyUmXcXcdf33UaSNEXjnkn8I/CX\nPH5jbFNmrgBk5iHgtGJ9C/BgX9zBYm0LcKBv/UCxpiewuLg47RRaw1pIzRm5SUTE7wErmfl1ehOU\nKk6aJamjNoxx2xcBF0XEq4EnASdFxIeBQxGxKTNXiq2kh4r4g8AZfbc/vVirWl/T0tISCwsLAGzc\nuJHt27f/4i/J1b3peTju34dvQz7TPF5da0s+Tfz7elaPF9c5HjR+dW29+2v6mHV+Pmr86tq4+Y36\n+E3Hr66t9fNl4NrieIFx1PIS2Ih4CfAXmXlRRLwH+GFmvjsi3gmckpm7isH1R4AX0NtO+jzwrMzM\niLgTeDtwN/AZ4H2Zecsaj+NLYAvLy8tusxRmvRa+BHZeYpvNY9SXwDbRJE4FbqR3drAf2JGZPy7i\ndgOXAUeAyzPz1mL9+fTa3gnAzZl5ecXj2CQ0d2wS8xLbbB5TbRKTYpPQPLJJzEtss3n4AX9zxvcG\nlKyF1BybhCSpkttNUsu53TQvsc3m4XaTJKl2NomOch++ZC2k5tgkJEmVnElILedMYl5im83DmYTU\nIX7bnLrCJtFR7sOXuliLlZX99P4KHOQiTY9NQpJUyZmENAXdmzO0Z299dmObzcOZhCSpdjaJjuri\nPnxTrIXUHJuEJKmSMwlpCpxJGDvpPJxJSJJqZ5PoKPfhS9ZCao5NQpJUyZmENAXOJIyddB7OJCRJ\ntbNJdJT78CVrITXHJiFJquRMQpoCZxLGTjoPZxKSpNrZJDrKffiStZCaY5OQauK3zWkWOZOQajLb\nc4b27K3PbmyzeTiTkCTVzibRUe7Dl6yF1BybhCSpkjMJqSbOJNqWR9dim83DmYQkqXY2iY5yH75k\nLaTm2CQkSZWcSUg1cSbRtjy6FttsHs4kJEm1G7lJRMTpEXF7RHwzIr4REW8v1k+JiFsj4v6I+FxE\nnNx3m90RsTci7ouIV/atnxcR90bEAxFx5Xj/pPngPnzJWkjNGedM4lHgzzPz2cBvA2+LiHOAXcBt\nmXk2cDuwGyAizgV2ANuAC4GrovwQm6uByzJzK7A1Ii4YIy+pNn4ek+ZdbTOJiPhX4P3F5SWZuRIR\nm4HlzDwnInYBmZnvLuI/C/wtsB+4PTPPLdZ3Frd/yxqP4UxCE+WcYZTYtuTRtdhm85jqTCIiFoDt\nwJ3ApsxcAcjMQ8BpRdgW4MG+mx0s1rYAB/rWDxRrkqQp2zDuHUTErwCfAC7PzJ9ExNGtrdY//ZeW\nllhYWABg48aNbN++ncXFRaDcm56H4/59+DbkM83j1bUm779n9Xix4nh1rernox4P+vjDxq+ujZvf\nuMes8/NR41fXxs1v1MdvOn51ba2fLwPXFscLjGOs7aaI2AB8GvhsZr63WLsPWOzbbrojM7etsd10\nC3AFve2mOzJzW7HudtMAlpeXf/FkNu+arIXbTaPEtiWPrsU2m8e0tpv+Bdiz2iAKNwFLxfU3Ap/q\nW98ZEcdHxFnAM4G7ii2pwxFxfjHIfkPfbVTBBlGyFlJzRj6TiIgXAV8AvkGvnSXw18BdwI3AGfTO\nEnZk5o+L2+wGLgOO0NueurVYfz69c6MTgJsz8/KKx/RMQhPlmcQosW3Jo2uxzeYx6pmE77juKLeb\nSm43tS22LXl0LbbZPHzHtSSpdp5JSE/AM4lRYtuSR9dim83DMwlJUu1sEh3l5xWVrIXUHJuEJKmS\nMwnpCTiTGCW2LXl0LbbZPJxJSJJqZ5PoKPfhS8PUYpiP/vbjv6UaPuBP6pKVlf0Mf/ovzS9nEpor\nw80YoB171V2LbUseXYttNg9nEpKk2tkkOsqZRMlaSM2xSUiSKjmT0FxxJjGJ2Lbk0bXYZvNwJiFJ\nqp1NoqPchy+deupm3/cgNcT3SajzHn54heFO0SUNypmEOq+5z1caNt7YduXRtdhm83AmIUmqnU2i\no5xJSJoEm4QkqZIzCbXS5s0LxYfxDap7e8SzG9uWPLoW22weo84kbBJqpXZ82U+T9z3LsW3Jo2ux\nzebh4HrOOJOQNAk2CUlSJbeb1EpuN3U5ti15dC222TzcblLrDfPVoZLawSbRUV2cSZRfHTrIRVIb\n2CQ0Fs8OpNnmTEJjacfsoJt7xLMb25Y8uhbbbB7OJCRJtbNJdNQwM4lhtoQigmOPfbJbSJIAv09i\nLpQD48E89tiwp7ySZpVNokWG+byiY445kcce+9+GM5I072wSDWvqg+r8a1/SJLRmJhERr4qIb0XE\nAxHxzmnnUxffGyCpy1rRJCLiGOD9wAXAs4HXRsQ5081KktSKJgGcD+zNzP2ZeQS4Abh4yjlV8g1k\nkuZFW2YSW4AH+44P0Gscv+Tw4cMD3eFxxx3HiSeeOHACzX7JjSR1U1uaxMA2btw4cOxppz2dhx76\n3hD37hO/JPVrS5M4CDy97/j0Ym0swzUIGO7Jf5Zj25JHG2LbkkfXYtuSR9dim77v4bXis5si4ljg\nfuDlwH8DdwGvzcz7ppqYJM25VpxJZObPI+JPgFvpDdM/aIOQpOlrxZmEJKmd2vIS2McZ5I11EfG+\niNgbEV+PiO2TznFS1qtFRLwuIu4pLl+MiOdMI89JGPQNlxHxWxFxJCJeM8n8JmnA35HFiPhaRPxn\nRNwx6RwnZYDfkadGxGeL54pvRMTSFNJsXER8MCJWIuLeJ4gZ/nkzM1t1ode4vg2cCRwHfB0456iY\nC4HPFNdfANw57bynWIsXAicX1181z7Xoi/s34NPAa6ad9xT/vzgZ+CawpTh+2rTznmItrgDetVoH\n4IfAhmnn3kAtXgxsB+6t+PlIz5ttPJMY5I11FwPXA2TmfwAnR8SmyaY5EevWIjPvzMzVN4/cSe89\nJ7No0Ddc/inwCeChSSY3YYPU4nXAJzPzIEBm/mDCOU7KILU4BJxUXD8J+GFmPjrBHCciM78IPPwE\nISM9b7axSaz1xrqjn/iOjjm4RswsGKQW/f4I+GyjGU3PurWIiF8FLsnMq5ntN7MM8v/FVuDUiLgj\nIu6OiNdPLLvJGqQW/ww8OyK+D9wDXD6h3NpmpOfNVry6SeOLiJcCb6J3yjmvrgT696RnuVGsZwNw\nHvAy4MnAlyLiS5n57emmNRW7gXsy86UR8WvA5yPiuZn5k2kn1gVtbBKDvLHuIHDGOjGzYKA3GUbE\nc4FrgFdl5hOdbnbZILX4TeCG6H1o1tOACyPiSGbeNKEcJ2WQWhwAfpCZPwN+FhFfAJ5Hb/9+lgxS\nixcBfweQmd+JiP8CzgG+PJEM22Ok5802bjfdDTwzIs6MiOOBncDRv+Q3AW8AiIgXAj/OzJXJpjkR\n69YiIp4OfBJ4fWZ+Zwo5Tsq6tcjMZxSXs+jNJd46gw0CBvsd+RTw4og4NiJOpDeonMX3Hg1Si/uA\nVwAUe/Bbge9ONMvJCarPoEd63mzdmURWvLEuIv649+O8JjNvjohXR8S3gZ/S22aZOYPUAvgb4FTg\nquIv6COZueaHI3bZgLV43E0mnuSEDPg78q2I+BxwL/Bz4JrM3DPFtBsx4P8X7wI+FBH30HsC/avM\n/NH0sm5GRHwUWASeGhHfo/eqruMZ83nTN9NJkiq1cbtJktQSNglJUiWbhCSpkk1CklTJJiFJqmST\nkCRVsklIkirZJCRJlf4fwcMSWFC+d9gAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f7ed49fd390>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "submission=pd.read_csv('data/sample_submission.csv')\n",
    "submission.index=submission.ID\n",
    "submission.PredictedProb=np.array(preds).mean(axis=0)\n",
    "submission.to_csv('submission/et_ln_bag1.csv', index=False)\n",
    "submission.PredictedProb.hist(bins=30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load in \n",
    "\n",
    " # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
    "\n",
    "from subprocess import check_output\n",
    "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n",
    "\n",
    "# Any results you write to the current directory are saved as output.\n",
    "\n",
    "from sklearn import ensemble, metrics, linear_model\n",
    "import random\n",
    "\n",
    "#Some parameters to play with\n",
    "rnd=12\n",
    "random.seed(rnd)\n",
    "n_ft=20 #Number of features to add\n",
    "max_elts=3 #Maximum size of a group of linear features\n",
    "\n",
    "class addNearestNeighbourLinearFeatures:\n",
    "    \n",
    "    def __init__(self, n_neighbours=1, max_elts=None, verbose=True, random_state=None):\n",
    "        self.rnd=random_state\n",
    "        self.n=n_neighbours\n",
    "        self.max_elts=max_elts\n",
    "        self.verbose=verbose\n",
    "        self.neighbours=[]\n",
    "        self.clfs=[]\n",
    "        \n",
    "    def fit(self,train,y):\n",
    "        if self.rnd!=None:\n",
    "            random.seed(rnd)\n",
    "        if self.max_elts==None:\n",
    "            self.max_elts=len(train.columns)\n",
    "        list_vars=list(train.columns)\n",
    "        random.shuffle(list_vars)\n",
    "        \n",
    "        lastscores=np.zeros(self.n)+1e15\n",
    "\n",
    "        for elt in list_vars[:self.n]:\n",
    "            self.neighbours.append([elt])\n",
    "        list_vars=list_vars[self.n:]\n",
    "        \n",
    "        for elt in list_vars:\n",
    "            indice=0\n",
    "            scores=[]\n",
    "            for elt2 in self.neighbours:\n",
    "                if len(elt2)<self.max_elts:\n",
    "                    clf=linear_model.LinearRegression(fit_intercept=False, normalize=True, copy_X=True, n_jobs=-1) \n",
    "                    clf.fit(train[elt2+[elt]], y)\n",
    "                    scores.append(metrics.log_loss(y,clf.predict(train[elt2 + [elt]])))\n",
    "                    indice=indice+1\n",
    "                else:\n",
    "                    scores.append(lastscores[indice])\n",
    "                    indice=indice+1\n",
    "            gains=lastscores-scores\n",
    "            if gains.max()>0:\n",
    "                temp=gains.argmax()\n",
    "                lastscores[temp]=scores[temp]\n",
    "                self.neighbours[temp].append(elt)\n",
    "\n",
    "        indice=0\n",
    "        for elt in self.neighbours:\n",
    "            clf=linear_model.LinearRegression(fit_intercept=False, normalize=True, copy_X=True, n_jobs=-1) \n",
    "            clf.fit(train[elt], y)\n",
    "            self.clfs.append(clf)\n",
    "            if self.verbose:\n",
    "                print(indice, lastscores[indice], elt)\n",
    "            indice=indice+1\n",
    "                    \n",
    "    def transform(self, train):\n",
    "        indice=0\n",
    "        for elt in self.neighbours:\n",
    "            train['_'.join(pd.Series(elt).sort_values().values)]=self.clfs[indice].predict(train[elt])\n",
    "            indice=indice+1\n",
    "        return train\n",
    "    \n",
    "    def fit_transform(self, train, y):\n",
    "        self.fit(train, y)\n",
    "        return self.transform(train)\n",
    "    \n",
    "    \n",
    "train = pd.read_csv(\"../input/train.csv\")\n",
    "target = train['target'].values\n",
    "test = pd.read_csv(\"../input/test.csv\")\n",
    "id_test = test['ID'].values\n",
    "\n",
    "train['v22-1']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[0]))\n",
    "test['v22-1']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[0]))\n",
    "train['v22-2']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[1]))\n",
    "test['v22-2']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[1]))\n",
    "train['v22-3']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[2]))\n",
    "test['v22-3']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[2]))\n",
    "train['v22-4']=train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[3]))\n",
    "test['v22-4']=test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[3]))\n",
    "\n",
    "drop_list=['v91','v1', 'v8', 'v10', 'v15', 'v17', 'v25', 'v29', 'v34', 'v41', 'v46', 'v54', 'v64', 'v67', 'v97', 'v105', 'v111', 'v122']\n",
    "train = train.drop(['ID','target'] + drop_list,axis=1).fillna(-999)\n",
    "test = test.drop(['ID'] + drop_list,axis=1).fillna(-999)\n",
    "\n",
    "refcols=list(train.columns)\n",
    "\n",
    "for elt in refcols:\n",
    "    if train[elt].dtype=='O':\n",
    "        train[elt], temp = pd.factorize(train[elt])\n",
    "        test[elt]=temp.get_indexer(test[elt])\n",
    "    else:\n",
    "        train[elt]=train[elt].round(5)\n",
    "        test[elt]=test[elt].round(5)\n",
    "        \n",
    "a=addNearestNeighbourLinearFeatures(n_neighbours=n_ft, max_elts=max_elts, verbose=True, random_state=rnd)\n",
    "a.fit(train, target)\n",
    "\n",
    "train = a.transform(train)\n",
    "test = a.transform(test)\n",
    "\n",
    "clf = ensemble.ExtraTreesClassifier(n_estimators=750,max_features=50,criterion= 'entropy',min_samples_split= 4,\n",
    "                        max_depth= 35, min_samples_leaf= 2, n_jobs = -1, random_state=rnd)\n",
    "\n",
    "clf.fit(train,target)\n",
    "pred_et=clf.predict_proba(test)\n",
    "\n",
    "submission=pd.read_csv('../input/sample_submission.csv')\n",
    "submission.index=submission.ID\n",
    "submission.PredictedProb=pred_et[:,1]\n",
    "submission.to_csv('./addNNLinearFt.csv', index=False)\n",
    "submission.PredictedProb.hist(bins=30)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}