Novelty.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import glob\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import fit_curve as fc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def transform_mesh_match(x):\n",
    "    return x.replace(\" \", \"\")\n",
    "\n",
    "df_mesh = set(pd.read_csv(\"data/mtrees2015.bin\", sep=\";\", header=None)[0].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "27453\n"
     ]
    }
   ],
   "source": [
    "mesh_correct = dict()\n",
    "for m in df_mesh:\n",
    "    mesh_correct[transform_mesh_match(m)] = m\n",
    "    \n",
    "mesh_correct_bc = sc.broadcast(mesh_correct)\n",
    "print len(mesh_correct)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate data for calculating scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def pmid_mesh(x, mesh_c=dict()):\n",
    "    # Row to pmid year mesh\n",
    "    x = x.split(\"\\t\")\n",
    "    return (x[0], int(x[1]),\n",
    "            tuple(set([mesh_c.get(transform_mesh_match(k), \"__UNKNOWN__\") for k in x[2].split('|')])))\n",
    "\n",
    "def exploded_names(x):\n",
    "    # Explode the mesh parents column into into list\n",
    "    x = x.split('\\t')\n",
    "    return (x[0], tuple(x[1].split('|'))) # Return exploded mesh terms as tuple\n",
    "\n",
    "mesh_exploded = sc.broadcast(sc.textFile(\"out/exploded_mhash.tsv\").map(exploded_names).collectAsMap())\n",
    "def get_exploded(k):\n",
    "    # Get the parents of a mesh term.\n",
    "    # If mesh is not in dict simply return the mesh\n",
    "    try:\n",
    "        return mesh_exploded.value[k]\n",
    "    except:\n",
    "        return (k,)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_exploded_counts(x):\n",
    "    # Return list of exploded mesh year count tuples\n",
    "    total_mesh = set(reduce(lambda a, b: a+b, map(get_exploded, x[2])))\n",
    "    return (((k,x[1]),1) for k in total_mesh)\n",
    "\n",
    "def get_exploded_pair_counts(x):\n",
    "    # Return list of exploded mesh pair year count tuples\n",
    "    total_mesh = set(reduce(lambda a, b: a+b, map(get_exploded, x[2])))\n",
    "    total_mesh = sorted(total_mesh, key=lambda x: x.lower)\n",
    "    mesh_pairs = [(k1,k2) for i,k1 in enumerate(total_mesh) for k2 in total_mesh[i+1:]]\n",
    "    return (((k1,k2,x[1]),1) for k1,k2 in mesh_pairs)\n",
    "\n",
    "# LOAD DATA FROM THE FILE\n",
    "pmid_data = sc.textFile(\"data/pmid_year_mesh.noheader.tsv\").map(lambda x: pmid_mesh(x, mesh_c=mesh_correct_bc.value))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# WRITE DATA FOR INDIVIDUAL MESH TERMS. ONLY EXPLODED COUNTS\n",
    "mesh_exploded_year_count = pmid_data.flatMap(get_exploded_counts).reduceByKey(lambda x,y: x+y)\n",
    "csv_data = mesh_exploded_year_count.map(lambda x: \"%s\\t%s\\t%s\" % (x[0][0], x[0][1], x[1]))\n",
    "csv_data.saveAsTextFile(\"out/mesh_exploded_year\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# WRITE DATA FOR PAIRWISE MESH TERMS. ONLY EXPLODED COUNTS\n",
    "meshpair_exploded_year_count = pmid_data.flatMap(get_exploded_pair_counts).reduceByKey(lambda x,y: x+y)\n",
    "csv_data = meshpair_exploded_year_count.map(lambda x: \"%s\\t%s\\t%s\\t%s\" % (x[0][0], x[0][1], x[0][2], x[1]))\n",
    "csv_data.saveAsTextFile(\"out/meshpair_exploded_year\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Individual Novelty Scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pmid_per_year = pd.read_csv(\"out/PMID_PER_YEAR.tsv\", sep=\"\\t\", index_col=\"year\", dtype = {'Year':int, 'TotalPMID':float})\n",
    "pmid_per_year_norm = pmid_per_year / pmid_per_year.TotalPMID.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# CALCULATE MODEL AND EMPIRICAL SCORES FOR INDIVIDUAL MESH TERMS, ONLY EXPLODED COUNTS\n",
    "def mesh_scores(x):\n",
    "    year, counts = zip(*sorted(x, key=lambda k: k[0]))\n",
    "    # Calculate Empirical scores\n",
    "    \n",
    "    #df = pd.DataFrame(data=list(counts), index=list(year), dtype=float)\n",
    "    #df.columns = [\"y\"]\n",
    "    # Collect all year and counts from spelling errors\n",
    "    df = pd.DataFrame({\"x\": list(year), \"y\": list(counts)})\n",
    "    df = df.groupby(\"x\").sum()\n",
    "    year, counts = df.index.values, df[\"y\"].values\n",
    "\n",
    "    # Start processing\n",
    "    df[\"y\"] = df[\"y\"].astype(\"float\")\n",
    "    TFirstP = df.index - df.index.min()\n",
    "    VolFirstP = df[\"y\"].values.cumsum()\n",
    "    # Calculate model scores\n",
    "    #year_min, year_max = df.index.min() - 5, df.index.max() + 1\n",
    "    popt, pcov = None, None\n",
    "    for offset in [0,2,5,10]:\n",
    "        #year_min, year_max = df.index.min()-offset, min(df.index.max() + 1, 2015)\n",
    "        modified_min_year = max(df.index.min(), 1966)\n",
    "        year_min, year_max = modified_min_year-offset, min(df.index.max() + 1, 2015)\n",
    "        #print offset\n",
    "        #year_range = np.arange(year_min, year_max)\n",
    "        #year_range = range(year_min, df.index.min()) + df.index[df.index <= year_max].values.tolist()\n",
    "        year_range = range(year_min, modified_min_year) + df.index[(df.index <= year_max) & (df.index>=modified_min_year)].values.tolist()\n",
    "        df_y = pd.DataFrame(data=np.zeros_like(year_range), index=year_range, dtype=float)\n",
    "        normalizer = pmid_per_year_norm.ix[df.index, \"TotalPMID\"]\n",
    "        df_counts = np.log((df[df.index.isin(year_range)][\"y\"] / normalizer) + 1) # LOG shoud be base exp as per formula\n",
    "        df_y[\"x\"] = np.array(year_range) - year_min\n",
    "        df_y[\"y\"] = df_counts\n",
    "        df_y[\"counts\"] = df[df.index.isin(year_range)][\"y\"]\n",
    "        df_y = df_y.fillna(0)\n",
    "        x, y = df_y[\"x\"], df_y[\"y\"]\n",
    "        # We now have x and y and we need to fit the curve.\n",
    "        if df.shape[0] < 2:\n",
    "            # Too few instances to fit\n",
    "            #print \"Too few instances\"\n",
    "            break\n",
    "        if len(year_range) < 3:\n",
    "            #print \"Too small year range\"\n",
    "            continue\n",
    "        try:\n",
    "            #print df_y\n",
    "            popt,pcov = fc.getFitCurve(x,y)\n",
    "            #print popt\n",
    "            if popt[0]*popt[2] > 0:\n",
    "                break\n",
    "        except RuntimeError:\n",
    "            #print \"Runtime error\"\n",
    "            popt, pcov = None, None\n",
    "    # After fitting the curve we only need values for the original years\n",
    "    #x_org = df_y.ix[df.index, \"x\"].values\n",
    "    #predicted_val = fc.func(x_org, popt[0],popt[1],popt[2])\n",
    "    #velocity = fc.f_der1(x_org, popt)\n",
    "    df[\"x_fit\"] = df.index - year_min\n",
    "    if popt is not None and popt[0]*popt[2] > 0:\n",
    "        #df[\"y_norm\"] = np.log((df[\"y\"] / normalizer) + 1) # LOG shoud be base exp as per formula\n",
    "        x_inflex = int(popt[1] + year_min + 1)\n",
    "        #print popt, x_inflex\n",
    "        df[\"pred\"] = fc.func(df[\"x_fit\"], popt[0],popt[1],popt[2])\n",
    "        df[\"pred_actual\"] = (np.exp(df[\"pred\"]) - 1) * normalizer\n",
    "        df[\"velocity\"] = fc.f_der1(df[\"x_fit\"], popt)\n",
    "        df[\"acceleration\"] = fc.f_der2(df[\"x_fit\"], popt)\n",
    "        # Fill NAN Velocity values with 0\n",
    "        df[\"velocity\"] = df[\"velocity\"].fillna(0)\n",
    "        # Least precision of float in MySQL around 1e-20 and -1e-20\n",
    "        df.ix[(df[\"x_fit\"] <= x_inflex) & (df[\"acceleration\"].isnull()), \"acceleration\"] = 0 # Fill positive acceleration\n",
    "        df.ix[(df[\"x_fit\"] > x_inflex) & (df[\"acceleration\"].isnull()), \"acceleration\"] = -1e-20 # Fill negative acceleration\n",
    "        df.ix[(df[\"acceleration\"] == -0.0) & (df[\"x_fit\"] > x_inflex), \"acceleration\"] = -1e-20 # Set all -0 values to -1e-20\n",
    "        # Truncate all values less than 0 which have precision less than 1e-20 to -1e-20\n",
    "        df.ix[(df[\"acceleration\"] < 0) & (np.abs(df[\"acceleration\"]) < 1e-20), \"acceleration\"] = -1e-20\n",
    "    else:\n",
    "        df[\"pred\"] = np.nan\n",
    "        df[\"pred_actual\"] = np.nan\n",
    "        df[\"velocity\"] = np.nan\n",
    "        df[\"acceleration\"] = np.nan\n",
    "    df = df.fillna(\"\\\\N\")\n",
    "    return ((year[i], counts[i], TFirstP[i], VolFirstP[i],\\\n",
    "        df[\"pred_actual\"].values[i], df[\"velocity\"].values[i], df[\"acceleration\"].values[i]) for i in xrange(len(year)))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def read_mesh(x, mesh_c=dict()):\n",
    "    x = x.split(\"\\t\")\n",
    "    correct_mesh = mesh_c.get(transform_mesh_match(x[0]), \"__UNKNOWN__\")\n",
    "    return (correct_mesh, int(x[1]), int(x[2]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "mesh_exploded_year_count = sc.textFile(\",\".join(glob.glob(\"out/mesh_exploded_year/part-*\")))\\\n",
    "    .map(lambda x: read_mesh(x, mesh_c=mesh_correct_bc.value))\n",
    "mesh_scores_data = mesh_exploded_year_count.map(lambda x: ((x[0]), (x[1], x[2]))).groupByKey().flatMapValues(mesh_scores)\n",
    "csv_data = mesh_scores_data.map(lambda x: \"%s\\t%s\\t%s\\t%s\\t%s\\t%s\\t%s\\t%s\" \\\n",
    "    % (x[0], x[1][0], x[1][1], x[1][2], x[1][3], x[1][4], x[1][5], x[1][6]))\n",
    "csv_data.saveAsTextFile(\"out/mesh_exploded_scores\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mesh_exploded_year_count = sc.textFile(\",\".join(glob.glob(\"out/mesh_exploded_year/part-*\")))\\\n",
    "    .map(lambda x: read_mesh(x, mesh_c=mesh_correct_bc.value))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "t = mesh_exploded_year_count.map(lambda x: ((x[0]), (x[1], x[2]))).filter(lambda x: x[0] == \"Lactose Tolerance Test\").groupByKey().collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[  1.60844116   0.05634948  28.40997984] -0.00198343960915\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1967</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.141488e+01</td>\n",
       "      <td>-9.134554e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1973</td>\n",
       "      <td>11</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>6</td>\n",
       "      <td>3.754980</td>\n",
       "      <td>4.032914e-73</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1974</td>\n",
       "      <td>24</td>\n",
       "      <td>7</td>\n",
       "      <td>36</td>\n",
       "      <td>7</td>\n",
       "      <td>3.820877</td>\n",
       "      <td>1.850638e-85</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1975</td>\n",
       "      <td>19</td>\n",
       "      <td>8</td>\n",
       "      <td>55</td>\n",
       "      <td>8</td>\n",
       "      <td>4.057935</td>\n",
       "      <td>8.492275e-98</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1976</td>\n",
       "      <td>17</td>\n",
       "      <td>9</td>\n",
       "      <td>72</td>\n",
       "      <td>9</td>\n",
       "      <td>4.142468</td>\n",
       "      <td>3.896965e-110</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1977</td>\n",
       "      <td>21</td>\n",
       "      <td>10</td>\n",
       "      <td>93</td>\n",
       "      <td>10</td>\n",
       "      <td>4.251472</td>\n",
       "      <td>1.788253e-122</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1978</td>\n",
       "      <td>19</td>\n",
       "      <td>11</td>\n",
       "      <td>112</td>\n",
       "      <td>11</td>\n",
       "      <td>4.414603</td>\n",
       "      <td>8.206002e-135</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1979</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>123</td>\n",
       "      <td>12</td>\n",
       "      <td>4.559864</td>\n",
       "      <td>3.765599e-147</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1980</td>\n",
       "      <td>22</td>\n",
       "      <td>13</td>\n",
       "      <td>145</td>\n",
       "      <td>13</td>\n",
       "      <td>4.530140</td>\n",
       "      <td>1.727972e-159</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1981</td>\n",
       "      <td>19</td>\n",
       "      <td>14</td>\n",
       "      <td>164</td>\n",
       "      <td>14</td>\n",
       "      <td>4.563006</td>\n",
       "      <td>7.929379e-172</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1982</td>\n",
       "      <td>8</td>\n",
       "      <td>15</td>\n",
       "      <td>172</td>\n",
       "      <td>15</td>\n",
       "      <td>4.740916</td>\n",
       "      <td>3.638662e-184</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1983</td>\n",
       "      <td>10</td>\n",
       "      <td>16</td>\n",
       "      <td>182</td>\n",
       "      <td>16</td>\n",
       "      <td>4.970942</td>\n",
       "      <td>1.669722e-196</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1984</td>\n",
       "      <td>12</td>\n",
       "      <td>17</td>\n",
       "      <td>194</td>\n",
       "      <td>17</td>\n",
       "      <td>5.119112</td>\n",
       "      <td>7.662081e-209</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1985</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>196</td>\n",
       "      <td>18</td>\n",
       "      <td>5.287929</td>\n",
       "      <td>3.516003e-221</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1986</td>\n",
       "      <td>5</td>\n",
       "      <td>19</td>\n",
       "      <td>201</td>\n",
       "      <td>19</td>\n",
       "      <td>5.493534</td>\n",
       "      <td>1.613436e-233</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1987</td>\n",
       "      <td>14</td>\n",
       "      <td>20</td>\n",
       "      <td>215</td>\n",
       "      <td>20</td>\n",
       "      <td>5.785153</td>\n",
       "      <td>7.403794e-246</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1988</td>\n",
       "      <td>10</td>\n",
       "      <td>21</td>\n",
       "      <td>225</td>\n",
       "      <td>21</td>\n",
       "      <td>6.077934</td>\n",
       "      <td>3.397479e-258</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1989</td>\n",
       "      <td>2</td>\n",
       "      <td>22</td>\n",
       "      <td>227</td>\n",
       "      <td>22</td>\n",
       "      <td>6.340328</td>\n",
       "      <td>1.559048e-270</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1990</td>\n",
       "      <td>5</td>\n",
       "      <td>23</td>\n",
       "      <td>232</td>\n",
       "      <td>23</td>\n",
       "      <td>6.453038</td>\n",
       "      <td>7.154213e-283</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1991</td>\n",
       "      <td>8</td>\n",
       "      <td>24</td>\n",
       "      <td>240</td>\n",
       "      <td>24</td>\n",
       "      <td>6.463146</td>\n",
       "      <td>3.282951e-295</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1992</td>\n",
       "      <td>2</td>\n",
       "      <td>25</td>\n",
       "      <td>242</td>\n",
       "      <td>25</td>\n",
       "      <td>6.513467</td>\n",
       "      <td>1.506492e-307</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1993</td>\n",
       "      <td>4</td>\n",
       "      <td>26</td>\n",
       "      <td>246</td>\n",
       "      <td>26</td>\n",
       "      <td>6.618148</td>\n",
       "      <td>6.908520e-320</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>1994</td>\n",
       "      <td>6</td>\n",
       "      <td>27</td>\n",
       "      <td>252</td>\n",
       "      <td>27</td>\n",
       "      <td>6.771338</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1995</td>\n",
       "      <td>5</td>\n",
       "      <td>28</td>\n",
       "      <td>257</td>\n",
       "      <td>28</td>\n",
       "      <td>6.923564</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1996</td>\n",
       "      <td>2</td>\n",
       "      <td>29</td>\n",
       "      <td>259</td>\n",
       "      <td>29</td>\n",
       "      <td>7.013018</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1997</td>\n",
       "      <td>4</td>\n",
       "      <td>30</td>\n",
       "      <td>263</td>\n",
       "      <td>30</td>\n",
       "      <td>7.182134</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>1999</td>\n",
       "      <td>5</td>\n",
       "      <td>32</td>\n",
       "      <td>268</td>\n",
       "      <td>32</td>\n",
       "      <td>7.642004</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>2000</td>\n",
       "      <td>5</td>\n",
       "      <td>33</td>\n",
       "      <td>273</td>\n",
       "      <td>33</td>\n",
       "      <td>8.069839</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>2001</td>\n",
       "      <td>2</td>\n",
       "      <td>34</td>\n",
       "      <td>275</td>\n",
       "      <td>34</td>\n",
       "      <td>8.407040</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>2002</td>\n",
       "      <td>2</td>\n",
       "      <td>35</td>\n",
       "      <td>277</td>\n",
       "      <td>35</td>\n",
       "      <td>8.666641</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>2003</td>\n",
       "      <td>4</td>\n",
       "      <td>36</td>\n",
       "      <td>281</td>\n",
       "      <td>36</td>\n",
       "      <td>9.125097</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>2004</td>\n",
       "      <td>5</td>\n",
       "      <td>37</td>\n",
       "      <td>286</td>\n",
       "      <td>37</td>\n",
       "      <td>9.620642</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>2005</td>\n",
       "      <td>6</td>\n",
       "      <td>38</td>\n",
       "      <td>292</td>\n",
       "      <td>38</td>\n",
       "      <td>10.125862</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>2006</td>\n",
       "      <td>4</td>\n",
       "      <td>39</td>\n",
       "      <td>296</td>\n",
       "      <td>39</td>\n",
       "      <td>10.541130</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>2007</td>\n",
       "      <td>8</td>\n",
       "      <td>40</td>\n",
       "      <td>304</td>\n",
       "      <td>40</td>\n",
       "      <td>10.921803</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>2008</td>\n",
       "      <td>11</td>\n",
       "      <td>41</td>\n",
       "      <td>315</td>\n",
       "      <td>41</td>\n",
       "      <td>11.396800</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>2009</td>\n",
       "      <td>2</td>\n",
       "      <td>42</td>\n",
       "      <td>317</td>\n",
       "      <td>42</td>\n",
       "      <td>11.747650</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>2010</td>\n",
       "      <td>5</td>\n",
       "      <td>43</td>\n",
       "      <td>322</td>\n",
       "      <td>43</td>\n",
       "      <td>12.190131</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>2011</td>\n",
       "      <td>5</td>\n",
       "      <td>44</td>\n",
       "      <td>327</td>\n",
       "      <td>44</td>\n",
       "      <td>12.725540</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>2012</td>\n",
       "      <td>5</td>\n",
       "      <td>45</td>\n",
       "      <td>332</td>\n",
       "      <td>45</td>\n",
       "      <td>13.403051</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>2013</td>\n",
       "      <td>5</td>\n",
       "      <td>46</td>\n",
       "      <td>337</td>\n",
       "      <td>46</td>\n",
       "      <td>13.925477</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>2014</td>\n",
       "      <td>2</td>\n",
       "      <td>47</td>\n",
       "      <td>339</td>\n",
       "      <td>47</td>\n",
       "      <td>11.029260</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       0   1   2    3   4          5              6             7\n",
       "0   1967   1   0    1   0   1.000000   1.141488e+01 -9.134554e+00\n",
       "1   1973  11   6   12   6   3.754980   4.032914e-73 -1.000000e-20\n",
       "2   1974  24   7   36   7   3.820877   1.850638e-85 -1.000000e-20\n",
       "3   1975  19   8   55   8   4.057935   8.492275e-98 -1.000000e-20\n",
       "4   1976  17   9   72   9   4.142468  3.896965e-110 -1.000000e-20\n",
       "5   1977  21  10   93  10   4.251472  1.788253e-122 -1.000000e-20\n",
       "6   1978  19  11  112  11   4.414603  8.206002e-135 -1.000000e-20\n",
       "7   1979  11  12  123  12   4.559864  3.765599e-147 -1.000000e-20\n",
       "8   1980  22  13  145  13   4.530140  1.727972e-159 -1.000000e-20\n",
       "9   1981  19  14  164  14   4.563006  7.929379e-172 -1.000000e-20\n",
       "10  1982   8  15  172  15   4.740916  3.638662e-184 -1.000000e-20\n",
       "11  1983  10  16  182  16   4.970942  1.669722e-196 -1.000000e-20\n",
       "12  1984  12  17  194  17   5.119112  7.662081e-209 -1.000000e-20\n",
       "13  1985   2  18  196  18   5.287929  3.516003e-221 -1.000000e-20\n",
       "14  1986   5  19  201  19   5.493534  1.613436e-233 -1.000000e-20\n",
       "15  1987  14  20  215  20   5.785153  7.403794e-246 -1.000000e-20\n",
       "16  1988  10  21  225  21   6.077934  3.397479e-258 -1.000000e-20\n",
       "17  1989   2  22  227  22   6.340328  1.559048e-270 -1.000000e-20\n",
       "18  1990   5  23  232  23   6.453038  7.154213e-283 -1.000000e-20\n",
       "19  1991   8  24  240  24   6.463146  3.282951e-295 -1.000000e-20\n",
       "20  1992   2  25  242  25   6.513467  1.506492e-307 -1.000000e-20\n",
       "21  1993   4  26  246  26   6.618148  6.908520e-320 -1.000000e-20\n",
       "22  1994   6  27  252  27   6.771338   0.000000e+00 -1.000000e-20\n",
       "23  1995   5  28  257  28   6.923564   0.000000e+00 -1.000000e-20\n",
       "24  1996   2  29  259  29   7.013018   0.000000e+00 -1.000000e-20\n",
       "25  1997   4  30  263  30   7.182134   0.000000e+00 -1.000000e-20\n",
       "26  1999   5  32  268  32   7.642004   0.000000e+00 -1.000000e-20\n",
       "27  2000   5  33  273  33   8.069839   0.000000e+00 -1.000000e-20\n",
       "28  2001   2  34  275  34   8.407040   0.000000e+00 -1.000000e-20\n",
       "29  2002   2  35  277  35   8.666641   0.000000e+00 -1.000000e-20\n",
       "30  2003   4  36  281  36   9.125097   0.000000e+00 -1.000000e-20\n",
       "31  2004   5  37  286  37   9.620642   0.000000e+00 -1.000000e-20\n",
       "32  2005   6  38  292  38  10.125862   0.000000e+00 -1.000000e-20\n",
       "33  2006   4  39  296  39  10.541130   0.000000e+00 -1.000000e-20\n",
       "34  2007   8  40  304  40  10.921803   0.000000e+00 -1.000000e-20\n",
       "35  2008  11  41  315  41  11.396800   0.000000e+00 -1.000000e-20\n",
       "36  2009   2  42  317  42  11.747650   0.000000e+00 -1.000000e-20\n",
       "37  2010   5  43  322  43  12.190131   0.000000e+00 -1.000000e-20\n",
       "38  2011   5  44  327  44  12.725540   0.000000e+00 -1.000000e-20\n",
       "39  2012   5  45  332  45  13.403051   0.000000e+00 -1.000000e-20\n",
       "40  2013   5  46  337  46  13.925477   0.000000e+00 -1.000000e-20\n",
       "41  2014   2  47  339  47  11.029260   0.000000e+00 -1.000000e-20"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tt = pd.DataFrame(list(mesh_scores(list(t[0][1]))))\n",
    "tt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "t = mesh_exploded_year_count.map(lambda x: ((x[0]), (x[1], x[2]))).filter(\n",
    "    lambda x: x[0] == \"Fluorescent Antibody Technique, Direct\").groupByKey().collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[   3.90247023 -524.76194476   18.73388139] 28.0113839713\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1966</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1970</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1976</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.103602e-145</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1977</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.877298e-137</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1980</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>7</td>\n",
       "      <td>14</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.362898e-113</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1983</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>8</td>\n",
       "      <td>17</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.884138e-88</td>\n",
       "      <td>3.529722e-87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1984</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>9</td>\n",
       "      <td>18</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.577116e-80</td>\n",
       "      <td>4.827939e-79</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1989</td>\n",
       "      <td>1</td>\n",
       "      <td>23</td>\n",
       "      <td>10</td>\n",
       "      <td>23</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.233788e-39</td>\n",
       "      <td>2.311364e-38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1994</td>\n",
       "      <td>8</td>\n",
       "      <td>28</td>\n",
       "      <td>18</td>\n",
       "      <td>28</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>1.807085e+01</td>\n",
       "      <td>3.596302e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1995</td>\n",
       "      <td>113</td>\n",
       "      <td>29</td>\n",
       "      <td>131</td>\n",
       "      <td>29</td>\n",
       "      <td>84.095579</td>\n",
       "      <td>6.615555e-07</td>\n",
       "      <td>-1.239350e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1996</td>\n",
       "      <td>209</td>\n",
       "      <td>30</td>\n",
       "      <td>340</td>\n",
       "      <td>30</td>\n",
       "      <td>85.182113</td>\n",
       "      <td>4.836654e-15</td>\n",
       "      <td>-9.060929e-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1997</td>\n",
       "      <td>245</td>\n",
       "      <td>31</td>\n",
       "      <td>585</td>\n",
       "      <td>31</td>\n",
       "      <td>87.236244</td>\n",
       "      <td>3.536093e-23</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1998</td>\n",
       "      <td>214</td>\n",
       "      <td>32</td>\n",
       "      <td>799</td>\n",
       "      <td>32</td>\n",
       "      <td>90.225466</td>\n",
       "      <td>2.585249e-31</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1999</td>\n",
       "      <td>145</td>\n",
       "      <td>33</td>\n",
       "      <td>944</td>\n",
       "      <td>33</td>\n",
       "      <td>92.821953</td>\n",
       "      <td>1.890084e-39</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>2000</td>\n",
       "      <td>162</td>\n",
       "      <td>34</td>\n",
       "      <td>1106</td>\n",
       "      <td>34</td>\n",
       "      <td>98.018562</td>\n",
       "      <td>1.381846e-47</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>2001</td>\n",
       "      <td>109</td>\n",
       "      <td>35</td>\n",
       "      <td>1215</td>\n",
       "      <td>35</td>\n",
       "      <td>102.114305</td>\n",
       "      <td>1.010272e-55</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>2002</td>\n",
       "      <td>102</td>\n",
       "      <td>36</td>\n",
       "      <td>1317</td>\n",
       "      <td>36</td>\n",
       "      <td>105.267485</td>\n",
       "      <td>7.386133e-64</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>2003</td>\n",
       "      <td>119</td>\n",
       "      <td>37</td>\n",
       "      <td>1436</td>\n",
       "      <td>37</td>\n",
       "      <td>110.836031</td>\n",
       "      <td>5.400025e-72</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>2004</td>\n",
       "      <td>135</td>\n",
       "      <td>38</td>\n",
       "      <td>1571</td>\n",
       "      <td>38</td>\n",
       "      <td>116.855060</td>\n",
       "      <td>3.947976e-80</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>2005</td>\n",
       "      <td>114</td>\n",
       "      <td>39</td>\n",
       "      <td>1685</td>\n",
       "      <td>39</td>\n",
       "      <td>122.991607</td>\n",
       "      <td>2.886378e-88</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>2006</td>\n",
       "      <td>142</td>\n",
       "      <td>40</td>\n",
       "      <td>1827</td>\n",
       "      <td>40</td>\n",
       "      <td>128.035565</td>\n",
       "      <td>2.110240e-96</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>2007</td>\n",
       "      <td>161</td>\n",
       "      <td>41</td>\n",
       "      <td>1988</td>\n",
       "      <td>41</td>\n",
       "      <td>132.659328</td>\n",
       "      <td>1.542803e-104</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>2008</td>\n",
       "      <td>206</td>\n",
       "      <td>42</td>\n",
       "      <td>2194</td>\n",
       "      <td>42</td>\n",
       "      <td>138.428784</td>\n",
       "      <td>1.127948e-112</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>2009</td>\n",
       "      <td>146</td>\n",
       "      <td>43</td>\n",
       "      <td>2340</td>\n",
       "      <td>43</td>\n",
       "      <td>142.690302</td>\n",
       "      <td>8.246467e-121</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>2010</td>\n",
       "      <td>122</td>\n",
       "      <td>44</td>\n",
       "      <td>2462</td>\n",
       "      <td>44</td>\n",
       "      <td>148.064803</td>\n",
       "      <td>6.029019e-129</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>2011</td>\n",
       "      <td>101</td>\n",
       "      <td>45</td>\n",
       "      <td>2563</td>\n",
       "      <td>45</td>\n",
       "      <td>154.568036</td>\n",
       "      <td>4.407835e-137</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>2012</td>\n",
       "      <td>67</td>\n",
       "      <td>46</td>\n",
       "      <td>2630</td>\n",
       "      <td>46</td>\n",
       "      <td>162.797280</td>\n",
       "      <td>3.222582e-145</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>2013</td>\n",
       "      <td>81</td>\n",
       "      <td>47</td>\n",
       "      <td>2711</td>\n",
       "      <td>47</td>\n",
       "      <td>169.142813</td>\n",
       "      <td>2.356040e-153</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>2014</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>2759</td>\n",
       "      <td>48</td>\n",
       "      <td>133.964538</td>\n",
       "      <td>1.722509e-161</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>2015</td>\n",
       "      <td>3</td>\n",
       "      <td>49</td>\n",
       "      <td>2762</td>\n",
       "      <td>49</td>\n",
       "      <td>17.817206</td>\n",
       "      <td>1.259332e-169</td>\n",
       "      <td>-1.000000e-20</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       0    1   2     3   4           5              6             7\n",
       "0   1966    1   0     1   0    0.000000   0.000000e+00  0.000000e+00\n",
       "1   1970    2   4     3   4    0.000000   0.000000e+00  0.000000e+00\n",
       "2   1976    1  10     4  10    0.000000  2.103602e-145  0.000000e+00\n",
       "3   1977    2  11     6  11    0.000000  2.877298e-137  0.000000e+00\n",
       "4   1980    1  14     7  14    0.000000  7.362898e-113  0.000000e+00\n",
       "5   1983    1  17     8  17    0.000000   1.884138e-88  3.529722e-87\n",
       "6   1984    1  18     9  18    0.000000   2.577116e-80  4.827939e-79\n",
       "7   1989    1  23    10  23    0.000000   1.233788e-39  2.311364e-38\n",
       "8   1994    8  28    18  28    8.000000   1.807085e+01  3.596302e+01\n",
       "9   1995  113  29   131  29   84.095579   6.615555e-07 -1.239350e-05\n",
       "10  1996  209  30   340  30   85.182113   4.836654e-15 -9.060929e-14\n",
       "11  1997  245  31   585  31   87.236244   3.536093e-23 -1.000000e-20\n",
       "12  1998  214  32   799  32   90.225466   2.585249e-31 -1.000000e-20\n",
       "13  1999  145  33   944  33   92.821953   1.890084e-39 -1.000000e-20\n",
       "14  2000  162  34  1106  34   98.018562   1.381846e-47 -1.000000e-20\n",
       "15  2001  109  35  1215  35  102.114305   1.010272e-55 -1.000000e-20\n",
       "16  2002  102  36  1317  36  105.267485   7.386133e-64 -1.000000e-20\n",
       "17  2003  119  37  1436  37  110.836031   5.400025e-72 -1.000000e-20\n",
       "18  2004  135  38  1571  38  116.855060   3.947976e-80 -1.000000e-20\n",
       "19  2005  114  39  1685  39  122.991607   2.886378e-88 -1.000000e-20\n",
       "20  2006  142  40  1827  40  128.035565   2.110240e-96 -1.000000e-20\n",
       "21  2007  161  41  1988  41  132.659328  1.542803e-104 -1.000000e-20\n",
       "22  2008  206  42  2194  42  138.428784  1.127948e-112 -1.000000e-20\n",
       "23  2009  146  43  2340  43  142.690302  8.246467e-121 -1.000000e-20\n",
       "24  2010  122  44  2462  44  148.064803  6.029019e-129 -1.000000e-20\n",
       "25  2011  101  45  2563  45  154.568036  4.407835e-137 -1.000000e-20\n",
       "26  2012   67  46  2630  46  162.797280  3.222582e-145 -1.000000e-20\n",
       "27  2013   81  47  2711  47  169.142813  2.356040e-153 -1.000000e-20\n",
       "28  2014   48  48  2759  48  133.964538  1.722509e-161 -1.000000e-20\n",
       "29  2015    3  49  2762  49   17.817206  1.259332e-169 -1.000000e-20"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tt = pd.DataFrame(list(mesh_scores(list(t[0][1]))))\n",
    "tt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "all_data = sc.textFile(\",\".join(glob.glob(\"out/mesh_exploded_scores/part-*\")))\\\n",
    "    .map(lambda x: x.split(\"\\t\")).map(lambda x: (x[0], int(x[2])))\\\n",
    "    .reduceByKey(lambda x, y: x+y).map(lambda x: x[1]).collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "all_data = np.array(all_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([  194.,   786.,  2847.,  6018.,  7625.,  5794.,  2798.,   934.,\n",
       "          222.,    32.]),\n",
       " array([ 0.        ,  0.72649239,  1.45298478,  2.17947716,  2.90596955,\n",
       "         3.63246194,  4.35895433,  5.08544672,  5.81193911,  6.53843149,\n",
       "         7.26492388]),\n",
       " <a list of 10 Patch objects>)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEACAYAAABfxaZOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFw5JREFUeJzt3X+snNV95/H3hzhgIMb1totvYydASi8xUbvBrEy2tGIq\nUhO3qs1frrPdBYqzlYBdUFaqYkda+fLPtv6jCql2Qco2BXtFQgxahLvxGmOZ2SrVJnYCqUns2FaR\nje3ENxvReJdFLXbz2T/mGAbn/pjrzJ1nfM/nJVn3mXPPM+c81zOfe+73eWZGtomIiDpc0vQEIiJi\ncBL6EREVSehHRFQkoR8RUZGEfkRERRL6EREV6Sn0JX1a0nck7Zf0pKRLJS2StEvSIUnPS1rY1X+j\npCOSDkpa2dW+vNzHYUmPzMYBRUTE5KYNfUnvB/4dsNz2rwLzgE8CG4Ddtm8A9gAbS/8bgbXAMmAV\n8Kgklbt7DFhvexQYlXRHn48nIiKm0Gt55z3AlZLmAZcDJ4E1wJby/S3AnWV7NfCU7bO2jwJHgBWS\nRoAFtveVflu79omIiAGYNvRtfx/4U+A1OmF/2vZuYLHt8dLnFHB12WUJcLzrLk6WtiXAia72E6Ut\nIiIGpJfyzs/RWdVfA7yfzor/94Hz378h7+cQETHk5vXQ5+PAq7ZfB5D0LPBrwLikxbbHS+nmh6X/\nSeADXfsvLW2Ttf8USfkFEhFxAWxrqu/3UtN/DfiYpPnlhOztwAFgO3BP6XM38FzZ3g6sK1f4XAdc\nD+wtJaDTklaU+7mra5+JJj70/zZt2tT4HObCHDPPzHPY/10s8+zFtCt923slPQO8DJwpX78ALAC2\nSboXOEbnih1sH5C0rfxiOAPc73dm8wDwBDAf2GF7Z0+zjIiIvuilvIPth4GHz2t+nU7pZ6L+fwz8\n8QTt3wJ+ZYZzjIiIPskrcn8GrVar6SlM62KYI2Se/ZZ59tfFMs9eqNc60CBJ8jDOKyJimEnCfTiR\nGxERc0RCPyKiIgn9iIiKJPQjIiqS0I9GjIxci6RG/o2MXNv04Uc0JlfvRCM6L8pu6v9YPb96MeJi\nkqt3IiLiXRL6EREVSehHRFQkoR8RUZGEfkRERRL6EREVSehHRFQkoR8RUZGEfkRERRL6EREVSehH\nRFRk2tCXNCrpZUkvla+nJT0oaZGkXZIOSXpe0sKufTZKOiLpoKSVXe3LJe2XdFjSI7N1UBERMbFp\nQ9/2Yds32V4O3Az8P+BZYAOw2/YNwB5gI4CkG4G1wDJgFfCoOu+uBfAYsN72KDAq6Y5+H1BERExu\npuWdjwN/a/s4sAbYUtq3AHeW7dXAU7bP2j4KHAFWSBoBFtjeV/pt7donIiIGYKah/3vAl8r2Ytvj\nALZPAVeX9iXA8a59Tpa2JcCJrvYTpS0iIgak59CX9F46q/inS9P5b0ieNyiPiBhy82bQdxXwLds/\nKrfHJS22PV5KNz8s7SeBD3Ttt7S0TdY+obGxsbe3W60WrVZrBlONiJj72u027XZ7Rvv0/MlZkr4M\n7LS9pdzeDLxue7OkzwCLbG8oJ3KfBG6hU755Afhl25b0deBBYB/wVeDPbO+cYKx8ctYcl0/Oiui/\nXj45q6fQl3QFcAz4kO3/W9r+CbCNzur9GLDW9o/L9zYC64EzwEO2d5X2m4EngPnADtsPTTJeQn+O\nS+hH9F/fQn/QEvpzX0I/ov/yGbkREfEuCf2IiIok9CMiKpLQj4ioSEI/IqIiCf2IiIok9CMiKpLQ\nj4ioSEI/IqIiCf2IiIok9CMiKpLQj4ioSEI/IqIiCf2IiIok9CMiKpLQj4ioSEI/IqIiCf2IiIok\n9CMiKtJT6EtaKOlpSQclfVfSLZIWSdol6ZCk5yUt7Oq/UdKR0n9lV/tySfslHZb0yGwcUERETK7X\nlf7ngR22lwH/DPgesAHYbfsGYA+wEUDSjcBaYBmwCnhUnU/BBngMWG97FBiVdEffjiQiIqY1behL\nugr4DduPA9g+a/s0sAbYUrptAe4s26uBp0q/o8ARYIWkEWCB7X2l39aufSIiYgB6WelfB/xI0uOS\nXpL0BUlXAIttjwPYPgVcXfovAY537X+ytC0BTnS1nyhtERExIPN67LMceMD2NyV9jk5px+f1O//2\nz2RsbOzt7VarRavV6ufdRzEyci3j48eankZEXIB2u0273Z7RPrKnzmpJi4H/ZftD5fav0wn9XwJa\ntsdL6eZF28skbQBse3PpvxPYBBw716e0rwNus33fBGN6unlFf3ROtzTxs25q3M7YeXzFXCQJ25qq\nz7TlnVLCOS5ptDTdDnwX2A7cU9ruBp4r29uBdZIulXQdcD2wt5SATktaUU7s3tW1T0REDEAv5R2A\nB4EnJb0XeBX4A+A9wDZJ99JZxa8FsH1A0jbgAHAGuL9r2f4A8AQwn87VQDv7dSARETG9acs7TUh5\nZ3BS3omYO/pS3omIiLmj1/JOxBxyGe+8XnBwFi++hlOnjg583IhuKe9UrtbyTlPHnMd1zKaUdyIi\n4l0S+hERFUnoR0RUJKEfEVGRhH5EREUS+hERFUnoR0RUJKEfEVGRhH5EREUS+hERFUnoR0RUJKEf\nEVGRhH5EREUS+hERFUnoR0RUJKEfEVGRnkJf0lFJfyPpZUl7S9siSbskHZL0vKSFXf03Sjoi6aCk\nlV3tyyXtl3RY0iP9P5yIiJhKryv9nwAt2zfZXlHaNgC7bd8A7AE2Aki6EVgLLANWAY/qnc+mewxY\nb3sUGJV0R5+OIyIietBr6GuCvmuALWV7C3Bn2V4NPGX7rO2jwBFghaQRYIHtfaXf1q59IiJiAHoN\nfQMvSNon6VOlbbHtcQDbp4CrS/sS4HjXvidL2xLgRFf7idIWEREDMq/Hfrfa/oGkfwrsknSIn/5k\n6b5+4vPY2Njb261Wi1ar1c+7j4i46LXbbdrt9oz2kT2zrJa0CXgD+BSdOv94Kd28aHuZpA2AbW8u\n/XcCm4Bj5/qU9nXAbbbvm2AMz3RecWE6p1ua+Fk3NW6TY4s8rmM2ScK2puozbXlH0hWS3le2rwRW\nAq8A24F7Sre7gefK9nZgnaRLJV0HXA/sLSWg05JWlBO7d3XtExERA9BLeWcx8Kwkl/5P2t4l6ZvA\nNkn30lnFrwWwfUDSNuAAcAa4v2vZ/gDwBDAf2GF7Z1+PJiIipjTj8s4gpLwzOCnvDHbcPK5jNvWl\nvBMREXNHQj8ioiIJ/YiIiiT0IyIqktCPiKhIQj8ioiIJ/YiIiiT0IyIqktCPiKhIQj8ioiIJ/YiI\niiT0IyIqktCPiKhIQj8ioiIJ/YiIiiT0IyIqktCPiKhIQj8ioiIJ/YiIivQc+pIukfSSpO3l9iJJ\nuyQdkvS8pIVdfTdKOiLpoKSVXe3LJe2XdFjSI/09lIiImM5MVvoPAQe6bm8Adtu+AdgDbASQdCOw\nFlgGrAIeVefTtwEeA9bbHgVGJd3xM84/IiJmoKfQl7QU+G3gz7ua1wBbyvYW4M6yvRp4yvZZ20eB\nI8AKSSPAAtv7Sr+tXftERMQA9LrS/xzwR4C72hbbHgewfQq4urQvAY539TtZ2pYAJ7raT5S2iIgY\nkHnTdZD0O8C47W9Lak3R1VN8b8bGxsbe3m61WrRaUw0dEVGfdrtNu92e0T6yp85qSf8R+FfAWeBy\nYAHwLPDPgZbt8VK6edH2MkkbANveXPbfCWwCjp3rU9rXAbfZvm+CMT3dvKI/OqdbmvhZNzVuk2OL\nPK5jNknCtqbqM215x/ZnbX/Q9oeAdcAe2/8a+EvgntLtbuC5sr0dWCfpUknXAdcDe0sJ6LSkFeXE\n7l1d+0RExABMW96Zwp8A2yTdS2cVvxbA9gFJ2+hc6XMGuL9r2f4A8AQwH9hhe+fPMH5ERMzQtOWd\nJqS8Mzgp7wx23DyuYzb1pbwTERFzR0I/IqIiCf2IiIok9CMiKpLQj4ioSEI/IqIiCf2IiIok9CMi\nKpLQj4ioSEI/IqIiCf2IiIok9CMiKpLQj4ioSEI/IqIiCf2IiIok9CMiKpLQj4ioSEI/IqIiCf2I\niIpMG/qSLpP0DUkvS3pF0qbSvkjSLkmHJD0vaWHXPhslHZF0UNLKrvblkvZLOizpkdk5pIiImMy0\noW/7H4DftH0T8FFglaQVwAZgt+0bgD3ARgBJNwJrgWXAKuBRdT59G+AxYL3tUWBU0h39PqCIiJhc\nT+Ud22+WzcuAeYCBNcCW0r4FuLNsrwaesn3W9lHgCLBC0giwwPa+0m9r1z4RETEAPYW+pEskvQyc\nAl4owb3Y9jiA7VPA1aX7EuB41+4nS9sS4ERX+4nSFhERAzKvl062fwLcJOkq4FlJH6Gz2n9Xt35O\nbGxs7O3tVqtFq9Xq591HRFz02u027XZ7RvvInllWS/oPwJvAp4CW7fFSunnR9jJJGwDb3lz67wQ2\nAcfO9Snt64DbbN83wRie6bziwnROtzTxs25q3CbHFnlcx2yShG1N1aeXq3d+4dyVOZIuB34LOAhs\nB+4p3e4Gnivb24F1ki6VdB1wPbC3lIBOS1pRTuze1bVPREQMQC/lnV8Etki6hM4via/Y3iHp68A2\nSffSWcWvBbB9QNI24ABwBri/a9n+APAEMB/YYXtnX48mIiKmNOPyziDUWN4ZGbmW8fFjDY2e8s6g\nxq3tcR2D1Ut5J6E/JOqrrSf0I/qtl9Dv6eqdiOiHy3jndYqDtXjxNZw6dbSRsWO4ZKU/JLLSr2Hs\nZo+5tudUjfpy9U5ERMwdCf2IiIok9CMiKpLQj4ioSEI/IqIiCf2IiIok9CMiKpLQj4ioSEI/IqIi\nCf2IiIok9CMiKpLQj4ioSEI/IqIiCf2IiIok9CMiKpLQj4ioyLShL2mppD2SvivpFUkPlvZFknZJ\nOiTpeUkLu/bZKOmIpIOSVna1L5e0X9JhSY/MziFFRMRkelnpnwX+ve2PAP8CeEDSh4ENwG7bNwB7\ngI0Akm4E1gLLgFXAo3rnM+IeA9bbHgVGJd3R16OJiIgpTRv6tk/Z/nbZfgM4CCwF1gBbSrctwJ1l\nezXwlO2zto8CR4AVkkaABbb3lX5bu/aJiIgBmFFNX9K1wEeBrwOLbY9D5xcDcHXptgQ43rXbydK2\nBDjR1X6itEVExIDM67WjpPcBzwAP2X5D0vmfstzXT10eGxt7e7vVatFqtfp59xERF712u0273Z7R\nPrKnz2pJ84D/DvwP258vbQeBlu3xUrp50fYySRsA295c+u0ENgHHzvUp7euA22zfN8F47mVec0nn\ntEcTx1zbuE2O3ewx1/acqpEkbGuqPr2Wd/4COHAu8IvtwD1l+27gua72dZIulXQdcD2wt5SATkta\nUU7s3tW1T0REDMC0K31JtwJ/BbxCZ5li4LPAXmAb8AE6q/i1tn9c9tkIrAfO0CkH7SrtNwNPAPOB\nHbYfmmTMrPQHN3Jl4zY5dlb6Mbt6Wen3VN4ZtIT+QEeubNwmx07ox+zqZ3knIiLmgIR+RERFEvoR\nERVJ6EdEVCShHxFRkYR+RERFEvoRERVJ6EdEVCShHxFRkYR+RERFEvoRERVJ6EdEVCShHxFRkYR+\nRERFEvoRERVJ6EdEVCShHxFRkYR+RERFpg19SV+UNC5pf1fbIkm7JB2S9LykhV3f2yjpiKSDklZ2\ntS+XtF/SYUmP9P9QIiJiOr2s9B8H7jivbQOw2/YNwB5gI4CkG4G1wDJgFfCoOh/+CvAYsN72KDAq\n6fz7jIiIWTZt6Nv+GvB35zWvAbaU7S3AnWV7NfCU7bO2jwJHgBWSRoAFtveVflu79omIiAG50Jr+\n1bbHAWyfAq4u7UuA4139Tpa2JcCJrvYTpS0iIgaoXydy3af7iYiIWTTvAvcbl7TY9ngp3fywtJ8E\nPtDVb2lpm6x9UmNjY29vt1otWq3WBU41ImJuarfbtNvtGe0je/pFuqRrgb+0/Svl9mbgddubJX0G\nWGR7QzmR+yRwC53yzQvAL9u2pK8DDwL7gK8Cf2Z75yTjuZd5zSWd891NHHNt4zY5drPHXNtzqkaS\nsK2p+ky70pf0JaAF/Lyk14BNwJ8AT0u6FzhG54odbB+QtA04AJwB7u9K7weAJ4D5wI7JAj8iZsNl\nvHMh3eAsXnwNp04dHfi4MbmeVvqDlpX+QEeubNwmx67zmGt7Ljepl5V+XpEbEVGRhH5EREUu9Oqd\nOWlk5FrGx481PY2IiFmTmv67x6XGmmtd4zY5dp3HPIwZM1elph8REe+S0I+IqEhCPyKiIgn9iIiK\nJPQjIiqS0I+IqEhCPyKiIgn9iIiKJPQjIiqS0I+IqEhCPyKiIgn9iIiKJPQjIioytG+tfPz48aan\nEBEx5wztWytfeeXSgY555swbvPXWj6nxrW/rGrfJsWs85vnAPwx81Fo/m7eXt1YeeOhL+gTwCJ3S\n0hdtb56gjwf/AP0vwB9S35OytnGbHDvHPMhxh3FBO9uG7v30JV0C/CfgDuAjwCclfXiQc+ivdtMT\n6EG76Qn0qN30BHrUbnoCPWo3PYEetZueQE/a7XbTU+ibQZ/IXQEcsX3M9hngKWDNgOfQR+2mJ9CD\ndtMT6FG76Qn0qN30BHrUbnoCPWo3PYGeJPQv3BKg+wztidIWEREDMLRX71x11e8OdLy33jrG3//9\nQIeMiFlzWfnM6/55+OGHe+o37CeRB3oiV9LHgDHbnyi3NwA+/2Ru50RuRETM1FBdvSPpPcAh4Hbg\nB8Be4JO2Dw5sEhERFRtoecf2P0r6t8Au3rlkM4EfETEgQ/nirIiImB1D9d47kj4h6XuSDkv6TNPz\nmYikL0oal7S/6blMRdJSSXskfVfSK5IebHpOE5F0maRvSHq5zHNT03OajKRLJL0kaXvTc5mKpKOS\n/qb8TPc2PZ+JSFoo6WlJB8tj9Jam53Q+SaPlZ/hS+Xp6iJ9Hn5b0HUn7JT0p6dJJ+w7LSr+8cOsw\nnXr/94F9wDrb32t0YueR9OvAG8BW27/a9HwmI2kEGLH9bUnvA74FrBm2nyeApCtsv1nO+fw18KDt\noQsrSZ8Gbgausr266flMRtKrwM22/67puUxG0hPA/7T9uKR5wBW2/0/D05pUyacTwC22h+qNwSS9\nH/ga8GHbb0n6CvBV21sn6j9MK/2L4oVbtr8GDO2T6Rzbp2x/u2y/ARxkSF8TYfvNsnkZnfNMw7ES\n6SJpKfDbwJ83PZceiOF6br+LpKuA37D9OIDts8Mc+MXHgb8dtsDv8h7gynO/QOksnCc0TA+MvHBr\nlki6Fvgo8I1mZzKxUjZ5GTgFvGB7X9NzmsDngD9iCH8hTcDAC5L2Sfo3TU9mAtcBP5L0eCmdfEHS\n5U1Pahq/B3y56UlMxPb3gT8FXgNOAj+2vXuy/sMU+jELSmnnGeChsuIfOrZ/YvsmYClwi6Qbm55T\nN0m/A4yXv5xU/g2zW20vp/OXyQOlJDlM5gHLgf9c5vkmsKHZKU1O0nuB1cDTTc9lIpJ+jk5V5Brg\n/cD7JP3LyfoPU+ifBD7YdXtpaYsLVP7Uewb4r7afa3o+0yl/4r8IfKLpuZznVmB1qZV/GfhNSRPW\nS4eB7R+Ur/8beJZO6XSYnACO2/5muf0MnV8Cw2oV8K3y8xxGHwdetf267X8E/hvwa5N1HqbQ3wdc\nL+macuZ5HTCsV0lcDKs9gL8ADtj+fNMTmYykX5C0sGxfDvwWMFQnm21/1vYHbX+IzuNyj+27mp7X\nRCRdUf66Q9KVwErgO83O6t1sjwPHJY2WptuBAw1OaTqfZEhLO8VrwMckzVfnvSdup3MOb0JD8947\nF8sLtyR9CWgBPy/pNWDTuRNSw0TSrcDvA6+UermBz9re2ezMfsovAlvK1RGXAF+xvaPhOV3MFgPP\nlrcymQc8aXtXw3OayIPAk6V08irwBw3PZ0KSrqCzkv7DpucyGdt7JT0DvAycKV+/MFn/oblkMyIi\nZt8wlXciImKWJfQjIiqS0I+IqEhCPyKiIgn9iIiKJPQjIiqS0I+IqEhCPyKiIv8fdNyMP7yV6KsA\nAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f303ac09310>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(np.log10(all_data), bins=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2842"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(all_data < 100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Pairwise novelty scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def mesh_pair_scores(x):\n",
    "    year, counts = zip(*sorted(x, key=lambda k: k[0]))\n",
    "    df = pd.DataFrame({\"x\": list(year), \"y\": list(counts)})\n",
    "    df = df.groupby(\"x\").sum()\n",
    "    year, counts = df.index.values, df[\"y\"].values\n",
    "    TFirstP = year - year.min()\n",
    "    VolFirstP = counts.cumsum()\n",
    "    return ((year[i], counts[i], TFirstP[i], VolFirstP[i]) for i in xrange(len(year)))\n",
    "\n",
    "\n",
    "def read_mesh_pair(x, mesh_c=dict()):\n",
    "    x = x.split(\"\\t\")\n",
    "    correct_mesh1 = mesh_c.get(transform_mesh_match(x[0]), \"__UNKNOWN__\")\n",
    "    correct_mesh2 = mesh_c.get(transform_mesh_match(x[1]), \"__UNKNOWN__\")\n",
    "    return (correct_mesh1, correct_mesh2, int(x[2]), int(x[3]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "meshpair_exploded_year_count = sc.textFile(\",\".join(glob.glob(\"out/meshpair_exploded_year/part-*\")))\\\n",
    "    .map(lambda x: read_mesh_pair(x, mesh_c=mesh_correct_bc.value))\n",
    "\n",
    "meshpair_scores_data = meshpair_exploded_year_count.map(lambda x: ((x[0], x[1]), (x[2], x[3]))).groupByKey()\\\n",
    "    .flatMapValues(mesh_pair_scores).filter(lambda x: (x[0][0] != \"__UNKNOWN__\") and (x[0][1] != \"__UNKNOWN__\"))\n",
    "\n",
    "csv_data = meshpair_scores_data.map(lambda x: \"%s\\t%s\\t%s\\t%s\\t%s\\t%s\" % (x[0][0], x[0][1], x[1][0], x[1][1], x[1][2], x[1][3]))\n",
    "csv_data.saveAsTextFile(\"out/meshpair_exploded_scores\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate novelty scores for all papers\n",
    "\n",
    "Make sure all the data is filled in the database \n",
    "and the database connection properties are correctly set in `settings.json`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## LOAD THE score helpers module for connecting to the database. \n",
    "import score_helpers as sch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('111', '2000', 6377, 66, 99376.0, '\\\\N', '\\\\N', 0.021203, 0.021203)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sch.gen_scores(\"111\\t2000\\tBreastNeoplasms\", mesh_c=mesh_correct)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('111', '2000', '\\\\N', '\\\\N', '\\\\N')"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sch.gen_pair_scores(\"111\\t2000\\tBreastNeoplasms\", mesh_c=mesh_correct)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# RUN INDIVIDUAL SCORES\n",
    "sc.textFile(\"data/pmid_year_mesh.noheader.tsv\").map(lambda x: sch.gen_scores(x, mesh_c=mesh_correct_bc.value))\\\n",
    "    .filter(lambda x: x is not None).map(lambda x: \"\\t\".join([\"%s\"]*len(x)) % x)\\\n",
    "    .saveAsTextFile(\"out/pmid_novelty_scores\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# RUN PAIR SCORES\n",
    "sc.textFile(\"data/pmid_year_mesh.noheader.tsv\").map(lambda x: sch.gen_pair_scores(x, mesh_c=mesh_correct_bc.value))\\\n",
    "    .filter(lambda x: x is not None).map(lambda x: \"\\t\".join([\"%s\"]*len(x)) % x)\\\n",
    "    .saveAsTextFile(\"out/pmid_novelty_pair_scores\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Merge Scores\n",
    "\n",
    "Merge individual and pairwise scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "individual_scores = sc.textFile(\"out/pmid_novelty_scores\").map(lambda x: x.split()).map(lambda x: (int(x[0]), tuple(x)))\n",
    "pair_scores = sc.textFile(\"out/pmid_novelty_pair_scores\").map(lambda x: x.split()).map(lambda x: (int(x[0]), tuple(x[2:])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "individual_scores.leftOuterJoin(pair_scores).map(lambda x: x[1][0] + x[1][1])\\\n",
    "  .map(lambda x: \"\\t\".join([\"%s\"]*len(x)) % x)\\\n",
    "  .saveAsTextFile(\"out/pmid_novelty_all_scores\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## Add mesh counts to the output\n",
    "\n",
    "mesh_count = sc.textFile(\"data/pmid_year_mesh.noheader.tsv\").map(\n",
    "    lambda x: sch.get_mesh_counts(x, mesh_c=mesh_correct_bc.value, mesh_exp=mesh_exploded.value))\n",
    "sc.textFile(\"out/pmid_novelty_all_scores\").map(lambda x: x.split(\"\\t\"))\\\n",
    "    .map(lambda x: (int(x[0]), tuple(x))).leftOuterJoin(mesh_count)\\\n",
    "    .map(lambda x: x[1][0] + x[1][1])\\\n",
    "    .map(lambda x: \"\\t\".join([\"%s\"]*len(x)) % x)\\\n",
    "    .saveAsTextFile(\"out/pmid_novelty_all_scores_mesh_c\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}