Sequence_novelty_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sample\n",
    "from __future__ import division"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Get sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "sampler = sample.Sampler(\"I will build a \")\n",
    "sample_raw = sampler.get_sample()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Trunate sample to align to [blog post](https://eightportions.com/2016-11-03-Trump-bot/#eval) Sample B"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I will build a wall. She's got the greatest victory that I've ever seen. He had a big problem. I think it was no beautiful. But I said, Let's go to the wall and the people are going to say, We want to have a show that they're going to pay for the wall. We're going to build a wall. We're going to build a wall. We're going to go. It's very hard. I love everybody. I'm not doing a great job for the vets. I'm one of the most important, you know, in the whole world, that are going to be more than what's going on. And I want to make sure that's the best that I've ever seen negotiated. I've gotten a lot of money outside of the world - the primary two days - I'm not a conservative on my own, and we're going to make America great again. I'm a messenger until we're going to be the good people in the world. I think it's amazing to say, I'm a conservative. I love you. I will tell you, I will say, I will not tell you I'm going to win because we're going to win. I'm going to make a lot of money. But everybody, that's a big story, that was the wall just have read it because I was the right. I've always been a very good person. But I don't want to use it... I'd love to thank you to see what happens because I'm not going to win this. I'm going to tell you very well. But I'm a messenger. I'm a conservative conservative. I'm the only one that had a tremendous people that I've ever seen for a long time. A couple of weeks ago, I'm a very conservative. This is the most of the greatest. And I was so honored by the way, the most thing we need. You understand it.\n"
     ]
    }
   ],
   "source": [
    "sample = sample_raw[:1563]\n",
    "print sample"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Read all speeches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open(\"data/trump/input_norm_caps/input.txt\", \"r\") as f:\n",
    "    data = f.read()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Find percent of n word sequences that are unique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def count_new_combos(sample, n_words, verbose=False):\n",
    "    words = sample.split()\n",
    "    word_count = len(words)\n",
    "    count_new = 0\n",
    "    for i in range(word_count - n_words):\n",
    "        next_words_3 = ' '.join(words[i: i+ n_words])\n",
    "        new_sequence = next_words_3 in data\n",
    "        if not new_sequence:\n",
    "            count_new += 1\n",
    "        if verbose:\n",
    "            print '{}: {}, {}'.format(i, new_sequence, next_words_3)\n",
    "    return count_new/word_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_pct_new_list(verbose=False):\n",
    "    percent_new_list = {}\n",
    "    for j in range(10):\n",
    "        percent_new = count_new_combos(sample, j)\n",
    "        if verbose:\n",
    "            print \"{0:.1%} of sequences of length {1} in the sample are new sequences\".\\\n",
    "                format(percent_new, j)\n",
    "        percent_new_list[j] = percent_new\n",
    "    return percent_new_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "percent_new_list = get_pct_new_list()\n",
    "df_transpose = zip(*percent_new_list.items())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot percent new by sequence length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loaded nvd3 IPython extension\n",
      "run nvd3.ipynb.initialize_javascript() to set up the notebook\n",
      "help(nvd3.ipynb.initialize_javascript) for options\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<link media=\"all\" href=\"https://cdnjs.cloudflare.com/ajax/libs/nvd3/1.7.0/nv.d3.min.css\" type=\"text/css\"\n",
       "                        rel=\"stylesheet\"/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "$.getScript(\"https://cdnjs.cloudflare.com/ajax/libs/nvd3/1.7.0/nv.d3.min.js\")"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "$.getScript(\"https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js\", function() {\n",
       "                              $.getScript(\"https://cdnjs.cloudflare.com/ajax/libs/nvd3/1.7.0/nv.d3.min.js\", function() {})});"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<script src=\"https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js\"></script>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<script src=\"https://cdnjs.cloudflare.com/ajax/libs/nvd3/1.7.0/nv.d3.min.js\"></script>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import nvd3\n",
    "nvd3.ipynb.initialize_javascript(use_remote=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "chart = nvd3.lineChart(name=\"lineChart\", height=450, width=750, show_legend=False)\n",
    "\n",
    "xdata = df_transpose[0]\n",
    "ydata = df_transpose[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<!DOCTYPE html>\n",
       "<html lang=\"en\">\n",
       "    <head>\n",
       "        <meta charset=\"utf-8\" />\n",
       "        <link href=\"https://cdnjs.cloudflare.com/ajax/libs/nvd3/1.7.0/nv.d3.min.css\" rel=\"stylesheet\" />\n",
       "        <script src=\"https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js\"></script>\n",
       "        <script src=\"https://cdnjs.cloudflare.com/ajax/libs/nvd3/1.7.0/nv.d3.min.js\"></script>\n",
       "    </head>\n",
       "    <body>\n",
       "        \n",
       "    <div id=\"linechart\"><svg style=\"width:750px;height:450px;\"></svg></div>\n",
       "\n",
       "\n",
       "    <script>\n",
       "\n",
       "\n",
       "\n",
       "            data_linechart=[{\"values\": [{\"y\": 0.0, \"x\": 0}, {\"y\": 0.10625, \"x\": 1}, {\"y\": 0.265625, \"x\": 2}, {\"y\": 0.534375, \"x\": 3}, {\"y\": 0.79375, \"x\": 4}, {\"y\": 0.915625, \"x\": 5}, {\"y\": 0.9625, \"x\": 6}, {\"y\": 0.975, \"x\": 7}, {\"y\": 0.975, \"x\": 8}, {\"y\": 0.971875, \"x\": 9}], \"key\": \"Percent new\", \"yAxis\": \"1\"}];\n",
       "\n",
       "\n",
       "        nv.addGraph(function() {\n",
       "        var chart = nv.models.lineChart();\n",
       "\n",
       "        chart.margin({top: 30, right: 60, bottom: 20, left: 60});\n",
       "\n",
       "        var datum = data_linechart;\n",
       "\n",
       "\n",
       "\n",
       "                chart.xAxis\n",
       "                .tickFormat(d3.format(',r'));\n",
       "            chart.yAxis\n",
       "                .tickFormat(d3.format(',.02f'));\n",
       "\n",
       "\n",
       "\n",
       "          chart.showLegend(false);\n",
       "\n",
       "\n",
       "    \n",
       "\n",
       "        \n",
       "\n",
       "\n",
       "\n",
       "            d3.select('#linechart svg')\n",
       "            .datum(datum)\n",
       "            .transition().duration(500)\n",
       "            .attr('width', 750)\n",
       "            .attr('height', 450)\n",
       "            .call(chart);\n",
       "\n",
       "\n",
       "        });\n",
       "\n",
       "\n",
       "\n",
       "    </script>\n",
       "\n",
       "    </body>\n",
       "</html>"
      ],
      "text/plain": [
       "<nvd3.lineChart.lineChart at 0x104b1fd90>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extra_serie = {\"tooltip\": {\"y_start\": \"\", \"y_end\": \" min\"}}\n",
    "chart.add_serie(y=ydata, x=xdata, name='Percent new', extra=extra_serie)\n",
    "chart.create_x_axis('Sequence length')\n",
    "chart.create_y_axis('Percent of all sequences new')\n",
    "chart"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('percent_new_list.html', 'w') as f:\n",
    "    f.write(chart.htmlcontent)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}