Twitter Highcharts.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import json\n",
    "path='C:\\\\Users\\\\Shahidhya\\\\OneDrive\\\\I524 Project\\\\3. Twitter Streaming\\\\set 1\\\\'\n",
    "tweetFile='Twitter_data2.txt'\n",
    "import pandas as pd\n",
    "statedata=pd.read_csv('C:\\\\Users\\\\Shahidhya\\\\OneDrive\\\\I524 Project\\\\3. Twitter Streaming\\\\'+\"states.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tweets_data = []\n",
    "tweets_data_path=path+tweetFile\n",
    "tweets_file = open(tweets_data_path, \"r\")\n",
    "for line in tweets_file:\n",
    "    try:\n",
    "        tweet = json.loads(line)\n",
    "        tweets_data.append(tweet)\n",
    "    except:\n",
    "        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from textblob import TextBlob\n",
    "count=0\n",
    "tweets = pd.DataFrame(index=range(len(tweets_data)), columns=['text','created_at','lang','retweeted','location','state','sentiment','sentiment_cat','country_code','lat','lon'])\n",
    "\n",
    "for i in range(len(tweets_data)):\n",
    "    try:\n",
    "        tweets['text'][i] = tweets_data[i]['text']\n",
    "    except:\n",
    "        tweets['text'][i] = \"\"\n",
    "    try:\n",
    "        tweets['lang'][i]=tweets_data[i]['lang']\n",
    "    except:\n",
    "        tweets['lang'][i]='NA'\n",
    "    try:\n",
    "        tweets['retweeted'][i]=tweets_data[i]['retweeted']\n",
    "    except:\n",
    "        tweets['lang'][i]='NA'\n",
    "    try:\n",
    "        tweets['location'][i]=tweets_data[i]['user']['location']\n",
    "    except:\n",
    "        tweets['location'][i]='NA'\n",
    "    try:\n",
    "        tweets['country_code'][i]=tweets_data[i]['place']['country_code']\n",
    "    except:\n",
    "        tweets['country_code'][i]=''\n",
    "    try:\n",
    "        tweets['lon'][i]=tweets_data[i]['place']['bounding_box']['coordinates'][0][0][0]\n",
    "    except:\n",
    "        tweets['lon'][i]='NA'\n",
    "    try:\n",
    "        tweets['lat'][i]=tweets_data[i]['place']['bounding_box']['coordinates'][0][0][1]\n",
    "    except:\n",
    "        tweets['lat'][i]='NA'\n",
    "    try:\n",
    "        tweets['created_at'][i]=tweets_data[i]['created_at']\n",
    "    except:\n",
    "        tweets['created_at'][i]='NA'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000  Tweets processed\n",
      "2000  Tweets processed\n",
      "3000  Tweets processed\n",
      "4000  Tweets processed\n",
      "5000  Tweets processed\n",
      "6000  Tweets processed\n",
      "7000  Tweets processed\n",
      "8000  Tweets processed\n",
      "9000  Tweets processed\n",
      "10000  Tweets processed\n",
      "--- 693.4969084262848 seconds ---\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "import zipcode\n",
    "start_time = time.time()\n",
    "count=0\n",
    "for i in range(len(tweets)):\n",
    "    blob = TextBlob(tweets['text'][i])\n",
    "    try:\n",
    "        sentence=blob.sentences[0]\n",
    "        tweets['sentiment'][i]=sentence.sentiment.polarity\n",
    "    except:\n",
    "        tweets['sentiment'][i]=0\n",
    "    if tweets['sentiment'][i] < 0:\n",
    "        tweets['sentiment_cat'][i] = 'Neg'\n",
    "    else:\n",
    "        if tweets['sentiment'][i] > 0:\n",
    "            tweets['sentiment_cat'][i] = 'Pos'\n",
    "        else:\n",
    "            tweets['sentiment_cat'][i] = 'Neu'  \n",
    "    try:\n",
    "        stateFromData=tweets['location'][i].split(',')[1]\n",
    "    except:\n",
    "        stateFromData=''\n",
    "    if len(stateFromData)==2:\n",
    "        tweets['state'][i]=stateFromData.upper()\n",
    "    else:\n",
    "        if tweets['lat'][i] !='NA':\n",
    "            radius=10\n",
    "            incre=10\n",
    "            zips=zipcode.isinradius((tweets['lat'][i],tweets['lon'][i]),radius)\n",
    "            while len(zips)==0:\n",
    "                radius=radius+incre\n",
    "                zips=zipcode.isinradius((tweets['lat'][i],tweets['lon'][i]),radius)\n",
    "                incre=incre+10\n",
    "            myzip = zipcode.isequal(str(zips[0].zip))\n",
    "            tweets['state'][i]=myzip.state\n",
    "        else:\n",
    "            tweets['state'][i]='NA'\n",
    "    count+=1\n",
    "    if count%1000==0:\n",
    "        print(count,\" Tweets processed\")\n",
    "print(\"--- %s seconds ---\" % (time.time() - start_time))\n",
    "tweetsFinal=pd.merge(tweets, statedata, how='left',left_on=\"state\",right_on=\"Abbreviation\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import json\n",
    "#a = tweets.groupby(['state'],as_index=False).count()\n",
    "output=pd.DataFrame({'value' : tweetsFinal.groupby( [ \"State\"] ).size()}).reset_index()\n",
    "outputJsonList=output.to_json(orient = \"records\")\n",
    "finalOutput=outputJsonList[33:len(outputJsonList)-1].upper().replace(\"\\\"STATE\\\"\",\"ucName\").replace(\"\\\"VALUE\\\"\",\"value\")\n",
    "print(finalOutput)\n",
    "\n",
    "with open('C:\\\\Users\\\\Shahidhya\\\\OneDrive\\\\I524 Project\\\\3. Twitter Streaming\\\\usStates-tweetCount.json', 'w') as outfile:\n",
    "    outfile.write(finalOutput)\n",
    "#-------------------------------------\n",
    "zips=zipcode.isinradius((tweets['lat'][0],tweets['lon'][0]),radius)\n",
    "print (zips[0].zip)\n",
    "myzip = zipcode.isequal(str(81507))\n",
    "print (myzip.statename)\n",
    "#-------------------------------------------\n",
    "output2=pd.DataFrame({'value' : tweetsFinal.groupby( [ \"State\",\"sentiment_cat\"] ).size()}).reset_index()\n",
    "output2.head()\n",
    "#-------------------------------------------\n",
    "import numpy as np\n",
    "outData=pd.pivot_table(output2,values='value', index=['State'], columns=['sentiment_cat'], aggfunc=np.sum)\n",
    "outData=outData.fillna(0)\n",
    "outData.head()\n",
    "#-------------------------------------------\n",
    "outData['sum']=outData[['Neg', 'Neu', 'Pos']].sum(axis=1)\n",
    "outData['max']=outData['maxFinal']=outData[['Neg', 'Neu', 'Pos']].idxmax(axis=1)\n",
    "#-------------------------------------------\n",
    "for i in range(len(outData)):\n",
    "    if outData['max'][i] ==\"Pos\":\n",
    "        outData['maxFinal'][i] = '1'\n",
    "    else:\n",
    "        if outData['max'][i] ==\"Neu\":\n",
    "            outData['maxFinal'][i] = '-1'\n",
    "        else:\n",
    "            outData['maxFinal'][i] = '2'\n",
    "\n",
    "outData\n",
    "#-------------------------------------------\n",
    "outData['state']=outData.index\n",
    "outData.reset_index()\n",
    "#-------------------------------------------\n",
    "d=\"var data =[\\n\"\n",
    "for i in range(len(outData)):\n",
    "    row=outData.ix[i]\n",
    "    d += \"[\\'\"+row['state']+\"\\',\"+\",\".join([str(i) for i in row[:5]])+\"],\\n\"\n",
    "    \n",
    "d+=']'\n",
    "#-------------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from time import strptime\n",
    "\n",
    "td1 = pd.DataFrame({'value' : tweets.groupby( [ \"created_at\"] ).size()}).reset_index()\n",
    "timedata = td1[td1.created_at != 'NA']\n",
    "data1 ={}\n",
    "data = [\"var data=[\"]\n",
    "for i in range(1,len(timedata)+1):\n",
    "\n",
    "    year = timedata['created_at'][i][-4:]\n",
    "    if (timedata['created_at'][i][4:7] == 'Jan'):\n",
    "        mon = '1'\n",
    "    else:\n",
    "        if (timedata['created_at'][i][4:7] == 'Feb'):\n",
    "            mon = '2'\n",
    "        else:\n",
    "            if (timedata['created_at'][i][4:7] == 'Mar'):\n",
    "                mon = '3'\n",
    "            else:\n",
    "                if (timedata['created_at'][i][4:7] == 'Apr'):\n",
    "                    mon = '4'\n",
    "                else:\n",
    "                    if (timedata['created_at'][i][4:7] == 'May'):\n",
    "                        mon = '5'\n",
    "                    else:\n",
    "                        if (timedata['created_at'][i][4:7] == 'Jun'):\n",
    "                            mon = '6'\n",
    "                        else:\n",
    "                            if (timedata['created_at'][i][4:7] == 'Jul'):\n",
    "                                mon = '7'\n",
    "                            else:\n",
    "                                if (timedata['created_at'][i][4:7] == 'Jul'):\n",
    "                                    mon = '8'\n",
    "                                else:\n",
    "                                    if (timedata['created_at'][i][4:7] == 'Jul'):\n",
    "                                        mon = '9'\n",
    "                                    else:\n",
    "                                        if (timedata['created_at'][i][4:7] == 'Jul'):\n",
    "                                            mon = '10'\n",
    "                                        else:\n",
    "                                            if (timedata['created_at'][i][4:7] == 'Jul'):\n",
    "                                                mon = '11'\n",
    "                                            else:\n",
    "                                                mon = '12'\n",
    "    date = timedata['created_at'][i][7:10]\n",
    "    hour = timedata['created_at'][i][10:13]\n",
    "    minu = timedata['created_at'][i][14:16]\n",
    "    sec = timedata['created_at'][i][17:20]\n",
    "    value = timedata['value'][i]\n",
    "    data1 = (\"[Date.UTC(\"+str(year)+\",\"+str(mon)+\",\"+str(date)+\",\"+str(hour)+\",\"+str(minu)+\",\"+str(sec)+\"),\"+str(value)+\"]\")\n",
    "    if (len(timedata)):\n",
    "        data.append\n",
    "    data.append(data1)\n",
    "data = \",\\n\".join(data)+\"\\n]\"\n",
    "data = data.replace(\"[,\",\"[\")\n",
    "with open('C:\\\\Users\\\\Shahidhya\\\\OneDrive\\\\I524 Project\\\\3. Twitter Streaming\\\\tweet_cnt.json', 'w') as outfile:\n",
    "    outfile.write(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Tokenizer:\n",
    "\n",
    "import re\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "#nltk.download('stopwords')\n",
    "\n",
    "stop = stopwords.words('english') + ['and']\n",
    "emoticons_str = r\"\"\"\n",
    "    (?:\n",
    "        [:=;] # Eyes\n",
    "        [oO\\-]? # Nose (optional)\n",
    "        [D\\)\\]\\(\\]/\\\\OpP] # Mouth\n",
    "    )\"\"\"\n",
    " \n",
    "regex_str = [\n",
    "    emoticons_str,\n",
    "    r'<[^>]+>', # HTML tags\n",
    "    r'(?:@[\\w_]+)', # @-mentions\n",
    "    r\"(?:\\#+[\\w_]+[\\w\\'_\\-]*[\\w_]+)\", # hash-tags\n",
    "    r'http[s]?://(?:[a-z]|[0-9]|[$-_@.&amp;+]|[!*\\(\\),]|(?:%[0-9a-f][0-9a-f]))+', # URLs\n",
    " \n",
    "    r'(?:(?:\\d+,?)+(?:\\.?\\d+)?)', # numbers\n",
    "    r\"(?:[a-z][a-z'\\-_]+[a-z])\", # words with - and '\n",
    "    r'(?:[\\w_]+)', # other words\n",
    "    r'(?:\\S)' # anything else\n",
    "]\n",
    "    \n",
    "tokens_re = re.compile(r'('+'|'.join(regex_str)+')', re.VERBOSE | re.IGNORECASE)\n",
    "emoticon_re = re.compile(r'^'+emoticons_str+'$', re.VERBOSE | re.IGNORECASE)\n",
    " \n",
    "def tokenize(s):\n",
    "    tokens=tokens_re.findall(s)\n",
    "    return [ x for x in tokens if 'http' not in x and len(x)>1 and x.lower() not in stop]\n",
    " \n",
    "def preprocess(s, lowercase=True):\n",
    "    tokens = tokenize(s)\n",
    "    if lowercase:\n",
    "        tokens = [token if emoticon_re.search(token) else token.lower() for token in tokens]\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Co-occurrence:\n",
    "\n",
    "from itertools import combinations\n",
    "from collections import Counter\n",
    "def collect_pairs(lines):\n",
    "    pair_counter = Counter()\n",
    "    for line in lines:\n",
    "        unique_tokens = sorted(set(line))  # exclude duplicates in same line and sort to ensure one word is always before other\n",
    "        combos = combinations(unique_tokens, 2)\n",
    "        pair_counter += Counter(combos)\n",
    "    return pair_counter\n",
    "t2 = []\n",
    "t1 =tweets['text']\n",
    "for t in range(len(t1)):\n",
    "    t2.append(preprocess(t1[t]))\n",
    "              \n",
    "pairs = collect_pairs(t2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "nptp = np.array(top_pairs)\n",
    "maxtp = np.max(nptp[:,1])\n",
    "\n",
    "top_pairs = pairs.most_common(200)\n",
    "nodes={}\n",
    "links=[\"\\\"links\\\":[\"]\n",
    "count =0\n",
    "len_top=len(top_pairs)\n",
    "for p in range(len(top_pairs)):\n",
    "    for i in range(2):\n",
    "        if top_pairs[p][0][i] not in nodes:\n",
    "            nodes[top_pairs[p][0][i]] = count\n",
    "            count+=1\n",
    "    link=\"{ \\\"source\\\":\"+str(nodes[top_pairs[p][0][0]])+\",\\\"target\\\":\"+str(nodes[top_pairs[p][0][1]])+\",\\\"value\\\":\"+str(round(top_pairs[p][1]*10/maxtp))+\"}\"\n",
    "    links.append(link)\n",
    "links=\",\\n\".join(links)+\"\\n]\"\n",
    "links=links.replace(\"[,\",\"[\")\n",
    "nodes = sorted(nodes.items(), key=lambda x: x[1])\n",
    "nodes1=[\"\\\"nodes\\\":[\"]\n",
    "for p in range(len(nodes)):\n",
    "    nodes1.append(\"{ \\\"name\\\":\\\"\"+nodes[p][0]+\"\\\",\\\"group\\\":\"+\"0}\")\n",
    "nodes1=\",\\n\".join(nodes1)+\"\\n]\"\n",
    "nodes1=nodes1.replace(\"[,\",\"[\")\n",
    "with open('C:\\\\Users\\\\Shahidhya\\\\OneDrive\\\\I524 Project\\\\3. Twitter Streaming\\\\cooccur_word.json', 'w') as outfile:\n",
    "    outfile.write(\"{\\n\"+nodes1+\",\\n\"+links+\"}\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Wordcloud:\n",
    "\n",
    "from os import path\n",
    "from wordcloud import WordCloud\n",
    "\n",
    "#d = path.dirname(__file__)\n",
    "\n",
    "# Read the whole text.\n",
    "#text = open(path.join(d, 'constitution.txt')).read()\n",
    "textpos = tweets[tweets.sentiment_cat == 'Pos']\n",
    "textneg = tweets[tweets.sentiment_cat == 'Neg']\n",
    "textp = preprocess(textpos['text'])\n",
    "textn = preprocess(textneg['text'])\n",
    "wordcloudp = WordCloud(font_path='/Users/kunal/Library/Fonts/sans-serif.ttf',\n",
    "                          stopwords=STOPWORDS,\n",
    "                          background_color='white',\n",
    "                          width=1200,\n",
    "                          height=1000).generate(textp)\n",
    "wordcloudn = WordCloud(font_path='/Users/kunal/Library/Fonts/sans-serif.ttf',\n",
    "                          stopwords=STOPWORDS,\n",
    "                          background_color='white',\n",
    "                          width=1200,\n",
    "                          height=1000).generate(textn)\n",
    "import matplotlib.pyplot as plt\n",
    "plt.imshow(wordcloudp, interpolation='bilinear')\n",
    "plt.axis(\"off\")\n",
    "plt.show()\n",
    "plt.imshow(wordcloudn, interpolation='bilinear')\n",
    "plt.axis(\"off\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dataset initialized\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "from geopy.geocoders import Nominatim\n",
    "geolocator = Nominatim()\n",
    "path='C:\\\\Users\\\\Shahidhya\\\\OneDrive\\\\I524 Project\\\\'\n",
    "tweetFile='WorldTweets-'\n",
    "import pandas as pd\n",
    "import zipcode\n",
    "from textblob import TextBlob\n",
    "#from random import shuffle\n",
    "import sys\n",
    "tweets_data = []\n",
    "fileno=1\n",
    "tweets_data_path=path+tweetFile+str(fileno)+\".txt\"\n",
    "tweets_file = open(tweets_data_path, \"r\")\n",
    "for line in tweets_file:\n",
    "    try:\n",
    "        tweet = json.loads(line)\n",
    "        tweets_data.append(tweet)\n",
    "    except:\n",
    "        continue\n",
    "\n",
    "tweets = pd.DataFrame(index=range(len(tweets_data)), columns=['text','lang','created_at','hour','retweet','country_code','state','sentiment','lat','lon'])\n",
    "print (\"dataset initialized\")\t\n",
    "#count=0\n",
    "#shuffle(data)\n",
    "#tweets_data=data[:200]\n",
    "for i in range(len(tweets_data)):\n",
    "    try:\n",
    "        tweets['text'][i] = tweets_data[i]['text']\n",
    "    except:\n",
    "        tweets['text'][i] = \"\"\n",
    "    try:\n",
    "        tweets['lang'][i]=tweets_data[i]['lang']\n",
    "    except:\n",
    "        tweets['lang'][i]='NA'\n",
    "    try:\n",
    "        tweets['retweet'][i]='0' if \"RT @\" not in tweets['text'][i] else '1'\n",
    "    except:\n",
    "        tweets['lang'][i]='NA'\n",
    "    try:\n",
    "        tweets['country_code'][i]=str(tweets_data[i]['place']['country_code']).upper()\n",
    "    except:\n",
    "        tweets['country_code'][i]=''\n",
    "    try:\n",
    "        tweets['lon'][i]=tweets_data[i]['place']['bounding_box']['coordinates'][0][0][0]\n",
    "    except:\n",
    "        tweets['lon'][i]='NA'\n",
    "    try:\n",
    "        tweets['lat'][i]=tweets_data[i]['place']['bounding_box']['coordinates'][0][0][1]\n",
    "    except:\n",
    "        tweets['lat'][i]='NA'\n",
    "    try:\n",
    "        tweets['hour'][i]=tweets_data[i]['created_at'][11:13]\n",
    "    except:\n",
    "        tweets['hour'][i]='NA'\n",
    "    try:\n",
    "        tweets['created_at'][i]=tweets_data[i]['created_at']\n",
    "    except:\n",
    "            tweets['created_at'][i]='NA'\n",
    "    blob = TextBlob(tweets['text'][i])\n",
    "    try:\n",
    "        sentence=blob.sentences[0]\n",
    "        tweets['sentiment'][i]=sentence.sentiment.polarity\n",
    "    except:\n",
    "        tweets['sentiment'][i]=0\n",
    "    try:\n",
    "        location=tweets_data[i]['user']['location']\n",
    "    except:\n",
    "        location=\"NA\"\n",
    "    try:\n",
    "        coor=geolocator.geocode(location)\n",
    "        coor=coor.raw\n",
    "        address = geolocator.reverse(coor['lat']+\",\"+coor['lon']).raw\n",
    "        country_code=str(address['address']['country_code']).upper()\n",
    "    except:\n",
    "        country=\"\"\n",
    "    if len(tweets['country_code']) <= 1:\n",
    "        tweets['country_code']=country_code\n",
    "    try:\n",
    "        stateFromData=tweets['location'][i].split(',')[1]\n",
    "    except:\n",
    "        stateFromData=''\n",
    "    if len(stateFromData)==2:\n",
    "        tweets['state'][i]=stateFromData\n",
    "    else:\n",
    "        if tweets['lat'][i] !='NA':\n",
    "            radius=10\n",
    "            incre=10\n",
    "            zips=zipcode.isinradius((tweets['lat'][i],tweets['lon'][i]),radius)\n",
    "            while len(zips)==0:\n",
    "                radius=radius+incre\n",
    "                zips=zipcode.isinradius((tweets['lat'][i],tweets['lon'][i]),radius)\n",
    "                incre=incre+10\n",
    "            myzip = zipcode.isequal(str(zips[0].zip))\n",
    "            tweets['state'][i]=myzip.state\n",
    "        else:\n",
    "            tweets['state'][i]='NA'\n",
    "tweets.to_csv(path+\"tweets-output/\"+tweetFile+str(fileno)+\".csv\",encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Sentiment matrix:\n",
    "\n",
    "td1 = pd.DataFrame({'value' : tweets.groupby( [ \"created_at\"],['country'],['country_code'] ).size()}).reset_index()\n",
    "timedata = td1[td1.created_at != 'NA']\n"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}