Data cleaning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "\"\"\"\n",
    "data_collection.ipynb\n",
    "Automatically generated by Colaboratory.\n",
    "Original file is located at:\n",
    "    https://colab.research.google.com/drive/1auiMh_kYPUusGn0avpANgkqYQD3mD2Xf\n",
    "\"\"\"\n",
    "\n",
    "import os\n",
    "import json\n",
    "\n",
    "def dw_config(filename):\n",
    "    '''\n",
    "    Args:\n",
    "        filename: the JSON file containing the authentication token\n",
    "\n",
    "    Return:\n",
    "        None\n",
    "\n",
    "    Load a JSON config file at `filename` containing the authentication and \n",
    "    other information. Before running this cell, make sure that `filename` has \n",
    "    been uploaded into Colab, or is at least in the current working directory of\n",
    "    this notebook. The file's contents should have the following format:\n",
    "\n",
    "    `\n",
    "    {\"token\": <YOUR_TOKEN>}\n",
    "    `\n",
    "\n",
    "    In this file, `<YOUR_TOKEN>` is the authentication token string for the \n",
    "    Python integration with data.world. Once the authentication token is \n",
    "    obtained, set it as an environment variable so that it can be used by the \n",
    "    datadotworld library, which is used to import datasets automatically. The \n",
    "    specific environment variable is `DW_AUTH_TOKEN`, as specified in the docs\n",
    "    for `datadotworld`. \n",
    "    \n",
    "    For information on obtaining this token and how it is being used in this \n",
    "    method, see the links below. You will need to create an account and generate\n",
    "    your own authentication token to run this code. The first link is to the\n",
    "    integration page. Once you have enabled the integration, go to the Manage\n",
    "    tab and you will see you authentication token. The second link is to help\n",
    "    understand how the library is configured and how datasets are imported.\n",
    "\n",
    "    https://data.world/integrations/python \n",
    "    https://help.data.world/hc/en-us/articles/360039429733-Python-SDK\n",
    "    '''\n",
    "    with open(filename) as f:\n",
    "        config = json.load(f)\n",
    "        token = config['auth_token']\n",
    "        os.environ['DW_AUTH_TOKEN'] = token\n",
    "\n",
    "import datadotworld as dw\n",
    "\n",
    "def get_twitter_dataframes():\n",
    "    '''\n",
    "    Args:\n",
    "        None\n",
    "\n",
    "    Return:\n",
    "        till2017_df: the dataframe for the dataset containg Tweets till 2017\n",
    "        from2017_df: the dataframe for the dataset containg Tweets till 2017\n",
    "\n",
    "    A function that uses the `datadotworld` library, to load the two datasets\n",
    "    used in this project, and the dataframes in this datasets are extracted.\n",
    "    Below are links to the datasets being used. \n",
    "    \n",
    "    https://data.world/adamhelsinger/elon-musk-tweets-until-4-6-17 (2010-2017)\n",
    "    https://data.world/barbaramaseda/elon-musk-tweets (2017 onwards)\n",
    "\n",
    "    The object returned from `dw.load_dataset()` contains a dataframe that needs\n",
    "    to be extracted from a dictionary of other dataframes. Because there is only\n",
    "    one dataframe in each dataset, it is extracted using set-list conversion, \n",
    "    avoiding the need to use a key to look it up.\n",
    "    '''\n",
    "    till2017 = dw.load_dataset('adamhelsinger/elon-musk-tweets-until-4-6-17')\n",
    "    from2017 = dw.load_dataset('barbaramaseda/elon-musk-tweets')\n",
    "\n",
    "    till2017_df = list(till2017.dataframes.values())[0]\n",
    "    from2017_df = list(from2017.dataframes.values())[0]\n",
    "    return till2017_df, from2017_df\n",
    "\n",
    "\"\"\"\n",
    "From here, the next step is cleaning the data. `from2017_df` contains the full Tweet URL, and the Tweet ID is at the end of the URL.\n",
    "`till2017_df` uses these same IDs, so it is possible to extract IDs from the URLs in `from2017_df`.\n",
    "Once these IDs are extracted, they can be used to index Tweets.\n",
    "This makes it easier to identify and remove duplicate Tweets when combining the two dataframes into one.\n",
    "We can also remove Retweets by finding all Tweets with the prefix \"RT\".\n",
    "This helps identify which Tweets are not actually authored by Musk specifically.\n",
    "\n",
    "Other next steps may involve removing formatting characters, retweets, and URLs from the Tweets as they appear.\n",
    "Mentions of other Twitter users should be left in the message with the \"@\" character removed so that the sentence still makes sense, and so that those words can be identified as referring to other users.\n",
    "After this is complete, columns relating to TSLA stock valuations can be added to this merged dataframe.\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "pd.set_option('display.max_rows', None)\n",
    "pd.set_option('display.max_colwidth', None)\n",
    "\n",
    "def clean_data(till2017_df_orig, from2017_df_orig):\n",
    "    '''\n",
    "    Args:\n",
    "        till2017_df_orig: the original dataframe containing data till 2017\n",
    "        from2017_df_orig: the original dataframe containing data from 2017\n",
    "\n",
    "    Return:\n",
    "        df: the cleaned dataframe containing the Tweets and other information\n",
    "    \n",
    "    Clean the dataframes obtained from `get_dataframes()`. Those dataframes \n",
    "    should be stored somewhere before calling this function in case changes need\n",
    "    to be reverted. The cleaning process is detailed below. \n",
    "    '''\n",
    "    # Make a copy of the original, remove binary string identifiers, rename columns, set index\n",
    "    till2017_df = till2017_df_orig.copy()\n",
    "    till2017_df['text'] = till2017_df['text'].str.strip('b\\'\\\"')\n",
    "    till2017_df = till2017_df.rename(columns={'created_at': 'timestamp'})\n",
    "    till2017_df = till2017_df.reindex(columns=['id', 'timestamp', 'text'])\n",
    "    till2017_df = till2017_df.set_index('id')\n",
    "\n",
    "    # Make a copy of the original, extract and format ids, drop and rename some columns, set index\n",
    "    from2017_df = from2017_df_orig.copy()\n",
    "    from2017_df['linktotweet'] = from2017_df['linktotweet'].str.strip('http://twitter.com/elonmusk/status/')\n",
    "    from2017_df['linktotweet'] = from2017_df['linktotweet'].astype(int)\n",
    "    from2017_df = from2017_df.drop(labels=['username', 'tweetembedcode'], axis=1)\n",
    "    from2017_df = from2017_df.rename(columns={'createdat': 'timestamp', 'linktotweet': 'id'})\n",
    "    from2017_df = from2017_df.reindex(columns=['id', 'timestamp', 'text'])\n",
    "    from2017_df = from2017_df.set_index('id')\n",
    "\n",
    "    # Concatenate dataframes, drop duplicates\n",
    "    tweets_df = pd.concat([till2017_df, from2017_df])\n",
    "    tweets_df = tweets_df.drop_duplicates()\n",
    "\n",
    "    # Separate timestamp into date and time, so that stock data can be found\n",
    "    tweets_df['date'] = pd.to_datetime(tweets_df['timestamp']).dt.date\n",
    "    tweets_df['time'] = pd.to_datetime(tweets_df['timestamp']).dt.time\n",
    "\n",
    "    # Reset index, remove id and timestamp columns, index by date column\n",
    "    tweets_df = tweets_df.reset_index()\n",
    "    tweets_df = tweets_df.drop(labels=['id', 'timestamp'], axis=1)\n",
    "    tweets_df = tweets_df.reindex(columns=['date', 'time', 'text'])\n",
    "    tweets_df = tweets_df.set_index('date')\n",
    "    tweets_df = tweets_df.sort_index()\n",
    "\n",
    "    '''\n",
    "    TODO:\n",
    "        Add more code here to do any word processing on the text column\n",
    "    '''\n",
    "    # Remove retweets\n",
    "    #tweets_df = tweets_df[~tweets_df['text'].str.startswith('RT')]\n",
    "\n",
    "    # Remove mentions of other users\n",
    "    #tweets_df = tweets_df[~df.text.str.contains(\"@\", na=False)]\n",
    "\n",
    "    # Export tweets to CSV\n",
    "    tweets_df.to_csv('tweets.csv')\n",
    "    print(tweets_df)\n",
    "    return tweets_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting datadotworld\n",
      "  Downloading datadotworld-1.7.0-py2.py3-none-any.whl (158 kB)\n",
      "\u001b[K     |████████████████████████████████| 158 kB 1.2 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting configparser<4.0a,>=3.5.0\n",
      "  Downloading configparser-3.8.1-py2.py3-none-any.whl (22 kB)\n",
      "Collecting tabulator>=1.22.0\n",
      "  Downloading tabulator-1.52.5-py2.py3-none-any.whl (71 kB)\n",
      "\u001b[K     |████████████████████████████████| 71 kB 946 kB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: click<7.0a,>=6.0 in /Users/anshuduggal/anaconda3/lib/python3.7/site-packages (from datadotworld) (6.7)\n",
      "Requirement already satisfied: urllib3<2.0a,>=1.15 in /Users/anshuduggal/anaconda3/lib/python3.7/site-packages (from datadotworld) (1.25.8)\n",
      "Requirement already satisfied: python-dateutil<3.0a,>=2.6.0 in /Users/anshuduggal/anaconda3/lib/python3.7/site-packages (from datadotworld) (2.8.1)\n",
      "Collecting datapackage<2.0a,>=1.6.2\n",
      "  Downloading datapackage-1.15.1-py2.py3-none-any.whl (85 kB)\n",
      "\u001b[K     |████████████████████████████████| 85 kB 753 kB/s eta 0:00:01:01\n",
      "\u001b[?25hRequirement already satisfied: six<2.0a,>=1.5.0 in /Users/anshuduggal/anaconda3/lib/python3.7/site-packages (from datadotworld) (1.14.0)\n",
      "Collecting tableschema<2.0a,>=1.5.2\n",
      "  Downloading tableschema-1.20.0-py2.py3-none-any.whl (68 kB)\n",
      "\u001b[K     |████████████████████████████████| 68 kB 1.1 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: requests<3.0a,>=2.0.0 in /Users/anshuduggal/anaconda3/lib/python3.7/site-packages (from datadotworld) (2.22.0)\n",
      "Requirement already satisfied: certifi>=2017.04.17 in /Users/anshuduggal/anaconda3/lib/python3.7/site-packages (from datadotworld) (2020.4.5.1)\n",
      "Requirement already satisfied: xlrd>=1.0 in /Users/anshuduggal/anaconda3/lib/python3.7/site-packages (from tabulator>=1.22.0->datadotworld) (1.2.0)\n",
      "Collecting boto3>=1.9\n",
      "  Downloading boto3-1.16.10-py2.py3-none-any.whl (129 kB)\n",
      "\u001b[K     |████████████████████████████████| 129 kB 959 kB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: chardet>=3.0 in /Users/anshuduggal/anaconda3/lib/python3.7/site-packages (from tabulator>=1.22.0->datadotworld) (3.0.4)\n",
      "Requirement already satisfied: sqlalchemy>=0.9.6 in /Users/anshuduggal/anaconda3/lib/python3.7/site-packages (from tabulator>=1.22.0->datadotworld) (1.3.13)\n",
      "Requirement already satisfied: unicodecsv>=0.14 in /Users/anshuduggal/anaconda3/lib/python3.7/site-packages (from tabulator>=1.22.0->datadotworld) (0.14.1)\n",
      "Requirement already satisfied: openpyxl>=2.6 in /Users/anshuduggal/anaconda3/lib/python3.7/site-packages (from tabulator>=1.22.0->datadotworld) (3.0.3)\n",
      "Collecting linear-tsv>=1.0\n",
      "  Downloading linear-tsv-1.1.0.tar.gz (9.6 kB)\n",
      "Collecting ijson>=3.0.3\n",
      "  Downloading ijson-3.1.2.post0-cp37-cp37m-macosx_10_9_x86_64.whl (53 kB)\n",
      "\u001b[K     |████████████████████████████████| 53 kB 699 kB/s eta 0:00:01\n",
      "\u001b[?25hCollecting jsonlines>=1.1\n",
      "  Downloading jsonlines-1.2.0-py2.py3-none-any.whl (7.6 kB)\n",
      "Requirement already satisfied: jsonschema>=2.5 in /Users/anshuduggal/anaconda3/lib/python3.7/site-packages (from datapackage<2.0a,>=1.6.2->datadotworld) (3.2.0)\n",
      "Collecting jsonpointer>=1.10\n",
      "  Downloading jsonpointer-2.0-py2.py3-none-any.whl (7.6 kB)\n",
      "Collecting cached-property>=1.5\n",
      "  Downloading cached_property-1.5.2-py2.py3-none-any.whl (7.6 kB)\n",
      "Collecting isodate>=0.5.4\n",
      "  Downloading isodate-0.6.0-py2.py3-none-any.whl (45 kB)\n",
      "\u001b[K     |████████████████████████████████| 45 kB 2.0 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting rfc3986>=1.1.0\n",
      "  Downloading rfc3986-1.4.0-py2.py3-none-any.whl (31 kB)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in /Users/anshuduggal/anaconda3/lib/python3.7/site-packages (from requests<3.0a,>=2.0.0->datadotworld) (2.8)\n",
      "Collecting botocore<1.20.0,>=1.19.10\n",
      "  Downloading botocore-1.19.10-py2.py3-none-any.whl (6.7 MB)\n",
      "\u001b[K     |████████████████████████████████| 6.7 MB 1.1 MB/s eta 0:00:01     |███▎                            | 686 kB 947 kB/s eta 0:00:07     |██████████▊                     | 2.3 MB 1.5 MB/s eta 0:00:04     |███████████████████▏            | 4.0 MB 505 kB/s eta 0:00:06     |███████████████████████         | 4.8 MB 830 kB/s eta 0:00:03     |█████████████████████████▉      | 5.4 MB 830 kB/s eta 0:00:02\n",
      "\u001b[?25hCollecting s3transfer<0.4.0,>=0.3.0\n",
      "  Downloading s3transfer-0.3.3-py2.py3-none-any.whl (69 kB)\n",
      "\u001b[K     |████████████████████████████████| 69 kB 3.5 MB/s eta 0:00:011\n",
      "\u001b[?25hCollecting jmespath<1.0.0,>=0.7.1\n",
      "  Downloading jmespath-0.10.0-py2.py3-none-any.whl (24 kB)\n",
      "Requirement already satisfied: et-xmlfile in /Users/anshuduggal/anaconda3/lib/python3.7/site-packages (from openpyxl>=2.6->tabulator>=1.22.0->datadotworld) (1.0.1)\n",
      "Requirement already satisfied: jdcal in /Users/anshuduggal/anaconda3/lib/python3.7/site-packages (from openpyxl>=2.6->tabulator>=1.22.0->datadotworld) (1.4.1)\n",
      "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /Users/anshuduggal/anaconda3/lib/python3.7/site-packages (from jsonschema>=2.5->datapackage<2.0a,>=1.6.2->datadotworld) (1.5.0)\n",
      "Requirement already satisfied: pyrsistent>=0.14.0 in /Users/anshuduggal/anaconda3/lib/python3.7/site-packages (from jsonschema>=2.5->datapackage<2.0a,>=1.6.2->datadotworld) (0.15.7)\n",
      "Requirement already satisfied: attrs>=17.4.0 in /Users/anshuduggal/anaconda3/lib/python3.7/site-packages (from jsonschema>=2.5->datapackage<2.0a,>=1.6.2->datadotworld) (19.3.0)\n",
      "Requirement already satisfied: setuptools in /Users/anshuduggal/anaconda3/lib/python3.7/site-packages (from jsonschema>=2.5->datapackage<2.0a,>=1.6.2->datadotworld) (46.0.0.post20200309)\n",
      "Requirement already satisfied: zipp>=0.5 in /Users/anshuduggal/anaconda3/lib/python3.7/site-packages (from importlib-metadata; python_version < \"3.8\"->jsonschema>=2.5->datapackage<2.0a,>=1.6.2->datadotworld) (2.2.0)\n",
      "Building wheels for collected packages: linear-tsv\n",
      "  Building wheel for linear-tsv (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for linear-tsv: filename=linear_tsv-1.1.0-py3-none-any.whl size=7382 sha256=33304589b709e90b96af24280b5c0bbbd61e70594dde955ebbad973fe6512b52\n",
      "  Stored in directory: /Users/anshuduggal/Library/Caches/pip/wheels/dd/0a/62/49ee84fab92057fb3e303fb1a59e75ccac6899b417f441305a\n",
      "Successfully built linear-tsv\n",
      "Installing collected packages: configparser, jmespath, botocore, s3transfer, boto3, linear-tsv, ijson, jsonlines, tabulator, cached-property, isodate, rfc3986, tableschema, jsonpointer, datapackage, datadotworld\n",
      "Successfully installed boto3-1.16.10 botocore-1.19.10 cached-property-1.5.2 configparser-3.8.1 datadotworld-1.7.0 datapackage-1.15.1 ijson-3.1.2.post0 isodate-0.6.0 jmespath-0.10.0 jsonlines-1.2.0 jsonpointer-2.0 linear-tsv-1.1.0 rfc3986-1.4.0 s3transfer-0.3.3 tableschema-1.20.0 tabulator-1.52.5\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "!{sys.executable} -m pip install datadotworld"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "Configuration file not found at /Users/anshuduggal/.dw/config.To fix this issue, run dw configure",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-15-5f08e38583fa>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mget_twitter_dataframes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-14-a3bf405fb491>\u001b[0m in \u001b[0;36mget_twitter_dataframes\u001b[0;34m()\u001b[0m\n\u001b[1;32m     72\u001b[0m     \u001b[0mavoiding\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mneed\u001b[0m \u001b[0mto\u001b[0m \u001b[0muse\u001b[0m \u001b[0ma\u001b[0m \u001b[0mkey\u001b[0m \u001b[0mto\u001b[0m \u001b[0mlook\u001b[0m \u001b[0mit\u001b[0m \u001b[0mup\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     73\u001b[0m     '''\n\u001b[0;32m---> 74\u001b[0;31m     \u001b[0mtill2017\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdw\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'adamhelsinger/elon-musk-tweets-until-4-6-17'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     75\u001b[0m     \u001b[0mfrom2017\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdw\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'barbaramaseda/elon-musk-tweets'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     76\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/datadotworld/__init__.py\u001b[0m in \u001b[0;36mload_dataset\u001b[0;34m(dataset_key, force_update, auto_update, profile, **kwargs)\u001b[0m\n\u001b[1;32m     96\u001b[0m     \u001b[0;34m[\u001b[0m\u001b[0;34m'changelog'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'datadotworldbballstats'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'datadotworldbballteam'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     97\u001b[0m     \"\"\"\n\u001b[0;32m---> 98\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_get_instance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprofile\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     99\u001b[0m         load_dataset(dataset_key,\n\u001b[1;32m    100\u001b[0m                      \u001b[0mforce_update\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mforce_update\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/datadotworld/__init__.py\u001b[0m in \u001b[0;36m_get_instance\u001b[0;34m(profile, **kwargs)\u001b[0m\n\u001b[1;32m     58\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mprofile\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'default'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m                 else FileConfig(profile=profile))\n\u001b[0;32m---> 60\u001b[0;31m         \u001b[0minstance\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDataDotWorld\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconfig_param\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     61\u001b[0m         \u001b[0m__instances\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mprofile\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minstance\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0minstance\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/datadotworld/datadotworld.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, config)\u001b[0m\n\u001b[1;32m     55\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     56\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_config\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconfig\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mChainedConfig\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 57\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapi_client\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mRestApiClient\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_config\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     58\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mquery\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdataset_key\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mquery\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mquery_type\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"sql\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparameters\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/datadotworld/client/api.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, config)\u001b[0m\n\u001b[1;32m     54\u001b[0m             \u001b[0mhost\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_host\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     55\u001b[0m             \u001b[0mheader_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Authorization'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 56\u001b[0;31m             header_value='Bearer {}'.format(self._config.auth_token))\n\u001b[0m\u001b[1;32m     57\u001b[0m         \u001b[0mswagger_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muser_agent\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_user_agent\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     58\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/datadotworld/config.py\u001b[0m in \u001b[0;36m__getattribute__\u001b[0;34m(self, item)\u001b[0m\n\u001b[1;32m    212\u001b[0m         return object.__getattribute__(self, '_first_not_none')(\n\u001b[1;32m    213\u001b[0m             \u001b[0mobject\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getattribute__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'_config_chain'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 214\u001b[0;31m             lambda c: c.__getattribute__(item))\n\u001b[0m\u001b[1;32m    215\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    216\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mstaticmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/datadotworld/config.py\u001b[0m in \u001b[0;36m_first_not_none\u001b[0;34m(seq, supplier_func)\u001b[0m\n\u001b[1;32m    225\u001b[0m         \"\"\"\n\u001b[1;32m    226\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mseq\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 227\u001b[0;31m             \u001b[0mobj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msupplier_func\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    228\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mobj\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    229\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/datadotworld/config.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(c)\u001b[0m\n\u001b[1;32m    212\u001b[0m         return object.__getattribute__(self, '_first_not_none')(\n\u001b[1;32m    213\u001b[0m             \u001b[0mobject\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getattribute__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'_config_chain'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 214\u001b[0;31m             lambda c: c.__getattribute__(item))\n\u001b[0m\u001b[1;32m    215\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    216\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mstaticmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/datadotworld/config.py\u001b[0m in \u001b[0;36mauth_token\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    123\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    124\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mauth_token\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 125\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__validate_config\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    126\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_config_parser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_section\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'auth_token'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    127\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/datadotworld/config.py\u001b[0m in \u001b[0;36m__validate_config\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    148\u001b[0m                 \u001b[0;34m'Configuration file not found at {}.'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    149\u001b[0m                 'To fix this issue, run dw configure'.format(\n\u001b[0;32m--> 150\u001b[0;31m                     self._config_file_path))\n\u001b[0m\u001b[1;32m    151\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_config_parser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhas_option\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_section\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'auth_token'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    152\u001b[0m             raise RuntimeError(\n",
      "\u001b[0;31mRuntimeError\u001b[0m: Configuration file not found at /Users/anshuduggal/.dw/config.To fix this issue, run dw configure"
     ]
    }
   ],
   "source": [
    "get_twitter_dataframes()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}