Munging_SQuAD_with_splits_without_title_id.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "import json\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from nltk.tokenize import word_tokenize\n",
    "from itertools import chain\n",
    "from sklearn.model_selection import StratifiedShuffleSplit\n",
    "from argparse import Namespace\n",
    "from tqdm import tqdm_notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "args = Namespace(\n",
    "    path=\"data/SQuAd/BatchSQuAD\",\n",
    "    train_proportion=0.7,\n",
    "    val_proportion=0.15,\n",
    "    test_proportion=0.15,\n",
    "    output_munged_csv=\"data/SQuAd/SQuAD_with_splits_without_title_id.csv\",\n",
    "    passage_text='data/SQuAd/passage_text',\n",
    "    seed=1337\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(args.seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4a2958f2d7f247cdae552a698448a533",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='paragraph', max=45, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b1d55eb126504d11a9575e6f00580a73",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "QADataset = []\n",
    "passage = []\n",
    "bar = tqdm_notebook(desc='paragraph',\n",
    "                   total = 45,\n",
    "                    position=1,\n",
    "                    leave=True)\n",
    "for i in tqdm_notebook(range(10)):\n",
    "    with open(args.path+\"/batch_{}.json\".format(i+1)) as fp:\n",
    "        batch = json.load(fp)\n",
    "        for para in batch:\n",
    "            title = para['title']\n",
    "            for par in para['paragraphs']:\n",
    "                text = {}\n",
    "                text['passage'] = par['context'].lower()\n",
    "                passage.append(text)\n",
    "                for question in par['qas']:\n",
    "                    sample = {}\n",
    "                    sample['title'] =  title\n",
    "                    sample['context'] = par['context'].lower()\n",
    "                    sample['question'] = question['question'].lower()\n",
    "                    sample['is_impossible'] = question['is_impossible']\n",
    "                    if not question['is_impossible']:\n",
    "                        sample['answer_start'] = question['answers'][0]['answer_start']\n",
    "                        sample['answer_text'] = question['answers'][0]['text'].lower()\n",
    "                    else:\n",
    "                        sample['answer_start'] = -1\n",
    "                        sample['answer_text'] = ''\n",
    "                    QADataset.append(sample)\n",
    "            bar.update()\n",
    "            \n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "QADataset = pd.DataFrame(QADataset)\n",
    "QADataset['split'] = 0\n",
    "passage = pd.DataFrame(passage)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "list indices must be integers or slices, not str",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-64-d9d782e882f1>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0msplit\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mStratifiedShuffleSplit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mn_splits\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtest_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtest_proportion\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mval_proportion\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrandom_state\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mseed\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[1;32mfor\u001b[0m \u001b[0mtrain_index\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtest_index\u001b[0m \u001b[1;32min\u001b[0m \u001b[0msplit\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mQADataset\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mQADataset\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'title'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m     \u001b[0mQADataset\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mtrain_index\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'split'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'train'\u001b[0m\u001b[1;31m#修改值的常规方式，其他方式可能会在copy上修改，没用\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m     \u001b[0mQADataset\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mtest_index\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'split'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'test'\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTypeError\u001b[0m: list indices must be integers or slices, not str"
     ]
    }
   ],
   "source": [
    "\n",
    "split = StratifiedShuffleSplit(n_splits=1, test_size=args.test_proportion+args.val_proportion, random_state=args.seed)\n",
    "\n",
    "for train_index, test_index in split.split(QADataset, QADataset['title']):\n",
    "    QADataset[train_index]['split'] = 'train'#修改值的常规方式，其他方式可能会在copy上修改，没用\n",
    "    QADataset[test_index]['split'] = 'test'\n",
    "    first_index = test_index\n",
    "                                           \n",
    "split = StratifiedShuffleSplit(n_splits=1, test_size=0.5, random_state=args.seed)\n",
    "\n",
    "for _, test_index in split.split(QADataset[first_index], QADataset[first_index]['title']):\n",
    "    QADataset[test_index]['split'] = 'val'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "#由Title分层采样  放弃使用 适用于df\n",
    "split = StratifiedShuffleSplit(n_splits=1, test_size=args.test_proportion+args.val_proportion, random_state=args.seed)\n",
    "\n",
    "for train_index, test_index in split.split(QADataset, QADataset.title):\n",
    "    QADataset.iloc[train_index,-1] = 'train'#修改值的常规方式，其他方式可能会在copy上修改，没用\n",
    "    QADataset.iloc[test_index,-1] = 'test'\n",
    "                                           \n",
    "split = StratifiedShuffleSplit(n_splits=1, test_size=0.5, random_state=args.seed)\n",
    "\n",
    "for _, test_index in split.split(QADataset[QADataset.split=='train'], QADataset[QADataset.split=='train'].title):\n",
    "    QADataset.iloc[test_index,-1] = 'val'\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>answer_start</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>130319.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>212.727384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>264.883218</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>-1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>110.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>357.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>3126.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        answer_start\n",
       "count  130319.000000\n",
       "mean      212.727384\n",
       "std       264.883218\n",
       "min        -1.000000\n",
       "25%        -1.000000\n",
       "50%       110.000000\n",
       "75%       357.000000\n",
       "max      3126.000000"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "QADataset.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_dataset = QADataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>answer_start</th>\n",
       "      <th>answer_text</th>\n",
       "      <th>context</th>\n",
       "      <th>is_impossible</th>\n",
       "      <th>question</th>\n",
       "      <th>title</th>\n",
       "      <th>split</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>269</td>\n",
       "      <td>in the late 1990s</td>\n",
       "      <td>beyoncé giselle knowles-carter (/biːˈjɒnseɪ/ b...</td>\n",
       "      <td>False</td>\n",
       "      <td>when did beyonce start becoming popular?</td>\n",
       "      <td>Beyoncé</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>207</td>\n",
       "      <td>singing and dancing</td>\n",
       "      <td>beyoncé giselle knowles-carter (/biːˈjɒnseɪ/ b...</td>\n",
       "      <td>False</td>\n",
       "      <td>what areas did beyonce compete in when she was...</td>\n",
       "      <td>Beyoncé</td>\n",
       "      <td>val</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>526</td>\n",
       "      <td>2003</td>\n",
       "      <td>beyoncé giselle knowles-carter (/biːˈjɒnseɪ/ b...</td>\n",
       "      <td>False</td>\n",
       "      <td>when did beyonce leave destiny's child and bec...</td>\n",
       "      <td>Beyoncé</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>166</td>\n",
       "      <td>houston, texas</td>\n",
       "      <td>beyoncé giselle knowles-carter (/biːˈjɒnseɪ/ b...</td>\n",
       "      <td>False</td>\n",
       "      <td>in what city and state did beyonce  grow up?</td>\n",
       "      <td>Beyoncé</td>\n",
       "      <td>val</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>276</td>\n",
       "      <td>late 1990s</td>\n",
       "      <td>beyoncé giselle knowles-carter (/biːˈjɒnseɪ/ b...</td>\n",
       "      <td>False</td>\n",
       "      <td>in which decade did beyonce become famous?</td>\n",
       "      <td>Beyoncé</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   answer_start          answer_text  \\\n",
       "0           269    in the late 1990s   \n",
       "1           207  singing and dancing   \n",
       "2           526                 2003   \n",
       "3           166       houston, texas   \n",
       "4           276           late 1990s   \n",
       "\n",
       "                                             context  is_impossible  \\\n",
       "0  beyoncé giselle knowles-carter (/biːˈjɒnseɪ/ b...          False   \n",
       "1  beyoncé giselle knowles-carter (/biːˈjɒnseɪ/ b...          False   \n",
       "2  beyoncé giselle knowles-carter (/biːˈjɒnseɪ/ b...          False   \n",
       "3  beyoncé giselle knowles-carter (/biːˈjɒnseɪ/ b...          False   \n",
       "4  beyoncé giselle knowles-carter (/biːˈjɒnseɪ/ b...          False   \n",
       "\n",
       "                                            question    title  split  \n",
       "0           when did beyonce start becoming popular?  Beyoncé  train  \n",
       "1  what areas did beyonce compete in when she was...  Beyoncé    val  \n",
       "2  when did beyonce leave destiny's child and bec...  Beyoncé  train  \n",
       "3      in what city and state did beyonce  grow up?   Beyoncé    val  \n",
       "4         in which decade did beyonce become famous?  Beyoncé  train  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "w\n",
      "h\n",
      "e\n",
      "n\n",
      " \n",
      "d\n",
      "i\n",
      "d\n",
      " \n",
      "b\n",
      "e\n",
      "y\n",
      "o\n",
      "n\n",
      "c\n",
      "e\n",
      " \n",
      "s\n",
      "t\n",
      "a\n",
      "r\n",
      "t\n",
      " \n",
      "b\n",
      "e\n",
      "c\n",
      "o\n",
      "m\n",
      "i\n",
      "n\n",
      "g\n",
      " \n",
      "p\n",
      "o\n",
      "p\n",
      "u\n",
      "l\n",
      "a\n",
      "r\n",
      "?\n",
      "w\n",
      "h\n",
      "a\n",
      "t\n",
      " \n",
      "a\n",
      "r\n",
      "e\n",
      "a\n",
      "s\n",
      " \n",
      "d\n",
      "i\n",
      "d\n",
      " \n",
      "b\n",
      "e\n",
      "y\n",
      "o\n",
      "n\n",
      "c\n",
      "e\n",
      " \n",
      "c\n",
      "o\n",
      "m\n",
      "p\n",
      "e\n",
      "t\n",
      "e\n",
      " \n",
      "i\n",
      "n\n",
      " \n",
      "w\n",
      "h\n",
      "e\n",
      "n\n",
      " \n",
      "s\n",
      "h\n",
      "e\n",
      " \n",
      "w\n",
      "a\n",
      "s\n",
      " \n",
      "g\n",
      "r\n",
      "o\n",
      "w\n",
      "i\n",
      "n\n",
      "g\n",
      " \n",
      "u\n",
      "p\n",
      "?\n",
      "w\n",
      "h\n",
      "e\n",
      "n\n",
      " \n",
      "d\n",
      "i\n",
      "d\n",
      " \n",
      "b\n",
      "e\n",
      "y\n",
      "o\n",
      "n\n",
      "c\n",
      "e\n",
      " \n",
      "l\n",
      "e\n",
      "a\n",
      "v\n",
      "e\n",
      " \n",
      "d\n",
      "e\n",
      "s\n",
      "t\n",
      "i\n",
      "n\n",
      "y\n",
      "'\n",
      "s\n",
      " \n",
      "c\n",
      "h\n",
      "i\n",
      "l\n",
      "d\n",
      " \n",
      "a\n",
      "n\n",
      "d\n",
      " \n",
      "b\n",
      "e\n",
      "c\n",
      "o\n",
      "m\n",
      "e\n",
      " \n",
      "a\n",
      " \n",
      "s\n",
      "o\n",
      "l\n",
      "o\n",
      " \n",
      "s\n",
      "i\n",
      "n\n",
      "g\n",
      "e\n",
      "r\n",
      "?\n",
      "i\n",
      "n\n",
      " \n",
      "w\n",
      "h\n",
      "a\n",
      "t\n",
      " \n",
      "c\n",
      "i\n",
      "t\n",
      "y\n",
      " \n",
      "a\n",
      "n\n",
      "d\n",
      " \n",
      "s\n",
      "t\n",
      "a\n",
      "t\n",
      "e\n",
      " \n",
      "d\n",
      "i\n",
      "d\n",
      " \n",
      "b\n",
      "e\n",
      "y\n",
      "o\n",
      "n\n",
      "c\n",
      "e\n",
      " \n",
      " \n",
      "g\n",
      "r\n",
      "o\n",
      "w\n",
      " \n",
      "u\n",
      "p\n",
      "?\n",
      " \n",
      "i\n",
      "n\n",
      " \n",
      "w\n",
      "h\n",
      "i\n",
      "c\n",
      "h\n",
      " \n",
      "d\n",
      "e\n",
      "c\n",
      "a\n",
      "d\n",
      "e\n",
      " \n",
      "d\n",
      "i\n",
      "d\n",
      " \n",
      "b\n",
      "e\n",
      "y\n",
      "o\n",
      "n\n",
      "c\n",
      "e\n",
      " \n",
      "b\n",
      "e\n",
      "c\n",
      "o\n",
      "m\n",
      "e\n",
      " \n",
      "f\n",
      "a\n",
      "m\n",
      "o\n",
      "u\n",
      "s\n",
      "?\n"
     ]
    }
   ],
   "source": [
    "for i in final_dataset.head().question:\n",
    "    for word in i:\n",
    "        print(word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "train    59325\n",
       "val      45612\n",
       "test     25382\n",
       "Name: split, dtype: int64"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_dataset.split.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_dataset.to_csv(args.output_munged_csv, index=False)\n",
    "passage.to_csv(args.passage_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>answer_start</th>\n",
       "      <th>answer_text</th>\n",
       "      <th>context</th>\n",
       "      <th>is_impossible</th>\n",
       "      <th>question</th>\n",
       "      <th>title</th>\n",
       "      <th>split</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>269</td>\n",
       "      <td>in the late 1990s</td>\n",
       "      <td>beyoncé giselle knowles-carter (/biːˈjɒnseɪ/ b...</td>\n",
       "      <td>False</td>\n",
       "      <td>when did beyonce start becoming popular?</td>\n",
       "      <td>Beyoncé</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>207</td>\n",
       "      <td>singing and dancing</td>\n",
       "      <td>beyoncé giselle knowles-carter (/biːˈjɒnseɪ/ b...</td>\n",
       "      <td>False</td>\n",
       "      <td>what areas did beyonce compete in when she was...</td>\n",
       "      <td>Beyoncé</td>\n",
       "      <td>val</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>526</td>\n",
       "      <td>2003</td>\n",
       "      <td>beyoncé giselle knowles-carter (/biːˈjɒnseɪ/ b...</td>\n",
       "      <td>False</td>\n",
       "      <td>when did beyonce leave destiny's child and bec...</td>\n",
       "      <td>Beyoncé</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>166</td>\n",
       "      <td>houston, texas</td>\n",
       "      <td>beyoncé giselle knowles-carter (/biːˈjɒnseɪ/ b...</td>\n",
       "      <td>False</td>\n",
       "      <td>in what city and state did beyonce  grow up?</td>\n",
       "      <td>Beyoncé</td>\n",
       "      <td>val</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>276</td>\n",
       "      <td>late 1990s</td>\n",
       "      <td>beyoncé giselle knowles-carter (/biːˈjɒnseɪ/ b...</td>\n",
       "      <td>False</td>\n",
       "      <td>in which decade did beyonce become famous?</td>\n",
       "      <td>Beyoncé</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   answer_start          answer_text  \\\n",
       "0           269    in the late 1990s   \n",
       "1           207  singing and dancing   \n",
       "2           526                 2003   \n",
       "3           166       houston, texas   \n",
       "4           276           late 1990s   \n",
       "\n",
       "                                             context  is_impossible  \\\n",
       "0  beyoncé giselle knowles-carter (/biːˈjɒnseɪ/ b...          False   \n",
       "1  beyoncé giselle knowles-carter (/biːˈjɒnseɪ/ b...          False   \n",
       "2  beyoncé giselle knowles-carter (/biːˈjɒnseɪ/ b...          False   \n",
       "3  beyoncé giselle knowles-carter (/biːˈjɒnseɪ/ b...          False   \n",
       "4  beyoncé giselle knowles-carter (/biːˈjɒnseɪ/ b...          False   \n",
       "\n",
       "                                            question    title  split  \n",
       "0           when did beyonce start becoming popular?  Beyoncé  train  \n",
       "1  what areas did beyonce compete in when she was...  Beyoncé    val  \n",
       "2  when did beyonce leave destiny's child and bec...  Beyoncé  train  \n",
       "3      in what city and state did beyonce  grow up?   Beyoncé    val  \n",
       "4         in which decade did beyonce become famous?  Beyoncé  train  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "k = final_dataset.head().to_json(orient='split', force_ascii = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"data/SQuAd/k.josn\", 'w+') as fp:\n",
    "    json.dump(k, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"data/SQuAd/k.josn\") as fp:\n",
    "    s = json.load(fp)\n",
    "    s = json.loads(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pandas.io.json import json_normalize\n",
    "S = json_normalize(s)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>columns</th>\n",
       "      <th>data</th>\n",
       "      <th>index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[answer_start, answer_text, context, is_imposs...</td>\n",
       "      <td>[[269, in the late 1990s, beyoncé giselle know...</td>\n",
       "      <td>[0, 1, 2, 3, 4]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             columns  \\\n",
       "0  [answer_start, answer_text, context, is_imposs...   \n",
       "\n",
       "                                                data            index  \n",
       "0  [[269, in the late 1990s, beyoncé giselle know...  [0, 1, 2, 3, 4]  "
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "S"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}