concatenation_datasets.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "import pandas as pd\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "education = pd.read_csv('Education -tweets.csv' , error_bad_lines=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Education1939</td>\n",
       "      <td>This past week The Staff Recognition Committee...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>alexanderrusso</td>\n",
       "      <td>RT @UNICEFmedia: \"Evidence shows that schools ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TaleamSystems</td>\n",
       "      <td>Taleam Systems' CEO implemented the Student e-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>javrsda</td>\n",
       "      <td>Ya quiero que salga la nueva temporada de Sex ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Calpe19</td>\n",
       "      <td>RT @LizzyJPrice: @JonnyGeller A great loss. Wh...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             user                                               Text\n",
       "0   Education1939  This past week The Staff Recognition Committee...\n",
       "1  alexanderrusso  RT @UNICEFmedia: \"Evidence shows that schools ...\n",
       "2   TaleamSystems  Taleam Systems' CEO implemented the Student e-...\n",
       "3         javrsda  Ya quiero que salga la nueva temporada de Sex ...\n",
       "4         Calpe19  RT @LizzyJPrice: @JonnyGeller A great loss. Wh..."
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "education.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "fashion = pd.read_csv('Fashion -tweets.csv' , error_bad_lines=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>UyghurProject</td>\n",
       "      <td>\"Whether it be via football or fashion, the re...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>mustucos_12</td>\n",
       "      <td>RT @fashionpressnet: M·A·C“金魚”モチーフの限定コスメ「ムーン マ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NicoleZizi</td>\n",
       "      <td>RT @ComplexStyle: .@bibbygregory built his Twi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>denimedlife</td>\n",
       "      <td>It's time to slow down and consider the true c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>JulieSwann14</td>\n",
       "      <td>@CanadianPenny1 You look like a fashion model!...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            user                                               Text\n",
       "0  UyghurProject  \"Whether it be via football or fashion, the re...\n",
       "1    mustucos_12  RT @fashionpressnet: M·A·C“金魚”モチーフの限定コスメ「ムーン マ...\n",
       "2     NicoleZizi  RT @ComplexStyle: .@bibbygregory built his Twi...\n",
       "3    denimedlife  It's time to slow down and consider the true c...\n",
       "4   JulieSwann14  @CanadianPenny1 You look like a fashion model!..."
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fashion.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "mechanical = pd.read_csv('Mechanical -tweets.csv' , error_bad_lines=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LALOLEM17</td>\n",
       "      <td>RT @Rainmaker1973: This low temperature differ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SentaiJullianna</td>\n",
       "      <td>RT @LouisatheLast: Hey all, as you’re buying h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>MAlkaitis</td>\n",
       "      <td>RT @MartinLandray: Latest results from RECOVER...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>BenReilly149</td>\n",
       "      <td>@RDMacQ @DDKeyD True, but the movies also use ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Kitlocks</td>\n",
       "      <td>Did you know we’ve expanded our Marine by Code...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              user                                               Text\n",
       "0        LALOLEM17  RT @Rainmaker1973: This low temperature differ...\n",
       "1  SentaiJullianna  RT @LouisatheLast: Hey all, as you’re buying h...\n",
       "2        MAlkaitis  RT @MartinLandray: Latest results from RECOVER...\n",
       "3     BenReilly149  @RDMacQ @DDKeyD True, but the movies also use ...\n",
       "4         Kitlocks  Did you know we’ve expanded our Marine by Code..."
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mechanical.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "technology = pd.read_csv('Technology -tweets.csv' , error_bad_lines=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>saraswathi1redy</td>\n",
       "      <td>RT @SantoshMishraB4: किसानों को नए बाजार, विकल...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Chris16965568</td>\n",
       "      <td>@PepsiMaxx I'm sorry if my cutting edge techno...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>dr_fadenstyle</td>\n",
       "      <td>@Nobodyexx I'm tweeting from my Samsung Galaxy...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1sundaywoman</td>\n",
       "      <td>Our Top Ten Technology Gifts with awards for @...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>SyahirMarof</td>\n",
       "      <td>RT @AdDien90: ABC, Sos, Titik, Lumba kereta. K...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              user                                               Text\n",
       "0  saraswathi1redy  RT @SantoshMishraB4: किसानों को नए बाजार, विकल...\n",
       "1    Chris16965568  @PepsiMaxx I'm sorry if my cutting edge techno...\n",
       "2    dr_fadenstyle  @Nobodyexx I'm tweeting from my Samsung Galaxy...\n",
       "3     1sundaywoman  Our Top Ten Technology Gifts with awards for @...\n",
       "4      SyahirMarof  RT @AdDien90: ABC, Sos, Titik, Lumba kereta. K..."
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "technology.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "politics = pd.read_csv('Politics-tweets.csv' , error_bad_lines=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ThomasDillhunt</td>\n",
       "      <td>RT @RepsForBiden: How’s this even legal?\\n#Loc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>avataylor02</td>\n",
       "      <td>@JujuSlick @ScottAdamsSays Your point is??? Sh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Harryy_Pawter</td>\n",
       "      <td>RT @OpIndia_in: बीजेपी प्रवक्ता ने कहा कि नए क...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>memykidsandeye</td>\n",
       "      <td>RT @freep: BREAKING: @GovWhitmer named co-chai...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>iza_edit</td>\n",
       "      <td>【イザ：政治ニュース】公明・山口代表「日程ありきではない」　７５歳以上医療費の１５日閣議決定...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4995</th>\n",
       "      <td>WelshYearian</td>\n",
       "      <td>https://t.co/uQy5O9j55N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4996</th>\n",
       "      <td>stalinone</td>\n",
       "      <td>RT @europechinese: 英歐貿易談判仍然停滯，英國首相莊漢生將訪問布魯塞爾，會...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4997</th>\n",
       "      <td>snidelines</td>\n",
       "      <td>RT @CertainDays: You can also buy a 2021 Certa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4998</th>\n",
       "      <td>cathbalmain</td>\n",
       "      <td>@SallyLawry @scottmorrison My guess is Morriso...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4999</th>\n",
       "      <td>Vinayak0710</td>\n",
       "      <td>RT @Republic_Bharat: कपिल मिश्रा ने योगेंद्र य...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                user                                               Text\n",
       "0     ThomasDillhunt  RT @RepsForBiden: How’s this even legal?\\n#Loc...\n",
       "1        avataylor02  @JujuSlick @ScottAdamsSays Your point is??? Sh...\n",
       "2      Harryy_Pawter  RT @OpIndia_in: बीजेपी प्रवक्ता ने कहा कि नए क...\n",
       "3     memykidsandeye  RT @freep: BREAKING: @GovWhitmer named co-chai...\n",
       "4           iza_edit  【イザ：政治ニュース】公明・山口代表「日程ありきではない」　７５歳以上医療費の１５日閣議決定...\n",
       "...              ...                                                ...\n",
       "4995    WelshYearian                            https://t.co/uQy5O9j55N\n",
       "4996       stalinone  RT @europechinese: 英歐貿易談判仍然停滯，英國首相莊漢生將訪問布魯塞爾，會...\n",
       "4997      snidelines  RT @CertainDays: You can also buy a 2021 Certa...\n",
       "4998     cathbalmain  @SallyLawry @scottmorrison My guess is Morriso...\n",
       "4999     Vinayak0710  RT @Republic_Bharat: कपिल मिश्रा ने योगेंद्र य...\n",
       "\n",
       "[5000 rows x 2 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "politics.head(5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "sport = pd.read_csv('Sport-tweets.csv' , error_bad_lines=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>dante3346</td>\n",
       "      <td>@dannyb0y6 @trucifer_x I don't know how old yo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>67dmy19</td>\n",
       "      <td>@NewsNationNow So not a sport.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Lautaro_Arribas</td>\n",
       "      <td>RT @MarianoReyOK: Matías Ledesma en \"La 94 Spo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AileenAdrienne</td>\n",
       "      <td>RT @BBCSport: Paul Pogba \"has to change teams\"...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>mali_hussein</td>\n",
       "      <td>RT @SunChelsea: Meet Marina Granovskaia, the m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2995</th>\n",
       "      <td>Aaliyah_ys2</td>\n",
       "      <td>RT @donachena: Mon hygiène de vie est horrible...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2996</th>\n",
       "      <td>ALEXANDREDECAS6</td>\n",
       "      <td>RT @anderbatist: COM O VASCO NO Z-4, SPORT x C...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2997</th>\n",
       "      <td>witho68</td>\n",
       "      <td>@wisey_9 No less a sport than gymnastics, sync...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2998</th>\n",
       "      <td>sportscardex</td>\n",
       "      <td>@SGCFinests @darrenrovell @Sothebys @GoldinAuc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2999</th>\n",
       "      <td>djliddle75</td>\n",
       "      <td>All \"sport\" hunters are cowards . . . https://...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 user                                               Text\n",
       "0           dante3346  @dannyb0y6 @trucifer_x I don't know how old yo...\n",
       "1             67dmy19                     @NewsNationNow So not a sport.\n",
       "2     Lautaro_Arribas  RT @MarianoReyOK: Matías Ledesma en \"La 94 Spo...\n",
       "3      AileenAdrienne  RT @BBCSport: Paul Pogba \"has to change teams\"...\n",
       "4        mali_hussein  RT @SunChelsea: Meet Marina Granovskaia, the m...\n",
       "...               ...                                                ...\n",
       "2995      Aaliyah_ys2  RT @donachena: Mon hygiène de vie est horrible...\n",
       "2996  ALEXANDREDECAS6  RT @anderbatist: COM O VASCO NO Z-4, SPORT x C...\n",
       "2997          witho68  @wisey_9 No less a sport than gymnastics, sync...\n",
       "2998     sportscardex  @SGCFinests @darrenrovell @Sothebys @GoldinAuc...\n",
       "2999       djliddle75  All \"sport\" hunters are cowards . . . https://...\n",
       "\n",
       "[3000 rows x 2 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sport.head(3000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Education1939</td>\n",
       "      <td>This past week The Staff Recognition Committee...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>alexanderrusso</td>\n",
       "      <td>RT @UNICEFmedia: \"Evidence shows that schools ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TaleamSystems</td>\n",
       "      <td>Taleam Systems' CEO implemented the Student e-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>javrsda</td>\n",
       "      <td>Ya quiero que salga la nueva temporada de Sex ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Calpe19</td>\n",
       "      <td>RT @LizzyJPrice: @JonnyGeller A great loss. Wh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19995</th>\n",
       "      <td>Aaliyah_ys2</td>\n",
       "      <td>RT @donachena: Mon hygiène de vie est horrible...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19996</th>\n",
       "      <td>ALEXANDREDECAS6</td>\n",
       "      <td>RT @anderbatist: COM O VASCO NO Z-4, SPORT x C...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19997</th>\n",
       "      <td>witho68</td>\n",
       "      <td>@wisey_9 No less a sport than gymnastics, sync...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19998</th>\n",
       "      <td>sportscardex</td>\n",
       "      <td>@SGCFinests @darrenrovell @Sothebys @GoldinAuc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19999</th>\n",
       "      <td>djliddle75</td>\n",
       "      <td>All \"sport\" hunters are cowards . . . https://...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  user                                               Text\n",
       "0        Education1939  This past week The Staff Recognition Committee...\n",
       "1       alexanderrusso  RT @UNICEFmedia: \"Evidence shows that schools ...\n",
       "2        TaleamSystems  Taleam Systems' CEO implemented the Student e-...\n",
       "3              javrsda  Ya quiero que salga la nueva temporada de Sex ...\n",
       "4              Calpe19  RT @LizzyJPrice: @JonnyGeller A great loss. Wh...\n",
       "...                ...                                                ...\n",
       "19995      Aaliyah_ys2  RT @donachena: Mon hygiène de vie est horrible...\n",
       "19996  ALEXANDREDECAS6  RT @anderbatist: COM O VASCO NO Z-4, SPORT x C...\n",
       "19997          witho68  @wisey_9 No less a sport than gymnastics, sync...\n",
       "19998     sportscardex  @SGCFinests @darrenrovell @Sothebys @GoldinAuc...\n",
       "19999       djliddle75  All \"sport\" hunters are cowards . . . https://...\n",
       "\n",
       "[20000 rows x 2 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "import glob\n",
    "import pandas as pd\n",
    "datasets = pd.concat([education, fashion, mechanical, technology, politics, sport],ignore_index=True)\n",
    "\n",
    "datasets.head(20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets.to_csv( \"data.csv\", index=False, encoding='utf-8-sig')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_datasets = pd.read_csv('data.csv' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Education1939</td>\n",
       "      <td>This past week The Staff Recognition Committee...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>alexanderrusso</td>\n",
       "      <td>RT @UNICEFmedia: \"Evidence shows that schools ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TaleamSystems</td>\n",
       "      <td>Taleam Systems' CEO implemented the Student e-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>javrsda</td>\n",
       "      <td>Ya quiero que salga la nueva temporada de Sex ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Calpe19</td>\n",
       "      <td>RT @LizzyJPrice: @JonnyGeller A great loss. Wh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19995</th>\n",
       "      <td>Aaliyah_ys2</td>\n",
       "      <td>RT @donachena: Mon hygiène de vie est horrible...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19996</th>\n",
       "      <td>ALEXANDREDECAS6</td>\n",
       "      <td>RT @anderbatist: COM O VASCO NO Z-4, SPORT x C...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19997</th>\n",
       "      <td>witho68</td>\n",
       "      <td>@wisey_9 No less a sport than gymnastics, sync...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19998</th>\n",
       "      <td>sportscardex</td>\n",
       "      <td>@SGCFinests @darrenrovell @Sothebys @GoldinAuc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19999</th>\n",
       "      <td>djliddle75</td>\n",
       "      <td>All \"sport\" hunters are cowards . . . https://...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  user                                               Text\n",
       "0        Education1939  This past week The Staff Recognition Committee...\n",
       "1       alexanderrusso  RT @UNICEFmedia: \"Evidence shows that schools ...\n",
       "2        TaleamSystems  Taleam Systems' CEO implemented the Student e-...\n",
       "3              javrsda  Ya quiero que salga la nueva temporada de Sex ...\n",
       "4              Calpe19  RT @LizzyJPrice: @JonnyGeller A great loss. Wh...\n",
       "...                ...                                                ...\n",
       "19995      Aaliyah_ys2  RT @donachena: Mon hygiène de vie est horrible...\n",
       "19996  ALEXANDREDECAS6  RT @anderbatist: COM O VASCO NO Z-4, SPORT x C...\n",
       "19997          witho68  @wisey_9 No less a sport than gymnastics, sync...\n",
       "19998     sportscardex  @SGCFinests @darrenrovell @Sothebys @GoldinAuc...\n",
       "19999       djliddle75  All \"sport\" hunters are cowards . . . https://...\n",
       "\n",
       "[20000 rows x 2 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_datasets.head(20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}