Reddit_Scraper.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "509f3c69",
   "metadata": {},
   "source": [
    "Scrape Reddit for Posts, Comments, or other Information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "55ec39bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import praw\n",
    "import pandas as pd\n",
    "import datetime as dt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "45a0bef0",
   "metadata": {},
   "outputs": [],
   "source": [
    "#used for authentication, values stored in other file\n",
    "reddit = praw.Reddit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "ac2161c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "#can change which subreddit to scrape from here\n",
    "subreddit = reddit.subreddit(\"explainlikeimfive\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "d32a375b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#create dictionary to store values\n",
    "info_dict = {\n",
    "    \"id\" : [],\n",
    "    \"score\" : [],\n",
    "    \"title\" : [],\n",
    "    \"url\" : []\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "0f9210d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "#fill dictionary with desired information\n",
    "# possible tags are: .hot .top .new .top .controversial .gilded\n",
    "top = subreddit.top(limit=200, time_filter=\"all\")\n",
    "for submission in top:\n",
    "    info_dict[\"id\"].append(submission.id)\n",
    "    info_dict[\"score\"].append(submission.score)\n",
    "    info_dict[\"title\"].append(submission.title)\n",
    "    info_dict[\"url\"].append(submission.url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "008f36b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "#put information into Data Frame\n",
    "data = pd.DataFrame(info_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "c32753db",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>score</th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6efne8</td>\n",
       "      <td>45791</td>\n",
       "      <td>ELI5:How after 5000 years of humanity survivin...</td>\n",
       "      <td>https://www.reddit.com/r/explainlikeimfive/com...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>pbkst9</td>\n",
       "      <td>42260</td>\n",
       "      <td>We call upon Reddit to take action against the...</td>\n",
       "      <td>/r/vaxxhappened/comments/pbe8nj/we_call_upon_r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>cxiwhl</td>\n",
       "      <td>41138</td>\n",
       "      <td>ELI5: How did we get to the point where laptop...</td>\n",
       "      <td>https://www.reddit.com/r/explainlikeimfive/com...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>l7ckr5</td>\n",
       "      <td>40931</td>\n",
       "      <td>ELI5: Stock Market Megathread</td>\n",
       "      <td>https://www.reddit.com/r/explainlikeimfive/com...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>g29knw</td>\n",
       "      <td>40551</td>\n",
       "      <td>ELI5: So how does the the human brain ignore t...</td>\n",
       "      <td>https://www.reddit.com/r/explainlikeimfive/com...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>195</th>\n",
       "      <td>856chn</td>\n",
       "      <td>17958</td>\n",
       "      <td>ELI5:Why can't our teeth heal like our skin an...</td>\n",
       "      <td>https://www.reddit.com/r/explainlikeimfive/com...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>196</th>\n",
       "      <td>c71z22</td>\n",
       "      <td>17960</td>\n",
       "      <td>ELI5: Why is that if you fall asleep hungry yo...</td>\n",
       "      <td>https://www.reddit.com/r/explainlikeimfive/com...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>197</th>\n",
       "      <td>iaxlrj</td>\n",
       "      <td>17943</td>\n",
       "      <td>ELI5: Why do some forests have undergrowth so ...</td>\n",
       "      <td>https://www.reddit.com/r/explainlikeimfive/com...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>198</th>\n",
       "      <td>6s2qzu</td>\n",
       "      <td>17939</td>\n",
       "      <td>ELI5: How did Salt and Pepper become the chose...</td>\n",
       "      <td>https://www.reddit.com/r/explainlikeimfive/com...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199</th>\n",
       "      <td>mza4id</td>\n",
       "      <td>17879</td>\n",
       "      <td>ELi5: Can plants be “overweight” if they produ...</td>\n",
       "      <td>https://www.reddit.com/r/explainlikeimfive/com...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>200 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         id  score                                              title  \\\n",
       "0    6efne8  45791  ELI5:How after 5000 years of humanity survivin...   \n",
       "1    pbkst9  42260  We call upon Reddit to take action against the...   \n",
       "2    cxiwhl  41138  ELI5: How did we get to the point where laptop...   \n",
       "3    l7ckr5  40931                      ELI5: Stock Market Megathread   \n",
       "4    g29knw  40551  ELI5: So how does the the human brain ignore t...   \n",
       "..      ...    ...                                                ...   \n",
       "195  856chn  17958  ELI5:Why can't our teeth heal like our skin an...   \n",
       "196  c71z22  17960  ELI5: Why is that if you fall asleep hungry yo...   \n",
       "197  iaxlrj  17943  ELI5: Why do some forests have undergrowth so ...   \n",
       "198  6s2qzu  17939  ELI5: How did Salt and Pepper become the chose...   \n",
       "199  mza4id  17879  ELi5: Can plants be “overweight” if they produ...   \n",
       "\n",
       "                                                   url  \n",
       "0    https://www.reddit.com/r/explainlikeimfive/com...  \n",
       "1    /r/vaxxhappened/comments/pbe8nj/we_call_upon_r...  \n",
       "2    https://www.reddit.com/r/explainlikeimfive/com...  \n",
       "3    https://www.reddit.com/r/explainlikeimfive/com...  \n",
       "4    https://www.reddit.com/r/explainlikeimfive/com...  \n",
       "..                                                 ...  \n",
       "195  https://www.reddit.com/r/explainlikeimfive/com...  \n",
       "196  https://www.reddit.com/r/explainlikeimfive/com...  \n",
       "197  https://www.reddit.com/r/explainlikeimfive/com...  \n",
       "198  https://www.reddit.com/r/explainlikeimfive/com...  \n",
       "199  https://www.reddit.com/r/explainlikeimfive/com...  \n",
       "\n",
       "[200 rows x 4 columns]"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#view data to ensure proper operation\n",
    "data"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "0dea3a21",
   "metadata": {},
   "source": [
    "Export Data to Google Spreadsheet for further use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "5199acea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Worksheet 'explainlikeimfive' id:200247213>"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#import necessary modules and authenticate\n",
    "from df2gspread import df2gspread as d2g\n",
    "import pandas as pd\n",
    "from oauth2client.service_account import ServiceAccountCredentials\n",
    "\n",
    "scope = ['https://spreadsheets.google.com/feeds',\"https://www.googleapis.com/auth/drive\"]\n",
    "credentials = ServiceAccountCredentials.from_json_keyfile_name('./google_key.json', scope)\n",
    "\n",
    "#declare spreadsheet key, can only be view with express permissions\n",
    "spreadsheet = 'XXX'\n",
    "\n",
    "#change name of sheet to name of subreddit for coherence\n",
    "wks_name = \"explainlikeimfive\"\n",
    "\n",
    "#upload to spreadsheet\n",
    "d2g.upload(data, spreadsheet, wks_name, credentials=credentials)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a7a9624",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}