ecuador_news_elcomercio.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# El Comercio daily scraper\n",
    "\n",
    "This one takes the links from the rss feeds and downloads the contents. In this version, it is mostly meant as a demonstation.\n",
    "\n",
    "### To do/limitations\n",
    "\n",
    "- Check if we are missing articles that are in the front page.\n",
    "- This is only a daily scraper. Older articles can be scraped from pdf files in the archives.\n",
    "- Check if it really works with all types of articles (does not leaves out stuff we want or include stuff we don't want)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import datetime\n",
    "import os\n",
    "import unidecode\n",
    "import re\n",
    "import json\n",
    "\n",
    "import sqlite3 # I'm putting the article list in a database. But you can use whatever you're comfortable with"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`unidecode` is purely for translating titles into ascii for file naming. It needs to be installed with `pip install unidecode`. Alternatives also here: <https://stackoverflow.com/questions/517923/what-is-the-best-way-to-remove-accents-in-a-python-unicode-string#2633310>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "rss_feeds = [\n",
    "    \"http://www.elcomercio.com/rss\",\n",
    "    \"https://www.elcomercio.com/rss/actualidad\",\n",
    "    \"http://www.elcomercio.com/rss/tendencias\",\n",
    "    \"http://www.elcomercio.com/rss/deportes\",\n",
    "    \"http://www.elcomercio.com/rss/opinion\"\n",
    "]\n",
    "output_dir = \"./elcomercio/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert dates from rss to simple YYYYMMDD dates\n",
    "\n",
    "ectimefmt = \"%a, %d %b %Y %H:%M:%S %z\"\n",
    "def ecdate2mydate(d):\n",
    "    return datetime.datetime.strptime(d, ectimefmt).strftime(\"%Y%M%d\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<sqlite3.Cursor at 0x110beb730>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cxn = sqlite3.connect(\"ecuador_news_elcomercio_urlqueue.sqlite\")\n",
    "cs = cxn.cursor()\n",
    "\n",
    "cs.execute(\"\"\"\n",
    "create table if not exists articles\n",
    "(\n",
    "    guid text,\n",
    "    url text,\n",
    "    author text,\n",
    "    title text,\n",
    "    pubdate text,\n",
    "    fetched integer\n",
    ")\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download item list\n",
    "\n",
    "The feeds only list the last 20 entries. Which might be alright for monitoring, but then it would make sense to have a script that keeps downloading them and putting them in a database."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item duplicated: 'Incidentes aislados y una gran concentración en Chile'\n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "'NoneType' object has no attribute 'get_text'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-12-32365c0a308f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     10\u001b[0m         \u001b[0mitemtitle\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnewsitem\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"title\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m         \u001b[0mitemurl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnewsitem\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"link\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m         \u001b[0mitemauthor\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnewsitem\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"author\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     13\u001b[0m         \u001b[0mitemguid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnewsitem\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"guid\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Is identical to link, but we'll use as identifier\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0mitempubdate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mecdate2mydate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewsitem\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mre\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"pub[dD]ate\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'NoneType' object has no attribute 'get_text'"
     ]
    }
   ],
   "source": [
    "articles = {}\n",
    "\n",
    "for feed in rss_feeds:\n",
    "    for i in range(5):\n",
    "        req = requests.get(feed)\n",
    "        if req.ok: break\n",
    "    for newsitem in BeautifulSoup(req.content, \"xml\").find_all(\"item\"):\n",
    "        \n",
    "        # For each news item, get the metadata\n",
    "        itemtitle = newsitem.find(\"title\").get_text()\n",
    "        itemurl = newsitem.find(\"link\").get_text()\n",
    "        itemauthor = newsitem.find(\"author\").get_text()\n",
    "        itemguid = newsitem.find(\"guid\").get_text() # Is identical to link, but we'll use as identifier\n",
    "        itempubdate = ecdate2mydate(newsitem.find(re.compile(\"pub[dD]ate\")).get_text())\n",
    "        \n",
    "        # check if exists\n",
    "        if cs.execute(\"select count(*) from articles where guid=?\", (itemguid,)).fetchone()[0] > 0:\n",
    "            print(\"Item duplicated: '%s'\" % itemtitle)\n",
    "            continue\n",
    "        \n",
    "        # save it\n",
    "        cs.execute(\n",
    "            \"insert into articles (guid, url, author, title, pubdate, fetched) values (?,?,?,?,?,0)\",\n",
    "            (itemguid, itemurl, itemauthor, itemtitle, itempubdate)\n",
    "        )\n",
    "\n",
    "cxn.commit()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Downloading the actual articles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_contents(soup):\n",
    "    \"Gets the content of an El Comercio article, from the soup.\"\n",
    "    return \"\\n\\n\".join(p.get_text() for p in soup.find(class_=\"paragraphs\").find_all(\"p\"))\n",
    "\n",
    "def get_todays_date():\n",
    "    \"Gets todays date, outputs it in YYYYMMDD format.\"\n",
    "    return datetime.datetime.now().strftime(\"%Y%M%d\")\n",
    "\n",
    "def title2ascii(title):\n",
    "    \"Converts a unicode title into something that can conveniently be added in a file name.\"\n",
    "    r = unidecode.unidecode(title).lower().replace(\" \", \"_\")\n",
    "    return re.sub(r\"\\W\", \"\", r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "lastdate = \"\"\n",
    "datecounter = 0\n",
    "\n",
    "# Fetch the articles we haven't processed yet\n",
    "cs.execute(\"select url, guid, title, author, pubdate from articles where fetched=0\")\n",
    "\n",
    "# Roll through them\n",
    "for itemurl, itemguid, itemtitle, itemauthor, itempubdate in cs.fetchall():\n",
    "    \n",
    "    # Get what's missing\n",
    "    itemcontents = get_contents(BeautifulSoup(requests.get(itemurl).content))\n",
    "    itemretrdate = get_todays_date()\n",
    "    \n",
    "    # Stuff we need for the file name\n",
    "    asciititle = title2ascii(itemtitle)\n",
    "    if lastdate == itempubdate: datecounter += 1\n",
    "    else:\n",
    "        datecounter = 1\n",
    "        lastdate = itempubdate\n",
    "    \n",
    "    filenameroot = \"%s%02d_%s\" % (lastdate, datecounter, asciititle)\n",
    "    \n",
    "    # Save text\n",
    "    with open(filenameroot + \".txt\", \"w\") as f:\n",
    "        f.write(itemcontents)\n",
    "    \n",
    "    # Save metadata\n",
    "    with open(filenameroot + \".json\", \"w\") as f:\n",
    "        json.dump(\n",
    "            {\n",
    "                \"title\": itemtitle,\n",
    "                \"author\": itemauthor,\n",
    "                \"url\": itemurl,\n",
    "                \"date_published\": itempubdate,\n",
    "                \"date_retrieved\": itemretrdate\n",
    "            },\n",
    "            f\n",
    "        )\n",
    "    \n",
    "    # Mark it down in the database, so that if something fails, we don't need to redo the whole url list.\n",
    "    cs.execute(\"update articles set fetched=1 where guid=?\", (itemguid,))\n",
    "    cxn.commit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}