3. Augment data with Zefix.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import traceback\n",
    "\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "import re\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "from helpers.zefix_scraper import scrape_company\n",
    "\n",
    "from helpers.language import stringify_list, sanitize_pipeline\n",
    "\n",
    "def string_pipeline(s):\n",
    "    return stringify_list(sanitize_pipeline(s))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data_source</th>\n",
       "      <th>first_name</th>\n",
       "      <th>index</th>\n",
       "      <th>interest_name</th>\n",
       "      <th>last_name</th>\n",
       "      <th>position</th>\n",
       "      <th>interest_name_clean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1980</th>\n",
       "      <td>geneva_parliament</td>\n",
       "      <td>Gabriel</td>\n",
       "      <td>1</td>\n",
       "      <td>Commission administrative du Foyer Eynard-Fati...</td>\n",
       "      <td>Barrillier</td>\n",
       "      <td>Membre</td>\n",
       "      <td>Commission administrative Foyer Eynard-Fatio ,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1981</th>\n",
       "      <td>geneva_parliament</td>\n",
       "      <td>Gabriel</td>\n",
       "      <td>1</td>\n",
       "      <td>Conseil de fondation de la SGIPA              ...</td>\n",
       "      <td>Barrillier</td>\n",
       "      <td>Membre</td>\n",
       "      <td>Conseil fondation SGIPA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1982</th>\n",
       "      <td>geneva_parliament</td>\n",
       "      <td>Gabriel</td>\n",
       "      <td>1</td>\n",
       "      <td>Conseil de la Fondation Pro Juventute</td>\n",
       "      <td>Barrillier</td>\n",
       "      <td></td>\n",
       "      <td>Conseil Fondation Pro Juventute</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1989</th>\n",
       "      <td>geneva_parliament</td>\n",
       "      <td>David</td>\n",
       "      <td>2</td>\n",
       "      <td>Association Nicolas Bogueret                  ...</td>\n",
       "      <td>Amsler</td>\n",
       "      <td>Membre du comité</td>\n",
       "      <td>Association Nicolas Bogueret</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1983</th>\n",
       "      <td>geneva_parliament</td>\n",
       "      <td>David</td>\n",
       "      <td>2</td>\n",
       "      <td>Amsler Bombeli et associés SA                 ...</td>\n",
       "      <td>Amsler</td>\n",
       "      <td>Administrateur</td>\n",
       "      <td>Amsler Bombeli associés</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            data_source first_name  index  \\\n",
       "1980  geneva_parliament    Gabriel      1   \n",
       "1981  geneva_parliament    Gabriel      1   \n",
       "1982  geneva_parliament    Gabriel      1   \n",
       "1989  geneva_parliament      David      2   \n",
       "1983  geneva_parliament      David      2   \n",
       "\n",
       "                                          interest_name   last_name  \\\n",
       "1980  Commission administrative du Foyer Eynard-Fati...  Barrillier   \n",
       "1981  Conseil de fondation de la SGIPA              ...  Barrillier   \n",
       "1982              Conseil de la Fondation Pro Juventute  Barrillier   \n",
       "1989  Association Nicolas Bogueret                  ...      Amsler   \n",
       "1983  Amsler Bombeli et associés SA                 ...      Amsler   \n",
       "\n",
       "              position                                interest_name_clean  \n",
       "1980            Membre  Commission administrative Foyer Eynard-Fatio ,...  \n",
       "1981            Membre                            Conseil fondation SGIPA  \n",
       "1982                                      Conseil Fondation Pro Juventute  \n",
       "1989  Membre du comité                       Association Nicolas Bogueret  \n",
       "1983    Administrateur                            Amsler Bombeli associés  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "interests = pd.read_json('data/all_interests.json')\n",
    "interests['interest_name_clean'] = interests.interest_name.apply(string_pipeline)\n",
    "interests = interests.sort_values(by=\"index\")\n",
    "interests.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n",
      "2\n",
      "2\n",
      "2\n",
      "2\n",
      "2\n"
     ]
    }
   ],
   "source": [
    "scraped_companies = []\n",
    "failed = []\n",
    "for _, row in interests.iterrows():\n",
    "    try:\n",
    "        data = scrape_company(row.interest_name_clean) \n",
    "        if data is None:\n",
    "            continue\n",
    "        data[\"source\"] = {\n",
    "            \"first_name\": row.first_name,\n",
    "            \"last_name\": row.last_name,\n",
    "            \"interest_name\": row.interest_name,\n",
    "            \"data_source\": row.data_source,\n",
    "            \"index\": row[\"index\"]\n",
    "        }\n",
    "        print(row[\"index\"])    \n",
    "        scraped_companies.append(data)\n",
    "    except Exception as e:\n",
    "        print(\"Error: \" + str(e))\n",
    "        traceback.print_exc()\n",
    "        failed.append(row[\"index\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "130348"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "open(\"scraped_companies_2.json\", \"w\").write(json.dumps(scraped_companies2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "154"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(scraped_companies2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:adaenv]",
   "language": "python",
   "name": "conda-env-adaenv-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}