Datazymes Sivaram K New Jersey.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing Library\n",
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "from urllib.request import Request, urlopen\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "URL = \"https://health.usnews.com/doctors/city-index/new-jersey\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "headers = {'User-Agent': 'Chrome/68.0.3440.106'} # To encounter 403 error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "page = requests.get(URL, headers=headers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "200"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#check if the page was downloaded - status code should be 200\n",
    "page.status_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Parse the HTML content from page using Beautiful Soup\n",
    "soup=BeautifulSoup(page.content,'html.parser')\n",
    "#soup = BeautifulSoup(page.text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Links of all the location in New Jersey (Page - 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = soup.find_all('li',class_= 'List__ListItem-xbc1kt-1 kkayKp')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "location_links=[]\n",
    "for i in a:\n",
    "    b = i.find('a')\n",
    "    location_links.append(b.get('href'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/doctors/specialists-index/new-jersey/absecon'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "location_links[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "588"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(location_links)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/doctors/specialists-index/new-jersey/absecon'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "location_links[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "page2 = []\n",
    "for i in location_links:\n",
    "    url = (\"https://health.usnews.com\"+ str(i))\n",
    "    page2.append(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "588"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(page2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://health.usnews.com/doctors/specialists-index/new-jersey/absecon'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "page2[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Doctors specialize in a particular area of medicine (Page - 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For city Absecon\n",
    "city1 = page2[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "page_2 = requests.get(city1, headers=headers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "200"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "page_2.status_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup_2 = BeautifulSoup(page_2.content,'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = soup_2.find_all('li',class_= 'List__ListItem-xbc1kt-1 kkayKp')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_spec=[]\n",
    "for i in a:\n",
    "    b = i.find('a')\n",
    "    doc_spec.append(b.get('href'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/doctors/anesthesiologists/new-jersey/absecon'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc_spec[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "page3 = []\n",
    "for i in doc_spec:\n",
    "    url = (\"https://health.usnews.com\"+ str(i))\n",
    "    page3.append(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://health.usnews.com/doctors/anesthesiologists/new-jersey/absecon'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "page3[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "116"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(page3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Anesthesiologists (Page - 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc = page3[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://health.usnews.com/doctors/anesthesiologists/new-jersey/absecon'"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "page_3 = requests.get(doc, headers=headers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "200"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "page_3.status_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup_3 = BeautifulSoup(page_3.content,'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = soup_3.find_all('a',class_= 'search-result-link bar-tighter')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_links=[]\n",
    "for i in a:\n",
    "    doc_links.append(i.get('href'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/doctors/kenneth-abrams-406792'"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc_links[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "page4 = []\n",
    "for i in doc_links:\n",
    "    url = (\"https://health.usnews.com\"+ str(i))\n",
    "    page4.append(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://health.usnews.com/doctors/kenneth-abrams-406792'"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "page4[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(page4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scrap the data from the website"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "overview=[]\n",
    "name=[]\n",
    "year_in_practice=[]\n",
    "language=[]\n",
    "location=[]\n",
    "affiliation=[]\n",
    "specialties=[]\n",
    "sub_specialties=[]\n",
    "education_training=[]\n",
    "\n",
    "for i in page4:\n",
    "    page_4 = requests.get(i, headers=headers)\n",
    "    soup_4=BeautifulSoup(page_4.content,'html.parser')\n",
    "    #OverView\n",
    "    overview.append(str(soup_4.find('div',{'class' :'block-normal clearfix'}).getText()))\n",
    "    \n",
    "    #Name Of the Doctor\n",
    "    name.append(str(soup_4.find('h1',{'class':'hero-heading flex-media-heading block-tight doctor-name '}).getText()))\n",
    "    \n",
    "    #Years In Practice\n",
    "    year_in_practice.append(soup_4.find_all('span',class_='text-large heading-normal-for-small-only right-for-medium-up')[1].getText())\n",
    "    \n",
    "    #Language\n",
    "    language.append(soup_4.find('span',class_='text-large heading-normal-for-small-only right-for-medium-up text-right showmore').getText())\n",
    "    \n",
    "    #Location\n",
    "    location.append(soup_4.find('span',attrs={'data-js-id':'doctor-address'}).getText())\n",
    "    \n",
    "    #Affiliation\n",
    "    affiliation.append(soup_4.find('p',class_='text-muted').getText())\n",
    "    \n",
    "    #Specialties\n",
    "    specialties.append(soup_4.find('a',class_='text-large').getText())\n",
    "    \n",
    "    #sub_specialties\n",
    "    s_speciality = soup_4.find_all('p',class_='text-large block-tight')\n",
    "    s = []\n",
    "    for i in range(len(s_speciality)):\n",
    "        s.append(s_speciality[i].get_text())\n",
    "    sub_specialties.append(s)\n",
    "    \n",
    "    #Education   \n",
    "    education = soup_4.findAll('section',class_='block-loosest')\n",
    "    for i in range(len(education)):\n",
    "        try:\n",
    "            if education[i].find('h2',class_=\"heading-larger block-normal\").get_text() == \"Education & Medical Training\":\n",
    "                a = education[i].find_all('li',class_ = 'block-tight')\n",
    "                edu_lst = []\n",
    "                for j in range(len(a)):\n",
    "                    edu_lst.append(education[i].find_all('li',class_ = 'block-tight')[j].get_text())\n",
    "                education_training.append(edu_lst)\n",
    "                break\n",
    "        except:\n",
    "            education_training.append('NaN')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "Certification_licensure=[]\n",
    "for i in page4:\n",
    "    page_4 = requests.get(i, headers=headers)\n",
    "    soup_4=BeautifulSoup(page_4.content,'html.parser') \n",
    "    #Education   \n",
    "    education = soup_4.findAll('section',class_='block-loosest')\n",
    "    for i in range(len(education)):\n",
    "        try:\n",
    "            if education[i].find('h2',class_=\"heading-larger block-normal\").get_text() == \"Certifications & Licensure\":\n",
    "                b = education[i].find_all('li',class_ = 'block-tight')\n",
    "                cet = []\n",
    "                for j in range(len(b)):\n",
    "                    cet.append(education[i].find_all('li',class_ = 'block-tight')[j].get_text())\n",
    "                Certification_licensure.append(cet)\n",
    "                break\n",
    "        except:\n",
    "            Certification_licensure.append('NaN')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating DataFrame & Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.DataFrame(columns = ['overview','name','year_in_practice','language','location','affiliation','specialties','sub_specialties','education_training','Certification_licensure'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>overview</th>\n",
       "      <th>name</th>\n",
       "      <th>year_in_practice</th>\n",
       "      <th>language</th>\n",
       "      <th>location</th>\n",
       "      <th>affiliation</th>\n",
       "      <th>specialties</th>\n",
       "      <th>sub_specialties</th>\n",
       "      <th>education_training</th>\n",
       "      <th>Certification_licensure</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [overview, name, year_in_practice, language, location, affiliation, specialties, sub_specialties, education_training, Certification_licensure]\n",
       "Index: []"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['overview'] = overview\n",
    "data['name'] = name\n",
    "data['year_in_practice'] = year_in_practice\n",
    "data['language'] = language\n",
    "data['location'] = location\n",
    "data['affiliation'] = affiliation\n",
    "data['specialties'] = specialties\n",
    "data['sub_specialties'] = sub_specialties\n",
    "data['education_training'] = education_training\n",
    "data['Certification_licensure']=Certification_licensure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "data1 = data.replace('\\n','', regex=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "data1 = data1.replace(\"\\t\", \"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(data1)):\n",
    "    for j in range(len(data1.education_training[i])):\n",
    "        data1.education_training[i][j].strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(data1)):\n",
    "    for j in range(len(Certification_licensure[i])):\n",
    "        data1.Certification_licensure[i][j].strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "data1.overview = data1.overview.str.strip()\n",
    "data1.name = data1.name.str.strip()\n",
    "data1.location = data1.location.str.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>overview</th>\n",
       "      <th>name</th>\n",
       "      <th>year_in_practice</th>\n",
       "      <th>language</th>\n",
       "      <th>location</th>\n",
       "      <th>affiliation</th>\n",
       "      <th>specialties</th>\n",
       "      <th>sub_specialties</th>\n",
       "      <th>education_training</th>\n",
       "      <th>Certification_licensure</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Dr. Kenneth Abrams is an anesthesiologist in P...</td>\n",
       "      <td>Dr. Kenneth Abrams                            ...</td>\n",
       "      <td>21+</td>\n",
       "      <td>English</td>\n",
       "      <td>65 Jimmie Leeds Road                Department...</td>\n",
       "      <td>Dr. Abrams does not have any hospital affiliat...</td>\n",
       "      <td>Anesthesiology</td>\n",
       "      <td>[General Anesthesiology]</td>\n",
       "      <td>[\\n            Albert Einstein College of Medi...</td>\n",
       "      <td>[\\n            American Board of Anesthesiolog...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Dr. Mohammad Chohan is an anesthesiologist in ...</td>\n",
       "      <td>Dr. Mohammad Chohan                           ...</td>\n",
       "      <td>21+</td>\n",
       "      <td>English</td>\n",
       "      <td>65 Jimmie Leeds Road                Pomona, NJ...</td>\n",
       "      <td>Dr. Chohan does not have any insurances listed.</td>\n",
       "      <td>Anesthesiology</td>\n",
       "      <td>[General Anesthesiology]</td>\n",
       "      <td>[\\n            George Washington University\\n ...</td>\n",
       "      <td>[\\n            American Board of Anesthesiolog...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Dr. Olawale Fadugba is an anesthesiologist in ...</td>\n",
       "      <td>Dr. Olawale Fadugba                           ...</td>\n",
       "      <td>21+</td>\n",
       "      <td>English</td>\n",
       "      <td>65 Jimmie Leeds Road                Department...</td>\n",
       "      <td>Dr. Fadugba does not have any insurances listed.</td>\n",
       "      <td>Anesthesiology</td>\n",
       "      <td>[General Anesthesiology]</td>\n",
       "      <td>[\\n            New York Medical College\\n     ...</td>\n",
       "      <td>[\\n            American Board of Anesthesiolog...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Dr. Nicholas Incandela is an anesthesiologist ...</td>\n",
       "      <td>Dr. Nicholas Incandela                        ...</td>\n",
       "      <td>21+</td>\n",
       "      <td>English</td>\n",
       "      <td>65 Jimmie Leeds Rd                Dept Of Anes...</td>\n",
       "      <td>Dr. Incandela does not have any hospital affil...</td>\n",
       "      <td>Anesthesiology</td>\n",
       "      <td>[General Anesthesiology]</td>\n",
       "      <td>[\\n            Albert Einstein College of Medi...</td>\n",
       "      <td>[\\n            American Board of Anesthesiolog...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Dr. Merceditas Lagmay is an anesthesiologist i...</td>\n",
       "      <td>Dr. Merceditas Lagmay                         ...</td>\n",
       "      <td>1 - 2</td>\n",
       "      <td>English</td>\n",
       "      <td>65 Jimmie Leeds Road                Pomona, NJ...</td>\n",
       "      <td>Dr. Lagmay does not have any insurances listed.</td>\n",
       "      <td>Anesthesiology</td>\n",
       "      <td>[General Anesthesiology]</td>\n",
       "      <td>[\\n            Mount Sinai School of Medicine\\...</td>\n",
       "      <td>[\\n            American Board of Anesthesiolog...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Dr. Thomas Long is an anesthesiologist in Pomo...</td>\n",
       "      <td>Dr. Thomas Long                               ...</td>\n",
       "      <td>21+</td>\n",
       "      <td>English</td>\n",
       "      <td>65 Jimmie Leeds Road                Pomona, NJ...</td>\n",
       "      <td>Dr. Long does not have any insurances listed.</td>\n",
       "      <td>Anesthesiology</td>\n",
       "      <td>[General Anesthesiology, Critical Care Medicine]</td>\n",
       "      <td>[\\n            Albert Einstein College of Medi...</td>\n",
       "      <td>[\\n            American Board of Anesthesiolog...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Dr. Denise Marable is an anesthesiologist in P...</td>\n",
       "      <td>Dr. Denise Marable                            ...</td>\n",
       "      <td>21+</td>\n",
       "      <td>English</td>\n",
       "      <td>65 Jimmie Leeds Road                Pomona, NJ...</td>\n",
       "      <td>Dr. Marable does not have any insurances listed.</td>\n",
       "      <td>Anesthesiology</td>\n",
       "      <td>[General Anesthesiology]</td>\n",
       "      <td>[\\n            Detroit Medical Center - Wayne ...</td>\n",
       "      <td>[\\n            American Board of Anesthesiolog...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Dr. Sudha Shah is an anesthesiologist in Pomon...</td>\n",
       "      <td>Dr. Sudha Shah                                ...</td>\n",
       "      <td>21+</td>\n",
       "      <td>English</td>\n",
       "      <td>65 Jimmie Leeds Road                Pomona, NJ...</td>\n",
       "      <td>Dr. Shah does not have any insurances listed.</td>\n",
       "      <td>Anesthesiology</td>\n",
       "      <td>[General Anesthesiology]</td>\n",
       "      <td>[\\n            Albert Einstein Healthcare Netw...</td>\n",
       "      <td>[\\n            American Board of Anesthesiolog...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Dr. Qin Wang is an anesthesiologist in Pomona,...</td>\n",
       "      <td>Dr. Qin Wang                                  ...</td>\n",
       "      <td>21+</td>\n",
       "      <td>English</td>\n",
       "      <td>65 Jimmie Leeds Road                Pomona, NJ...</td>\n",
       "      <td>Dr. Wang does not have any insurances listed.</td>\n",
       "      <td>Anesthesiology</td>\n",
       "      <td>[General Anesthesiology]</td>\n",
       "      <td>[\\n            Wannan Medical College\\n       ...</td>\n",
       "      <td>[\\n            American Board of Anesthesiolog...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Dr. Su-Lin Yao is an anesthesiologist in Pomon...</td>\n",
       "      <td>Dr. Su-Lin Yao                                ...</td>\n",
       "      <td>1 - 2</td>\n",
       "      <td>English</td>\n",
       "      <td>65 Jimmie Leeds Road                Pomona, NJ...</td>\n",
       "      <td>Dr. Yao does not have any insurances listed.</td>\n",
       "      <td>Anesthesiology</td>\n",
       "      <td>[General Anesthesiology]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[\\n            American Board of Anesthesiolog...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Dr. Richard Zalman is an anesthesiologist in P...</td>\n",
       "      <td>Dr. Richard Zalman                            ...</td>\n",
       "      <td>1 - 2</td>\n",
       "      <td>English</td>\n",
       "      <td>65 Jimmie Leeds Road                Pomona, NJ...</td>\n",
       "      <td>Dr. Zalman does not have any insurances listed.</td>\n",
       "      <td>Anesthesiology</td>\n",
       "      <td>[General Anesthesiology]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[\\n            American Board of Anesthesiolog...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Dr. Nina Singh is an anesthesiologist in Pomon...</td>\n",
       "      <td>Dr. Nina Singh                                ...</td>\n",
       "      <td>1 - 2</td>\n",
       "      <td>English</td>\n",
       "      <td>65 W Jimmie Leeds Rd                Pomona, NJ...</td>\n",
       "      <td>Dr. Singh does not have any insurances listed.</td>\n",
       "      <td>Anesthesiology</td>\n",
       "      <td>[General Anesthesiology]</td>\n",
       "      <td>[\\n            Kaiser Permanente Southern Cali...</td>\n",
       "      <td>[\\n            American Board of Anesthesiolog...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Dr. Jessica Costabile is an anesthesiologist i...</td>\n",
       "      <td>Dr. Jessica Costabile                         ...</td>\n",
       "      <td>11 - 20</td>\n",
       "      <td>English</td>\n",
       "      <td>65 Jimmie Leeds Road                Pomona, NJ...</td>\n",
       "      <td>Dr. Costabile does not have any insurances lis...</td>\n",
       "      <td>Anesthesiology</td>\n",
       "      <td>[General Anesthesiology]</td>\n",
       "      <td>[\\n            Thomas Jefferson University\\n  ...</td>\n",
       "      <td>[\\n            American Board of Anesthesiolog...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Dr. Mark Davidov is an anesthesiologist in Pom...</td>\n",
       "      <td>Dr. Mark Davidov                              ...</td>\n",
       "      <td>11 - 20</td>\n",
       "      <td>English</td>\n",
       "      <td>65 W Jimmie Leeds Rd                Pomona, NJ...</td>\n",
       "      <td>Dr. Davidov does not have any insurances listed.</td>\n",
       "      <td>Anesthesiology</td>\n",
       "      <td>[General Anesthesiology, Cardiothoracic Anesth...</td>\n",
       "      <td>[\\n            University of Toledo Medical Ce...</td>\n",
       "      <td>[\\n            American Board of Anesthesiolog...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Dr. David Gayeski is an anesthesiologist in Po...</td>\n",
       "      <td>Dr. David Gayeski                             ...</td>\n",
       "      <td>11 - 20</td>\n",
       "      <td>English</td>\n",
       "      <td>65 Jimmie Leeds Road                Pomona, NJ...</td>\n",
       "      <td>Dr. Gayeski does not have any insurances listed.</td>\n",
       "      <td>Anesthesiology</td>\n",
       "      <td>[General Anesthesiology]</td>\n",
       "      <td>[\\n            Drexel University - Hahnemann U...</td>\n",
       "      <td>[\\n            American Board of Anesthesiolog...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Dr. Romeo Pericic is an anesthesiologist in Po...</td>\n",
       "      <td>Dr. Romeo Pericic                             ...</td>\n",
       "      <td>21+</td>\n",
       "      <td>English</td>\n",
       "      <td>65 Jimmie Leeds Road                Pomona, NJ...</td>\n",
       "      <td>Dr. Pericic does not have any insurances listed.</td>\n",
       "      <td>Anesthesiology</td>\n",
       "      <td>[General Anesthesiology]</td>\n",
       "      <td>[\\n            Montefiore New Rochelle Hospita...</td>\n",
       "      <td>[\\n            American Board of Anesthesiolog...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Dr. Elizabeth Riccobono is an anesthesiologist...</td>\n",
       "      <td>Dr. Elizabeth Riccobono                       ...</td>\n",
       "      <td>1 - 2</td>\n",
       "      <td>English</td>\n",
       "      <td>65 W Jimmie Leeds Rd                Pomona, NJ...</td>\n",
       "      <td>Dr. Riccobono does not have any insurances lis...</td>\n",
       "      <td>Anesthesiology</td>\n",
       "      <td>[Critical Care Medicine]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[\\n            American Board of Anesthesiolog...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Dr. Yun Zhu is an anesthesiologist in Pomona, ...</td>\n",
       "      <td>Dr. Yun Zhu                                   ...</td>\n",
       "      <td>21+</td>\n",
       "      <td>English</td>\n",
       "      <td>65 Jimmie Leeds Road                Pomona, NJ...</td>\n",
       "      <td>Dr. Zhu does not have any insurances listed.</td>\n",
       "      <td>Anesthesiology</td>\n",
       "      <td>[General Anesthesiology]</td>\n",
       "      <td>[\\n            Thomas Jefferson University\\n  ...</td>\n",
       "      <td>[\\n            American Board of Anesthesiolog...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Dr. John Fisgus is an anesthesiologist in Pomo...</td>\n",
       "      <td>Dr. John Fisgus                               ...</td>\n",
       "      <td>21+</td>\n",
       "      <td>English</td>\n",
       "      <td>65 West Jimmie Leeds Road                Pomon...</td>\n",
       "      <td>Dr. Fisgus does not have any insurances listed.</td>\n",
       "      <td>Anesthesiology</td>\n",
       "      <td>[General Anesthesiology]</td>\n",
       "      <td>[\\n            University at Buffalo, School o...</td>\n",
       "      <td>[\\n            American Board of Anesthesiolog...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Dr. Patricia Kasica is an anesthesiologist in ...</td>\n",
       "      <td>Dr. Patricia Kasica                           ...</td>\n",
       "      <td>21+</td>\n",
       "      <td>English</td>\n",
       "      <td>76 W Jim Leeds Rd                Park Center, ...</td>\n",
       "      <td>Dr. Kasica does not have any insurances listed.</td>\n",
       "      <td>Anesthesiology</td>\n",
       "      <td>[General Anesthesiology, Pain Medicine]</td>\n",
       "      <td>[\\n            UMDNJ-Robert Wood Johnson Medic...</td>\n",
       "      <td>[\\n            DE State Medical License\\nActiv...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             overview  \\\n",
       "0   Dr. Kenneth Abrams is an anesthesiologist in P...   \n",
       "1   Dr. Mohammad Chohan is an anesthesiologist in ...   \n",
       "2   Dr. Olawale Fadugba is an anesthesiologist in ...   \n",
       "3   Dr. Nicholas Incandela is an anesthesiologist ...   \n",
       "4   Dr. Merceditas Lagmay is an anesthesiologist i...   \n",
       "5   Dr. Thomas Long is an anesthesiologist in Pomo...   \n",
       "6   Dr. Denise Marable is an anesthesiologist in P...   \n",
       "7   Dr. Sudha Shah is an anesthesiologist in Pomon...   \n",
       "8   Dr. Qin Wang is an anesthesiologist in Pomona,...   \n",
       "9   Dr. Su-Lin Yao is an anesthesiologist in Pomon...   \n",
       "10  Dr. Richard Zalman is an anesthesiologist in P...   \n",
       "11  Dr. Nina Singh is an anesthesiologist in Pomon...   \n",
       "12  Dr. Jessica Costabile is an anesthesiologist i...   \n",
       "13  Dr. Mark Davidov is an anesthesiologist in Pom...   \n",
       "14  Dr. David Gayeski is an anesthesiologist in Po...   \n",
       "15  Dr. Romeo Pericic is an anesthesiologist in Po...   \n",
       "16  Dr. Elizabeth Riccobono is an anesthesiologist...   \n",
       "17  Dr. Yun Zhu is an anesthesiologist in Pomona, ...   \n",
       "18  Dr. John Fisgus is an anesthesiologist in Pomo...   \n",
       "19  Dr. Patricia Kasica is an anesthesiologist in ...   \n",
       "\n",
       "                                                 name year_in_practice  \\\n",
       "0   Dr. Kenneth Abrams                            ...              21+   \n",
       "1   Dr. Mohammad Chohan                           ...              21+   \n",
       "2   Dr. Olawale Fadugba                           ...              21+   \n",
       "3   Dr. Nicholas Incandela                        ...              21+   \n",
       "4   Dr. Merceditas Lagmay                         ...            1 - 2   \n",
       "5   Dr. Thomas Long                               ...              21+   \n",
       "6   Dr. Denise Marable                            ...              21+   \n",
       "7   Dr. Sudha Shah                                ...              21+   \n",
       "8   Dr. Qin Wang                                  ...              21+   \n",
       "9   Dr. Su-Lin Yao                                ...            1 - 2   \n",
       "10  Dr. Richard Zalman                            ...            1 - 2   \n",
       "11  Dr. Nina Singh                                ...            1 - 2   \n",
       "12  Dr. Jessica Costabile                         ...          11 - 20   \n",
       "13  Dr. Mark Davidov                              ...          11 - 20   \n",
       "14  Dr. David Gayeski                             ...          11 - 20   \n",
       "15  Dr. Romeo Pericic                             ...              21+   \n",
       "16  Dr. Elizabeth Riccobono                       ...            1 - 2   \n",
       "17  Dr. Yun Zhu                                   ...              21+   \n",
       "18  Dr. John Fisgus                               ...              21+   \n",
       "19  Dr. Patricia Kasica                           ...              21+   \n",
       "\n",
       "   language                                           location  \\\n",
       "0   English  65 Jimmie Leeds Road                Department...   \n",
       "1   English  65 Jimmie Leeds Road                Pomona, NJ...   \n",
       "2   English  65 Jimmie Leeds Road                Department...   \n",
       "3   English  65 Jimmie Leeds Rd                Dept Of Anes...   \n",
       "4   English  65 Jimmie Leeds Road                Pomona, NJ...   \n",
       "5   English  65 Jimmie Leeds Road                Pomona, NJ...   \n",
       "6   English  65 Jimmie Leeds Road                Pomona, NJ...   \n",
       "7   English  65 Jimmie Leeds Road                Pomona, NJ...   \n",
       "8   English  65 Jimmie Leeds Road                Pomona, NJ...   \n",
       "9   English  65 Jimmie Leeds Road                Pomona, NJ...   \n",
       "10  English  65 Jimmie Leeds Road                Pomona, NJ...   \n",
       "11  English  65 W Jimmie Leeds Rd                Pomona, NJ...   \n",
       "12  English  65 Jimmie Leeds Road                Pomona, NJ...   \n",
       "13  English  65 W Jimmie Leeds Rd                Pomona, NJ...   \n",
       "14  English  65 Jimmie Leeds Road                Pomona, NJ...   \n",
       "15  English  65 Jimmie Leeds Road                Pomona, NJ...   \n",
       "16  English  65 W Jimmie Leeds Rd                Pomona, NJ...   \n",
       "17  English  65 Jimmie Leeds Road                Pomona, NJ...   \n",
       "18  English  65 West Jimmie Leeds Road                Pomon...   \n",
       "19  English  76 W Jim Leeds Rd                Park Center, ...   \n",
       "\n",
       "                                          affiliation     specialties  \\\n",
       "0   Dr. Abrams does not have any hospital affiliat...  Anesthesiology   \n",
       "1     Dr. Chohan does not have any insurances listed.  Anesthesiology   \n",
       "2    Dr. Fadugba does not have any insurances listed.  Anesthesiology   \n",
       "3   Dr. Incandela does not have any hospital affil...  Anesthesiology   \n",
       "4     Dr. Lagmay does not have any insurances listed.  Anesthesiology   \n",
       "5       Dr. Long does not have any insurances listed.  Anesthesiology   \n",
       "6    Dr. Marable does not have any insurances listed.  Anesthesiology   \n",
       "7       Dr. Shah does not have any insurances listed.  Anesthesiology   \n",
       "8       Dr. Wang does not have any insurances listed.  Anesthesiology   \n",
       "9        Dr. Yao does not have any insurances listed.  Anesthesiology   \n",
       "10    Dr. Zalman does not have any insurances listed.  Anesthesiology   \n",
       "11     Dr. Singh does not have any insurances listed.  Anesthesiology   \n",
       "12  Dr. Costabile does not have any insurances lis...  Anesthesiology   \n",
       "13   Dr. Davidov does not have any insurances listed.  Anesthesiology   \n",
       "14   Dr. Gayeski does not have any insurances listed.  Anesthesiology   \n",
       "15   Dr. Pericic does not have any insurances listed.  Anesthesiology   \n",
       "16  Dr. Riccobono does not have any insurances lis...  Anesthesiology   \n",
       "17       Dr. Zhu does not have any insurances listed.  Anesthesiology   \n",
       "18    Dr. Fisgus does not have any insurances listed.  Anesthesiology   \n",
       "19    Dr. Kasica does not have any insurances listed.  Anesthesiology   \n",
       "\n",
       "                                      sub_specialties  \\\n",
       "0                            [General Anesthesiology]   \n",
       "1                            [General Anesthesiology]   \n",
       "2                            [General Anesthesiology]   \n",
       "3                            [General Anesthesiology]   \n",
       "4                            [General Anesthesiology]   \n",
       "5    [General Anesthesiology, Critical Care Medicine]   \n",
       "6                            [General Anesthesiology]   \n",
       "7                            [General Anesthesiology]   \n",
       "8                            [General Anesthesiology]   \n",
       "9                            [General Anesthesiology]   \n",
       "10                           [General Anesthesiology]   \n",
       "11                           [General Anesthesiology]   \n",
       "12                           [General Anesthesiology]   \n",
       "13  [General Anesthesiology, Cardiothoracic Anesth...   \n",
       "14                           [General Anesthesiology]   \n",
       "15                           [General Anesthesiology]   \n",
       "16                           [Critical Care Medicine]   \n",
       "17                           [General Anesthesiology]   \n",
       "18                           [General Anesthesiology]   \n",
       "19            [General Anesthesiology, Pain Medicine]   \n",
       "\n",
       "                                   education_training  \\\n",
       "0   [\\n            Albert Einstein College of Medi...   \n",
       "1   [\\n            George Washington University\\n ...   \n",
       "2   [\\n            New York Medical College\\n     ...   \n",
       "3   [\\n            Albert Einstein College of Medi...   \n",
       "4   [\\n            Mount Sinai School of Medicine\\...   \n",
       "5   [\\n            Albert Einstein College of Medi...   \n",
       "6   [\\n            Detroit Medical Center - Wayne ...   \n",
       "7   [\\n            Albert Einstein Healthcare Netw...   \n",
       "8   [\\n            Wannan Medical College\\n       ...   \n",
       "9                                                 NaN   \n",
       "10                                                NaN   \n",
       "11  [\\n            Kaiser Permanente Southern Cali...   \n",
       "12  [\\n            Thomas Jefferson University\\n  ...   \n",
       "13  [\\n            University of Toledo Medical Ce...   \n",
       "14  [\\n            Drexel University - Hahnemann U...   \n",
       "15  [\\n            Montefiore New Rochelle Hospita...   \n",
       "16                                                NaN   \n",
       "17  [\\n            Thomas Jefferson University\\n  ...   \n",
       "18  [\\n            University at Buffalo, School o...   \n",
       "19  [\\n            UMDNJ-Robert Wood Johnson Medic...   \n",
       "\n",
       "                              Certification_licensure  \n",
       "0   [\\n            American Board of Anesthesiolog...  \n",
       "1   [\\n            American Board of Anesthesiolog...  \n",
       "2   [\\n            American Board of Anesthesiolog...  \n",
       "3   [\\n            American Board of Anesthesiolog...  \n",
       "4   [\\n            American Board of Anesthesiolog...  \n",
       "5   [\\n            American Board of Anesthesiolog...  \n",
       "6   [\\n            American Board of Anesthesiolog...  \n",
       "7   [\\n            American Board of Anesthesiolog...  \n",
       "8   [\\n            American Board of Anesthesiolog...  \n",
       "9   [\\n            American Board of Anesthesiolog...  \n",
       "10  [\\n            American Board of Anesthesiolog...  \n",
       "11  [\\n            American Board of Anesthesiolog...  \n",
       "12  [\\n            American Board of Anesthesiolog...  \n",
       "13  [\\n            American Board of Anesthesiolog...  \n",
       "14  [\\n            American Board of Anesthesiolog...  \n",
       "15  [\\n            American Board of Anesthesiolog...  \n",
       "16  [\\n            American Board of Anesthesiolog...  \n",
       "17  [\\n            American Board of Anesthesiolog...  \n",
       "18  [\\n            American Board of Anesthesiolog...  \n",
       "19  [\\n            DE State Medical License\\nActiv...  "
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "data1.to_csv('data_New-Jersey.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Elasticsearch & Python "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "from elasticsearch import Elasticsearch,helpers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "es = Elasticsearch(HOST=\"http://localhost\",PORT=9200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'error': {'root_cause': [{'type': 'resource_already_exists_exception',\n",
       "    'reason': 'index [first_index/0J7zr8_CReO1WZpegZh5Yg] already exists',\n",
       "    'index_uuid': '0J7zr8_CReO1WZpegZh5Yg',\n",
       "    'index': 'first_index'}],\n",
       "  'type': 'resource_already_exists_exception',\n",
       "  'reason': 'index [first_index/0J7zr8_CReO1WZpegZh5Yg] already exists',\n",
       "  'index_uuid': '0J7zr8_CReO1WZpegZh5Yg',\n",
       "  'index': 'first_index'},\n",
       " 'status': 400}"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "es.indices.create(index='first_index',ignore=400)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "es.indices.exists(index=\"first_index\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method Elasticsearch.info of <Elasticsearch([{}])>>"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "es.info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "#with open('data.csv') as a:\n",
    "    #reader = csv.DictReader(a)\n",
    "    #helpers.bulk(es, reader,index='my-index', doc_type='my-type')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}