ExtractImdbReleaseDatesLocalGlobal.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\niles\\Anaconda3\\lib\\site-packages\\pandas\\util\\_decorators.py:118: FutureWarning: The `sheetname` keyword is deprecated, use `sheet_name` instead\n",
      "  return func(*args, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 6 Bullets 17 November 2012 (Japan)\n",
      "\n",
      " Canada 11 September 2012\n",
      "1\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup\n",
    "import urllib\n",
    "import csv\n",
    "\n",
    "moviesList_file = pd.read_excel('C:/Users/niles/.spyder-py3/MovieData.xls', sheetname='Sheet1')\n",
    "y=1\n",
    "with open('example2.csv', 'a',newline='') as myFile:\n",
    "    writer = csv.writer(myFile)\n",
    "    flag=True\n",
    "    \n",
    "    for i in moviesList_file.index:\n",
    "        flag2=True\n",
    "        if i==1:\n",
    "            flag=True\n",
    "        else:\n",
    "            flag=False\n",
    "        movie=moviesList_file['position'][i]\n",
    "            #print(movie,i)\n",
    "        home_url = 'http://www.imdb.com/search/title?title=' + urllib.parse.quote_plus(str(movie))+\"&&title_type=feature&release_date=2005-01-01,\"\n",
    "        html = urllib.request.urlopen(home_url).read()\n",
    "            \n",
    "        soup_home_html = BeautifulSoup(html, \"lxml\")\n",
    "            #movie_list=soup_home_html.find('table',attrs={'class':'findList'})\n",
    "        movieList=[]\n",
    "        movie_refList=[]\n",
    "        movie_ref_dict={}\n",
    "        movie_text=soup_home_html.findAll('h3',attrs={'class':'lister-item-header'})\n",
    "        #print(movie_text[1])\n",
    "        for div in movie_text:\n",
    "            #print(div.find('a')['href'])\n",
    "            #print(div.find('a').contents[0])\n",
    "            movieList.append(div.find('a').contents[0])\n",
    "            movie_refList.append(div.find('a')['href'])\n",
    "            movie_ref_dict.update({div.find('a').contents[0]:div.find('a')['href']})  \n",
    "            if movie in movieList:\n",
    "                break\n",
    "        movieList=list(set(movieList))\n",
    "        movie_refList=list(set(movie_refList))\n",
    "            #To print the key-value pairs of movie and references\n",
    "            #print(movie_ref_dict)\n",
    "            #print(movie_refList)\n",
    "            #print(html)\n",
    "            #print(movieList)\n",
    "            #print(movie)\n",
    "            # To get the correct movie passed as input\n",
    "        \n",
    "        for i in (movieList):\n",
    "            if i==movie:\n",
    "                movie_selected=i\n",
    "                break\n",
    "            else:\n",
    "                flag2=False\n",
    "                   #print(i)\n",
    "        #print(movie_selected)\n",
    "        #print(movie_selected)\n",
    "        if flag2==True:\n",
    "            movie_ref_url = movie_ref_dict[movie_selected]\n",
    "            #print(movie_ref_url)\n",
    "                ## To print the key value pairs of Movie and its reference\n",
    "                ##for k,v in movie_ref_dict.items():\n",
    "                ##    print(k,v)\n",
    "            sub_url = 'http://www.imdb.com' + movie_ref_url\n",
    "                #print(sub_url)\n",
    "            sub_html = urllib.request.urlopen(sub_url).read()\n",
    "                #print(sub_html)\n",
    "            soup_sub_html=BeautifulSoup(sub_html,'html.parser')\n",
    "            nav = soup_sub_html.find('div',attrs={'class':'subtext'}).findAll('a')\n",
    "            #for i in nav:\n",
    "                    #print(i.find('a')['href'])\n",
    "            #    print(i)\n",
    "                \n",
    "            release_url=movie_ref_url.split('?')\n",
    "                #print(release_url[0]+'releaseinfo?ref_=tt_ov_inf')\n",
    "            release=release_url[0]+'releaseinfo?ref_=tt_ov_inf'\n",
    "            nav = soup_sub_html.find('div',attrs={'class':'subtext'}).find('a',title=\"See more release dates\").find('meta')['content']\n",
    "            release_date_from_imdb=nav\n",
    "            nav = soup_sub_html.find('div',attrs={'class':'subtext'}).find('a',title=\"See more release dates\")\n",
    "                #print(nav.get_text(strip=True))\n",
    "            nav = soup_sub_html.find('div',attrs={'class':'subtext'}).find('a',title=\"See more release dates\").text\n",
    "            #print(nav)\n",
    "                \n",
    "                #release_with_country = (nav.text.strip())\n",
    "                #print(release_with_country)\n",
    "                \n",
    "            sub_url2= 'http://www.imdb.com' + release\n",
    "                #print(sub_url2)\n",
    "                \n",
    "            sub_html2 = urllib.request.urlopen(sub_url2).read()\n",
    "                #print(sub_html2)\n",
    "            soup_release_countries=BeautifulSoup(sub_html2,'html.parser')\n",
    "                \n",
    "            release_dates = soup_release_countries.find('table')\n",
    "            #print(release_dates.text)\n",
    "            list1=release_dates.text\n",
    "                \n",
    "            first_release_country_href=release_dates.find('a').attrs    \n",
    "            #print(first_release_country_href)\n",
    "            \n",
    "            \n",
    "            first_release_country=release_dates.find('a',first_release_country_href).text\n",
    "            \n",
    "            first_release_country_date=release_dates.find('td',attrs={'class':'release_date'})\n",
    "            first_release_country_date_text=first_release_country_date.text\n",
    "            print(\" \"+movie_selected+ \" \"+nav +\" \"+ first_release_country+\" \"+first_release_country_date.text)\n",
    "            #print(nav)\n",
    "            #print(first_release_country)\n",
    "            #print(first_release_country_date.text)\n",
    "            print(y)\n",
    "            y=y+1\n",
    "            #print(first_release_country_date.text)\n",
    "            #f = csv.writer(open('MovieReleaseDates2.csv', 'w'))\n",
    "            #f.writerows(['Movie', 'USA Release Date','Overseas Release Country','Overseas Release Date'])\n",
    "            #f.writerows([movie_selected, nav,first_release_country,first_release_country_date_text])    \n",
    "            #if flag==True:\n",
    "            #    arraylist=[['Movie', 'USA Release Date','Overseas Release Country','Overseas Release Date'],[movie_selected, nav,first_release_country,first_release_country_date_text]]\n",
    "            #else:\n",
    "            arraylist=[[movie_selected, nav,first_release_country,first_release_country_date_text]]\n",
    "            writer.writerows(arraylist)\n",
    "        else:\n",
    "            print(movie)\n",
    "            print(y)\n",
    "            y=y+1\n",
    "            arraylist=[[movie]]\n",
    "            writer.writerows(arraylist)\n",
    "\n",
    "#print(movie_release_date.text)\n",
    "#print(html)\n",
    "#print(df1)\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}