scraping info.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5bdd1cd4-c687-437e-a3c5-c0dd52badb88",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import libs\n",
    "import urllib\n",
    "import numpy as np\n",
    "import re\n",
    "import pandas as pd\n",
    "\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "60191cb7-796b-4f1e-a392-67ac3f4b1f17",
   "metadata": {},
   "outputs": [],
   "source": [
    "regex_link = r\"(?:\\/movie\\/)(.+?summary)\"\n",
    "regex_moviename = r\"(?:tab=summary\\\">)(.+)(?:<\\/a>)\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0aa5845a-f314-46be-8657-d8a39ba19e62",
   "metadata": {},
   "outputs": [],
   "source": [
    "# index of pages:  the last being 6301\n",
    "page_index = np.linspace(1, 6301, num = 64).astype(int)\n",
    "save = pd.DataFrame(columns =['moviename', 'link'])\n",
    "\n",
    "# storing data\n",
    "dataset = pd.DataFrame(columns = ['number', 'moviename', 'Budget', 'Domestic', 'Worldwide'])\n",
    "for index in page_index:\n",
    "    # create url\n",
    "    url = \"https://www.the-numbers.com/movie/budgets/all/\" + str(index)\n",
    "    # open page\n",
    "    page = urllib.request.Request(url, headers={'User-Agent' : \"Magic Browser\"})\n",
    "    page = urllib.request.urlopen(page)\n",
    "    # read page\n",
    "    html_bytes = page.read()\n",
    "    html = html_bytes.decode(\"utf-8\")\n",
    "    \n",
    "    # get the part between the table tags\n",
    "    start_index = html.find(\"<table\")\n",
    "    end_index = html.find(\"</table\")\n",
    "    html = html[start_index:end_index]\n",
    "    \n",
    "    moviename = re.findall(regex_moviename, html)\n",
    "    link = re.findall(regex_link, html)\n",
    "    \n",
    "    # create dataframe\n",
    "    df_1 = pd.DataFrame(list(zip(moviename, link)), columns =['moviename', 'link'])\n",
    "    save = pd.concat([save, df_1])\n",
    "\n",
    "save = save.reset_index(drop = True)\n",
    "save.to_csv('name_and_links.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b8cd9c03-f001-46a7-b05a-eedcb7e88ed1",
   "metadata": {},
   "outputs": [],
   "source": [
    "save.to_csv('name_and_links.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16d723c9-dddd-4df4-8197-85bb60cbd267",
   "metadata": {},
   "outputs": [],
   "source": [
    "##################################################################################\n",
    "# Extract Information\n",
    "##################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "543e8c47-9fa3-4f1f-b1a5-0a7d6d9dc56c",
   "metadata": {},
   "outputs": [],
   "source": [
    "save = pd.read_csv('name_and_links.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "fd7bb2c7-8556-40cf-9923-9d3616b75179",
   "metadata": {},
   "outputs": [],
   "source": [
    "regex_gen = r\"(?:Genre.+?\\\">)(.+?)(?:<\\/a)\"\n",
    "regex_com = r\"(?:production-company.+?\\\">)(.+?)(?:<\\/a>)\"\n",
    "regex_cou = r\"(?:/movies\\\">)(.+?)(?:</a>)\"\n",
    "regex_lan = r\"(?:language.+?\\\">)(.+?)(?:<\\/a>)\"\n",
    "regex_key = r\"(?:keywords.+?\\\">)(.+?)(?:<\\/a>)\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "5f4387ea-1fd3-4e45-b950-221bfdb9621f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract(link):\n",
    "\n",
    "    tempurl = \"https://www.the-numbers.com/movie/\" + link\n",
    "    page = urllib.request.Request(tempurl, headers={'User-Agent' : \"Magic Browser\"})\n",
    "    page = urllib.request.urlopen(page)\n",
    "    # read page\n",
    "    html_bytes = page.read()\n",
    "    html = html_bytes.decode(\"utf-8\")\n",
    "    \n",
    "    start_index = html.find(\"2>Movie Details\")\n",
    "    end_index = html.find(\"Ranking on other Records and Milestones\")\n",
    "    html = html[start_index:end_index]\n",
    "    \n",
    "    temp = pd.DataFrame(columns = ['link', 'genre', 'company', 'country', 'language', 'keywords'])\n",
    "\n",
    "    temp['genre'] = [re.findall(regex_gen, html)]\n",
    "    temp['company'] = [re.findall(regex_com, html)]\n",
    "    country = re.findall(regex_cou, html)\n",
    "    if country:\n",
    "        country = [country[0]]\n",
    "    else:\n",
    "        country = [\" \"]\n",
    "    temp['country'] = country\n",
    "    temp['language'] = [re.findall(regex_lan, html)]\n",
    "    temp['keywords'] = [re.findall(regex_key, html)]\n",
    "    temp['link'] = link\n",
    "    \n",
    "    return temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "f00689a9-2d77-4837-b86b-70b0e377411a",
   "metadata": {},
   "outputs": [],
   "source": [
    "tqdm.pandas()\n",
    "# df_test = save['link'].progress_apply(extract)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "5abbaa6a-b76b-4c01-9138-b0e781e8306b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [32:33<00:00,  1.95s/it]\n"
     ]
    }
   ],
   "source": [
    "df_test = save['link'][0:1000].progress_apply(extract)\n",
    "df_test = pd.concat(df_test.tolist())\n",
    "df_test.to_csv('info_p1.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "3bcc1638-149b-4fb8-9e3d-313caa787480",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [29:09<00:00,  1.75s/it]\n"
     ]
    }
   ],
   "source": [
    "df_test = save['link'][1000:2000].progress_apply(extract)\n",
    "df_test = pd.concat(df_test.tolist())\n",
    "df_test.to_csv('info_p2.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "6ea2aad4-4314-45ee-8411-67b46afbac4e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [27:31<00:00,  1.65s/it]\n"
     ]
    }
   ],
   "source": [
    "df_test = save['link'][2000:3000].progress_apply(extract)\n",
    "df_test = pd.concat(df_test.tolist())\n",
    "df_test.to_csv('info_p3.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "f6d9b475-1b6b-4250-8d25-1c05e1836faf",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [25:24<00:00,  1.52s/it]\n"
     ]
    }
   ],
   "source": [
    "df_test = save['link'][3000:4000].progress_apply(extract)\n",
    "df_test = pd.concat(df_test.tolist())\n",
    "df_test.to_csv('info_p4.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "2b1396a5-52e4-4460-b112-b0d7a4d93608",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [22:59<00:00,  1.38s/it]\n"
     ]
    }
   ],
   "source": [
    "df_test = save['link'][4000:5000].progress_apply(extract)\n",
    "df_test = pd.concat(df_test.tolist())\n",
    "df_test.to_csv('info_p5.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "974ed0c0-bb6c-4b38-afca-539c4c0f12d1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [20:21<00:00,  1.22s/it]\n"
     ]
    }
   ],
   "source": [
    "df_test = save['link'][5000:6000].progress_apply(extract)\n",
    "df_test = pd.concat(df_test.tolist())\n",
    "df_test.to_csv('info_p6.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "8d4c1830-8686-4ca1-a00f-963d3a83bb2e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 381/381 [06:41<00:00,  1.05s/it]\n"
     ]
    }
   ],
   "source": [
    "df_test = save['link'][6000:].progress_apply(extract)\n",
    "df_test = pd.concat(df_test.tolist())\n",
    "df_test.to_csv('info_p7.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "id": "cf69aab5-e9d6-4294-a2f6-df1e595b3033",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df_test' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_16704\\1186045382.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mdf_test\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'df_test' is not defined"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "ff61e43a-0eba-4db4-8de4-041dbc461156",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_1 = pd.read_csv('info_p1.csv')\n",
    "df_2 = pd.read_csv('info_p2.csv')\n",
    "df_3 = pd.read_csv('info_p3.csv')\n",
    "df_4 = pd.read_csv('info_p4.csv')\n",
    "df_5 = pd.read_csv('info_p5.csv')\n",
    "df_6 = pd.read_csv('info_p6.csv')\n",
    "df_7 = pd.read_csv('info_p7.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "dcf0f459-b0cc-4d49-8171-875e8fe520e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.concat([df_1, df_2, df_3, df_4, df_5, df_6, df_7])\n",
    "data = data.reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "819d8216-d9be-40bc-bc5e-a78d990811e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_csv('movie_info.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "e4305247-3ba8-4d89-b316-fa39cf9721cc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>link</th>\n",
       "      <th>genre</th>\n",
       "      <th>company</th>\n",
       "      <th>country</th>\n",
       "      <th>language</th>\n",
       "      <th>keywords</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Avatar-The-Way-of-Water-(2022)#tab=summary</td>\n",
       "      <td>['Action']</td>\n",
       "      <td>['Lightstorm Entertainment', '20th Century Stu...</td>\n",
       "      <td>United States</td>\n",
       "      <td>['English']</td>\n",
       "      <td>['Action Adventure', 'Delayed Sequel', 'Humans...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Avengers-Endgame-(2019)#tab=summary</td>\n",
       "      <td>['Action']</td>\n",
       "      <td>['Marvel Studios']</td>\n",
       "      <td>United States</td>\n",
       "      <td>['English']</td>\n",
       "      <td>['Ensemble', 'Marvel Comics', 'Animal Lead', '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Pirates-of-the-Caribbean-On-Stranger-Tides#tab...</td>\n",
       "      <td>['Adventure']</td>\n",
       "      <td>['Walt Disney Pictures']</td>\n",
       "      <td>United States</td>\n",
       "      <td>['English']</td>\n",
       "      <td>['Pirates', 'Zombies', 'Visual Effects', 'Roma...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Avengers-Age-of-Ultron#tab=summary</td>\n",
       "      <td>['Action']</td>\n",
       "      <td>['Marvel Studios']</td>\n",
       "      <td>United States</td>\n",
       "      <td>['English']</td>\n",
       "      <td>['Marvel Comics', 'Robot', 'End of the World',...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Fast-X-(2023)#tab=summary</td>\n",
       "      <td>['Action', 'Action']</td>\n",
       "      <td>['Universal Pictures', 'Original Film', 'One R...</td>\n",
       "      <td>United States</td>\n",
       "      <td>['English', 'English']</td>\n",
       "      <td>['Car Chase', 'Gangs', 'Organized Crime', 'Seq...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>Kangaroo-Jack#tab=summary</td>\n",
       "      <td>['Adventure']</td>\n",
       "      <td>[]</td>\n",
       "      <td>Australia</td>\n",
       "      <td>['English']</td>\n",
       "      <td>['Mafia', 'Animal Lead']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>Happening-The#tab=summary</td>\n",
       "      <td>['Thriller/Suspense']</td>\n",
       "      <td>['Twentieth Century Fox']</td>\n",
       "      <td>France</td>\n",
       "      <td>['English']</td>\n",
       "      <td>['End of the World', 'Survival Thriller', 'Tea...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>London-Has-Fallen#tab=summary</td>\n",
       "      <td>['Thriller/Suspense']</td>\n",
       "      <td>['Nu Image', 'Millennium Films']</td>\n",
       "      <td>United States</td>\n",
       "      <td>['English']</td>\n",
       "      <td>['Set in London', 'Set in United Kingdom', 'Po...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>Rules-of-Engagement#tab=summary</td>\n",
       "      <td>['Drama']</td>\n",
       "      <td>['Richard D. Zanuck/Scott Rudin Production']</td>\n",
       "      <td>United States</td>\n",
       "      <td>['English']</td>\n",
       "      <td>['Courtroom Drama', 'Lawyers', 'Political Prot...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>Shaggy-Dog-The-(2006)#tab=summary</td>\n",
       "      <td>['Comedy']</td>\n",
       "      <td>['Walt Disney Pictures', 'Mandeville Films', '...</td>\n",
       "      <td>United States</td>\n",
       "      <td>['English']</td>\n",
       "      <td>['Talking Animals', 'Family Comedy', 'Animal L...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  link                  genre  \\\n",
       "0           Avatar-The-Way-of-Water-(2022)#tab=summary             ['Action']   \n",
       "1                  Avengers-Endgame-(2019)#tab=summary             ['Action']   \n",
       "2    Pirates-of-the-Caribbean-On-Stranger-Tides#tab...          ['Adventure']   \n",
       "3                   Avengers-Age-of-Ultron#tab=summary             ['Action']   \n",
       "4                            Fast-X-(2023)#tab=summary   ['Action', 'Action']   \n",
       "..                                                 ...                    ...   \n",
       "995                          Kangaroo-Jack#tab=summary          ['Adventure']   \n",
       "996                          Happening-The#tab=summary  ['Thriller/Suspense']   \n",
       "997                      London-Has-Fallen#tab=summary  ['Thriller/Suspense']   \n",
       "998                    Rules-of-Engagement#tab=summary              ['Drama']   \n",
       "999                  Shaggy-Dog-The-(2006)#tab=summary             ['Comedy']   \n",
       "\n",
       "                                               company        country  \\\n",
       "0    ['Lightstorm Entertainment', '20th Century Stu...  United States   \n",
       "1                                   ['Marvel Studios']  United States   \n",
       "2                             ['Walt Disney Pictures']  United States   \n",
       "3                                   ['Marvel Studios']  United States   \n",
       "4    ['Universal Pictures', 'Original Film', 'One R...  United States   \n",
       "..                                                 ...            ...   \n",
       "995                                                 []      Australia   \n",
       "996                          ['Twentieth Century Fox']         France   \n",
       "997                   ['Nu Image', 'Millennium Films']  United States   \n",
       "998       ['Richard D. Zanuck/Scott Rudin Production']  United States   \n",
       "999  ['Walt Disney Pictures', 'Mandeville Films', '...  United States   \n",
       "\n",
       "                   language                                           keywords  \n",
       "0               ['English']  ['Action Adventure', 'Delayed Sequel', 'Humans...  \n",
       "1               ['English']  ['Ensemble', 'Marvel Comics', 'Animal Lead', '...  \n",
       "2               ['English']  ['Pirates', 'Zombies', 'Visual Effects', 'Roma...  \n",
       "3               ['English']  ['Marvel Comics', 'Robot', 'End of the World',...  \n",
       "4    ['English', 'English']  ['Car Chase', 'Gangs', 'Organized Crime', 'Seq...  \n",
       "..                      ...                                                ...  \n",
       "995             ['English']                           ['Mafia', 'Animal Lead']  \n",
       "996             ['English']  ['End of the World', 'Survival Thriller', 'Tea...  \n",
       "997             ['English']  ['Set in London', 'Set in United Kingdom', 'Po...  \n",
       "998             ['English']  ['Courtroom Drama', 'Lawyers', 'Political Prot...  \n",
       "999             ['English']  ['Talking Animals', 'Family Comedy', 'Animal L...  \n",
       "\n",
       "[1000 rows x 6 columns]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_1"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}