nomis mortality All causes only.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Extract data from NOMIS\r\n",
    "\r\n",
    "This notebook extracts data from the NOMIS API. The purpose of this notebook is to test the API interface and demonstrate missing data when extracting data with multiple dates and other parameters such as cause_of_death, gender and age.\r\n",
    "\r\n",
    "When the notebook was created no data is returned for the first date."
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "source": [
    "import pandas as pd\r\n",
    "from tqdm.notebook import tqdm\r\n",
    "from pyjstat import pyjstat\r\n",
    "from typing import List"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "source": [
    "nomis_la_codes = pd.read_csv(\"./A_Assumptions/nomis_la_codes.csv\")"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "source": [
    "def nomis_url(table_name: str, geography: str) -> str:\r\n",
    "    # tables:\r\n",
    "    # NM_161_1 - Deaths\r\n",
    "    # NM_31_1 - Population\r\n",
    "\r\n",
    "    url_base = f\"https://www.nomisweb.co.uk/api/v01/dataset/{table_name}.jsonstat.json?\"\r\n",
    "    url_geography_base=\"geography=\"\r\n",
    "    url_date_base=\"date=\"    \r\n",
    "\r\n",
    "    url_params = {}\r\n",
    "    url_params[\"NM_161_1\"] = \"&cause_of_death=0&gender=0&age=0&measure=1&measures=20100\"\r\n",
    "    url_params[\"NM_31_1\"] = \"&sex=5...7&age=0...19&measures=20100\"\r\n",
    "\r\n",
    "    dates = [\r\n",
    "        \"latest\",\r\n",
    "        \"latestMINUS1\",\r\n",
    "        \"latestMINUS2\",\r\n",
    "        \"latestMINUS3\",\r\n",
    "        \"latestMINUS4\",\r\n",
    "        \"latestMINUS5\",\r\n",
    "        \"latestMINUS6\",\r\n",
    "    ]\r\n",
    "    date_enc = \",\".join(dates)\r\n",
    "\r\n",
    "    url = (\r\n",
    "        url_base\r\n",
    "        + url_geography_base\r\n",
    "        + geography\r\n",
    "        + \"&\"\r\n",
    "        + url_date_base\r\n",
    "        + date_enc\r\n",
    "        + url_params[table_name]\r\n",
    "    )\r\n",
    "\r\n",
    "    return url\r\n"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "source": [
    "def write_list(output_list: List, output_filename: str) -> None:\r\n",
    "    with open(f\"./X_Output/{output_filename}\", \"w\") as textfile:\r\n",
    "        for el in output_list:\r\n",
    "            textfile.write(el + \"\\n\")"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "source": [
    "first = True\r\n",
    "geography_urls = []\r\n",
    "deaths = pd.DataFrame()\r\n",
    "for _, geography in tqdm(nomis_la_codes.iterrows()):\r\n",
    "    url = nomis_url(\"NM_161_1\", str(geography[\"GEOGRAPHY\"]))\r\n",
    "    geography_urls.append(url)\r\n",
    "    dataset = pyjstat.Dataset.read(url)\r\n",
    "    df:pd.DataFrame = dataset.write('dataframe')  # type: ignore\r\n",
    "    df['GEOGRAPHY'] = geography[\"GEOGRAPHY\"]\r\n",
    "    df['GEOGRAPHY_NAME'] = geography[\"GEOGRAPHY_NAME\"]\r\n",
    "    df['GEOGRAPHY_CODE'] = geography[\"GEOGRAPHY_CODE\"]\r\n",
    "    if first:\r\n",
    "        deaths = df\r\n",
    "        first = False\r\n",
    "        break  # NOTE: THIS WILL STOP THE LOOP\r\n",
    "    else:\r\n",
    "        deaths = pd.concat([deaths, df], axis=0)"
   ],
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "117bc61eec684d23969665000fcf9dd6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {}
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "source": [
    "deaths.to_csv(\"./X_Output/la_deaths_all_causes_only.csv\")\r\n",
    "write_list(geography_urls, \"mortality_urls_all_causes_only.txt\")"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [],
   "outputs": [],
   "metadata": {}
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "bec1da797f70239e7cfdf88cccc7d3f235eb824f3706317d51c20723117a6b25"
  },
  "kernelspec": {
   "display_name": "Python 3.7.10 64-bit ('nomis2021': conda)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "orig_nbformat": 2
 },
 "nbformat": 4,
 "nbformat_minor": 2
}