lab.ipynb

{
 "cells": [
  {
   "attachments": {
    "image.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAAABwAAAAaCAYAAACkVDyJAAAMTmlDQ1BJQ0MgUHJvZmlsZQAASImVVwdYU1cbPndkkrACEZAR9hJlCASQEcKKICBTEJWQBBJGjAlBxU0pVbBuEQG1olURResApE7UOovito6iFJVKLVZxofKfDNDafzz/l+fc+97vfOf9Rs659xwA9Gr5Mlk+qg9AgbRQnhAZypqUls4idQEd+DMCnsCSL1DIOPHxMQDK0P3v8uoGQFT3q24qrn/2/1cxEIoUAgCQeIizhApBAcQHAMBLBDJ5IQBENtTbziyUqXAGxEZyGCDEMhXO0eBSFc7S4Cq1TVICF+JdAJBpfL48BwDdFqhnFQlyII/uLYjdpUKJFAA9MsRBAjFfCHEUxKMKCqarMLQDTlmf8eT8jTNrmJPPzxnGmlzUQg6TKGT5/Nn/Zzn+txTkK4d8OMBGE8ujElQ5w7rdypsercI0iHulWbFxEBtC/EYiVNtDjFLFyqhkjT1qLlBwYc0AE2J3IT8sGmJziCOk+bExWn1WtiSCBzGcIegsSSEvSTt2sUgRnqjlrJVPT4gbwtlyLkc7tpEvV/tV2Z9S5iVztPy3xCLeEP/LYnFSKsRUADBqkSQlFmJdiI0UeYnRGhvMpljMjR2ykSsTVPHbQcwWSSNDNfxYRrY8IkFrLytQDOWLlYklvFgtrioUJ0Vp6oPtFPDV8ZtA3CSScpKHeESKSTFDuQhFYeGa3LF2kTRZmy92X1YYmqAd2yfLj9fa42RRfqRKbwOxmaIoUTsWH1cIJ6SGH4+RFcYnaeLEM3P54+M18eBFIAZwQRhgASVsWWA6yAWS9t7mXvik6YkAfCAHOUAE3LSaoRGp6h4pvCaCYvAHRCKgGB4Xqu4VgSKo/zCs1VzdQLa6t0g9Ig88grgARIN8+KxUj5IOe0sBv0GN5B/eBTDWfNhUff/UcaAmRqtRDvGy9IYsieHEMGIUMYLojJvhQXgAHgOvIbB54mzcbyjaT/aER4QOwkPCdUIn4fY0SYn8i1gmgE7IH6HNOOvzjHEHyOmNh+KBkB0y40zcDLjhY6EfDh4MPXtDLVcbtyp31r/JcziDz2qutaO4U1DKCEoIxenLkbouut7DLKqKfl4fTaxZw1XlDvd86Z/7WZ2F8B79pSW2GNuPncFOYOeww1gzYGHHsBbsInZEhYfn0G/qOTTkLUEdTx7kkfzDH1/rU1VJhXuDe4/7e20fKBTNUr0fAXe6bLZckiMuZHHgm1/E4kkFo0exPN09/ABQfUc0r6kXTPX3AWGe/6RbdBWAcfcGBwcPf9JF/wjA/qVwmd/8pHOG61VvHQBnqwVKeZFGh6suBPg20IMryhRYAlvgBDPyBD4gAISAcDAexIEkkAamwjqL4XyWg5lgLlgEykAFWAHWgmqwCWwBO8BusA80g8PgBPgJXACXwXVwB86fbvAU9IFXYABBEBJCRxiIKWKF2COuiCfCRoKQcCQGSUDSkEwkB5EiSmQu8hVSgaxCqpHNSD3yA3IIOYGcQzqQ28gDpAf5C3mHYigNNUItUAd0DMpGOWg0moROQXPQGWgxWoouQ6vQOnQX2oSeQC+g19FO9CnajwFMB2Ni1pgbxsa4WByWjmVjcmw+Vo5VYnVYI9YK/+mrWCfWi73FiTgDZ+FucA5H4cm4AJ+Bz8eX4tX4DrwJP4VfxR/gffhHAp1gTnAl+BN4hEmEHMJMQhmhkrCNcJBwGq6mbsIrIpHIJDoSfeFqTCPmEucQlxI3EPcQjxM7iF3EfhKJZEpyJQWS4kh8UiGpjLSetIt0jHSF1E16Q9YhW5E9yRHkdLKUXEKuJO8kHyVfIT8mD1D0KfYUf0ocRUiZTVlO2UpppVyidFMGqAZUR2ogNYmaS11EraI2Uk9T71Jf6Ojo2Oj46UzUkegs1KnS2atzVueBzluaIc2FxqVl0JS0ZbTttOO027QXdDrdgR5CT6cX0pfR6+kn6ffpb3QZuqN1ebpC3QW6NbpNuld0n+lR9Oz1OHpT9Yr1KvX2613S69Wn6Dvoc/X5+vP1a/QP6d/U7zdgGHgYxBkUGCw12GlwzuCJIcnQwTDcUGhYarjF8KRhFwNj2DK4DAHjK8ZWxmlGtxHRyNGIZ5RrVGG026jdqM/Y0HiscYrxLOMa4yPGnUyM6cDkMfOZy5n7mDeY70ZYjOCMEI1YMqJxxJURr01GmoSYiEzKTfaYXDd5Z8oyDTfNM11p2mx6zww3czGbaDbTbKPZabPekUYjA0YKRpaP3DfyF3PU3MU8wXyO+Rbzi+b9FpYWkRYyi/UWJy16LZmWIZa5lmssj1r2WDGsgqwkVmusjln9zjJmcVj5rCrWKVaftbl1lLXSerN1u/WAjaNNsk2JzR6be7ZUW7Zttu0a2zbbPjsruwl2c+0a7H6xp9iz7cX26+zP2L92cHRIdfjGodnhiaOJI8+x2LHB8a4T3SnYaYZTndM1Z6Iz2znPeYPzZRfUxdtF7FLjcskVdfVxlbhucO0YRRjlN0o6qm7UTTeaG8etyK3B7cFo5uiY0SWjm0c/G2M3Jn3MyjFnxnx093bPd9/qfsfD0GO8R4lHq8dfni6eAs8az2tedK8IrwVeLV7Px7qOFY3dOPaWN8N7gvc33m3eH3x8feQ+jT49vna+mb61vjfZRux49lL2WT+CX6jfAr/Dfm/9ffwL/ff5/xngFpAXsDPgyTjHcaJxW8d1BdoE8gM3B3YGsYIyg74L6gy2DuYH1wU/DLENEYZsC3nMcebkcnZxnoW6h8pDD4a+5vpz53GPh2FhkWHlYe3hhuHJ4dXh9yNsInIiGiL6Ir0j50QejyJERUetjLrJs+AJePW8vvG+4+eNPxVNi06Mro5+GOMSI49pnYBOGD9h9YS7sfax0tjmOBDHi1sddy/eMX5G/I8TiRPjJ9ZMfJTgkTA34UwiI3Fa4s7EV0mhScuT7iQ7JSuT21L0UjJS6lNep4alrkrtnDRm0rxJF9LM0iRpLemk9JT0ben9k8Mnr53cneGdUZZxY4rjlFlTzk01m5o/9cg0vWn8afszCZmpmTsz3/Pj+HX8/ixeVm1Wn4ArWCd4KgwRrhH2iAJFq0SPswOzV2U/yQnMWZ3TIw4WV4p7JVxJteR5blTuptzXeXF52/MG81Pz9xSQCzILDkkNpXnSU9Mtp8+a3iFzlZXJOmf4z1g7o08eLd+mQBRTFC2FRnDDflHppPxa+aAoqKim6M3MlJn7ZxnMks66ONtl9pLZj4sjir+fg88RzGmbaz130dwH8zjzNs9H5mfNb1tgu6B0QffCyIU7FlEX5S36ucS9ZFXJy69Sv2ottShdWNr1deTXDWW6ZfKym98EfLNpMb5Ysrh9ideS9Us+lgvLz1e4V1RWvF8qWHr+W49vq74dXJa9rH25z/KNK4grpCturAxeuWOVwariVV2rJ6xuWsNaU77m5dppa89Vjq3ctI66TrmusyqmqmW93foV699Xi6uv14TW7Kk1r11S+3qDcMOVjSEbGzdZbKrY9O47yXe3NkdubqpzqKvcQtxStOXR1pStZ75nf1+/zWxbxbYP26XbO3ck7DhV71tfv9N85/IGtEHZ0LMrY9fl3WG7WxrdGjfvYe6p2Av2Kvf+/kPmDzf2Re9r28/e33jA/kDtQcbB8iakaXZTX7O4ubMlraXj0PhDba0BrQd/HP3j9sPWh2uOGB9ZfpR6tPTo4LHiY/3HZcd7T+Sc6Gqb1nbn5KST105NPNV+Ovr02Z8ifjp5hnPm2NnAs4fP+Z87dJ59vvmCz4Wmi94XD/7s/fPBdp/2pku+l1ou+11u7RjXcfRK8JUTV8Ou/nSNd+3C9djrHTeSb9y6mXGz85bw1pPb+bef/1L0y8CdhXcJd8vv6d+rvG9+v+5X51/3dPp0HnkQ9uDiw8SHd7oEXU9/U/z2vrv0Ef1R5WOrx/VPPJ8c7onoufz75N+7n8qeDvSW/WHwR+0zp2cH/gz582LfpL7u5/Lng38tfWH6YvvLsS/b+uP7778qeDXwuvyN6Zsdb9lvz7xLffd4YOZ70vuqD84fWj9Gf7w7WDA4KOPL+eqtAAYbmp0NwF/bAaCnAcC4DPcPkzXnPLUgmrOpGoH/hDVnQbX4ANAIb6rtOvc4AHthcwyBRxLYVFv1pBCAenkNN60osr08NVw0eOIhvBkcfGEBAKkVgA/ywcGBDYODH7bCYG8DcHyG5nypEiI8G3yn5rjClK0AX8i/AMyYgBby3xJ6AAAAimVYSWZNTQAqAAAACAAEARoABQAAAAEAAAA+ARsABQAAAAEAAABGASgAAwAAAAEAAgAAh2kABAAAAAEAAABOAAAAAAAAAJAAAAABAAAAkAAAAAEAA5KGAAcAAAASAAAAeKACAAQAAAABAAAAHKADAAQAAAABAAAAGgAAAABBU0NJSQAAAFNjcmVlbnNob3Q8FnjdAAAACXBIWXMAABYlAAAWJQFJUiTwAAAB1GlUWHRYTUw6Y29tLmFkb2JlLnhtcAAAAAAAPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0iWE1QIENvcmUgNS40LjAiPgogICA8cmRmOlJERiB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiPgogICAgICA8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0iIgogICAgICAgICAgICB4bWxuczpleGlmPSJodHRwOi8vbnMuYWRvYmUuY29tL2V4aWYvMS4wLyI+CiAgICAgICAgIDxleGlmOlBpeGVsWERpbWVuc2lvbj4yODwvZXhpZjpQaXhlbFhEaW1lbnNpb24+CiAgICAgICAgIDxleGlmOlVzZXJDb21tZW50PlNjcmVlbnNob3Q8L2V4aWY6VXNlckNvbW1lbnQ+CiAgICAgICAgIDxleGlmOlBpeGVsWURpbWVuc2lvbj4yNjwvZXhpZjpQaXhlbFlEaW1lbnNpb24+CiAgICAgIDwvcmRmOkRlc2NyaXB0aW9uPgogICA8L3JkZjpSREY+CjwveDp4bXBtZXRhPgqrjGYFAAAAHGlET1QAAAACAAAAAAAAAA0AAAAoAAAADQAAAA0AAAD942C5QAAAAMlJREFUSA3skyEOhDAQRX8hgQQSDAkOgcRgK5AIHALTVDT0LL1E71RuwRnwsIgaAtkusKt2zE//JPPyZ1KyvAo/LPIHPr3t05XO84woiuB53rPM7YZHZYxZlFLLNE1H7cveacJxHKG1hu/7aNsWXdchCILbad8CLSHLMgzDgLIsrXVJnYHbdEII6roGYwxxHH8faAlJkoBzDkqptZz1o4T7qVVVQQiBNE33rdP3LeA2NQxD9H2PpmmcvtBtoI1SFAWklMjz3FqHugIAAP//84SijAAAAM9JREFU1ZMxCoQwFEQnChEU0gh2FpY2tiksLexS2AQL0bN4Ce+kt/AM9u5a2MgPJgoL+5uQ+cM8BhK2fwfELMuCaZqIjVnyfR91XUMpBc45aWR/C8yyDMMwIE1Tstkpvm4YBAGapkFVVfA878w1nq+ARVGg6zrEcWwEXBePgEIItG0LKeU17/buBGSMoSxLaK0RRdFtOGWwBiZJgr7vkec5lWOt3QJt/pY17TAe/5CaeZ73cRz3dV2p9WPN2HDbNoRhaPXUXRoagS4hLt6fAz/+c6AOkZXOYQAAAABJRU5ErkJggg=="
    }
   },
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Introduction\n",
    "\n",
    "Jupyter notebooks are divided into cells that can contain [markdown](https://daringfireball.net/projects/markdown/) or code that you can run interactively from the notebook interface. You can progress through the cells in the notebook by clicking the play button ![image.png](attachment:image.png) in the notebook tab's toolbar.\n",
    "\n",
    "Click the play button to advance to the next cell and continue on in the lab whenever you have completed a cell.\n",
    "\n",
    "After clicking the play button, the status in the left-hand side of the bottom statusbar will change from Idle to Busy. Wait for the status to change back to Idle before proceeding to the next cell."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Notebook Overview\n",
    "\n",
    "This notebook works through the main steps that are shared by most machine learning projects:\n",
    "\n",
    "- preparing data\n",
    "- training a model\n",
    "- hosting the model\n",
    "- Using the hosted model for inference\n",
    "\n",
    "You will use Python 3 as the programming language. This notebook is built on the SageMaker notebook conda_python3 environment. conda refers to [Anaconda](https://www.anaconda.com/distribution/) which is a data science platform. The environment comes with many common Python machine learning and data science libraries already installed. \n",
    "\n",
    "You will also use a built-in SageMaker algorithm for training the model as well as use SageMaker's model hosting capability to deploy the trained model to learn how to leverage SageMaker in your machine learning code."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Updating Python Dependencies\n",
    "\n",
    "The following will ensure you have versions of dependencies that are required in this lab. This command uses the Pip, the python package and dependency management tool."
    ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "!pip install awscli==1.27.43 boto3==1.26.43 sagemaker==2.127.0 \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Understanding the Machine Learning Objective\n",
    "\n",
    "In this notebook you will work with [US flight delay data](https://www.transtats.bts.gov/Tables.asp?DB_ID=120&DB_Name=Airline%20On-Time%20Performance%20Data&DB_Short_Name=On-Time) to develop a model that can forecast flight delays. You will create a regression model to forecast the number of minutes a given flight will be delayed."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparing the Data\n",
    "\n",
    "Raw flight delay data is provided to you in an Amazon S3 bucket. The data you will use is a sample of the total flight data available in order to keep the runtimes reasonably short for a more fluid lab experience."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "import urllib.request\n",
    "\n",
    "# Download the data from S3 to the notebook instance (~100MB of data)\n",
    "source = \"https://clouda-labs-assets.s3-us-west-2.amazonaws.com/sagemaker-notebooks/Flights.csv\"\n",
    "filepath = \"Flights.csv\"\n",
    "urllib.request.urlretrieve(source, filepath)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The data is in comma-separated value (CSV) format. You will now inspect a few rows of the data to understand the features and sample values in the raw data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(filepath) as file:\n",
    "   line = file.readline()\n",
    "   count = 1\n",
    "   while count <= 5:\n",
    "       print(\"Line {}: {}\".format(count, line))\n",
    "       line = file.readline()\n",
    "       count += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__Line 1__ of the CSV file is the header which gives the names of each column in the file. The arrival delay (__ARR_DELAY__) column is the target you want to predict. The other columns are a variety of features for identifying the flight including the origin (__ORIGIN__) and destination (__DEST__) airports, date features (__YEAR__, __QUARTER__, etc.). The following lines show sample values for the features.\n",
    "\n",
    "To get the most value from the data, you need to consider each feature's data type. The following dictionary \n",
    "\n",
    "You will use the [Python Data Analysis Library](https://pandas.pydata.org), or `pandas`, to load the CSV data into a `DataFrame` (`df`) that is easy to manipulate. To do so, you need to provide `pandas` with data types (`dtypes`) for each of the features. [`pandas` supports several data types](https://pandas.pydata.org/pandas-docs/stable/getting_started/basics.html#basics-dtypes). The following dictionary maps each feature to a data type."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "dtypes = {\n",
    "        \"YEAR\": np.int64,\n",
    "        \"QUARTER\": \"category\",\n",
    "        \"MONTH\": \"category\",\n",
    "        \"DAY_OF_MONTH\": \"category\",\n",
    "        \"DAY_OF_WEEK\": \"category\",\n",
    "        \"UNIQUE_CARRIER\": \"category\",\n",
    "        \"TAIL_NUM\": \"category\",\n",
    "        \"FL_NUM\": \"category\",\n",
    "        \"ORIGIN\": \"category\",\n",
    "        \"DEST\": \"category\",\n",
    "        \"CRS_DEP_TIME\": np.int64,\n",
    "        \"DEP_TIME\": np.int64,\n",
    "        \"DEP_DELAY\": np.float64,\n",
    "        \"DEP_DELAY_NEW\": np.float64,\n",
    "        \"DEP_DEL15\": np.int64,\n",
    "        \"DEP_DELAY_GROUP\": np.int64,\n",
    "        \"CRS_ARR_TIME\": np.int64,\n",
    "        \"ARR_DELAY\": np.float64,\n",
    "        \"CRS_ELAPSED_TIME\": np.float64,\n",
    "        \"DISTANCE\": np.float64,\n",
    "        \"DISTANCE_GROUP\": \"category\"\n",
    "    }"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It is not always obvious which data type to use for a feature. The provided types are reasonable but in practice it may be worth trying different data types to see which yields the best results when it is not clear.\n",
    "\n",
    "The [`read_csv`](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_csv.html) function can now create a DataFrame from the csv data. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "df = pd.read_csv(filepath, dtype=dtypes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can now explore the data to better understand each feature's statistics and distribution. For brevity, the following code only explores the target arrival delay. You can explore other features by extending the code if you desire."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "# Plot histogram of all arrival delays and arrival delays between -100 (100 minutes early) and 100\n",
    "fig, axes = plt.subplots(1, 2, figsize=(12,6))\n",
    "fig.suptitle('Arrival Delay Histograms')\n",
    "axes[0].hist(df['ARR_DELAY'], bins=50)\n",
    "axes[0].set(ylabel='Frequency', xlabel='Arrival Delay (Minutes)');\n",
    "\n",
    "delays_zoom = df.loc[(df['ARR_DELAY'] >= -100) & (df['ARR_DELAY'] <= 100), 'ARR_DELAY']\n",
    "axes[1].hist(delays_zoom, bins=50)\n",
    "axes[1].set(ylabel='Frequency', xlabel='Arrival Delay (Minutes)');\n",
    "\n",
    "# Print arrival delay statistics\n",
    "df['ARR_DELAY'].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There is a wide range of arrival delays with the largest delay being 1935 minutes. The average delay is 4.7 minutes. The majority of flights arrive early (50th percentile is -6 minutes).\n",
    "\n",
    "Some features will not be included in the model to reduce the training time for the sake of this lab and because some features are not expected to correlate with the target. For example, the __YEAR__ is always 2017 in the sample data so it has not value in predicting delay."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "drop_columns = [\"YEAR\", \"TAIL_NUM\", \"FL_NUM\", \"DEST\"]\n",
    "df = df.drop(columns=drop_columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The SageMaker algorithm you will use, linear-learner, only supports numerical feature values. You must encode the categorical features because of this restriction. There are several ways to encode categorical features into numerical features but you will use one-hot encoding which replaces a categorical feature with one feature for each possible value of the encoded feature. This does result in an increase in the number of features which can impact training times."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "categoricals = [column for (column, dtype) in dtypes.items() if dtype == \"category\" and column in df.columns]\n",
    "for column in categoricals:\n",
    "    df = pd.get_dummies(data=df, columns=[column])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After one-hot encoding the categorical features the number of features has increased to 402 from 21."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Had some of the columns not been dropped the number would be significantly higher. For example, the __TAIL_NUM__ feature is a unique number assigned to each aircraft. In the data set there are over 4000 aircrafts which would result in over 4000 features being added with one-hot encoding. For this lab you assume there is no correlation between specific aircrafts and arrival delay by dropping the column.\n",
    "\n",
    "The data is all numerical now but the the linear-learner SageMaker algorithm requires either [CSV or RecordIO protobuf formatted data](https://docs.aws.amazon.com/sagemaker/latest/dg/cdf-training.html). `pandas` has a [`to_csv`](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.to_csv.html) function that can be used for outputting the DataFrame as a CSV file. The SageMaker Python library includes a function for creating the [RecordIO formatted data](https://github.com/aws/sagemaker-python-sdk/blob/master/src/sagemaker/amazon/common.py#L132). You will use the CSV format. You will upload the training data to an S3 bucket created for you by the Cloud Academy lab environment. SageMaker can also access training data on AWS Elastic File Systems (EFS) or AWS FSx for Lustre (FSxLustre)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "import io\n",
    "import sagemaker.amazon.common as smac\n",
    "\n",
    "# Get the available S3 bucket created by the Cloud Academy lab environment\n",
    "bucket = boto3.client('s3').list_buckets()['Buckets'][0]['Name']\n",
    "print(bucket)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The data will be split into training and test sets. The test set will be used by SageMaker to test the performance of the trained model using data that isn't part of the training process. You will then write the CSV files.\n",
    "\n",
    "_Warning_: It takes 5-10 minutes to complete."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Place target in first column for SageMaker training with CSV files\n",
    "columns = df.columns.tolist()\n",
    "columns.remove(\"ARR_DELAY\")\n",
    "columns.insert(0, \"ARR_DELAY\")\n",
    "df = df[columns]\n",
    "\n",
    "# Split data into training and test sets (60%/40% deterministic split)\n",
    "train = df.sample(frac=0.6, random_state=1) # fix random_state for consistent results\n",
    "test = df.drop(train.index)\n",
    "del df\n",
    "\n",
    "# Write CSV files\n",
    "train.to_csv(path_or_buf='train.csv', header=False, index=False) # don't write header or row indexes (index=False)\n",
    "test.to_csv(path_or_buf='test.csv', header=False, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The files can now be uploaded to S3."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "import os\n",
    "\n",
    "prefix = 'FlightDelays'\n",
    "key = 'csv-data'\n",
    "\n",
    "# Upload data to S3\n",
    "boto3.resource('s3').Bucket(bucket).Object(os.path.join(prefix, 'train', key)).upload_file('train.csv', ExtraArgs={'ContentType': 'text/csv'})\n",
    "s3_train_data = 's3://{}/{}/train/{}'.format(bucket, prefix, key)\n",
    "print('uploaded training data location: {}'.format(s3_train_data))\n",
    "\n",
    "boto3.resource('s3').Bucket(bucket).Object(os.path.join(prefix, 'test', key)).upload_file('test.csv', ExtraArgs={'ContentType': 'text/csv'})\n",
    "s3_test_data = 's3://{}/{}/test/{}'.format(bucket, prefix, key)\n",
    "print('uploaded testing data location: {}'.format(s3_test_data))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training the Model\n",
    "\n",
    "Now that the training data is available on S3 in a format that SageMaker can use, you are ready to train the model. You will use the [linear-learner algorithm](https://docs.aws.amazon.com/sagemaker/latest/dg/linear-learner.html) for training. The linear-learner algorithm can be used for regression or classification problems. It scales to large data sets and is sufficient for demonstrating the use of built-in SageMaker algorithms using Python.\n",
    "\n",
    "SageMaker algorithms are available via container images. Each region that supports SageMaker has its own copy of the images. You will begin by retreiving the URI of the container image for the current session's region. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sagemaker import image_uris\n",
    "\n",
    "container = image_uris.retrieve('linear-learner', boto3.Session().region_name)\n",
    "container"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From the image URI you can see the images are stored in Elastic Container Registry (ECR). You can also make your own images to use with SageMaker and store them in ECR.\n",
    "\n",
    "You will configure a training job next."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sagemaker\n",
    "\n",
    "# Store the output in the S3 bucket\n",
    "s3_output = 's3://{}/{}/output'.format(bucket, prefix)\n",
    "\n",
    "# Retrieve the notebooks role and use it for model training (has access to S3, SageMaker)\n",
    "role = sagemaker.get_execution_role()\n",
    "\n",
    "session = sagemaker.Session()\n",
    "\n",
    "# Use the Estimator interface to configure a training job\n",
    "linear = sagemaker.estimator.Estimator(container,\n",
    "                                       role, \n",
    "                                       instance_count=1, \n",
    "                                       instance_type='ml.m4.xlarge',\n",
    "                                       output_path=s3_output,\n",
    "                                       sagemaker_session=session,\n",
    "                                       input_mode='Pipe' # stream training data from S3\n",
    "                                       )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Pipe mode is used for faster training and allows data to be streamed directly from S3 as opposed to File mode which downloads the entire data files before any training. It is [supported for CSV files](https://aws.amazon.com/blogs/machine-learning/now-use-pipe-mode-with-csv-datasets-for-faster-training-on-amazon-sagemaker-built-in-algorithms/) after it was originally only for RecordIO protobuf format.\n",
    "\n",
    "You must now configure the [hyperparameters for the linear-learner algorithm](https://docs.aws.amazon.com/sagemaker/latest/dg/ll_hyperparameters.html). You will only configure the required hyperparameters, however for production models it is worth tuning and considering other hyperparameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "linear.set_hyperparameters(feature_dim=len(train.columns) - 1, # minus 1 for the target\n",
    "                           predictor_type='regressor')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The training job can now begin training the model. The time to provision the training instance and perform the actual training together take around 8 minutes. You can monitor the progress of the training job as it is output below the following cell. Blue logs represent logs coming from the training container and summarize the progress of the training process. The job is finished when you see __Completed - Training job completed__."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "import time\n",
    "\n",
    "# Set the content_type to text/csv (default is application/x-recordio-protobuf)\n",
    "train_channel = sagemaker.session.TrainingInput(s3_train_data, content_type='text/csv')\n",
    "test_channel = sagemaker.session.TrainingInput(s3_test_data, content_type='text/csv')\n",
    "\n",
    "name = f\"flight-delays-{int(time.time())}\"\n",
    "\n",
    "linear.fit({'train': train_channel, 'test': test_channel}, job_name=name) # validation channel is also supported"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Near the end of the training container logs you can see the __#test_score__ logs which show the performance of the model on the test set. The mean-squared error (__mse__) is 164.38. Taking the square root gives 12.8 loosely meaning that the model had an error of around 12 minutes for its test set predictions on average."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Deploying the Trained Model\n",
    "\n",
    "SageMaker can host models through its hosting services. The model is accessible to client through a SageMaker endpoint. The hosted model, endpoint configuration resource, and endpoint are all created with a single function call `deploy()`. The process takes around 5 minutes including the time to create an instance to host the model. You will see a __!__ output when everything is finished."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "linear_predictor = linear.deploy(initial_instance_count=1,\n",
    "                                 instance_type='ml.t2.medium')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using the Endpoint for Inference\n",
    "\n",
    "The Endpoint is accessible over HTTPS but when using the SageMaker Python SDK you can use the `predict` function to abstract the complexity of HTTPS requests. You first configure how the predictor is to serialize requests and deserialize responses. You will use CSV to serialize requests and JSON to deserialize responses."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sagemaker.serializers import CSVSerializer\n",
    "from sagemaker.deserializers import JSONDeserializer\n",
    "\n",
    "linear_predictor.serializer = CSVSerializer()\n",
    "linear_predictor.deserializer = JSONDeserializer()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You will now use the model endpoint to make a prediciton using a sample vector from the test data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_vector = test.iloc[0][1:] # use all but the first column (first column is actual delay)\n",
    "actual_delay = test.iloc[0][0]\n",
    "\n",
    "response = linear_predictor.predict(test_vector)\n",
    "\n",
    "predicted_delay = response['predictions'][0]['score']\n",
    "print(f'Predicted {predicted_delay}, actual {actual_delay}, error {actual_delay-predicted_delay}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this case the prediction is within eight minutes of the actual delay."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summary\n",
    "\n",
    "You have now seen how to use Python to prepare data, use SageMaker to train and deploy a model, and make predictions with a model hosted in SageMaker. \n",
    "\n",
    "This concludes the notebook. Return to the Cloud Academy lab browser tab to complete the lab."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_amazonei_pytorch_latest_p37",
   "language": "python",
   "name": "conda_amazonei_pytorch_latest_p37"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}