notebooks/Confident Data Chapter 06.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"../images/CONFIDENTDATASCIENCE.png\" alt=\"An image that reads Confident Data Science: The Essential Skills of Data Science, Companion Jupyter Notebook + Coding Examples. Also shows book image.\"/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Chapter 6 *Data Manipulation + Preparation*\n",
    "\n",
    "To give citation to these coding examples give reference to an appropriate chapter and page of:\n",
    "\n",
    "*Confident Data Science: Discover the essential skills of data science* <br>\n",
    "(Confident Series, 15) 1st Edition by Adam Ross Nelson (Author) <br>\n",
    "Publisher : Kogan Page (September 26, 2023).\n",
    "\n",
    "\n",
    "## The Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import Panadas + Numpy libraries\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "# As a Python dictionary the data are as follows\n",
    "data = {'PackageID':list(range(1001,1011))+[1010],\n",
    "        'Length':[12.5,12.5,12.5,12.5,12.5,np.nan,12.5,12.5,\n",
    "                  np.nan,12.5,12.5],\n",
    "        'Width':[7]*3 + [8]*3 + [9]*4 + [9],\n",
    "        'Height':[4,5,6,5,6,4,np.nan,5,7,6]+[6],\n",
    "        'Liquid':[1,0,0,1,0,0,0,0,0,0]+[0],\n",
    "        'Perishable':[1] + [0]*9 + [0],\n",
    "        'Origin':['FL','FL','GA','SC','SC','GA','SC','SC',\n",
    "                  'GA','GA']+['GA'],\n",
    "        'Destination':['FL','NC','NC','SC','FL','NC','GA','SC',\n",
    "                       'NC','GA']+['GA'],\n",
    "        'Insurance':[100,100,200,150,10009,150,100,100,\n",
    "                     150,200] + [200],\n",
    "        'ShipCost':[19.99,19.99,24.99,21.99,19.99,\n",
    "                    21.99,19.99,19.99,21.99,24.99] + [24.99],\n",
    "        'ShipDate':pd.to_datetime(['1/6/2028','1/8/2028',\n",
    "                                   '1/7/2028','1/8/2028',\n",
    "                                   '1/9/2028','1/10/2028',\n",
    "                                   '1/11/2028','1/14/2028',\n",
    "                                   '1/12/2028','1/12/2028',\n",
    "                                   '1/12/2028']),\n",
    "        'ArriveDate':pd.to_datetime(['1/7/2028','1/6/2028',\n",
    "                                     '1/9/2028','1/9/2028',\n",
    "                                     '1/11/2028','1/15/2028',\n",
    "                                     '1/14/2028','1/11/2028',\n",
    "                                     '1/15/2028','1/13/2028',\n",
    "                                     '1/13/2028'])\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PackageID</th>\n",
       "      <th>Length</th>\n",
       "      <th>Width</th>\n",
       "      <th>Height</th>\n",
       "      <th>Liquid</th>\n",
       "      <th>Perishable</th>\n",
       "      <th>Origin</th>\n",
       "      <th>Destination</th>\n",
       "      <th>Insurance</th>\n",
       "      <th>ShipCost</th>\n",
       "      <th>ShipDate</th>\n",
       "      <th>ArriveDate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1001</td>\n",
       "      <td>12.5</td>\n",
       "      <td>7</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>FL</td>\n",
       "      <td>FL</td>\n",
       "      <td>100</td>\n",
       "      <td>19.99</td>\n",
       "      <td>2028-01-06</td>\n",
       "      <td>2028-01-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1002</td>\n",
       "      <td>12.5</td>\n",
       "      <td>7</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>FL</td>\n",
       "      <td>NC</td>\n",
       "      <td>100</td>\n",
       "      <td>19.99</td>\n",
       "      <td>2028-01-08</td>\n",
       "      <td>2028-01-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1003</td>\n",
       "      <td>12.5</td>\n",
       "      <td>7</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>GA</td>\n",
       "      <td>NC</td>\n",
       "      <td>200</td>\n",
       "      <td>24.99</td>\n",
       "      <td>2028-01-07</td>\n",
       "      <td>2028-01-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1004</td>\n",
       "      <td>12.5</td>\n",
       "      <td>8</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>SC</td>\n",
       "      <td>SC</td>\n",
       "      <td>150</td>\n",
       "      <td>21.99</td>\n",
       "      <td>2028-01-08</td>\n",
       "      <td>2028-01-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1005</td>\n",
       "      <td>12.5</td>\n",
       "      <td>8</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>SC</td>\n",
       "      <td>FL</td>\n",
       "      <td>10009</td>\n",
       "      <td>19.99</td>\n",
       "      <td>2028-01-09</td>\n",
       "      <td>2028-01-11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1006</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>GA</td>\n",
       "      <td>NC</td>\n",
       "      <td>150</td>\n",
       "      <td>21.99</td>\n",
       "      <td>2028-01-10</td>\n",
       "      <td>2028-01-15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1007</td>\n",
       "      <td>12.5</td>\n",
       "      <td>9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>SC</td>\n",
       "      <td>GA</td>\n",
       "      <td>100</td>\n",
       "      <td>19.99</td>\n",
       "      <td>2028-01-11</td>\n",
       "      <td>2028-01-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1008</td>\n",
       "      <td>12.5</td>\n",
       "      <td>9</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>SC</td>\n",
       "      <td>SC</td>\n",
       "      <td>100</td>\n",
       "      <td>19.99</td>\n",
       "      <td>2028-01-14</td>\n",
       "      <td>2028-01-11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1009</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>GA</td>\n",
       "      <td>NC</td>\n",
       "      <td>150</td>\n",
       "      <td>21.99</td>\n",
       "      <td>2028-01-12</td>\n",
       "      <td>2028-01-15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1010</td>\n",
       "      <td>12.5</td>\n",
       "      <td>9</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>GA</td>\n",
       "      <td>GA</td>\n",
       "      <td>200</td>\n",
       "      <td>24.99</td>\n",
       "      <td>2028-01-12</td>\n",
       "      <td>2028-01-13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1010</td>\n",
       "      <td>12.5</td>\n",
       "      <td>9</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>GA</td>\n",
       "      <td>GA</td>\n",
       "      <td>200</td>\n",
       "      <td>24.99</td>\n",
       "      <td>2028-01-12</td>\n",
       "      <td>2028-01-13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    PackageID  Length  Width  Height  Liquid  Perishable Origin Destination  \\\n",
       "0        1001    12.5      7     4.0       1           1     FL          FL   \n",
       "1        1002    12.5      7     5.0       0           0     FL          NC   \n",
       "2        1003    12.5      7     6.0       0           0     GA          NC   \n",
       "3        1004    12.5      8     5.0       1           0     SC          SC   \n",
       "4        1005    12.5      8     6.0       0           0     SC          FL   \n",
       "5        1006     NaN      8     4.0       0           0     GA          NC   \n",
       "6        1007    12.5      9     NaN       0           0     SC          GA   \n",
       "7        1008    12.5      9     5.0       0           0     SC          SC   \n",
       "8        1009     NaN      9     7.0       0           0     GA          NC   \n",
       "9        1010    12.5      9     6.0       0           0     GA          GA   \n",
       "10       1010    12.5      9     6.0       0           0     GA          GA   \n",
       "\n",
       "    Insurance  ShipCost   ShipDate ArriveDate  \n",
       "0         100     19.99 2028-01-06 2028-01-07  \n",
       "1         100     19.99 2028-01-08 2028-01-06  \n",
       "2         200     24.99 2028-01-07 2028-01-09  \n",
       "3         150     21.99 2028-01-08 2028-01-09  \n",
       "4       10009     19.99 2028-01-09 2028-01-11  \n",
       "5         150     21.99 2028-01-10 2028-01-15  \n",
       "6         100     19.99 2028-01-11 2028-01-14  \n",
       "7         100     19.99 2028-01-14 2028-01-11  \n",
       "8         150     21.99 2028-01-12 2028-01-15  \n",
       "9         200     24.99 2028-01-12 2028-01-13  \n",
       "10        200     24.99 2028-01-12 2028-01-13  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a Pandas df from the Python dictionary\n",
    "df = pd.DataFrame(data)\n",
    "\n",
    "# Evaluate the df to display data in tabular format\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Alternatively, this data loads from online. Note, the added requirement to convert the dates, stored in CSV as strings, to `datetime64[ns]` format.\n",
    "\n",
    "```Python\n",
    "# Specify the location, path, and file mame\n",
    "location = 'https://raw.githubusercontent.com/'\n",
    "path = 'adamrossnelson/confident/main/data/'\n",
    "fname = 'confident_ch6.csv'\n",
    "\n",
    "# Load the csv from online into a Pandas df\n",
    "df = pd.read_csv(location + path + fname)\n",
    "\n",
    "# Convert string dates to datetime64[ns] format\n",
    "df['ShipDate'] = pd.to_datetime(df['ShipDate'])\n",
    "df['ArriveDate'] = pd.to_datetime(df['ArriveDate'])\n",
    "\n",
    "# Evaluate the df to display data in tabular format\n",
    "df\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Replacing Missing Continuous Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Height</th>\n",
       "      <th>4.0</th>\n",
       "      <th>5.0</th>\n",
       "      <th>6.0</th>\n",
       "      <th>7.0</th>\n",
       "      <th>99.0</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Width</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Height  4.0   5.0   6.0   7.0   99.0\n",
       "Width                               \n",
       "7          1     1     1     0     0\n",
       "8          1     1     1     0     0\n",
       "9          0     1     2     1     1"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a cross-tabulation table from two columns of data\n",
    "# Fill missing values in 'Height' column with 99\n",
    "pd.crosstab(df['Width'],\n",
    "            df['Height'].fillna(99))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fill missing values in the 'Height' \n",
    "# with a default value of 6.0. \n",
    "# column of the DataFrame\n",
    "df['Height'] = df['Height'].fillna(6.0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Width</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Height</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4.0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5.0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6.0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7.0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Width   7  8  9\n",
       "Height         \n",
       "4.0     1  1  0\n",
       "5.0     1  1  1\n",
       "6.0     1  1  3\n",
       "7.0     0  0  1"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check manipulation performed as expected\n",
    "pd.crosstab(df['Height'], df['Width'])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Adjusting Outliers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>PackageID</th>\n",
       "      <td>11.0</td>\n",
       "      <td>1005.909091</td>\n",
       "      <td>3.176619</td>\n",
       "      <td>1001.00</td>\n",
       "      <td>1003.50</td>\n",
       "      <td>1006.00</td>\n",
       "      <td>1008.50</td>\n",
       "      <td>1010.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Length</th>\n",
       "      <td>9.0</td>\n",
       "      <td>12.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>12.50</td>\n",
       "      <td>12.50</td>\n",
       "      <td>12.50</td>\n",
       "      <td>12.50</td>\n",
       "      <td>12.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Width</th>\n",
       "      <td>11.0</td>\n",
       "      <td>8.181818</td>\n",
       "      <td>0.873863</td>\n",
       "      <td>7.00</td>\n",
       "      <td>7.50</td>\n",
       "      <td>8.00</td>\n",
       "      <td>9.00</td>\n",
       "      <td>9.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Height</th>\n",
       "      <td>11.0</td>\n",
       "      <td>5.454545</td>\n",
       "      <td>0.934199</td>\n",
       "      <td>4.00</td>\n",
       "      <td>5.00</td>\n",
       "      <td>6.00</td>\n",
       "      <td>6.00</td>\n",
       "      <td>7.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Liquid</th>\n",
       "      <td>11.0</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.404520</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Perishable</th>\n",
       "      <td>11.0</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>0.301511</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Insurance</th>\n",
       "      <td>11.0</td>\n",
       "      <td>1041.727273</td>\n",
       "      <td>2974.397892</td>\n",
       "      <td>100.00</td>\n",
       "      <td>100.00</td>\n",
       "      <td>150.00</td>\n",
       "      <td>200.00</td>\n",
       "      <td>10009.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ShipCost</th>\n",
       "      <td>11.0</td>\n",
       "      <td>21.899091</td>\n",
       "      <td>2.165851</td>\n",
       "      <td>19.99</td>\n",
       "      <td>19.99</td>\n",
       "      <td>21.99</td>\n",
       "      <td>23.49</td>\n",
       "      <td>24.99</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            count         mean          std      min      25%      50%  \\\n",
       "PackageID    11.0  1005.909091     3.176619  1001.00  1003.50  1006.00   \n",
       "Length        9.0    12.500000     0.000000    12.50    12.50    12.50   \n",
       "Width        11.0     8.181818     0.873863     7.00     7.50     8.00   \n",
       "Height       11.0     5.454545     0.934199     4.00     5.00     6.00   \n",
       "Liquid       11.0     0.181818     0.404520     0.00     0.00     0.00   \n",
       "Perishable   11.0     0.090909     0.301511     0.00     0.00     0.00   \n",
       "Insurance    11.0  1041.727273  2974.397892   100.00   100.00   150.00   \n",
       "ShipCost     11.0    21.899091     2.165851    19.99    19.99    21.99   \n",
       "\n",
       "                75%       max  \n",
       "PackageID   1008.50   1010.00  \n",
       "Length        12.50     12.50  \n",
       "Width          9.00      9.00  \n",
       "Height         6.00      7.00  \n",
       "Liquid         0.00      1.00  \n",
       "Perishable     0.00      1.00  \n",
       "Insurance    200.00  10009.00  \n",
       "ShipCost      23.49     24.99  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Observe summary statistics from the entire df\n",
    "df.describe().transpose()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>ShipCost</th>\n",
       "      <th>19.99</th>\n",
       "      <th>21.99</th>\n",
       "      <th>24.99</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Insurance</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>200</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10009</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "ShipCost   19.99  21.99  24.99\n",
       "Insurance                     \n",
       "100            4      0      0\n",
       "150            0      3      0\n",
       "200            0      0      3\n",
       "10009          1      0      0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Evaluate the outlier + extreme insurance value\n",
    "pd.crosstab(df['Insurance'],\n",
    "            df['ShipCost'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[100, 100, 200, 150, 100, 150, 100, 100, 150, 200, 200]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Update data with list comprehension with a conditional\n",
    "[x if x < 2000 else 100 for x in df['Insurance']]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([100, 100, 200, 150, 100, 150, 100, 100, 150, 200, 200])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Udpate data with np.where()\n",
    "np.where(df['Insurance'] < 2000, df['Insurance'], 100)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     100\n",
       "1     100\n",
       "2     200\n",
       "3     150\n",
       "4     100\n",
       "5     150\n",
       "6     100\n",
       "7     100\n",
       "8     150\n",
       "9     200\n",
       "10    200\n",
       "Name: Insurance, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Update data with .apply() method and a lambda \n",
    "df['Insurance'].apply(lambda x: x if x < 2000 else 100)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[100, 100, 200, 150, 100, 150, 100, 100, 150, 200, 200]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Updates specific entry with list comprehension and a conditional\n",
    "[x if x != 10009 else 100 for x in df['Insurance']]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     100\n",
       "1     100\n",
       "2     200\n",
       "3     150\n",
       "4     100\n",
       "5     150\n",
       "6     100\n",
       "7     100\n",
       "8     150\n",
       "9     200\n",
       "10    200\n",
       "Name: Insurance, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Update specific entry with .apply() method and a lambda function\n",
    "df['Insurance'].apply(lambda x: x if x != 10009 else 100)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Removing Duplicates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     False\n",
       "1     False\n",
       "2     False\n",
       "3     False\n",
       "4     False\n",
       "5     False\n",
       "6     False\n",
       "7     False\n",
       "8     False\n",
       "9     False\n",
       "10     True\n",
       "dtype: bool"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Review for duplicates\n",
    "df.duplicated()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove duplicates with the .drop_duplicates() method\n",
    "df.drop_duplicates(inplace=True)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Addressing Incorrect Date Entries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PackageID</th>\n",
       "      <th>Length</th>\n",
       "      <th>Width</th>\n",
       "      <th>Height</th>\n",
       "      <th>Liquid</th>\n",
       "      <th>Perishable</th>\n",
       "      <th>Origin</th>\n",
       "      <th>Destination</th>\n",
       "      <th>Insurance</th>\n",
       "      <th>ShipCost</th>\n",
       "      <th>ShipDate</th>\n",
       "      <th>ArriveDate</th>\n",
       "      <th>Sequence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1001</td>\n",
       "      <td>12.5</td>\n",
       "      <td>7</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>FL</td>\n",
       "      <td>FL</td>\n",
       "      <td>100</td>\n",
       "      <td>19.99</td>\n",
       "      <td>2028-01-06</td>\n",
       "      <td>2028-01-07</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1002</td>\n",
       "      <td>12.5</td>\n",
       "      <td>7</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>FL</td>\n",
       "      <td>NC</td>\n",
       "      <td>100</td>\n",
       "      <td>19.99</td>\n",
       "      <td>2028-01-08</td>\n",
       "      <td>2028-01-06</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1003</td>\n",
       "      <td>12.5</td>\n",
       "      <td>7</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>GA</td>\n",
       "      <td>NC</td>\n",
       "      <td>200</td>\n",
       "      <td>24.99</td>\n",
       "      <td>2028-01-07</td>\n",
       "      <td>2028-01-09</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1004</td>\n",
       "      <td>12.5</td>\n",
       "      <td>8</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>SC</td>\n",
       "      <td>SC</td>\n",
       "      <td>150</td>\n",
       "      <td>21.99</td>\n",
       "      <td>2028-01-08</td>\n",
       "      <td>2028-01-09</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1005</td>\n",
       "      <td>12.5</td>\n",
       "      <td>8</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>SC</td>\n",
       "      <td>FL</td>\n",
       "      <td>10009</td>\n",
       "      <td>19.99</td>\n",
       "      <td>2028-01-09</td>\n",
       "      <td>2028-01-11</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1006</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>GA</td>\n",
       "      <td>NC</td>\n",
       "      <td>150</td>\n",
       "      <td>21.99</td>\n",
       "      <td>2028-01-10</td>\n",
       "      <td>2028-01-15</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1007</td>\n",
       "      <td>12.5</td>\n",
       "      <td>9</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>SC</td>\n",
       "      <td>GA</td>\n",
       "      <td>100</td>\n",
       "      <td>19.99</td>\n",
       "      <td>2028-01-11</td>\n",
       "      <td>2028-01-14</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1008</td>\n",
       "      <td>12.5</td>\n",
       "      <td>9</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>SC</td>\n",
       "      <td>SC</td>\n",
       "      <td>100</td>\n",
       "      <td>19.99</td>\n",
       "      <td>2028-01-14</td>\n",
       "      <td>2028-01-11</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1009</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>GA</td>\n",
       "      <td>NC</td>\n",
       "      <td>150</td>\n",
       "      <td>21.99</td>\n",
       "      <td>2028-01-12</td>\n",
       "      <td>2028-01-15</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1010</td>\n",
       "      <td>12.5</td>\n",
       "      <td>9</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>GA</td>\n",
       "      <td>GA</td>\n",
       "      <td>200</td>\n",
       "      <td>24.99</td>\n",
       "      <td>2028-01-12</td>\n",
       "      <td>2028-01-13</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PackageID  Length  Width  Height  Liquid  Perishable Origin Destination  \\\n",
       "0       1001    12.5      7     4.0       1           1     FL          FL   \n",
       "1       1002    12.5      7     5.0       0           0     FL          NC   \n",
       "2       1003    12.5      7     6.0       0           0     GA          NC   \n",
       "3       1004    12.5      8     5.0       1           0     SC          SC   \n",
       "4       1005    12.5      8     6.0       0           0     SC          FL   \n",
       "5       1006     NaN      8     4.0       0           0     GA          NC   \n",
       "6       1007    12.5      9     6.0       0           0     SC          GA   \n",
       "7       1008    12.5      9     5.0       0           0     SC          SC   \n",
       "8       1009     NaN      9     7.0       0           0     GA          NC   \n",
       "9       1010    12.5      9     6.0       0           0     GA          GA   \n",
       "\n",
       "   Insurance  ShipCost   ShipDate ArriveDate  Sequence  \n",
       "0        100     19.99 2028-01-06 2028-01-07      True  \n",
       "1        100     19.99 2028-01-08 2028-01-06     False  \n",
       "2        200     24.99 2028-01-07 2028-01-09      True  \n",
       "3        150     21.99 2028-01-08 2028-01-09      True  \n",
       "4      10009     19.99 2028-01-09 2028-01-11      True  \n",
       "5        150     21.99 2028-01-10 2028-01-15      True  \n",
       "6        100     19.99 2028-01-11 2028-01-14      True  \n",
       "7        100     19.99 2028-01-14 2028-01-11     False  \n",
       "8        150     21.99 2028-01-12 2028-01-15      True  \n",
       "9        200     24.99 2028-01-12 2028-01-13      True  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Generate a new column that will evaluate data order\n",
    "df['Sequence'] = df['ArriveDate'] > df['ShipDate']\n",
    "\n",
    "# Display data with new column\n",
    "df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Perform a logic check to see if any values are False\n",
    "(df['ArriveDate'] > df['ShipDate']).sum() == len(df)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "An alternate logic check, not shown in the book may rely on the new `Sequence` column.\n",
    "```Python\n",
    "(df['Sequence']).sum() == len(df)\n",
    "\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True     8\n",
       "False    2\n",
       "Name: Sequence, dtype: int64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Perform a check to count the number of True and False values\n",
    "df['Sequence'].value_counts()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.25"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Make array of integers equal to number of days in transit.\n",
    "days = (df['ArriveDate'] - df['ShipDate']).dt.days\n",
    "\n",
    "# Keep only those observations above zero.\n",
    "days = np.array([x for x in days if x > 0])\n",
    "\n",
    "# Find the mean\n",
    "days_mean = days.mean()\n",
    "\n",
    "# Display the result\n",
    "days_mean\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PackageID</th>\n",
       "      <th>Length</th>\n",
       "      <th>Width</th>\n",
       "      <th>Height</th>\n",
       "      <th>Liquid</th>\n",
       "      <th>Perishable</th>\n",
       "      <th>Origin</th>\n",
       "      <th>Destination</th>\n",
       "      <th>Insurance</th>\n",
       "      <th>ShipCost</th>\n",
       "      <th>ShipDate</th>\n",
       "      <th>ArriveDate</th>\n",
       "      <th>Sequence</th>\n",
       "      <th>Corrected</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1001</td>\n",
       "      <td>12.5</td>\n",
       "      <td>7</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>FL</td>\n",
       "      <td>FL</td>\n",
       "      <td>100</td>\n",
       "      <td>19.99</td>\n",
       "      <td>2028-01-06</td>\n",
       "      <td>2028-01-07</td>\n",
       "      <td>True</td>\n",
       "      <td>2028-01-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1002</td>\n",
       "      <td>12.5</td>\n",
       "      <td>7</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>FL</td>\n",
       "      <td>NC</td>\n",
       "      <td>100</td>\n",
       "      <td>19.99</td>\n",
       "      <td>2028-01-08</td>\n",
       "      <td>2028-01-06</td>\n",
       "      <td>False</td>\n",
       "      <td>2028-01-10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1003</td>\n",
       "      <td>12.5</td>\n",
       "      <td>7</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>GA</td>\n",
       "      <td>NC</td>\n",
       "      <td>200</td>\n",
       "      <td>24.99</td>\n",
       "      <td>2028-01-07</td>\n",
       "      <td>2028-01-09</td>\n",
       "      <td>True</td>\n",
       "      <td>2028-01-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1004</td>\n",
       "      <td>12.5</td>\n",
       "      <td>8</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>SC</td>\n",
       "      <td>SC</td>\n",
       "      <td>150</td>\n",
       "      <td>21.99</td>\n",
       "      <td>2028-01-08</td>\n",
       "      <td>2028-01-09</td>\n",
       "      <td>True</td>\n",
       "      <td>2028-01-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1005</td>\n",
       "      <td>12.5</td>\n",
       "      <td>8</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>SC</td>\n",
       "      <td>FL</td>\n",
       "      <td>10009</td>\n",
       "      <td>19.99</td>\n",
       "      <td>2028-01-09</td>\n",
       "      <td>2028-01-11</td>\n",
       "      <td>True</td>\n",
       "      <td>2028-01-11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1006</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>GA</td>\n",
       "      <td>NC</td>\n",
       "      <td>150</td>\n",
       "      <td>21.99</td>\n",
       "      <td>2028-01-10</td>\n",
       "      <td>2028-01-15</td>\n",
       "      <td>True</td>\n",
       "      <td>2028-01-15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1007</td>\n",
       "      <td>12.5</td>\n",
       "      <td>9</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>SC</td>\n",
       "      <td>GA</td>\n",
       "      <td>100</td>\n",
       "      <td>19.99</td>\n",
       "      <td>2028-01-11</td>\n",
       "      <td>2028-01-14</td>\n",
       "      <td>True</td>\n",
       "      <td>2028-01-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1008</td>\n",
       "      <td>12.5</td>\n",
       "      <td>9</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>SC</td>\n",
       "      <td>SC</td>\n",
       "      <td>100</td>\n",
       "      <td>19.99</td>\n",
       "      <td>2028-01-14</td>\n",
       "      <td>2028-01-11</td>\n",
       "      <td>False</td>\n",
       "      <td>2028-01-16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1009</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>GA</td>\n",
       "      <td>NC</td>\n",
       "      <td>150</td>\n",
       "      <td>21.99</td>\n",
       "      <td>2028-01-12</td>\n",
       "      <td>2028-01-15</td>\n",
       "      <td>True</td>\n",
       "      <td>2028-01-15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1010</td>\n",
       "      <td>12.5</td>\n",
       "      <td>9</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>GA</td>\n",
       "      <td>GA</td>\n",
       "      <td>200</td>\n",
       "      <td>24.99</td>\n",
       "      <td>2028-01-12</td>\n",
       "      <td>2028-01-13</td>\n",
       "      <td>True</td>\n",
       "      <td>2028-01-13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PackageID  Length  Width  Height  Liquid  Perishable Origin Destination  \\\n",
       "0       1001    12.5      7     4.0       1           1     FL          FL   \n",
       "1       1002    12.5      7     5.0       0           0     FL          NC   \n",
       "2       1003    12.5      7     6.0       0           0     GA          NC   \n",
       "3       1004    12.5      8     5.0       1           0     SC          SC   \n",
       "4       1005    12.5      8     6.0       0           0     SC          FL   \n",
       "5       1006     NaN      8     4.0       0           0     GA          NC   \n",
       "6       1007    12.5      9     6.0       0           0     SC          GA   \n",
       "7       1008    12.5      9     5.0       0           0     SC          SC   \n",
       "8       1009     NaN      9     7.0       0           0     GA          NC   \n",
       "9       1010    12.5      9     6.0       0           0     GA          GA   \n",
       "\n",
       "   Insurance  ShipCost   ShipDate ArriveDate  Sequence  Corrected  \n",
       "0        100     19.99 2028-01-06 2028-01-07      True 2028-01-07  \n",
       "1        100     19.99 2028-01-08 2028-01-06     False 2028-01-10  \n",
       "2        200     24.99 2028-01-07 2028-01-09      True 2028-01-09  \n",
       "3        150     21.99 2028-01-08 2028-01-09      True 2028-01-09  \n",
       "4      10009     19.99 2028-01-09 2028-01-11      True 2028-01-11  \n",
       "5        150     21.99 2028-01-10 2028-01-15      True 2028-01-15  \n",
       "6        100     19.99 2028-01-11 2028-01-14      True 2028-01-14  \n",
       "7        100     19.99 2028-01-14 2028-01-11     False 2028-01-16  \n",
       "8        150     21.99 2028-01-12 2028-01-15      True 2028-01-15  \n",
       "9        200     24.99 2028-01-12 2028-01-13      True 2028-01-13  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a column with an imputed correct arrival date.\n",
    "df['Corrected'] = np.where(df['Sequence'], \n",
    "                           df['ArriveDate'], \n",
    "                           df['ShipDate'] + \n",
    "                           pd.Timedelta(\n",
    "                               days=round(\n",
    "                                   int(days_mean))))\n",
    "\n",
    "# Display the data with the new corrected column\n",
    "df\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Final Results + Checking The Work"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PackageID</th>\n",
       "      <th>Length</th>\n",
       "      <th>Width</th>\n",
       "      <th>Height</th>\n",
       "      <th>Liquid</th>\n",
       "      <th>Perishable</th>\n",
       "      <th>Origin</th>\n",
       "      <th>Destination</th>\n",
       "      <th>Insurance</th>\n",
       "      <th>ShipCost</th>\n",
       "      <th>ShipDate</th>\n",
       "      <th>ArriveDate</th>\n",
       "      <th>Corrected</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1001</td>\n",
       "      <td>12.5</td>\n",
       "      <td>7</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>FL</td>\n",
       "      <td>FL</td>\n",
       "      <td>100</td>\n",
       "      <td>19.99</td>\n",
       "      <td>2028-01-06</td>\n",
       "      <td>2028-01-07</td>\n",
       "      <td>2028-01-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1002</td>\n",
       "      <td>12.5</td>\n",
       "      <td>7</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>FL</td>\n",
       "      <td>NC</td>\n",
       "      <td>100</td>\n",
       "      <td>19.99</td>\n",
       "      <td>2028-01-08</td>\n",
       "      <td>2028-01-06</td>\n",
       "      <td>2028-01-10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1003</td>\n",
       "      <td>12.5</td>\n",
       "      <td>7</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>GA</td>\n",
       "      <td>NC</td>\n",
       "      <td>200</td>\n",
       "      <td>24.99</td>\n",
       "      <td>2028-01-07</td>\n",
       "      <td>2028-01-09</td>\n",
       "      <td>2028-01-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1004</td>\n",
       "      <td>12.5</td>\n",
       "      <td>8</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>SC</td>\n",
       "      <td>SC</td>\n",
       "      <td>150</td>\n",
       "      <td>21.99</td>\n",
       "      <td>2028-01-08</td>\n",
       "      <td>2028-01-09</td>\n",
       "      <td>2028-01-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1005</td>\n",
       "      <td>12.5</td>\n",
       "      <td>8</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>SC</td>\n",
       "      <td>FL</td>\n",
       "      <td>100</td>\n",
       "      <td>19.99</td>\n",
       "      <td>2028-01-09</td>\n",
       "      <td>2028-01-11</td>\n",
       "      <td>2028-01-11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1006</td>\n",
       "      <td>12.5</td>\n",
       "      <td>8</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>GA</td>\n",
       "      <td>NC</td>\n",
       "      <td>150</td>\n",
       "      <td>21.99</td>\n",
       "      <td>2028-01-10</td>\n",
       "      <td>2028-01-15</td>\n",
       "      <td>2028-01-15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1007</td>\n",
       "      <td>12.5</td>\n",
       "      <td>9</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>SC</td>\n",
       "      <td>GA</td>\n",
       "      <td>100</td>\n",
       "      <td>19.99</td>\n",
       "      <td>2028-01-11</td>\n",
       "      <td>2028-01-14</td>\n",
       "      <td>2028-01-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1008</td>\n",
       "      <td>12.5</td>\n",
       "      <td>9</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>SC</td>\n",
       "      <td>SC</td>\n",
       "      <td>100</td>\n",
       "      <td>19.99</td>\n",
       "      <td>2028-01-14</td>\n",
       "      <td>2028-01-11</td>\n",
       "      <td>2028-01-16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1009</td>\n",
       "      <td>12.5</td>\n",
       "      <td>9</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>GA</td>\n",
       "      <td>NC</td>\n",
       "      <td>150</td>\n",
       "      <td>21.99</td>\n",
       "      <td>2028-01-12</td>\n",
       "      <td>2028-01-15</td>\n",
       "      <td>2028-01-15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1010</td>\n",
       "      <td>12.5</td>\n",
       "      <td>9</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>GA</td>\n",
       "      <td>GA</td>\n",
       "      <td>200</td>\n",
       "      <td>24.99</td>\n",
       "      <td>2028-01-12</td>\n",
       "      <td>2028-01-13</td>\n",
       "      <td>2028-01-13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PackageID  Length  Width  Height  Liquid  Perishable Origin Destination  \\\n",
       "0       1001    12.5      7     4.0       1           1     FL          FL   \n",
       "1       1002    12.5      7     5.0       0           0     FL          NC   \n",
       "2       1003    12.5      7     6.0       0           0     GA          NC   \n",
       "3       1004    12.5      8     5.0       1           0     SC          SC   \n",
       "4       1005    12.5      8     6.0       0           0     SC          FL   \n",
       "5       1006    12.5      8     4.0       0           0     GA          NC   \n",
       "6       1007    12.5      9     6.0       0           0     SC          GA   \n",
       "7       1008    12.5      9     5.0       0           0     SC          SC   \n",
       "8       1009    12.5      9     7.0       0           0     GA          NC   \n",
       "9       1010    12.5      9     6.0       0           0     GA          GA   \n",
       "\n",
       "   Insurance  ShipCost   ShipDate ArriveDate  Corrected  \n",
       "0        100     19.99 2028-01-06 2028-01-07 2028-01-07  \n",
       "1        100     19.99 2028-01-08 2028-01-06 2028-01-10  \n",
       "2        200     24.99 2028-01-07 2028-01-09 2028-01-09  \n",
       "3        150     21.99 2028-01-08 2028-01-09 2028-01-09  \n",
       "4        100     19.99 2028-01-09 2028-01-11 2028-01-11  \n",
       "5        150     21.99 2028-01-10 2028-01-15 2028-01-15  \n",
       "6        100     19.99 2028-01-11 2028-01-14 2028-01-14  \n",
       "7        100     19.99 2028-01-14 2028-01-11 2028-01-16  \n",
       "8        150     21.99 2028-01-12 2028-01-15 2028-01-15  \n",
       "9        200     24.99 2028-01-12 2028-01-13 2028-01-13  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Replace missing values in the Length column.\n",
    "df['Length'] = df['Length'].fillna(12.5)\n",
    "\n",
    "# Replace missing value in the Height column.\n",
    "df['Height'] = df['Height'].fillna(6.0)\n",
    "\n",
    "# Replace the erroneous value in the Insurance column.\n",
    "df['Insurance'] = [x if x < 2000 else 100 for x in df['Insurance']]\n",
    "\n",
    "# Remove duplicates from the DataFrame.\n",
    "df.drop_duplicates(inplace=True)\n",
    "\n",
    "# Correct the erroneous date entries.\n",
    "df['Corrected'] = np.where(df['Sequence'], \n",
    "                           df['ArriveDate'], \n",
    "                           df['ShipDate'] + \n",
    "                           pd.Timedelta(\n",
    "                               days=round(\n",
    "                                   int(days_mean))))\n",
    "\n",
    "# Remove the Sequence column\n",
    "df.drop('Sequence', axis=1, inplace=True)\n",
    "\n",
    "# Display the results of these manipulations\n",
    "df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check that the length updated correctly\n",
    "(df['Length'] == 12.5).sum() == len(df['Length'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True    10\n",
       "Name: Length, dtype: int64"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check that the length updated correctly\n",
    "(df['Length'] == 12.5).value_counts()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Width</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Height</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4.0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5.0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6.0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7.0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Width   7  8  9  All\n",
       "Height              \n",
       "4.0     1  1  0    2\n",
       "5.0     1  1  1    3\n",
       "6.0     1  1  2    4\n",
       "7.0     0  0  1    1\n",
       "All     3  3  4   10"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check Height manipulation performed as expected\n",
    "pd.crosstab(df['Height'].fillna(99), \n",
    "            df['Width'].fillna(99), \n",
    "            margins=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>ShipCost</th>\n",
       "      <th>19.99</th>\n",
       "      <th>21.99</th>\n",
       "      <th>24.99</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Insurance</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>200</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "ShipCost   19.99  21.99  24.99  All\n",
       "Insurance                          \n",
       "100            5      0      0    5\n",
       "150            0      3      0    3\n",
       "200            0      0      2    2\n",
       "All            5      3      2   10"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check the insurance column updated correctly\n",
    "pd.crosstab(df['Insurance'].fillna(99),\n",
    "            df['ShipCost'].fillna(99),\n",
    "            margins=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Again check that the updated ArrivalDate updated correctly\n",
    "(df['Corrected'] > df['ShipDate']).sum() == len(df)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Simplifying + Preparing For Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PackageID</th>\n",
       "      <th>Length</th>\n",
       "      <th>Width</th>\n",
       "      <th>Height</th>\n",
       "      <th>Liquid</th>\n",
       "      <th>Perishable</th>\n",
       "      <th>Insurance</th>\n",
       "      <th>ShipCost</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>PackageID</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.943042</td>\n",
       "      <td>0.531816</td>\n",
       "      <td>-0.522233</td>\n",
       "      <td>-0.522233</td>\n",
       "      <td>0.289749</td>\n",
       "      <td>0.273724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Length</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Width</th>\n",
       "      <td>0.943042</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.472866</td>\n",
       "      <td>-0.361158</td>\n",
       "      <td>-0.441415</td>\n",
       "      <td>0.046241</td>\n",
       "      <td>0.025240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Height</th>\n",
       "      <td>0.531816</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.472866</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.490990</td>\n",
       "      <td>-0.509175</td>\n",
       "      <td>0.307339</td>\n",
       "      <td>0.320256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Liquid</th>\n",
       "      <td>-0.522233</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.361158</td>\n",
       "      <td>-0.490990</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>-0.128037</td>\n",
       "      <td>-0.157243</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Perishable</th>\n",
       "      <td>-0.522233</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.441415</td>\n",
       "      <td>-0.509175</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.298753</td>\n",
       "      <td>-0.279543</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Insurance</th>\n",
       "      <td>0.289749</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.046241</td>\n",
       "      <td>0.307339</td>\n",
       "      <td>-0.128037</td>\n",
       "      <td>-0.298753</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.993221</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ShipCost</th>\n",
       "      <td>0.273724</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.025240</td>\n",
       "      <td>0.320256</td>\n",
       "      <td>-0.157243</td>\n",
       "      <td>-0.279543</td>\n",
       "      <td>0.993221</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            PackageID  Length     Width    Height    Liquid  Perishable  \\\n",
       "PackageID    1.000000     NaN  0.943042  0.531816 -0.522233   -0.522233   \n",
       "Length            NaN     NaN       NaN       NaN       NaN         NaN   \n",
       "Width        0.943042     NaN  1.000000  0.472866 -0.361158   -0.441415   \n",
       "Height       0.531816     NaN  0.472866  1.000000 -0.490990   -0.509175   \n",
       "Liquid      -0.522233     NaN -0.361158 -0.490990  1.000000    0.666667   \n",
       "Perishable  -0.522233     NaN -0.441415 -0.509175  0.666667    1.000000   \n",
       "Insurance    0.289749     NaN  0.046241  0.307339 -0.128037   -0.298753   \n",
       "ShipCost     0.273724     NaN  0.025240  0.320256 -0.157243   -0.279543   \n",
       "\n",
       "            Insurance  ShipCost  \n",
       "PackageID    0.289749  0.273724  \n",
       "Length            NaN       NaN  \n",
       "Width        0.046241  0.025240  \n",
       "Height       0.307339  0.320256  \n",
       "Liquid      -0.128037 -0.157243  \n",
       "Perishable  -0.298753 -0.279543  \n",
       "Insurance    1.000000  0.993221  \n",
       "ShipCost     0.993221  1.000000  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Review the correlation matrix post-maipulations\n",
    "df.corr(numeric_only=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PackageID</th>\n",
       "      <th>Liquid</th>\n",
       "      <th>Perishable</th>\n",
       "      <th>Origin</th>\n",
       "      <th>Destination</th>\n",
       "      <th>ShipCost</th>\n",
       "      <th>ShipDate</th>\n",
       "      <th>ArriveDate</th>\n",
       "      <th>Corrected</th>\n",
       "      <th>Volume</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1001</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>FL</td>\n",
       "      <td>FL</td>\n",
       "      <td>19.99</td>\n",
       "      <td>2028-01-06</td>\n",
       "      <td>2028-01-07</td>\n",
       "      <td>2028-01-07</td>\n",
       "      <td>350.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1002</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>FL</td>\n",
       "      <td>NC</td>\n",
       "      <td>19.99</td>\n",
       "      <td>2028-01-08</td>\n",
       "      <td>2028-01-10</td>\n",
       "      <td>2028-01-10</td>\n",
       "      <td>437.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1003</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>GA</td>\n",
       "      <td>NC</td>\n",
       "      <td>24.99</td>\n",
       "      <td>2028-01-07</td>\n",
       "      <td>2028-01-09</td>\n",
       "      <td>2028-01-09</td>\n",
       "      <td>525.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1004</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>SC</td>\n",
       "      <td>SC</td>\n",
       "      <td>21.99</td>\n",
       "      <td>2028-01-08</td>\n",
       "      <td>2028-01-09</td>\n",
       "      <td>2028-01-09</td>\n",
       "      <td>500.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1005</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>SC</td>\n",
       "      <td>FL</td>\n",
       "      <td>19.99</td>\n",
       "      <td>2028-01-09</td>\n",
       "      <td>2028-01-11</td>\n",
       "      <td>2028-01-11</td>\n",
       "      <td>600.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1006</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>GA</td>\n",
       "      <td>NC</td>\n",
       "      <td>21.99</td>\n",
       "      <td>2028-01-10</td>\n",
       "      <td>2028-01-15</td>\n",
       "      <td>2028-01-15</td>\n",
       "      <td>400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1007</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>SC</td>\n",
       "      <td>GA</td>\n",
       "      <td>19.99</td>\n",
       "      <td>2028-01-11</td>\n",
       "      <td>2028-01-14</td>\n",
       "      <td>2028-01-14</td>\n",
       "      <td>675.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1008</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>SC</td>\n",
       "      <td>SC</td>\n",
       "      <td>19.99</td>\n",
       "      <td>2028-01-14</td>\n",
       "      <td>2028-01-16</td>\n",
       "      <td>2028-01-16</td>\n",
       "      <td>562.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1009</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>GA</td>\n",
       "      <td>NC</td>\n",
       "      <td>21.99</td>\n",
       "      <td>2028-01-12</td>\n",
       "      <td>2028-01-15</td>\n",
       "      <td>2028-01-15</td>\n",
       "      <td>787.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1010</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>GA</td>\n",
       "      <td>GA</td>\n",
       "      <td>24.99</td>\n",
       "      <td>2028-01-12</td>\n",
       "      <td>2028-01-13</td>\n",
       "      <td>2028-01-13</td>\n",
       "      <td>675.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PackageID  Liquid  Perishable Origin Destination  ShipCost   ShipDate  \\\n",
       "0       1001       1           1     FL          FL     19.99 2028-01-06   \n",
       "1       1002       0           0     FL          NC     19.99 2028-01-08   \n",
       "2       1003       0           0     GA          NC     24.99 2028-01-07   \n",
       "3       1004       1           0     SC          SC     21.99 2028-01-08   \n",
       "4       1005       0           0     SC          FL     19.99 2028-01-09   \n",
       "5       1006       0           0     GA          NC     21.99 2028-01-10   \n",
       "6       1007       0           0     SC          GA     19.99 2028-01-11   \n",
       "7       1008       0           0     SC          SC     19.99 2028-01-14   \n",
       "8       1009       0           0     GA          NC     21.99 2028-01-12   \n",
       "9       1010       0           0     GA          GA     24.99 2028-01-12   \n",
       "\n",
       "  ArriveDate  Corrected  Volume  \n",
       "0 2028-01-07 2028-01-07   350.0  \n",
       "1 2028-01-10 2028-01-10   437.5  \n",
       "2 2028-01-09 2028-01-09   525.0  \n",
       "3 2028-01-09 2028-01-09   500.0  \n",
       "4 2028-01-11 2028-01-11   600.0  \n",
       "5 2028-01-15 2028-01-15   400.0  \n",
       "6 2028-01-14 2028-01-14   675.0  \n",
       "7 2028-01-16 2028-01-16   562.5  \n",
       "8 2028-01-15 2028-01-15   787.5  \n",
       "9 2028-01-13 2028-01-13   675.0  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove Insurance (highly correlated with ShipCost)\n",
    "df = df.drop(columns=['Insurance'])\n",
    "\n",
    "# Create a Volume colum\n",
    "df['Volume'] = df['Length'] * \\\n",
    "               df['Width'] *  \\\n",
    "               df['Height']\n",
    "\n",
    "# Remove Length, Width, and Height columns\n",
    "df = df.drop(columns=['Length', 'Width', 'Height'])\n",
    "\n",
    "# Replace ArriveData with the Corrected column\n",
    "df['ArriveDate'] = df['Corrected']\n",
    "\n",
    "# Review the revised and updated data\n",
    "df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Origin</th>\n",
       "      <th>Destination</th>\n",
       "      <th>Origin_FL</th>\n",
       "      <th>Origin_GA</th>\n",
       "      <th>Origin_SC</th>\n",
       "      <th>Destination_FL</th>\n",
       "      <th>Destination_GA</th>\n",
       "      <th>Destination_NC</th>\n",
       "      <th>Destination_SC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>FL</td>\n",
       "      <td>FL</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>FL</td>\n",
       "      <td>NC</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>GA</td>\n",
       "      <td>NC</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>SC</td>\n",
       "      <td>SC</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>SC</td>\n",
       "      <td>FL</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>GA</td>\n",
       "      <td>NC</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>SC</td>\n",
       "      <td>GA</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>SC</td>\n",
       "      <td>SC</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>GA</td>\n",
       "      <td>NC</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>GA</td>\n",
       "      <td>GA</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Origin Destination  Origin_FL  Origin_GA  Origin_SC  Destination_FL  \\\n",
       "0     FL          FL          1          0          0               1   \n",
       "1     FL          NC          1          0          0               0   \n",
       "2     GA          NC          0          1          0               0   \n",
       "3     SC          SC          0          0          1               0   \n",
       "4     SC          FL          0          0          1               1   \n",
       "5     GA          NC          0          1          0               0   \n",
       "6     SC          GA          0          0          1               0   \n",
       "7     SC          SC          0          0          1               0   \n",
       "8     GA          NC          0          1          0               0   \n",
       "9     GA          GA          0          1          0               0   \n",
       "\n",
       "   Destination_GA  Destination_NC  Destination_SC  \n",
       "0               0               0               0  \n",
       "1               0               1               0  \n",
       "2               0               1               0  \n",
       "3               0               0               1  \n",
       "4               0               0               0  \n",
       "5               0               1               0  \n",
       "6               1               0               0  \n",
       "7               0               0               1  \n",
       "8               0               1               0  \n",
       "9               1               0               0  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Display dummy array (one-hot-encoding works)\n",
    "pd.concat([\n",
    "    df[['Origin','Destination']],\n",
    "    pd.get_dummies(df[['Origin','Destination']])], \n",
    "    axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PackageID</th>\n",
       "      <th>Liquid</th>\n",
       "      <th>Perishable</th>\n",
       "      <th>ShipCost</th>\n",
       "      <th>ShipDate</th>\n",
       "      <th>ArriveDate</th>\n",
       "      <th>Corrected</th>\n",
       "      <th>Volume</th>\n",
       "      <th>Origin_GA</th>\n",
       "      <th>Origin_SC</th>\n",
       "      <th>Destination_GA</th>\n",
       "      <th>Destination_NC</th>\n",
       "      <th>Destination_SC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1001</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>19.99</td>\n",
       "      <td>2028-01-06</td>\n",
       "      <td>2028-01-07</td>\n",
       "      <td>2028-01-07</td>\n",
       "      <td>350.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1002</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>19.99</td>\n",
       "      <td>2028-01-08</td>\n",
       "      <td>2028-01-10</td>\n",
       "      <td>2028-01-10</td>\n",
       "      <td>437.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1003</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>24.99</td>\n",
       "      <td>2028-01-07</td>\n",
       "      <td>2028-01-09</td>\n",
       "      <td>2028-01-09</td>\n",
       "      <td>525.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1004</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>21.99</td>\n",
       "      <td>2028-01-08</td>\n",
       "      <td>2028-01-09</td>\n",
       "      <td>2028-01-09</td>\n",
       "      <td>500.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1005</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>19.99</td>\n",
       "      <td>2028-01-09</td>\n",
       "      <td>2028-01-11</td>\n",
       "      <td>2028-01-11</td>\n",
       "      <td>600.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1006</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>21.99</td>\n",
       "      <td>2028-01-10</td>\n",
       "      <td>2028-01-15</td>\n",
       "      <td>2028-01-15</td>\n",
       "      <td>400.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1007</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>19.99</td>\n",
       "      <td>2028-01-11</td>\n",
       "      <td>2028-01-14</td>\n",
       "      <td>2028-01-14</td>\n",
       "      <td>675.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1008</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>19.99</td>\n",
       "      <td>2028-01-14</td>\n",
       "      <td>2028-01-16</td>\n",
       "      <td>2028-01-16</td>\n",
       "      <td>562.5</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1009</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>21.99</td>\n",
       "      <td>2028-01-12</td>\n",
       "      <td>2028-01-15</td>\n",
       "      <td>2028-01-15</td>\n",
       "      <td>787.5</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1010</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>24.99</td>\n",
       "      <td>2028-01-12</td>\n",
       "      <td>2028-01-13</td>\n",
       "      <td>2028-01-13</td>\n",
       "      <td>675.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PackageID  Liquid  Perishable  ShipCost   ShipDate ArriveDate  Corrected  \\\n",
       "0       1001       1           1     19.99 2028-01-06 2028-01-07 2028-01-07   \n",
       "1       1002       0           0     19.99 2028-01-08 2028-01-10 2028-01-10   \n",
       "2       1003       0           0     24.99 2028-01-07 2028-01-09 2028-01-09   \n",
       "3       1004       1           0     21.99 2028-01-08 2028-01-09 2028-01-09   \n",
       "4       1005       0           0     19.99 2028-01-09 2028-01-11 2028-01-11   \n",
       "5       1006       0           0     21.99 2028-01-10 2028-01-15 2028-01-15   \n",
       "6       1007       0           0     19.99 2028-01-11 2028-01-14 2028-01-14   \n",
       "7       1008       0           0     19.99 2028-01-14 2028-01-16 2028-01-16   \n",
       "8       1009       0           0     21.99 2028-01-12 2028-01-15 2028-01-15   \n",
       "9       1010       0           0     24.99 2028-01-12 2028-01-13 2028-01-13   \n",
       "\n",
       "   Volume  Origin_GA  Origin_SC  Destination_GA  Destination_NC  \\\n",
       "0   350.0          0          0               0               0   \n",
       "1   437.5          0          0               0               1   \n",
       "2   525.0          1          0               0               1   \n",
       "3   500.0          0          1               0               0   \n",
       "4   600.0          0          1               0               0   \n",
       "5   400.0          1          0               0               1   \n",
       "6   675.0          0          1               1               0   \n",
       "7   562.5          0          1               0               0   \n",
       "8   787.5          1          0               0               1   \n",
       "9   675.0          1          0               1               0   \n",
       "\n",
       "   Destination_SC  \n",
       "0               0  \n",
       "1               0  \n",
       "2               0  \n",
       "3               1  \n",
       "4               0  \n",
       "5               0  \n",
       "6               0  \n",
       "7               1  \n",
       "8               0  \n",
       "9               0  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Generate dummy arras and join with original\n",
    "df = pd.concat([\n",
    "     df.drop(columns=['Origin','Destination']),\n",
    "     pd.get_dummies(df[['Origin','Destination']])], \n",
    "     axis=1).drop(columns=['Origin_FL',\n",
    "                          'Destination_FL'])\n",
    "\n",
    "# Evaluate the results\n",
    "df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PackageID</th>\n",
       "      <th>Liquid</th>\n",
       "      <th>Perishable</th>\n",
       "      <th>ShipCost</th>\n",
       "      <th>Volume</th>\n",
       "      <th>Origin_GA</th>\n",
       "      <th>Origin_SC</th>\n",
       "      <th>Destination_GA</th>\n",
       "      <th>Destination_NC</th>\n",
       "      <th>Destination_SC</th>\n",
       "      <th>TransitTime</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1001</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>19.99</td>\n",
       "      <td>350.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1 days</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1002</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>19.99</td>\n",
       "      <td>437.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2 days</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1003</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>24.99</td>\n",
       "      <td>525.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2 days</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1004</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>21.99</td>\n",
       "      <td>500.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1 days</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1005</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>19.99</td>\n",
       "      <td>600.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2 days</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1006</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>21.99</td>\n",
       "      <td>400.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>5 days</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1007</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>19.99</td>\n",
       "      <td>675.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3 days</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1008</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>19.99</td>\n",
       "      <td>562.5</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2 days</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1009</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>21.99</td>\n",
       "      <td>787.5</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3 days</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1010</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>24.99</td>\n",
       "      <td>675.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1 days</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PackageID  Liquid  Perishable  ShipCost  Volume  Origin_GA  Origin_SC  \\\n",
       "0       1001       1           1     19.99   350.0          0          0   \n",
       "1       1002       0           0     19.99   437.5          0          0   \n",
       "2       1003       0           0     24.99   525.0          1          0   \n",
       "3       1004       1           0     21.99   500.0          0          1   \n",
       "4       1005       0           0     19.99   600.0          0          1   \n",
       "5       1006       0           0     21.99   400.0          1          0   \n",
       "6       1007       0           0     19.99   675.0          0          1   \n",
       "7       1008       0           0     19.99   562.5          0          1   \n",
       "8       1009       0           0     21.99   787.5          1          0   \n",
       "9       1010       0           0     24.99   675.0          1          0   \n",
       "\n",
       "   Destination_GA  Destination_NC  Destination_SC TransitTime  \n",
       "0               0               0               0      1 days  \n",
       "1               0               1               0      2 days  \n",
       "2               0               1               0      2 days  \n",
       "3               0               0               1      1 days  \n",
       "4               0               0               0      2 days  \n",
       "5               0               1               0      5 days  \n",
       "6               1               0               0      3 days  \n",
       "7               0               0               1      2 days  \n",
       "8               0               1               0      3 days  \n",
       "9               1               0               0      1 days  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Add column that represents transit time\n",
    "df['TransitTime'] = df['Corrected'] - df['ShipDate']\n",
    "\n",
    "# Drop ShipDate, ArrivalDate, and Corrected columns\n",
    "df = df.drop(['ShipDate','ArriveDate', \n",
    "              'Corrected'], axis=1)\n",
    "\n",
    "# Evaluate the results\n",
    "df\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Update + Change Log\n",
    "\n",
    "March, 2023 - Created notebook.<br>\n",
    "April, 2023 - Prepared for GitHub.<br>\n",
    "October, 2023 - Revised to match published book."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}