ML zoomcamp week 2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "08ea4643",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2fc7f5cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# New York City Airbnb Open Data\n",
    "data = pd.read_csv(\"https://raw.githubusercontent.com/alexeygrigorev/datasets/master/AB_NYC_2019.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5dcc7364",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>name</th>\n",
       "      <th>host_id</th>\n",
       "      <th>host_name</th>\n",
       "      <th>neighbourhood_group</th>\n",
       "      <th>neighbourhood</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>room_type</th>\n",
       "      <th>price</th>\n",
       "      <th>minimum_nights</th>\n",
       "      <th>number_of_reviews</th>\n",
       "      <th>last_review</th>\n",
       "      <th>reviews_per_month</th>\n",
       "      <th>calculated_host_listings_count</th>\n",
       "      <th>availability_365</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2539</td>\n",
       "      <td>Clean &amp; quiet apt home by the park</td>\n",
       "      <td>2787</td>\n",
       "      <td>John</td>\n",
       "      <td>Brooklyn</td>\n",
       "      <td>Kensington</td>\n",
       "      <td>40.64749</td>\n",
       "      <td>-73.97237</td>\n",
       "      <td>Private room</td>\n",
       "      <td>149</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>2018-10-19</td>\n",
       "      <td>0.21</td>\n",
       "      <td>6</td>\n",
       "      <td>365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2595</td>\n",
       "      <td>Skylit Midtown Castle</td>\n",
       "      <td>2845</td>\n",
       "      <td>Jennifer</td>\n",
       "      <td>Manhattan</td>\n",
       "      <td>Midtown</td>\n",
       "      <td>40.75362</td>\n",
       "      <td>-73.98377</td>\n",
       "      <td>Entire home/apt</td>\n",
       "      <td>225</td>\n",
       "      <td>1</td>\n",
       "      <td>45</td>\n",
       "      <td>2019-05-21</td>\n",
       "      <td>0.38</td>\n",
       "      <td>2</td>\n",
       "      <td>355</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3647</td>\n",
       "      <td>THE VILLAGE OF HARLEM....NEW YORK !</td>\n",
       "      <td>4632</td>\n",
       "      <td>Elisabeth</td>\n",
       "      <td>Manhattan</td>\n",
       "      <td>Harlem</td>\n",
       "      <td>40.80902</td>\n",
       "      <td>-73.94190</td>\n",
       "      <td>Private room</td>\n",
       "      <td>150</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3831</td>\n",
       "      <td>Cozy Entire Floor of Brownstone</td>\n",
       "      <td>4869</td>\n",
       "      <td>LisaRoxanne</td>\n",
       "      <td>Brooklyn</td>\n",
       "      <td>Clinton Hill</td>\n",
       "      <td>40.68514</td>\n",
       "      <td>-73.95976</td>\n",
       "      <td>Entire home/apt</td>\n",
       "      <td>89</td>\n",
       "      <td>1</td>\n",
       "      <td>270</td>\n",
       "      <td>2019-07-05</td>\n",
       "      <td>4.64</td>\n",
       "      <td>1</td>\n",
       "      <td>194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5022</td>\n",
       "      <td>Entire Apt: Spacious Studio/Loft by central park</td>\n",
       "      <td>7192</td>\n",
       "      <td>Laura</td>\n",
       "      <td>Manhattan</td>\n",
       "      <td>East Harlem</td>\n",
       "      <td>40.79851</td>\n",
       "      <td>-73.94399</td>\n",
       "      <td>Entire home/apt</td>\n",
       "      <td>80</td>\n",
       "      <td>10</td>\n",
       "      <td>9</td>\n",
       "      <td>2018-11-19</td>\n",
       "      <td>0.10</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     id                                              name  host_id  \\\n",
       "0  2539                Clean & quiet apt home by the park     2787   \n",
       "1  2595                             Skylit Midtown Castle     2845   \n",
       "2  3647               THE VILLAGE OF HARLEM....NEW YORK !     4632   \n",
       "3  3831                   Cozy Entire Floor of Brownstone     4869   \n",
       "4  5022  Entire Apt: Spacious Studio/Loft by central park     7192   \n",
       "\n",
       "     host_name neighbourhood_group neighbourhood  latitude  longitude  \\\n",
       "0         John            Brooklyn    Kensington  40.64749  -73.97237   \n",
       "1     Jennifer           Manhattan       Midtown  40.75362  -73.98377   \n",
       "2    Elisabeth           Manhattan        Harlem  40.80902  -73.94190   \n",
       "3  LisaRoxanne            Brooklyn  Clinton Hill  40.68514  -73.95976   \n",
       "4        Laura           Manhattan   East Harlem  40.79851  -73.94399   \n",
       "\n",
       "         room_type  price  minimum_nights  number_of_reviews last_review  \\\n",
       "0     Private room    149               1                  9  2018-10-19   \n",
       "1  Entire home/apt    225               1                 45  2019-05-21   \n",
       "2     Private room    150               3                  0         NaN   \n",
       "3  Entire home/apt     89               1                270  2019-07-05   \n",
       "4  Entire home/apt     80              10                  9  2018-11-19   \n",
       "\n",
       "   reviews_per_month  calculated_host_listings_count  availability_365  \n",
       "0               0.21                               6               365  \n",
       "1               0.38                               2               355  \n",
       "2                NaN                               1               365  \n",
       "3               4.64                               1               194  \n",
       "4               0.10                               1                 0  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d715a622",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id\n",
      "[2539 2595 3647 3831 5022]\n",
      "48895\n",
      "\n",
      "name\n",
      "['Clean & quiet apt home by the park' 'Skylit Midtown Castle'\n",
      " 'THE VILLAGE OF HARLEM....NEW YORK !' 'Cozy Entire Floor of Brownstone'\n",
      " 'Entire Apt: Spacious Studio/Loft by central park']\n",
      "47905\n",
      "\n",
      "host_id\n",
      "[2787 2845 4632 4869 7192]\n",
      "37457\n",
      "\n",
      "host_name\n",
      "['John' 'Jennifer' 'Elisabeth' 'LisaRoxanne' 'Laura']\n",
      "11452\n",
      "\n",
      "neighbourhood_group\n",
      "['Brooklyn' 'Manhattan' 'Queens' 'Staten Island' 'Bronx']\n",
      "5\n",
      "\n",
      "neighbourhood\n",
      "['Kensington' 'Midtown' 'Harlem' 'Clinton Hill' 'East Harlem']\n",
      "221\n",
      "\n",
      "latitude\n",
      "[40.64749 40.75362 40.80902 40.68514 40.79851]\n",
      "19048\n",
      "\n",
      "longitude\n",
      "[-73.97237 -73.98377 -73.9419  -73.95976 -73.94399]\n",
      "14718\n",
      "\n",
      "room_type\n",
      "['Private room' 'Entire home/apt' 'Shared room']\n",
      "3\n",
      "\n",
      "price\n",
      "[149 225 150  89  80]\n",
      "674\n",
      "\n",
      "minimum_nights\n",
      "[ 1  3 10 45  2]\n",
      "109\n",
      "\n",
      "number_of_reviews\n",
      "[  9  45   0 270  74]\n",
      "394\n",
      "\n",
      "last_review\n",
      "['2018-10-19' '2019-05-21' nan '2019-07-05' '2018-11-19']\n",
      "1764\n",
      "\n",
      "reviews_per_month\n",
      "[0.21 0.38  nan 4.64 0.1 ]\n",
      "937\n",
      "\n",
      "calculated_host_listings_count\n",
      "[6 2 1 4 3]\n",
      "47\n",
      "\n",
      "availability_365\n",
      "[365 355 194   0 129]\n",
      "366\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for col in data.columns:\n",
    "    print(col)\n",
    "    print(data[col].unique()[:5])\n",
    "    print(data[col].nunique())\n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6bdda66c",
   "metadata": {},
   "source": [
    "- Choose columns to input in lin reg\n",
    "    - categorical: neighbourhood_group, neighbourhood, room_type, (last_review: today - last_review)\n",
    "    - variable: number_of_reviews, minimum_nights, reviews_per_month, (availability_365)\n",
    "- Output variable: price"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5faee4a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6f58ef69",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='price', ylabel='Count'>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEGCAYAAABPdROvAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAaoUlEQVR4nO3df7Bf9V3n8eerSUvRFsqPwMSEbmiJnQKzDXKbTa27U5uuZFlH6A5oOtuSnYmmi1RbdXSIzqx1ZzIjuyoddKFG6BJoLURsl8iClYWq01026UUpEELkKlhuyZJUkKK7YJO+94/v59pvbr65uc2533tzk+dj5sz3fN/nfM79fKDtq+d8zvecVBWSJB2t18x1ByRJ85tBIknqxCCRJHVikEiSOjFIJEmdLJzrDsy2M888s5YtWzbX3ZCkeeXhhx/+elUtGrTthAuSZcuWMTo6OtfdkKR5JclfH26bl7YkSZ0MPUiSLEjy50nuad9PT3J/kqfa52l9+25MMpZkd5JL+uoXJ3msbbshSVr9pCR3tvr2JMuGPR5J0sFm44zko8Cuvu/XAg9U1XLggfadJOcDa4ELgDXAjUkWtDY3ARuA5W1Z0+rrgRer6jzgeuC64Q5FkjTZUIMkyVLgXwM395UvA7a09S3A5X31O6rq1ap6GhgDViZZDJxSVQ9V73kut01qM3Gsu4DVE2crkqTZMewzkk8AvwB8q692dlXtAWifZ7X6EuDZvv3GW21JW59cP6hNVe0HXgLOmNyJJBuSjCYZ3bdvX8chSZL6DS1IkvwwsLeqHp5ukwG1mqI+VZuDC1Wbq2qkqkYWLRp495ok6SgN8/bfdwM/kuRS4PXAKUk+DTyfZHFV7WmXrfa2/ceBc/raLwWea/WlA+r9bcaTLAROBV4Y1oAkSYca2hlJVW2sqqVVtYzeJPqDVfVBYBuwru22Dri7rW8D1rY7sc6lN6m+o13+ejnJqjb/cdWkNhPHuqL9DZ+LL0mzaC5+kPirwNYk64GvAlcCVNXOJFuBJ4D9wDVVdaC1uRq4FTgZuK8tALcAtycZo3cmsna2BiFJ6smJ9n/gR0ZG6mh/2f7BH//37Pn6S4fUF595Kp+++ZNduyZJx6wkD1fVyKBtJ9wjUrrY8/WXWHTpTx9av/eGOeiNJB0bfESKJKkTg0SS1IlBIknqxCCRJHVikEiSOjFIJEmdGCSSpE4MEklSJwaJJKkTg0SS1IlBIknqxCCRJHVikEiSOjFIJEmdGCSSpE4MEklSJ0MLkiSvT7IjyVeS7EzyK63+8SRfS/JIWy7ta7MxyViS3Uku6atfnOSxtu2G9u522vvd72z17UmWDWs8kqTBhnlG8irw3qp6B7ACWJNkVdt2fVWtaMu9AEnOp/fO9QuANcCNSRa0/W8CNgDL27Km1dcDL1bVecD1wHVDHI8kaYChBUn1/F37+tq2TPWC+MuAO6rq1ap6GhgDViZZDJxSVQ9V7wXztwGX97XZ0tbvAlZPnK1IkmbHUOdIkixI8giwF7i/qra3TR9J8miSTyU5rdWWAM/2NR9vtSVtfXL9oDZVtR94CThjQD82JBlNMrpv376ZGZwkCRhykFTVgapaASyld3ZxIb3LVG+ld7lrD/DrbfdBZxI1RX2qNpP7sbmqRqpqZNGiRd/RGCRJU5uVu7aq6m+BPwbWVNXzLWC+BfwOsLLtNg6c09dsKfBcqy8dUD+oTZKFwKnAC8MZhSRpkGHetbUoyZva+snA+4An25zHhPcDj7f1bcDadifWufQm1XdU1R7g5SSr2vzHVcDdfW3WtfUrgAfbPIokaZYsHOKxFwNb2p1XrwG2VtU9SW5PsoLeJahngA8DVNXOJFuBJ4D9wDVVdaAd62rgVuBk4L62ANwC3J5kjN6ZyNohjkeSNMDQgqSqHgUuGlD/0BRtNgGbBtRHgQsH1F8BruzWU0lSF/6yXZLUiUEiSerEIJEkdWKQSJI6MUgkSZ0YJJKkTgwSSVInBokkqRODRJLUiUEiSerEIJEkdWKQSJI6MUgkSZ0YJJKkTgwSSVInBokkqRODRJLUiUEiSepkaEGS5PVJdiT5SpKdSX6l1U9Pcn+Sp9rnaX1tNiYZS7I7ySV99YuTPNa23ZAkrX5SkjtbfXuSZcMajyRpsGGekbwKvLeq3gGsANYkWQVcCzxQVcuBB9p3kpwPrAUuANYANyZZ0I51E7ABWN6WNa2+Hnixqs4DrgeuG+J4JEkDDC1Iqufv2tfXtqWAy4Atrb4FuLytXwbcUVWvVtXTwBiwMsli4JSqeqiqCrhtUpuJY90FrJ44W5EkzY6hzpEkWZDkEWAvcH9VbQfOrqo9AO3zrLb7EuDZvubjrbakrU+uH9SmqvYDLwFnDOjHhiSjSUb37ds3Q6OTJMGQg6SqDlTVCmApvbOLC6fYfdCZRE1Rn6rN5H5srqqRqhpZtGjREXotSfpOzMpdW1X1t8Af05vbeL5drqJ97m27jQPn9DVbCjzX6ksH1A9qk2QhcCrwwjDGIEkabJh3bS1K8qa2fjLwPuBJYBuwru22Dri7rW8D1rY7sc6lN6m+o13+ejnJqjb/cdWkNhPHugJ4sM2jSJJmycIhHnsxsKXdefUaYGtV3ZPkIWBrkvXAV4ErAapqZ5KtwBPAfuCaqjrQjnU1cCtwMnBfWwBuAW5PMkbvTGTtEMcjSRpgaEFSVY8CFw2o/w2w+jBtNgGbBtRHgUPmV6rqFVoQSZLmhr9slyR1YpBIkjoxSCRJnRgkkqRODBJJUicGiSSpE4NEktSJQSJJ6sQgkSR1YpBIkjoxSCRJnRgkkqRODBJJUicGiSSpE4NEktSJQSJJ6sQgkSR1Msx3tp+T5ItJdiXZmeSjrf7xJF9L8khbLu1rszHJWJLdSS7pq1+c5LG27Yb27nba+93vbPXtSZYNazySpMGGeUayH/i5qno7sAq4Jsn5bdv1VbWiLfcCtG1rgQuANcCN7X3vADcBG4DlbVnT6uuBF6vqPOB64LohjkeSNMDQgqSq9lTVn7X1l4FdwJIpmlwG3FFVr1bV08AYsDLJYuCUqnqoqgq4Dbi8r82Wtn4XsHribEWSNDtmZY6kXXK6CNjeSh9J8miSTyU5rdWWAM/2NRtvtSVtfXL9oDZVtR94CThjwN/fkGQ0yei+fftmZlCSJGAWgiTJG4DfBz5WVd+gd5nqrcAKYA/w6xO7DmheU9SnanNwoWpzVY1U1ciiRYu+swFIkqY01CBJ8lp6IfKZqvocQFU9X1UHqupbwO8AK9vu48A5fc2XAs+1+tIB9YPaJFkInAq8MJzRSJIGGeZdWwFuAXZV1W/01Rf37fZ+4PG2vg1Y2+7EOpfepPqOqtoDvJxkVTvmVcDdfW3WtfUrgAfbPIokaZYsHOKx3w18CHgsySOt9ovAB5KsoHcJ6hngwwBVtTPJVuAJend8XVNVB1q7q4FbgZOB+9oCvaC6PckYvTORtUMcjyRpgKEFSVV9icFzGPdO0WYTsGlAfRS4cED9FeDKDt2UJHXkL9slSZ0YJJKkTgwSSVInBokkqRODRJLUiUEiSepkWkGS5N3TqUmSTjzTPSP5zWnWJEknmCl/kJjkXcD3A4uS/GzfplOABYNbSZJOJEf6ZfvrgDe0/d7YV/8GvWdbSZJOcFMGSVX9CfAnSW6tqr+epT5JkuaR6T5r66Qkm4Fl/W2q6r3D6JQkaf6YbpD8HvBJ4GbgwBH2lSSdQKYbJPur6qah9kSSNC9N9/bfP0jyk0kWJzl9YhlqzyRJ88J0z0gm3kL48321At4ys92RJM030wqSqjp32B2RJM1P0wqSJFcNqlfVbTPbHUnSfDPdOZJ39i3/HPg48CNTNUhyTpIvJtmVZGeSj7b66UnuT/JU+zytr83GJGNJdie5pK9+cZLH2rYbkqTVT0pyZ6tvT7LsOxm8JKm7aQVJVf1U3/ITwEX0fvU+lf3Az1XV24FVwDVJzgeuBR6oquXAA+07bdta4AJgDXBjkonHsNwEbACWt2VNq68HXqyq84DrgeumMx5J0sw52sfI/196/4N+WFW1p6r+rK2/DOwClgCXAVvabluAy9v6ZcAdVfVqVT0NjAErkywGTqmqh6qqgNsmtZk41l3A6omzFUnS7JjuHMkf0LtLC3oPa3w7sHW6f6RdcroI2A6cXVV7oBc2Sc5quy0B/ndfs/FW+2Zbn1yfaPNsO9b+JC8BZwBfn/T3N9A7o+HNb37zdLstSZqG6d7++2t96/uBv66q8cPt3C/JG4DfBz5WVd+Y4oRh0Iaaoj5Vm4MLVZuBzQAjIyOHbJckHb3pzpH8CfAkvScAnwb8w3TaJXktvRD5TFV9rpWfb5eraJ97W30cOKev+VLguVZfOqB+UJskC4FTgRem0zdJ0syY7hsSfxTYAVwJ/CiwPcmUj5FvcxW3ALuq6jf6Nm3j2z9wXAfc3Vdf2+7EOpfeHMyOdhns5SSr2jGvmtRm4lhXAA+2eRRJ0iyZ7qWtXwLeWVV7AZIsAv4HvQnuw3k38CHgsSSPtNovAr8KbE2yHvgqvXCiqnYm2Qo8Qe/y2TVVNfGAyKuBW4GTgfvaAr2guj3JGL0zkbXTHI8kaYZMN0heMxEizd9whLOZqvoSg+cwAFYfps0mYNOA+ihw4YD6K7QgkiTNjekGyR8m+QLw2fb9x4B7h9MlSdJ8cqR3tp9H73bdn0/yb4AfoHeW8RDwmVnonyTpGHekyfZPAC8DVNXnqupnq+pn6J2NfGK4XZMkzQdHCpJlVfXo5GKbs1g2lB5JkuaVIwXJ66fYdvJMdkSSND8dKUi+nOQnJhfbrbsPD6dLkqT55Eh3bX0M+HySf8u3g2OE3pN/3z/EfkmS5okpg6Sqnge+P8kP8u3fcfz3qnpw6D2TJM0L033V7heBLw65L5Kkeeho30ciSRJgkEiSOjJIJEmdGCSSpE4MEklSJwaJJKkTg0SS1IlBIknqZGhBkuRTSfYmebyv9vEkX0vySFsu7du2MclYkt1JLumrX5zksbbthvbedtq73e9s9e1Jlg1rLJKkwxvmGcmtwJoB9eurakVb7gVIcj69961f0NrcmGRB2/8mYAOwvC0Tx1wPvFhV5wHXA9cNayCSpMMbWpBU1Z8CL0xz98uAO6rq1ap6GhgDViZZDJxSVQ9VVQG3AZf3tdnS1u8CVk+crUiSZs9czJF8JMmj7dLXaa22BHi2b5/xVlvS1ifXD2pTVfuBl4AzBv3BJBuSjCYZ3bdv38yNRJI060FyE/BWYAWwB/j1Vh90JlFT1Kdqc2ixanNVjVTVyKJFi76jDkuSpjarQVJVz1fVgar6FvA7wMq2aRw4p2/XpcBzrb50QP2gNkkWAqcy/UtpkqQZMqtB0uY8JrwfmLijaxuwtt2JdS69SfUdVbUHeDnJqjb/cRVwd1+bdW39CuDBNo8iSZpF03ofydFI8lngPcCZScaBXwbek2QFvUtQzwAfBqiqnUm2Ak8A+4FrqupAO9TV9O4AOxm4ry0AtwC3JxmjdyaydlhjkSQd3tCCpKo+MKB8yxT7bwI2DaiP8u23M/bXXwGu7NJHSVJ3/rJdktSJQSJJ6sQgkSR1YpBIkjoxSCRJnRgkkqRODBJJUicGiSSpE4NEktSJQSJJ6sQgkSR1YpBIkjoxSCRJnRgkkqRODBJJUicGiSSpE4NEktSJQSJJ6mRoQZLkU0n2Jnm8r3Z6kvuTPNU+T+vbtjHJWJLdSS7pq1+c5LG27YYkafWTktzZ6tuTLBvWWCRJhzfMM5JbgTWTatcCD1TVcuCB9p0k5wNrgQtamxuTLGhtbgI2AMvbMnHM9cCLVXUecD1w3dBGIkk6rKEFSVX9KfDCpPJlwJa2vgW4vK9+R1W9WlVPA2PAyiSLgVOq6qGqKuC2SW0mjnUXsHribEWSNHtme47k7KraA9A+z2r1JcCzffuNt9qStj65flCbqtoPvAScMeiPJtmQZDTJ6L59+2ZoKJIkOHYm2wedSdQU9anaHFqs2lxVI1U1smjRoqPsoiRpkNkOkufb5Sra595WHwfO6dtvKfBcqy8dUD+oTZKFwKkceilNkjRksx0k24B1bX0dcHdffW27E+tcepPqO9rlr5eTrGrzH1dNajNxrCuAB9s8iiRpFi0c1oGTfBZ4D3BmknHgl4FfBbYmWQ98FbgSoKp2JtkKPAHsB66pqgPtUFfTuwPsZOC+tgDcAtyeZIzemcjaYY1FknR4QwuSqvrAYTatPsz+m4BNA+qjwIUD6q/QgkiSNHeOlcl2SdI8ZZBIkjoxSCRJnRgkkqRODBJJUicGiSSpk6Hd/nsi2fXETlZffujdzovPPJVP3/zJOeiRJM0eg2QGfLNew6JLf/qQ+p57b5iD3kjS7PLSliSpE4NEktSJQSJJ6sQgkSR1YpBIkjoxSCRJnRgkkqRODBJJUicGiSSpkzkJkiTPJHksySNJRlvt9CT3J3mqfZ7Wt//GJGNJdie5pK9+cTvOWJIb2nvdJUmzaC7PSH6wqlZU1Uj7fi3wQFUtBx5o30lyPr33sV8ArAFuTLKgtbkJ2AAsb8uaWey/JIlj69LWZcCWtr4FuLyvfkdVvVpVTwNjwMoki4FTquqhqirgtr42kqRZMldBUsAfJXk4yYZWO7uq9gC0z7NafQnwbF/b8VZb0tYn1w+RZEOS0SSj+/btm8FhSJLm6um/766q55KcBdyf5Mkp9h0071FT1A8tVm0GNgOMjIwM3EeSdHTm5Iykqp5rn3uBzwMrgefb5Sra5962+zhwTl/zpcBzrb50QF2SNItmPUiSfHeSN06sAz8EPA5sA9a13dYBd7f1bcDaJCclOZfepPqOdvnr5SSr2t1aV/W1kSTNkrm4tHU28Pl2p+5C4Her6g+TfBnYmmQ98FXgSoCq2plkK/AEsB+4pqoOtGNdDdwKnAzc1xZJ0iya9SCpqr8C3jGg/jfA6sO02QRsGlAfBS6c6T5KkqbvWLr9V5I0DxkkkqRODBJJUicGiSSpE4NEktSJQSJJ6sQgkSR1YpBIkjoxSCRJnRgkkqRODBJJUicGiSSpE4NEktSJQSJJ6sQgkSR1MlfvbD8h7HpiJ6sv/8DAbYvPPJVP3/zJWe6RJM08g2SIvlmvYdGlPz1w2557b5jl3kjScHhpS5LUybwPkiRrkuxOMpbk2rnujySdaOb1pa0kC4D/AvxLYBz4cpJtVfXE3PbsyA43f+LciaT5Zl4HCbASGKuqvwJIcgdwGXDMB8nh5k8e/LUPDwyYZ/7yL1j21u89pG7wSJprqaq57sNRS3IFsKaqfrx9/xDwz6rqI5P22wBsaF/fBuw+yj95JvD1o2w7XznmE4NjPjF0GfM/qapFgzbM9zOSDKgdkoxVtRnY3PmPJaNVNdL1OPOJYz4xOOYTw7DGPN8n28eBc/q+LwWem6O+SNIJab4HyZeB5UnOTfI6YC2wbY77JEknlHl9aauq9if5CPAFYAHwqaraOcQ/2fny2DzkmE8MjvnEMJQxz+vJdknS3Jvvl7YkSXPMIJEkdWKQTNPx8iiWJOck+WKSXUl2Jvloq5+e5P4kT7XP0/rabGzj3p3kkr76xUkea9tuSDLoduxjRpIFSf48yT3t+3E95iRvSnJXkifbv+93nQBj/pn2n+vHk3w2yeuPtzEn+VSSvUke76vN2BiTnJTkzlbfnmTZETtVVS5HWOhN5P8l8BbgdcBXgPPnul9HOZbFwPe19TcCfwGcD/wn4NpWvxa4rq2f38Z7EnBu++ewoG3bAbyL3u957gP+1VyP7whj/1ngd4F72vfjeszAFuDH2/rrgDcdz2MGlgBPAye371uBf3e8jRn4F8D3AY/31WZsjMBPAp9s62uBO4/Yp7n+hzIflvYP+wt93zcCG+e6XzM0trvpPatsN7C41RYDuweNld4dcu9q+zzZV/8A8NtzPZ4pxrkUeAB4L98OkuN2zMAp7X9UM6l+PI95CfAscDq9O1LvAX7oeBwzsGxSkMzYGCf2aesL6f0SPlP1x0tb0zPxH9AJ4602r7VT1ouA7cDZVbUHoH2e1XY73NiXtPXJ9WPVJ4BfAL7VVzuex/wWYB/wX9vlvJuTfDfH8Zir6mvArwFfBfYAL1XVH3Ecj7nPTI7xH9tU1X7gJeCMqf64QTI903oUy3yS5A3A7wMfq6pvTLXrgFpNUT/mJPlhYG9VPTzdJgNq82rM9P6f5PcBN1XVRcDf07vkcTjzfsxtXuAyepdwvgf47iQfnKrJgNq8GvM0HM0Yv+PxGyTTc1w9iiXJa+mFyGeq6nOt/HySxW37YmBvqx9u7ONtfXL9WPRu4EeSPAPcAbw3yac5vsc8DoxX1fb2/S56wXI8j/l9wNNVta+qvgl8Dvh+ju8xT5jJMf5jmyQLgVOBF6b64wbJ9Bw3j2Jpd2bcAuyqqt/o27QNWNfW19GbO5mor213cpwLLAd2tNPnl5Osase8qq/NMaWqNlbV0qpaRu/f3YNV9UGO7zH/H+DZJG9rpdX0Xq9w3I6Z3iWtVUm+q/V1NbCL43vME2ZyjP3HuoLef1+mPiOb60mj+bIAl9K7w+kvgV+a6/50GMcP0DtNfRR4pC2X0rsG+gDwVPs8va/NL7Vx76bv7hVgBHi8bfstjjAhdywswHv49mT7cT1mYAUw2v5d/zfgtBNgzL8CPNn6ezu9u5WOqzEDn6U3B/RNemcP62dyjMDrgd8Dxujd2fWWI/XJR6RIkjrx0pYkqRODRJLUiUEiSerEIJEkdWKQSJI6MUikY0CS/5jkfXPdD+loePuvNMeSLKiqA3PdD+loeUYiDVGSZe19IFuSPNreD/JdSZ5J8h+SfAm4MsmtSa5obd6Z5H8l+UqSHUnemN67VP5zki+343x4jocm/SODRBq+twGbq+qfAt+g974HgFeq6geq6o6JHdsjeO4EPlpV76D3/Kj/R+/Xyy9V1TuBdwI/0R55Ic05g0Qavmer6n+29U/Te0wN9AJjsrcBe6rqywBV9Y3qPcr7h4CrkjxC77H/Z9B7bpI05xbOdQekE8DkiciJ738/YN8M2H+i/lNV9YWZ7Jg0EzwjkYbvzUne1dY/AHxpin2fBL4nyTsB2vzIQnpvrbu6vQKAJN/bXlQlzTmDRBq+XcC6JI/Sew3sTYfbsar+Afgx4DeTfAW4n97TWG+m9xj4P0vyOPDbeEVBxwhv/5WGqL3O+J6qunCu+yINi2ckkqROPCORJHXiGYkkqRODRJLUiUEiSerEIJEkdWKQSJI6+f+PvFu1n/zPYQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.histplot(data.price, bins = 50)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "43097c16",
   "metadata": {},
   "source": [
    "data is in the shape \"long tail\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9460645b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='price', ylabel='Count'>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEGCAYAAACUzrmNAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAZzklEQVR4nO3df5DU9Z3n8edLYBBB8MdMLAJ4sLusF7VWPEePXbfustHbsL8Oty6ase4ilfVCYjDqnZsr9apuc1dFVeqM2d1sAheMRsyaEE5NSTyNK6ybVG6NMLpGRbSgotFZOJnG9WAJBwy874/+NHwdeubbM9Pd3+7p16Oqa7796e+3+0NP06/5fn59FRGYmZmN5rSiK2BmZq3PYWFmZrkcFmZmlsthYWZmuRwWZmaWa2rRFWiU7u7uWLhwYdHVMDNrK88//3wpInqGl0/asFi4cCH9/f1FV8PMrK1I+nm1cjdDmZlZLoeFmZnlcliYmVkuh4WZmeVyWJiZWS6HhZmZ5XJYmJlZLoeFmZnlmrST8sxaTURQKpUA6O7uRlLBNTKrnc8szJqkVCqxYs1mVqzZfCI0zNpFw8JC0umStkr6qaTtkv5rKj9H0tOSdqafZ2eOuVPSLkmvS/popvwySS+nx74i/0lmbapr1hy6Zs0puhpmY9bIM4vDwEci4hJgCbBM0lLgDmBLRCwGtqT7SLoQ6AMuApYBayRNSc+1FlgJLE63ZQ2st5mZDdOwsIiyf0x3p6VbAMuB9al8PXBN2l4ObIiIwxHxBrALuELSXGB2RDwb5QuGP5g5xszMmqChfRaSpkh6EdgLPB0RzwHnRcQegPTzA2n3ecDbmcMHUtm8tD28vNrrrZTUL6l/cHCwrv8WM7NO1tCwiIhjEbEEmE/5LOHiUXav1g8Ro5RXe711EdEbEb09Pacsx25mZuPUlNFQEfEe8DeU+xreSU1LpJ97024DwILMYfOB3al8fpVyMzNrkkaOhuqRdFbangFcDbwGbAJWpN1WAI+l7U1An6TpkhZR7sjempqqDkhamkZB3ZA5xszMmqCRk/LmAuvTiKbTgI0R8bikZ4GNkm4E3gKuBYiI7ZI2Aq8CQ8CqiDiWnusm4AFgBvBkupmZWZM0LCwi4iXg0irl+4CrRjhmNbC6Snk/MFp/h5mZNZBncJuZWS6HhZmZ5XJYmJlZLoeFmZnlcliYmVkuh4WZmeVyWJi1uIhgcHCQ8jqaZsVwWJi1uFKpRN/dj/iCSVYoh4VZG+g6Y3bRVbAO57AwM7NcDgszM8vlsDAzs1wOCzMzy+WwMDOzXA4LMzPL5bAwM7NcDgszM8vlsDAzs1wOCzMzy+WwMDOzXA4LMzPL5bAwM7NcDgszM8vlsDAzs1wOCzMzy9WwsJC0QNIzknZI2i7p1lT+BUl/L+nFdPvdzDF3Stol6XVJH82UXybp5fTYVySpUfU2M7NTTW3gcw8Bt0fEC5LOBJ6X9HR67E8j4kvZnSVdCPQBFwEfBDZL+tWIOAasBVYCPwGeAJYBTzaw7mZmltGwM4uI2BMRL6TtA8AOYN4ohywHNkTE4Yh4A9gFXCFpLjA7Ip6N8hXrHwSuaVS9zczsVE3ps5C0ELgUeC4V3SzpJUn3Szo7lc0D3s4cNpDK5qXt4eXVXmelpH5J/YODg/X8J5iZdbSGh4WkWcAjwG0RsZ9yk9IvA0uAPcA9lV2rHB6jlJ9aGLEuInojorenp2eiVTczs6ShYSFpGuWgeCgiHgWIiHci4lhEHAfuBa5Iuw8ACzKHzwd2p/L5VcrNzKxJGjkaSsB9wI6I+HKmfG5mtz8EXknbm4A+SdMlLQIWA1sjYg9wQNLS9Jw3AI81qt5mZnaqRo6GuhL4BPCypBdT2V3A9ZKWUG5KehP4NEBEbJe0EXiV8kiqVWkkFMBNwAPADMqjoDwSysysiRoWFhHxY6r3NzwxyjGrgdVVyvuBi+tXOzMzGwvP4DYzs1wOCzMzy+WwMDOzXA4LMzPL5bAwM7NcDgszM8vlsDAzs1wOCzMzy+WwMDOzXA4LMzPL5bAwM7NcDgszM8vlsDAzs1wOCzMzy+WwMDOzXA4LMzPL5bAwM7NcDgszM8vlsDAzs1wOCzMzy+WwMDOzXA4LMzPL5bAwM7NcDgszM8vlsDAzs1wNCwtJCyQ9I2mHpO2Sbk3l50h6WtLO9PPszDF3Stol6XVJH82UXybp5fTYVySpUfU2M7NTNfLMYgi4PSI+BCwFVkm6ELgD2BIRi4Et6T7psT7gImAZsEbSlPRca4GVwOJ0W9bAepu1pYhgcHCQiCi6KjYJNSwsImJPRLyQtg8AO4B5wHJgfdptPXBN2l4ObIiIwxHxBrALuELSXGB2RDwb5f8FD2aOMZtUJvKFXyqV6Lv7EUqlUgNqZp2uKX0WkhYClwLPAedFxB4oBwrwgbTbPODtzGEDqWxe2h5eXu11Vkrql9Q/ODhY13+DWTNM9Au/64zZda6RWVnDw0LSLOAR4LaI2D/arlXKYpTyUwsj1kVEb0T09vT0jL2yZi3AX/jWihoaFpKmUQ6KhyLi0VT8TmpaIv3cm8oHgAWZw+cDu1P5/CrlZmbWJI0cDSXgPmBHRHw589AmYEXaXgE8linvkzRd0iLKHdlbU1PVAUlL03PekDnGzMyaYGoDn/tK4BPAy5JeTGV3AV8ENkq6EXgLuBYgIrZL2gi8Snkk1aqIOJaOuwl4AJgBPJluZmbWJA0Li4j4MdX7GwCuGuGY1cDqKuX9wMX1q53Z2EUEpVKJ7u5uPNXHOo1ncJvVyENTrZM5LMzGwCOVrFM5LMzMLJfDwszMcjkszMwsl8PCzMxyOSzMzCyXw8LMzHLVFBaSrqylzMzMJqdazyz+osYys0mh1utK+IJD1ilGDQtJvy7pdqBH0n/M3L4ATBntWLN2Vutsbc/qtk6RtzZUFzAr7Xdmpnw/8LFGVcqsFdQ6W9uzuq0TjBoWEfFD4IeSHoiInzepTmZm1mJqXXV2uqR1wMLsMRHxkUZUyqydeXVam4xqDYv/CfwP4BvAsZx9zTpapR9jw+f/Db68r00WtYbFUESsbWhNzCYR92PYZFPr0NnvS/qspLmSzqncGlozszZTGUbrkVE2GdV6ZlG5ZvbnM2UB/FJ9q2PWvkqlEivWbObwwf2c1jWj6OqY1VVNYRERixpdEbPJoGvWHAIYOnq06KqY1VVNYSHphmrlEfFgfatjZmatqNZmqMsz26cDVwEvAA4LM7MOUGsz1Oey9yXNAb7VkBqZmVnLGe8S5b8AFtezImZm1rpq7bP4PuXRT1BeQPBDwMZGVcpssqjM5q5sm7WrWvssvpTZHgJ+HhEDox0g6X7g94G9EXFxKvsC8ClgMO12V0Q8kR67E7iR8gzxWyLiqVR+GfAAMAN4Arg1/L/O2kRlOC3APdctKbYyZhNQUzNUWlDwNcorz54NHKnhsAeAZVXK/zQilqRbJSguBPqAi9IxayRVlkBfC6yk3Oy1eITnNGtZXbPm0DVrTtHVMJuQWq+Udx2wFbgWuA54TtKoS5RHxI+Ad2usx3JgQ0Qcjog3gF3AFZLmArMj4tl0NvEgcE2Nz2lmZnVSazPUfwYuj4i9AJJ6gM3Aw+N4zZvTvI1+4PaI+AdgHvCTzD4Dqexo2h5ebmZmTVTraKjTKkGR7BvDsVlrgV8GlgB7gHtSebV1nGOU8qokrZTUL6l/cHBwpN3MWo7XlbJWV+uZxQ8kPQV8J93/OOXO5jGJiHcq25LuBR5PdweABZld5wO7U/n8KuUjPf86YB1Ab2+vO8GtbXhdKWt1edfg/hVJV0bE54GvA78GXAI8S/pSHovUB1Hxh8AraXsT0CdpuqRFlDuyt0bEHuCApKUqX0XmBuCxsb6uWTvomjWHrple2txaU96ZxZ8BdwFExKPAowCSetNjfzDSgZK+A3wY6JY0APwJ8GFJSyg3Jb0JfDo993ZJG4FXKQ/NXRURlYss3cTJobNPppuZmTVRXlgsjIiXhhdGRL+khaMdGBHXVym+b5T9VwOrq70WcHFOPc3MrIHyOqlPH+UxN6yamXWIvLDYJulTwwsl3Qg835gqmbU+j16yTpPXDHUb8D1J/5aT4dALdFHuoDbrSB69ZJ1m1LBIQ11/Q9JvcbLf4H9FxF83vGZmLc5XxbNOUuv1LJ4BnmlwXczMrEXVOinPzNpUdpn07u5uylOWzMZmvBc/MrMCVTrYa1mtv9K/smLNZnfI27g5LMzaUKlUou/uR2r+8vcy6TZRDguzNtV1hpcGseZxWJiZWS6HhZmZ5XJYmJlZLoeFmZnlcliYmVkuh4WZmeXyDG6zEQyf+TzeY80mA4eF2QiO/uIAt3x7G9OmTmP9Z68e17HHDx9ixjlzmTrN/9WsvfkTbMbIZxHTZ5417i/66TPP4tjUaXWpn1nRHBZmnFw/CRjzWYRZJ3BYmCVeO8lsZB4NZWZmuRwWZmaWy2FhVpCxXJPCrGgOC7OCjPWaFGZFcliYFcjXpLB20bCwkHS/pL2SXsmUnSPpaUk708+zM4/dKWmXpNclfTRTfpmkl9NjX5EvIGxm1nSNPLN4AFg2rOwOYEtELAa2pPtIuhDoAy5Kx6yRNCUdsxZYCSxOt+HPaWZmDdawsIiIHwHvDiteDqxP2+uBazLlGyLicES8AewCrpA0F5gdEc9GuRfwwcwxZh2tMuvcneTWDM2elHdeROwBiIg9kj6QyucBP8nsN5DKjqbt4eVVSVpJ+SyE888/v47VNms9E1m7ymysWqWDu1o/RIxSXlVErIuI3ojo7enpqVvlzFrV9Jlneea5NUWzw+Kd1LRE+rk3lQ8ACzL7zQd2p/L5VcrNxq0yv6GZzTcRwbvvDm+VHdvxbnKyIjU7LDYBK9L2CuCxTHmfpOmSFlHuyN6amqwOSFqaRkHdkDnGbFwqiwauWLO5aXMcjhzczx9/60cMHR0a1/GVJqdm1tksq2F9FpK+A3wY6JY0APwJ8EVgo6QbgbeAawEiYrukjcCrwBCwKiKOpae6ifLIqhnAk+lmNiFFNN10zThzQsdPZLl0s4lq2CcvIq4f4aGrRth/NbC6Snk/cHEdq2ZmyfDreHgak42kVTq4zawARTTJWXvyOa1Zh/NoKquFw8KsRVWaiPwXv7UCh4VZi6o0ER0+uJ+hobGNosr2RXiordWDw8KshXXNmkMAQ+/tG9Nx2WuK33PdkvpXzDqOw8KszWXPIrLcF2H15LAwa3NHDu7nlm9v4/jhQ5zWNaPo6tgk5bAwmwSmzzyLY1OnMXT0aNFVsUnK8yzMzCyXw8LMzHI5LMzMLJfDwszMcjkszMwsl8PCzMxyOSzMzCyX51mYFWik2ddmrcZhYVagyuVSjx8+xIxz5hZdHbMROSzMClaZfW3WytxnYWZmuXxmYdZi3I9hrchhYdZi3I9hrchhYR2h3a4c534MazUOC+sIvnKc2cQ4LKxj+MpxZuPnsDDLcOeyWXWFhIWkN4EDwDFgKCJ6JZ0DfBdYCLwJXBcR/5D2vxO4Me1/S0Q8VUC1rQNkL1E6NDRUdHXMWkaR8yx+KyKWRERvun8HsCUiFgNb0n0kXQj0ARcBy4A1kqYUUWHrDNNnnkXXzNlFV6MuIoJ333236GrYJNBKk/KWA+vT9nrgmkz5hog4HBFvALuAK5pfPbP6qDR1NaO568jB/fzxt37E0NHaz5IigsHBQQYHB9ti5Jg1R1F9FgH8laQAvh4R64DzImIPQETskfSBtO884CeZYwdS2SkkrQRWApx//vmNqrvZhDS7qatrxplj2j87cmz9Z6+mp6enEdWyNlNUWFwZEbtTIDwt6bVR9lWVsqp/7qTQWQfQ29vrP4msZVXmUQy9t29MxzWrA94jx2y4QsIiInann3slfY9ys9I7kuams4q5wN60+wCwIHP4fGB3Uyts1iLcAW9FaXqfhaSZks6sbAO/DbwCbAJWpN1WAI+l7U1An6TpkhYBi4Gtza21WeuYTB3w1j6KOLM4D/iepMrrfzsifiBpG7BR0o3AW8C1ABGxXdJG4FVgCFgVEccKqLeZWcdqelhExM+AS6qU7wOuGuGY1cDqBlfNrCNU+j26u7uLroq1kVYaOmvW8po57LVRSqUSfXc/0tb/Bms+L/dhNgaTZfnwrjPc52Fj47AwG6NWXj7ca1tZozgszCaRRp35ZEOou7ubNEDFOojDwmySacSZj2d1m8PCzGriWd2dzWFhk46bTPLVq2/D73XncFjYpDNak4mX7C4b3rcxddr4vgrcPNU5HBY2KY3UZFJesvslzl14EVOmTjnxV/FEluJu1xFI9erbcPNUZ3BYWMepLNldWZRv2tRp3HPdknE/X7vMvRgt1Dyr2/I4LKyjTZ951ribYIY/T6vOvagYbcXaIwf385l7t/DwnQ4Lq87LfVjH66R+jNFWrPWsbhuNw8I63nguPWrWadwMZcbYLz3aySrX6K5sW2dwWJjlaNfRTo2yb98+bt/4IsCEBgZYe3FY2KRW+St4Il/2nXwp05HePw+X7TwOC5vUKn8FHz64f0LDWiujnYbe21fH2rW2iGDnzp2s3vzWifevHiPHrD25g9smva5Zc3zN6nGodPyfNn2m3z9zWJjZyNzxbxU+pzQzoP4d+V5kcHJxWFhb8hdR/dWybMlYJjBWW2TQv7f25WYoa0uVL6IVazZTKpVOjNrJjvvvpJnZ9TLaDG8Y+wTGrllz3jdyKvt7GxwcPHEbab5Gtd+rFcNhYW0r+0VUKpX4+H9/mNdee+3EX66emd0YlX6MamE8UmhXAh3Kv7dpM2ezc+fO9wV+NaVSib67H/E8lxbgsLBJQ4hbvr2Nm795MiDcQds41cJ4pND+zL1b3veF/76RVjlzNrxmVWtwn4VNKu2w+utkkj3LqIRBJbSzfR/VvvAd5O2lbcJC0jLgz4EpwDci4osFV8maIHudhUpnaPaLyUtxtIbhs9zPHBbatfyesmtOVezbd+okyGqfiWqPw+id6O5sH5u2CAtJU4CvAf8KGAC2SdoUEa/W+7X8ARqf8bxvw9uyJZ1ybKVZ42t/9GHOPfdcoPwFsvJrjzNn/gUcO3ywY5fiaDWjzXIfbaRV5XNQKpVY+bXHOf3s8zh++BCnTZ/B8cOH0LTT3/c52bdvH6vu/5v3fSYqenp6TnSiRwT3XLfkfRd0yn7G8i6/Wy2Q6vH90KjvmLwQnai2CAvgCmBXRPwMQNIGYDlQ97CodKgBfPWT/9JXDqtRqVTi5m/+EKj9fascc+QXBzitawZTp0495dhSqcTRQ//IZ+7dwvEjhzitawbHj5SD4fDB9058qQAcOXTgZNnQUY4fPuSyBpeN5ZiuWeWmqGrHfubeN078Xoc7uG931d//KWVHh7j/tmtOHHfkFwe44YsPMbN77on9sp+x7JnO8LOeUqnEyq9+n3U3/8Epn8exfs5H+txP5DlGet6VX/0+j37hkw25FrraYUiapI8ByyLi36f7nwD+eUTcPGy/lcDKdPcC4PVxvmQ34LYNvw8Vfh/K/D6UTfb34Z9ExClp0y5nFtXOqU5JuYhYB6yb8ItJ/RHRO9HnaXd+H8r8PpT5fSjr1PehXYbODgALMvfnA7sLqouZWcdpl7DYBiyWtEhSF9AHbCq4TmZmHaMtmqEiYkjSzcBTlIfO3h8R2xv4khNuypok/D6U+X0o8/tQ1pHvQ1t0cJuZWbHapRnKzMwK5LAwM7NcDosMScskvS5pl6Q7iq5PUSQtkPSMpB2Stku6teg6FUXSFEl/J+nxoutSJElnSXpY0mvpc/HrRdepCJL+Q/o/8Yqk70g6veg6NYvDIsksKfI7wIXA9ZIuLLZWhRkCbo+IDwFLgVUd/F7cCuwouhIt4M+BH0TEPwUuoQPfE0nzgFuA3oi4mPJgm75ia9U8DouTTiwpEhFHgMqSIh0nIvZExAtp+wDlL4Z5xdaq+STNB34P+EbRdSmSpNnAvwDuA4iIIxHxXqGVKs5UYIakqcAZdNB8L4fFSfOAtzP3B+jAL8jhJC0ELgWeK7gqRfgz4D8BxwuuR9F+CRgEvpma5L4haWbRlWq2iPh74EvAW8Ae4P9GxF8VW6vmcVicVNOSIp1E0izgEeC2iNhfdH2aSdLvA3sj4vmi69ICpgL/DFgbEZcCB4GO69OTdDbl1oZFwAeBmZL+XbG1ah6HxUleUiRD0jTKQfFQRDxadH0KcCXwryW9SblJ8iOS/rLYKhVmABiIiMrZ5cOUw6PTXA28ERGDEXEUeBT4jYLr1DQOi5O8pEii8mL49wE7IuLLRdenCBFxZ0TMj4iFlD8Lfx0RHfNXZFZE/B/gbUkXpKKraMDlAdrAW8BSSWek/yNX0UEd/W2x3EczFLCkSCu7EvgE8LKkF1PZXRHxRHFVsoJ9Dngo/SH1M+CTBden6SLiOUkPAy9QHjH4d3TQ0h9e7sPMzHK5GcrMzHI5LMzMLJfDwszMcjkszMwsl8PCzMxyOSzMmkjSf5N0ddH1MBsrD501axJJUyLiWNH1MBsPn1mY1YGkhelaD+slvZSu/XCGpDcl/RdJPwaulfSApI+lYy6X9LeSfippq6Qz0/Uz7pa0LT3Ppwv+p5kBDguzeroAWBcRvwbsBz6byv9fRPxmRGyo7JhmQn8XuDUiLqG87tAh4EbKq5leDlwOfErSomb+I8yqcViY1c/bEfG/0/ZfAr+Ztr9bZd8LgD0RsQ0gIvZHxBDw28ANaZmV54BzgcUNrbVZDbw2lFn9DO8ArNw/WGVfVdm/Uv65iHiqnhUzmyifWZjVz/mZa1NfD/x4lH1fAz4o6XKA1F8xlfJCljelJeKR9KudeKEhaz0OC7P62QGskPQScA6wdqQd06V7Pw78haSfAk8Dp1O+hOurwAuSXgG+jlsArAV46KxZHaTLzz4eERcXXRezRvCZhZmZ5fKZhZmZ5fKZhZmZ5XJYmJlZLoeFmZnlcliYmVkuh4WZmeX6/29vkITqDfaSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.histplot(np.log1p(data.price))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2258e06",
   "metadata": {},
   "source": [
    "Features for homework: 'latitude',\n",
    "'longitude',\n",
    "'price',\n",
    "'minimum_nights',\n",
    "'number_of_reviews',\n",
    "'reviews_per_month',\n",
    "'calculated_host_listings_count',\n",
    "'availability_365'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3a1c4dff",
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = ['latitude',\n",
    "'longitude',\n",
    "'price',\n",
    "'minimum_nights',\n",
    "'number_of_reviews',\n",
    "'reviews_per_month',\n",
    "'calculated_host_listings_count',\n",
    "'availability_365']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "d1d92610",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "latitude                              0\n",
       "longitude                             0\n",
       "price                                 0\n",
       "minimum_nights                        0\n",
       "number_of_reviews                     0\n",
       "reviews_per_month                 10052\n",
       "calculated_host_listings_count        0\n",
       "availability_365                      0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Question 1:\n",
    "data_hw = data[columns]\n",
    "data_hw.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "2bfca9a1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.0"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Question 2:\n",
    "data_hw.minimum_nights.median()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "437ab327",
   "metadata": {},
   "outputs": [],
   "source": [
    "# data prep\n",
    "n = len(data_hw)\n",
    "n_train = int(n * 0.6)\n",
    "n_val = int(n * 0.2)\n",
    "n_test = n - n_train - n_val\n",
    "\n",
    "idx = np.arange(n)\n",
    "np.random.seed(42)\n",
    "np.random.shuffle(idx)\n",
    "\n",
    "df_train = data_hw.iloc[idx[:n_train]]\n",
    "df_val = data_hw.iloc[idx[n_train:n_train+n_val]]\n",
    "df_test = data_hw.iloc[idx[n_train+n_val:]]\n",
    "\n",
    "df_train = df_train.reset_index(drop=True)\n",
    "df_val = df_val.reset_index(drop=True)\n",
    "df_test = df_test.reset_index(drop=True)\n",
    "\n",
    "y_train = np.log1p(df_train[\"price\"])\n",
    "y_val = np.log1p(df_val[\"price\"])\n",
    "y_test = np.log1p(df_test[\"price\"])\n",
    "\n",
    "del df_train[\"price\"]\n",
    "del df_val[\"price\"]\n",
    "del df_test[\"price\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "bd2a9bcb",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_linear_regression(X, y):\n",
    "    ones = np.ones(X.shape[0])\n",
    "    X = np.column_stack([ones, X])\n",
    "\n",
    "    XTX = X.T.dot(X)\n",
    "    XTX_inv = np.linalg.inv(XTX)\n",
    "    w_full = XTX_inv.dot(X.T).dot(y)\n",
    "    \n",
    "    return w_full[0], w_full[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "450b2540",
   "metadata": {},
   "outputs": [],
   "source": [
    "def rmse(y, y_pred):\n",
    "    se = (y - y_pred) ** 2\n",
    "    mse = se.mean()\n",
    "    return np.sqrt(mse)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "618c835a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Question 3.Option 1\n",
    "df_train_zero = df_train.fillna(0)\n",
    "X_train_zero = df_train_zero.values\n",
    "\n",
    "w0_zero, w_zero = train_linear_regression(X_train_zero, y_train)\n",
    "\n",
    "#w0_zero, w_zero\n",
    "\n",
    "y_pred_zero = w0_zero + X_train_zero.dot(w_zero)\n",
    "\n",
    "rmse_zero = rmse(y_train, y_pred_zero)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "665c44f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Question 3.Option 2\n",
    "df_train_mean = df_train.fillna(data_hw[\"reviews_per_month\"].mean())\n",
    "X_train_mean = df_train_mean.values\n",
    "\n",
    "w0_mean, w_mean = train_linear_regression(X_train_mean, y_train)\n",
    "\n",
    "y_pred_mean = w0_mean + X_train_mean.dot(w_mean)\n",
    "\n",
    "rmse_mean = rmse(y_val, y_pred_mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "dfd2f0fe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE for zero fill: 0.64\n",
      "RMSE for mean fill: 0.64\n"
     ]
    }
   ],
   "source": [
    "print(\"RMSE for zero fill: {}\".format(round(rmse_zero, 2)))\n",
    "print(\"RMSE for mean fill: {}\".format(round(rmse_mean, 2)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "632d9141",
   "metadata": {},
   "source": [
    "no difference for option 1 and option 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "6edd5b3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_linear_regression_reg(X, y, r=0.001):\n",
    "    ones = np.ones(X.shape[0])\n",
    "    X = np.column_stack([ones, X])\n",
    "\n",
    "    XTX = X.T.dot(X)\n",
    "    XTX = XTX + r * np.eye(XTX.shape[0])\n",
    "\n",
    "    XTX_inv = np.linalg.inv(XTX)\n",
    "    w_full = XTX_inv.dot(X.T).dot(y)\n",
    "    \n",
    "    return w_full[0], w_full[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "30e65bd0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 0.64\n",
      "1e-06 0.64\n",
      "0.0001 0.64\n",
      "0.001 0.64\n",
      "0.01 0.66\n",
      "0.1 0.68\n",
      "1 0.68\n",
      "5 0.68\n",
      "10 0.68\n"
     ]
    }
   ],
   "source": [
    "# Question 4, use df_train_zero\n",
    "r_eval = [0, 0.000001, 0.0001, 0.001, 0.01, 0.1, 1, 5, 10]\n",
    "X_val_zero = df_val.fillna(0).values\n",
    "for r in r_eval:\n",
    "    w0, w = train_linear_regression_reg(X_train_zero, y_train, r)\n",
    "    y_pred_zero = w0 + X_val_zero.dot(w)\n",
    "    score = round(rmse(y_val, y_pred_zero), 2)\n",
    "    print(r, score)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b8501e0",
   "metadata": {},
   "source": [
    "Choose r = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "28719d2b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 0.6549779961155779\n",
      "1 0.6462523685753724\n",
      "2 0.6476558177126073\n",
      "3 0.6375145070991337\n",
      "4 0.6445809084147295\n",
      "5 0.6305809995663865\n",
      "6 0.6297851918604098\n",
      "7 0.6506184266834151\n",
      "8 0.648978035329433\n",
      "9 0.6437565168334167\n",
      "std: 0.008\n"
     ]
    }
   ],
   "source": [
    "# Question 5\n",
    "seed = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\n",
    "score_list = []\n",
    "for s in seed:\n",
    "    idx = np.arange(n)\n",
    "    np.random.seed(s)\n",
    "    np.random.shuffle(idx)\n",
    "\n",
    "    df_train = data_hw.iloc[idx[:n_train]]\n",
    "    df_val = data_hw.iloc[idx[n_train:n_train+n_val]]\n",
    "    df_test = data_hw.iloc[idx[n_train+n_val:]]\n",
    "\n",
    "    df_train = df_train.reset_index(drop=True)\n",
    "    df_val = df_val.reset_index(drop=True)\n",
    "    df_test = df_test.reset_index(drop=True)\n",
    "\n",
    "    y_train = np.log1p(df_train[\"price\"])\n",
    "    y_val = np.log1p(df_val[\"price\"])\n",
    "    y_test = np.log1p(df_test[\"price\"])\n",
    "\n",
    "    del df_train[\"price\"]\n",
    "    del df_val[\"price\"]\n",
    "    del df_test[\"price\"]\n",
    "    \n",
    "    # Train model\n",
    "    X_train_zero = df_train.fillna(0).values\n",
    "    w0_zero, w_zero = train_linear_regression(X_train_zero, y_train)\n",
    "    \n",
    "    # Validate\n",
    "    X_val_zero = df_val.fillna(0).values\n",
    "    y_pred_zero = w0_zero + X_val_zero.dot(w_zero)\n",
    "    rmse_zero = rmse(y_val, y_pred_zero)\n",
    "    \n",
    "    # Print output\n",
    "    print(s, rmse_zero)\n",
    "    score_list.append(rmse_zero)\n",
    "print(\"std: {}\".format(round(np.std(score_list), 3)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "a766181a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.65\n"
     ]
    }
   ],
   "source": [
    "# Question 6\n",
    "idx = np.arange(n)\n",
    "np.random.seed(9)\n",
    "np.random.shuffle(idx)\n",
    "\n",
    "df_train_and_val = data_hw.iloc[idx[:n_train+n_val]]\n",
    "df_test = data_hw.iloc[idx[n_train+n_val:]]\n",
    "\n",
    "y_train_and_val = np.log1p(df_train_and_val[\"price\"])\n",
    "y_test = np.log1p(df_test[\"price\"])\n",
    "\n",
    "del df_train_and_val[\"price\"]\n",
    "del df_test[\"price\"]\n",
    "\n",
    "# Train model\n",
    "X_train_and_val = df_train_and_val.values\n",
    "w0, w = train_linear_regression_reg(X_train_and_val, y_train_and_val, r = 0.001)\n",
    "\n",
    "# Validate\n",
    "X_test = df_test.values\n",
    "y_pred = w0 + X_test.dot(w)\n",
    "\n",
    "score = rmse(y_test, y_pred)\n",
    "print(round(score,2))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}