Predicting_house_sale_prices.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Solution: https://github.com/dataquestio/solutions/blob/master/Mission240Solutions.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 418,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.model_selection import KFold, cross_val_score\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 419,
   "metadata": {},
   "outputs": [],
   "source": [
    "TARGET_COLUMN = 'SalePrice'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get and transform data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 489,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_data(fname):\n",
    "    data = pd.read_csv(fname, sep='\\t')\n",
    "    return data\n",
    "\n",
    "\n",
    "def handle_missing_values(df, mv_cutoff=0.05):\n",
    "    mvalues = (df.isnull().sum() / df.shape[0])\n",
    "    # All columns with missing values\n",
    "    mv_columns = mvalues[mvalues > 0].index\n",
    "    # Drop columns with missing values > 5 %\n",
    "    columns_to_drop = (mvalues[mvalues > mv_cutoff]).index\n",
    "    df = df.drop(columns_to_drop, axis=1)\n",
    "    # Fill missing values for the remaining columns\n",
    "    columns = (df[mv_columns.difference(columns_to_drop)]\n",
    "              .columns)\n",
    "    for col in columns:\n",
    "        df[col] = df[col].fillna(value=df[col].mode()[0])\n",
    "    return df\n",
    "\n",
    "\n",
    "def add_new_features(dframe):\n",
    "    df = dframe.assign(\n",
    "    years_build_sold = dframe['Yr Sold'] - dframe['Year Built'],\n",
    "    years_mod_sold = dframe['Yr Sold'] - dframe['Year Remod/Add'],\n",
    "    years_build_mod = dframe['Year Remod/Add'] - dframe['Year Built']\n",
    "    )\n",
    "    return df\n",
    "\n",
    "\n",
    "def drop_not_useful_columns(dframe):\n",
    "    '''\n",
    "    Drops these columns:\n",
    "        Order (Discrete): Observation number\n",
    "        PID (Nominal): Parcel identification number  - can be used with city web site for parcel review.\n",
    "        Mo Sold (Discrete): Month Sold (MM)\n",
    "        Yr Sold (Discrete): Year Sold (YYYY)\n",
    "        Year Built (Discrete): Original construction date\n",
    "        Year Remod/Add (Discrete): Remodel date (same as construction date if no remodeling or additions)\n",
    "        Garage Yr Blt (Discrete): Year garage was built\n",
    "    '''\n",
    "    cols_to_drop = ['Order', 'PID', 'Yr Sold', 'Mo Sold',\n",
    "                    'Year Built', 'Year Remod/Add', 'Garage Yr Blt']\n",
    "    return dframe.drop(cols_to_drop, axis=1)\n",
    "\n",
    "\n",
    "def convert_object_to_categorical(dframe):\n",
    "    # Transform object to categorical data\n",
    "    df = dframe.copy()\n",
    "    for col in df.select_dtypes(include=['object']).columns:\n",
    "        df[col] = df[col].astype('category')\n",
    "    return df\n",
    "\n",
    "\n",
    "def convert_num_to_categorical(dframe):\n",
    "    '''\n",
    "    Convert numeric to categorical\n",
    "    MS SubClass (Nominal): Identifies the type of dwelling involved in the sale.\t\n",
    "\n",
    "           020\t1-STORY 1946 & NEWER ALL STYLES\n",
    "           030\t1-STORY 1945 & OLDER\n",
    "           040\t1-STORY W/FINISHED ATTIC ALL AGES\n",
    "           045\t1-1/2 STORY - UNFINISHED ALL AGES\n",
    "           050\t1-1/2 STORY FINISHED ALL AGES\n",
    "           060\t2-STORY 1946 & NEWER\n",
    "           070\t2-STORY 1945 & OLDER\n",
    "           075\t2-1/2 STORY ALL AGES\n",
    "           080\tSPLIT OR MULTI-LEVEL\n",
    "           085\tSPLIT FOYER\n",
    "           090\tDUPLEX - ALL STYLES AND AGES\n",
    "           120\t1-STORY PUD (Planned Unit Development) - 1946 & NEWER\n",
    "           150\t1-1/2 STORY PUD - ALL AGES\n",
    "           160\t2-STORY PUD - 1946 & NEWER\n",
    "           180\tPUD - MULTILEVEL - INCL SPLIT LEV/FOYER\n",
    "           190\t2 FAMILY CONVERSION - ALL STYLES AND AGES\n",
    "    '''\n",
    "    cols_num_to_cat = ['MS SubClass']\n",
    "    df = dframe.copy()\n",
    "    for col in cols_num_to_cat:\n",
    "        df[col] = df[col].astype('category')\n",
    "    return df\n",
    "\n",
    "\n",
    "def drop_cat_columns_unique_over_10(df):\n",
    "    cat_columns = df.select_dtypes(include='category').columns\n",
    "    for col in cat_columns:\n",
    "        if df[col].unique().shape[0] > 10:\n",
    "            df = df.drop(col, axis=1)\n",
    "    return df\n",
    "\n",
    "\n",
    "def normalize_data(df):\n",
    "    dfn = df.select_dtypes(include=['int', 'float']).drop(TARGET_COLUMN, axis=1)\n",
    "    df.loc[:, dfn.columns] = (dfn - dfn.min()) / (dfn.max() - dfn.min())\n",
    "    return df\n",
    "\n",
    "\n",
    "def convert_cat_to_dummy(df):\n",
    "    return pd.get_dummies(df)\n",
    "\n",
    "\n",
    "def drop_columns_with_low_variance(df, cutoff_num=0.0001, cutoff_cat=0.95):\n",
    "    def num_cols_to_drop():\n",
    "    # Drop numeric columns with low variance\n",
    "        dfn = df.select_dtypes(include=['int', 'float'])\n",
    "        res = dfn.var()\n",
    "        return res[res < cutoff_num].index\n",
    "    \n",
    "    def cat_cols_to_drop():\n",
    "    # Drop category columns where most of the values belong to a specific category\n",
    "        dfc = df.select_dtypes(include='category')\n",
    "        res = dfc.apply(lambda x: (x.value_counts() / x.shape[0] > cutoff_cat).any())\n",
    "        return res[res == True].index\n",
    "    \n",
    "    cols_to_drop = num_cols_to_drop().union(cat_cols_to_drop())\n",
    "    return df.drop(cols_to_drop, axis=1)\n",
    "\n",
    "\n",
    "def transform_features(df):\n",
    "    df = (df.pipe(add_new_features)\n",
    "            .pipe(drop_not_useful_columns)\n",
    "            .pipe(handle_missing_values)\n",
    "            .pipe(convert_object_to_categorical)\n",
    "            .pipe(convert_num_to_categorical)\n",
    "            .pipe(drop_cat_columns_unique_over_10)\n",
    "            .pipe(normalize_data)\n",
    "            .pipe(drop_columns_with_low_variance)\n",
    "            .pipe(convert_cat_to_dummy)\n",
    "         )\n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Select features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 539,
   "metadata": {},
   "outputs": [],
   "source": [
    "def select_all_features(df):\n",
    "    return df.drop(TARGET_COLUMN, axis=1).columns\n",
    "\n",
    "\n",
    "def select_highest_corr_features(df, corr_thresh=0.4):\n",
    "    corr = np.abs(df.select_dtypes(include=['int64', 'float64']).corr()[TARGET_COLUMN]).sort_values(ascending=False).drop(TARGET_COLUMN)\n",
    "    return corr[corr > corr_thresh].index\n",
    "\n",
    "\n",
    "def select_features(df, select_func, *args, **kwargs):\n",
    "    return select_func(df, *args, **kwargs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 540,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_and_test(select_func=None, k=0, *args, **kwargs):\n",
    "    data = get_data('data/AmesHousing.tsv')\n",
    "    data = transform_features(data)\n",
    "    selected_features = select_features(data, select_func, *args, **kwargs)\n",
    "#     data = select_features(data, n_features)\n",
    "    if k == 0:\n",
    "        train = data[:1460]\n",
    "        test = data[1460:]\n",
    "        lr = LinearRegression()\n",
    "        lr.fit(train[selected_features], train[TARGET_COLUMN])\n",
    "        rmse = mean_squared_error(test[TARGET_COLUMN], lr.predict(test[selected_features])) ** (0.5)\n",
    "        return rmse\n",
    "    elif k == 1:\n",
    "        data = data.sample(frac=1)\n",
    "        train = data[:1460]\n",
    "        test = data[1460:]\n",
    "        lr = LinearRegression()\n",
    "        lr.fit(train[selected_features], train[TARGET_COLUMN])\n",
    "        rmse = mean_squared_error(test[TARGET_COLUMN], lr.predict(test[selected_features])) ** (0.5)\n",
    "        return rmse\n",
    "    else:\n",
    "        kf = KFold(n_splits=k, shuffle=True, random_state=1)\n",
    "        lr = LinearRegression()\n",
    "        mses = cross_val_score(lr, data[selected_features], data[TARGET_COLUMN], scoring=\"neg_mean_squared_error\", cv=kf)\n",
    "                                        \n",
    "        return np.mean(np.sqrt(np.abs(mses)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 544,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "34807.0286910009"
      ]
     },
     "execution_count": 544,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_and_test(select_func=select_highest_corr_features, k=6, corr_thresh=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 537,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "uint8      229\n",
       "float64     14\n",
       "dtype: int64"
      ]
     },
     "execution_count": 537,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[select_all_features(data)].dtypes.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 538,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "uint8      229\n",
       "float64     14\n",
       "int64        1\n",
       "dtype: int64"
      ]
     },
     "execution_count": 538,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.dtypes.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}