Housing_Exercises.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1- Data Preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fdccec4b-e895-4992-96ca-082d94ef73ab",
   "metadata": {},
   "source": [
    "you can skip the following since it's just dataset preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "bd8d4279-3fbc-4eaa-947c-d6a437d02a9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import tarfile\n",
    "import urllib\n",
    "DOWNLOAD_ROOT = \"https://raw.githubusercontent.com/ageron/handson-ml2/master/\"\n",
    "HOUSING_PATH = os.path.join(\"datasets\", \"housing\")\n",
    "HOUSING_URL = DOWNLOAD_ROOT + \"datasets/housing/housing.tgz\"\n",
    "def fetch_housing_data(housing_url=HOUSING_URL, housing_path=HOUSING_PATH):\n",
    " os.makedirs(housing_path, exist_ok=True)\n",
    " tgz_path = os.path.join(housing_path, \"housing.tgz\")\n",
    " urllib.request.urlretrieve(housing_url, tgz_path)\n",
    " housing_tgz = tarfile.open(tgz_path)\n",
    " housing_tgz.extractall(path=housing_path)\n",
    " housing_tgz.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "2713f792-3401-480d-ad8d-f99c97e08ad2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "def load_housing_data(housing_path=HOUSING_PATH):\n",
    " csv_path = os.path.join(housing_path, \"housing.csv\")\n",
    " return pd.read_csv(csv_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "52bb81b1-ffa2-454f-922c-c299713ac71f",
   "metadata": {},
   "outputs": [],
   "source": [
    "fetch_housing_data() #fetching data\n",
    "housing = load_housing_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "21cbc45c-6a98-4c63-ae7d-a946a1707c8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "train_set, test_set = train_test_split(housing, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "5c839ef7-da75-4f0f-acb3-331cbb59948e",
   "metadata": {},
   "outputs": [],
   "source": [
    "housing[\"income_cat\"] = pd.cut(housing[\"median_income\"],\n",
    " bins=[0., 1.5, 3.0, 4.5, 6., np.inf],\n",
    " labels=[1, 2, 3, 4, 5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "9f93f88d-de20-4e12-98eb-e70c629d94d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import StratifiedShuffleSplit\n",
    "split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)\n",
    "for train_index, test_index in split.split(housing, housing[\"income_cat\"]):\n",
    " strat_train_set = housing.loc[train_index]\n",
    " strat_test_set = housing.loc[test_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d2a637e1-ec53-4f29-830f-ea298bb3b50b",
   "metadata": {},
   "outputs": [],
   "source": [
    "for set_ in (strat_train_set, strat_test_set):\n",
    " set_.drop(\"income_cat\", axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "3ea6d124-f715-4190-a8eb-1c38d122c384",
   "metadata": {},
   "outputs": [],
   "source": [
    "housing = strat_train_set.drop(\"median_house_value\", axis=1)\n",
    "housing_labels = strat_train_set[\"median_house_value\"].copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f161e874",
   "metadata": {},
   "source": [
    "## Custom Transformers"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "44f8c54d",
   "metadata": {},
   "source": [
    "Let's create a custom Transfomer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "83e93f41",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-121.46, 38.52, 29.0, ..., 706.0, 2.1736, 'INLAND'],\n",
       "       [-117.23, 33.09, 7.0, ..., 768.0, 6.3373, 'NEAR OCEAN'],\n",
       "       [-119.04, 35.37, 44.0, ..., 300.0, 2.875, 'INLAND'],\n",
       "       ...,\n",
       "       [-122.72, 38.44, 48.0, ..., 172.0, 3.1797, '<1H OCEAN'],\n",
       "       [-122.7, 38.31, 14.0, ..., 501.0, 4.1964, '<1H OCEAN'],\n",
       "       [-122.14, 39.97, 27.0, ..., 197.0, 3.1319, 'INLAND']], dtype=object)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "col_names = \"total_rooms\", \"total_bedrooms\", \"population\", \"households\"\n",
    "rooms_ix, bedrooms_ix, population_ix, households_ix = [housing.columns.get_loc(c) for c in col_names] ## dynamic name extraction\n",
    "\n",
    "class CombinedAttributesAdder(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, add_bedrooms_per_room = True): # no *args or **kargs , hyperparameters\n",
    "        self.add_bedrooms_per_room = add_bedrooms_per_room\n",
    "    def fit(self, X, y=None):\n",
    "        return self # nothing else to do\n",
    "    def transform(self, X):\n",
    "        rooms_per_household = X[:, rooms_ix] / X[:, households_ix]\n",
    "        population_per_household = X[:, population_ix] / X[:, households_ix]\n",
    "        if self.add_bedrooms_per_room:\n",
    "            bedrooms_per_room = X[:, bedrooms_ix] / X[:, rooms_ix]\n",
    "            return np.c_[X, rooms_per_household, population_per_household,bedrooms_per_room]\n",
    "        else:\n",
    "            return np.c_[X, rooms_per_household, population_per_household]\n",
    "attr_adder = CombinedAttributesAdder(add_bedrooms_per_room=False)\n",
    "housing_extra_attribs = attr_adder.transform(housing.values)\n",
    "housing.values"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b577730f",
   "metadata": {},
   "source": [
    "The previous code create a transformer that create different new features using the BaseEstimator and TransformMixin functions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8afa3100-3b47-40ef-b4e1-0bd3f1bbff40",
   "metadata": {},
   "source": [
    "## Transfomation Pipline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "8d3de89e-9bf9-4487-b580-6eb1480fa195",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "housing_num = housing.drop(\"ocean_proximity\", axis=1)\n",
    "\n",
    "\n",
    "num_pipeline = Pipeline([\n",
    " ('imputer', SimpleImputer(strategy=\"median\")),\n",
    " ('attribs_adder', CombinedAttributesAdder()),\n",
    " ('std_scaler', StandardScaler()),\n",
    " ])\n",
    "housing_num_tr = num_pipeline.fit_transform(housing_num)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "851cf967",
   "metadata": {},
   "source": [
    "There is a way to both treat numerical and categorical data in one shot "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "b5c84223",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "\n",
    "num_attribs = list(housing_num)\n",
    "cat_attribs = [\"ocean_proximity\"]\n",
    "full_pipeline = ColumnTransformer([\n",
    " (\"num\", num_pipeline, num_attribs),\n",
    " (\"cat\", OneHotEncoder(), cat_attribs),\n",
    " ])\n",
    "housing_prepared = full_pipeline.fit_transform(housing)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "93bb2148-5aef-47e9-9242-974efd1b37c6",
   "metadata": {},
   "source": [
    "# 2- Using SVM model on the data "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3b0057f9-e802-45ea-a5a7-6e6c46abcf48",
   "metadata": {},
   "source": [
    "## 2.1-GridSearchCV"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f84e30d-b4f5-4263-a098-9bfed64aecc6",
   "metadata": {},
   "source": [
    "Using the SVR estimator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "cda89ccd-aeb8-43eb-8b8a-eaa8535f7a18",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVR\n",
    "from sklearn.model_selection import GridSearchCV \n",
    "\n",
    "SVR_reg= SVR()\n",
    "params= [\n",
    "  {'C': [1, 10, 100, 1000], 'kernel': ['linear']},\n",
    "  {'C': [1, 10, 100, 1000], 'gamma': [0.001, 0.0001], 'kernel': ['rbf']},\n",
    " ]\n",
    "svm_grid_search = GridSearchCV(SVR_reg,params, cv=5 ,scoring=\"neg_mean_squared_error\",return_train_score=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Disclaimer: it may take an eternity lol!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "c0d45daf-c08b-4718-bafc-a2fc5a951b55",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>GridSearchCV(cv=5, estimator=SVR(),\n",
       "             param_grid=[{&#x27;C&#x27;: [1, 10, 100, 1000], &#x27;kernel&#x27;: [&#x27;linear&#x27;]},\n",
       "                         {&#x27;C&#x27;: [1, 10, 100, 1000], &#x27;gamma&#x27;: [0.001, 0.0001],\n",
       "                          &#x27;kernel&#x27;: [&#x27;rbf&#x27;]}],\n",
       "             return_train_score=True, scoring=&#x27;neg_mean_squared_error&#x27;)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">GridSearchCV</label><div class=\"sk-toggleable__content\"><pre>GridSearchCV(cv=5, estimator=SVR(),\n",
       "             param_grid=[{&#x27;C&#x27;: [1, 10, 100, 1000], &#x27;kernel&#x27;: [&#x27;linear&#x27;]},\n",
       "                         {&#x27;C&#x27;: [1, 10, 100, 1000], &#x27;gamma&#x27;: [0.001, 0.0001],\n",
       "                          &#x27;kernel&#x27;: [&#x27;rbf&#x27;]}],\n",
       "             return_train_score=True, scoring=&#x27;neg_mean_squared_error&#x27;)</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: SVR</label><div class=\"sk-toggleable__content\"><pre>SVR()</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SVR</label><div class=\"sk-toggleable__content\"><pre>SVR()</pre></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "GridSearchCV(cv=5, estimator=SVR(),\n",
       "             param_grid=[{'C': [1, 10, 100, 1000], 'kernel': ['linear']},\n",
       "                         {'C': [1, 10, 100, 1000], 'gamma': [0.001, 0.0001],\n",
       "                          'kernel': ['rbf']}],\n",
       "             return_train_score=True, scoring='neg_mean_squared_error')"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svm_grid_search.fit(housing_prepared, housing_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "de87fb63-66d0-40b0-a4f0-c2db862b1142",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'C': 1000, 'kernel': 'linear'}"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svm_grid_search.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "2b6f3861-3f72-4502-b7cb-42636392475f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-2 {color: black;}#sk-container-id-2 pre{padding: 0;}#sk-container-id-2 div.sk-toggleable {background-color: white;}#sk-container-id-2 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-2 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-2 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-2 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-2 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-2 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-2 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-2 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-2 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-2 div.sk-item {position: relative;z-index: 1;}#sk-container-id-2 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-2 div.sk-item::before, #sk-container-id-2 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-2 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-2 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-2 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-2 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-2 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-2 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-2 div.sk-label-container {text-align: center;}#sk-container-id-2 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-2 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>SVR(C=1000, kernel=&#x27;linear&#x27;)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-4\" type=\"checkbox\" checked><label for=\"sk-estimator-id-4\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SVR</label><div class=\"sk-toggleable__content\"><pre>SVR(C=1000, kernel=&#x27;linear&#x27;)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "SVR(C=1000, kernel='linear')"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svm_grid_search.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "ffa812a8-d81a-4fee-9d94-2242be5f2149",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "112571.06378605746 {'C': 1, 'kernel': 'linear'}\n",
      "84649.6069847477 {'C': 10, 'kernel': 'linear'}\n",
      "71635.55363120146 {'C': 100, 'kernel': 'linear'}\n",
      "70396.4975685597 {'C': 1000, 'kernel': 'linear'}\n",
      "118924.31070087965 {'C': 1, 'gamma': 0.001, 'kernel': 'rbf'}\n",
      "118938.30756431246 {'C': 1, 'gamma': 0.0001, 'kernel': 'rbf'}\n",
      "118795.07721129213 {'C': 10, 'gamma': 0.001, 'kernel': 'rbf'}\n",
      "118923.98270894501 {'C': 10, 'gamma': 0.0001, 'kernel': 'rbf'}\n",
      "117620.69007715455 {'C': 100, 'gamma': 0.001, 'kernel': 'rbf'}\n",
      "118792.37077576606 {'C': 100, 'gamma': 0.0001, 'kernel': 'rbf'}\n",
      "107354.76152040453 {'C': 1000, 'gamma': 0.001, 'kernel': 'rbf'}\n",
      "117591.85547157719 {'C': 1000, 'gamma': 0.0001, 'kernel': 'rbf'}\n"
     ]
    }
   ],
   "source": [
    "cvres = svm_grid_search.cv_results_\n",
    "for mean_score, params in zip(cvres[\"mean_test_score\"], cvres[\"params\"]):\n",
    "    print(np.sqrt(-mean_score), params)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3575db01-c514-421f-bf78-d8b7f50e7ff1",
   "metadata": {},
   "source": [
    "As we can see the best result given by the SVR estimator gives a 70369 RMSE error"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "63877380-2a41-4089-ae6f-1f7128fa1463",
   "metadata": {},
   "source": [
    "## 2.2-RandomizedSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "a93791ef-8d68-4abb-b605-1648087684d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "params = {\n",
    "    'C': np.logspace(-3, 2, 6),\n",
    "    'kernel': [\"linear\",\"rbf\"],\n",
    "    'gamma': np.logspace(-3, 2, 6)\n",
    "}\n",
    "random_search = RandomizedSearchCV(SVR_reg,params, cv=5 ,scoring=\"neg_mean_squared_error\",random_state=1,return_train_score=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Disclaimer: it may take an eternity lol!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "178d843e-d0c5-4a16-a5ee-647619bcf215",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-3 {color: black;}#sk-container-id-3 pre{padding: 0;}#sk-container-id-3 div.sk-toggleable {background-color: white;}#sk-container-id-3 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-3 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-3 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-3 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-3 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-3 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-3 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-3 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-3 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-3 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-3 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-3 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-3 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-3 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-3 div.sk-item {position: relative;z-index: 1;}#sk-container-id-3 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-3 div.sk-item::before, #sk-container-id-3 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-3 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-3 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-3 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-3 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-3 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-3 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-3 div.sk-label-container {text-align: center;}#sk-container-id-3 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-3 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-3\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>RandomizedSearchCV(cv=5, estimator=SVR(),\n",
       "                   param_distributions={&#x27;C&#x27;: array([1.e-03, 1.e-02, 1.e-01, 1.e+00, 1.e+01, 1.e+02]),\n",
       "                                        &#x27;gamma&#x27;: array([1.e-03, 1.e-02, 1.e-01, 1.e+00, 1.e+01, 1.e+02]),\n",
       "                                        &#x27;kernel&#x27;: [&#x27;linear&#x27;, &#x27;rbf&#x27;]},\n",
       "                   random_state=1, return_train_score=True,\n",
       "                   scoring=&#x27;neg_mean_squared_error&#x27;)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-5\" type=\"checkbox\" ><label for=\"sk-estimator-id-5\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomizedSearchCV</label><div class=\"sk-toggleable__content\"><pre>RandomizedSearchCV(cv=5, estimator=SVR(),\n",
       "                   param_distributions={&#x27;C&#x27;: array([1.e-03, 1.e-02, 1.e-01, 1.e+00, 1.e+01, 1.e+02]),\n",
       "                                        &#x27;gamma&#x27;: array([1.e-03, 1.e-02, 1.e-01, 1.e+00, 1.e+01, 1.e+02]),\n",
       "                                        &#x27;kernel&#x27;: [&#x27;linear&#x27;, &#x27;rbf&#x27;]},\n",
       "                   random_state=1, return_train_score=True,\n",
       "                   scoring=&#x27;neg_mean_squared_error&#x27;)</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-6\" type=\"checkbox\" ><label for=\"sk-estimator-id-6\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: SVR</label><div class=\"sk-toggleable__content\"><pre>SVR()</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-7\" type=\"checkbox\" ><label for=\"sk-estimator-id-7\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SVR</label><div class=\"sk-toggleable__content\"><pre>SVR()</pre></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "RandomizedSearchCV(cv=5, estimator=SVR(),\n",
       "                   param_distributions={'C': array([1.e-03, 1.e-02, 1.e-01, 1.e+00, 1.e+01, 1.e+02]),\n",
       "                                        'gamma': array([1.e-03, 1.e-02, 1.e-01, 1.e+00, 1.e+01, 1.e+02]),\n",
       "                                        'kernel': ['linear', 'rbf']},\n",
       "                   random_state=1, return_train_score=True,\n",
       "                   scoring='neg_mean_squared_error')"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random_search.fit(housing_prepared, housing_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "b5140365-5ce8-46b1-8024-e5a8bf043ecd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'kernel': 'linear', 'gamma': 10.0, 'C': 100.0}"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random_search.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "c629fc90-9ed3-4ca8-b99f-611562a28885",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-4 {color: black;}#sk-container-id-4 pre{padding: 0;}#sk-container-id-4 div.sk-toggleable {background-color: white;}#sk-container-id-4 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-4 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-4 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-4 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-4 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-4 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-4 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-4 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-4 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-4 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-4 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-4 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-4 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-4 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-4 div.sk-item {position: relative;z-index: 1;}#sk-container-id-4 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-4 div.sk-item::before, #sk-container-id-4 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-4 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-4 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-4 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-4 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-4 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-4 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-4 div.sk-label-container {text-align: center;}#sk-container-id-4 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-4 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-4\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>SVR(C=100.0, gamma=10.0, kernel=&#x27;linear&#x27;)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-8\" type=\"checkbox\" checked><label for=\"sk-estimator-id-8\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SVR</label><div class=\"sk-toggleable__content\"><pre>SVR(C=100.0, gamma=10.0, kernel=&#x27;linear&#x27;)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "SVR(C=100.0, gamma=10.0, kernel='linear')"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random_search.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's see How the estimator we got from randomized search "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "fe57cf34-81b4-41f9-a90f-aacd4968743d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "118939.4732834471 {'kernel': 'rbf', 'gamma': 1.0, 'C': 0.01}\n",
      "118591.6498917307 {'kernel': 'rbf', 'gamma': 1.0, 'C': 10.0}\n",
      "118931.92012214728 {'kernel': 'linear', 'gamma': 100.0, 'C': 0.001}\n",
      "71635.55363120146 {'kernel': 'linear', 'gamma': 10.0, 'C': 100.0}\n",
      "116181.25173057283 {'kernel': 'rbf', 'gamma': 0.1, 'C': 10.0}\n",
      "118258.82445795466 {'kernel': 'linear', 'gamma': 0.01, 'C': 0.1}\n",
      "118819.34364522224 {'kernel': 'rbf', 'gamma': 0.01, 'C': 1.0}\n",
      "84649.6069847477 {'kernel': 'linear', 'gamma': 1.0, 'C': 10.0}\n",
      "118939.81287990055 {'kernel': 'rbf', 'gamma': 10.0, 'C': 0.1}\n",
      "118938.89503192669 {'kernel': 'rbf', 'gamma': 100.0, 'C': 10.0}\n"
     ]
    }
   ],
   "source": [
    "cvres = random_search.cv_results_\n",
    "for mean_score, params in zip(cvres[\"mean_test_score\"], cvres[\"params\"]):\n",
    "    print(np.sqrt(-mean_score), params)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3340bdef-f4f1-40e9-b4b2-312903d39aee",
   "metadata": {},
   "source": [
    "As we can see the best estimator has RMSE equal to 71635 "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.3-Costum Transformer "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "\n",
    "def indices_of_top_k(arr, k):\n",
    "    return np.sort(np.argpartition(np.array(arr), -k)[-k:])\n",
    "\n",
    "class TopFeatureSelector(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, feature_importances, k):\n",
    "        self.feature_importances = feature_importances\n",
    "        self.k = k\n",
    "    def fit(self, X, y=None):\n",
    "        self.feature_indices_ = indices_of_top_k(self.feature_importances, self.k)\n",
    "        return self\n",
    "    def transform(self, X):\n",
    "        return X[:, self.feature_indices_]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The above transformer only does extract the k indices of the most important features of the dataset. It does not compute the feature importances by itself. Therefore, we need to create a pipeline that computes the feature importances first, then feeds them to the TopFeatureSelector:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since we require for this transfomer to work :features importances , we can use the *feature_importances_* attribute of decision tree or a random forest regressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5.89419698e-02, 5.63495205e-02, 4.42741728e-02, 1.32544942e-02,\n",
       "       1.25134341e-02, 1.20148068e-02, 1.22129089e-02, 4.73380378e-01,\n",
       "       2.74675612e-02, 1.22755489e-01, 2.27348164e-02, 1.19768261e-03,\n",
       "       1.39886532e-01, 7.05372062e-05, 8.00138228e-04, 2.14555876e-03])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "# Assume 'X' is your input features and 'y' is the target\n",
    "random_forest = RandomForestRegressor(random_state=1234)\n",
    "random_forest.fit(housing_prepared, housing_labels)\n",
    "# The feature importances\n",
    "importances = random_forest.feature_importances_\n",
    "importances"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Above you can see the importance score of each attribute in your dataset. Now let's use them in the cosutom transformer we created above."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Suppose we want to extarct the 5 most important features of the Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "k=5 \n",
    "top_k_feature_indices = indices_of_top_k(importances, k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "housing_data_extra= TopFeatureSelector(importances, k).fit_transform(housing_prepared)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "let's create a single pipeline that does the full data preparation and selects the top k features:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "create_and_select_k_best_features_pipeline = Pipeline([(\"full_pipeline\", full_pipeline),\n",
    "                                                      (\"feature_selection\", TopFeatureSelector(importances, k))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.94135046,  1.34743822, -0.8936472 ,  0.00622264,  1.        ],\n",
       "       [ 1.17178212, -1.19243966,  1.292168  , -0.04081077,  0.        ],\n",
       "       [ 0.26758118, -0.1259716 , -0.52543365, -0.07537122,  1.        ],\n",
       "       ...,\n",
       "       [-1.5707942 ,  1.31001828, -0.36547546, -0.03743619,  0.        ],\n",
       "       [-1.56080303,  1.2492109 ,  0.16826095, -0.05915604,  0.        ],\n",
       "       [-1.28105026,  2.02567448, -0.390569  ,  0.00657083,  1.        ]])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "housing_data_extra = create_and_select_k_best_features_pipeline.fit_transform(housing)\n",
    "housing_data_extra"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.4-A pipeline that does all the data preparation and prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-5 {color: black;}#sk-container-id-5 pre{padding: 0;}#sk-container-id-5 div.sk-toggleable {background-color: white;}#sk-container-id-5 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-5 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-5 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-5 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-5 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-5 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-5 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-5 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-5 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-5 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-5 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-5 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-5 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-5 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-5 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-5 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-5 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-5 div.sk-item {position: relative;z-index: 1;}#sk-container-id-5 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-5 div.sk-item::before, #sk-container-id-5 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-5 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-5 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-5 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-5 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-5 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-5 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-5 div.sk-label-container {text-align: center;}#sk-container-id-5 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-5 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-5\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>Pipeline(steps=[(&#x27;feature_selection&#x27;,\n",
       "                 Pipeline(steps=[(&#x27;full_pipeline&#x27;,\n",
       "                                  ColumnTransformer(transformers=[(&#x27;num&#x27;,\n",
       "                                                                   Pipeline(steps=[(&#x27;imputer&#x27;,\n",
       "                                                                                    SimpleImputer(strategy=&#x27;median&#x27;)),\n",
       "                                                                                   (&#x27;attribs_adder&#x27;,\n",
       "                                                                                    CombinedAttributesAdder()),\n",
       "                                                                                   (&#x27;std_scaler&#x27;,\n",
       "                                                                                    StandardScaler())]),\n",
       "                                                                   [&#x27;longitude&#x27;,\n",
       "                                                                    &#x27;latitude&#x27;,\n",
       "                                                                    &#x27;housing_median_age&#x27;,\n",
       "                                                                    &#x27;total_rooms&#x27;,\n",
       "                                                                    &#x27;total_bedrooms&#x27;,\n",
       "                                                                    &#x27;population&#x27;,\n",
       "                                                                    &#x27;households&#x27;,...\n",
       "                                  TopFeatureSelector(feature_importances=array([5.89419698e-02, 5.63495205e-02, 4.42741728e-02, 1.32544942e-02,\n",
       "       1.25134341e-02, 1.20148068e-02, 1.22129089e-02, 4.73380378e-01,\n",
       "       2.74675612e-02, 1.22755489e-01, 2.27348164e-02, 1.19768261e-03,\n",
       "       1.39886532e-01, 7.05372062e-05, 8.00138228e-04, 2.14555876e-03]),\n",
       "                                                     k=5))])),\n",
       "                (&#x27;Predictor&#x27;, SVR(C=100.0, gamma=10.0, kernel=&#x27;linear&#x27;))])</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-9\" type=\"checkbox\" ><label for=\"sk-estimator-id-9\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;feature_selection&#x27;,\n",
       "                 Pipeline(steps=[(&#x27;full_pipeline&#x27;,\n",
       "                                  ColumnTransformer(transformers=[(&#x27;num&#x27;,\n",
       "                                                                   Pipeline(steps=[(&#x27;imputer&#x27;,\n",
       "                                                                                    SimpleImputer(strategy=&#x27;median&#x27;)),\n",
       "                                                                                   (&#x27;attribs_adder&#x27;,\n",
       "                                                                                    CombinedAttributesAdder()),\n",
       "                                                                                   (&#x27;std_scaler&#x27;,\n",
       "                                                                                    StandardScaler())]),\n",
       "                                                                   [&#x27;longitude&#x27;,\n",
       "                                                                    &#x27;latitude&#x27;,\n",
       "                                                                    &#x27;housing_median_age&#x27;,\n",
       "                                                                    &#x27;total_rooms&#x27;,\n",
       "                                                                    &#x27;total_bedrooms&#x27;,\n",
       "                                                                    &#x27;population&#x27;,\n",
       "                                                                    &#x27;households&#x27;,...\n",
       "                                  TopFeatureSelector(feature_importances=array([5.89419698e-02, 5.63495205e-02, 4.42741728e-02, 1.32544942e-02,\n",
       "       1.25134341e-02, 1.20148068e-02, 1.22129089e-02, 4.73380378e-01,\n",
       "       2.74675612e-02, 1.22755489e-01, 2.27348164e-02, 1.19768261e-03,\n",
       "       1.39886532e-01, 7.05372062e-05, 8.00138228e-04, 2.14555876e-03]),\n",
       "                                                     k=5))])),\n",
       "                (&#x27;Predictor&#x27;, SVR(C=100.0, gamma=10.0, kernel=&#x27;linear&#x27;))])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-10\" type=\"checkbox\" ><label for=\"sk-estimator-id-10\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">feature_selection: Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;full_pipeline&#x27;,\n",
       "                 ColumnTransformer(transformers=[(&#x27;num&#x27;,\n",
       "                                                  Pipeline(steps=[(&#x27;imputer&#x27;,\n",
       "                                                                   SimpleImputer(strategy=&#x27;median&#x27;)),\n",
       "                                                                  (&#x27;attribs_adder&#x27;,\n",
       "                                                                   CombinedAttributesAdder()),\n",
       "                                                                  (&#x27;std_scaler&#x27;,\n",
       "                                                                   StandardScaler())]),\n",
       "                                                  [&#x27;longitude&#x27;, &#x27;latitude&#x27;,\n",
       "                                                   &#x27;housing_median_age&#x27;,\n",
       "                                                   &#x27;total_rooms&#x27;,\n",
       "                                                   &#x27;total_bedrooms&#x27;,\n",
       "                                                   &#x27;population&#x27;, &#x27;households&#x27;,\n",
       "                                                   &#x27;median_income&#x27;]),\n",
       "                                                 (&#x27;cat&#x27;, OneHotEncoder(),\n",
       "                                                  [&#x27;ocean_proximity&#x27;])])),\n",
       "                (&#x27;feature_selection&#x27;,\n",
       "                 TopFeatureSelector(feature_importances=array([5.89419698e-02, 5.63495205e-02, 4.42741728e-02, 1.32544942e-02,\n",
       "       1.25134341e-02, 1.20148068e-02, 1.22129089e-02, 4.73380378e-01,\n",
       "       2.74675612e-02, 1.22755489e-01, 2.27348164e-02, 1.19768261e-03,\n",
       "       1.39886532e-01, 7.05372062e-05, 8.00138228e-04, 2.14555876e-03]),\n",
       "                                    k=5))])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-11\" type=\"checkbox\" ><label for=\"sk-estimator-id-11\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">full_pipeline: ColumnTransformer</label><div class=\"sk-toggleable__content\"><pre>ColumnTransformer(transformers=[(&#x27;num&#x27;,\n",
       "                                 Pipeline(steps=[(&#x27;imputer&#x27;,\n",
       "                                                  SimpleImputer(strategy=&#x27;median&#x27;)),\n",
       "                                                 (&#x27;attribs_adder&#x27;,\n",
       "                                                  CombinedAttributesAdder()),\n",
       "                                                 (&#x27;std_scaler&#x27;,\n",
       "                                                  StandardScaler())]),\n",
       "                                 [&#x27;longitude&#x27;, &#x27;latitude&#x27;, &#x27;housing_median_age&#x27;,\n",
       "                                  &#x27;total_rooms&#x27;, &#x27;total_bedrooms&#x27;, &#x27;population&#x27;,\n",
       "                                  &#x27;households&#x27;, &#x27;median_income&#x27;]),\n",
       "                                (&#x27;cat&#x27;, OneHotEncoder(), [&#x27;ocean_proximity&#x27;])])</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-12\" type=\"checkbox\" ><label for=\"sk-estimator-id-12\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">num</label><div class=\"sk-toggleable__content\"><pre>[&#x27;longitude&#x27;, &#x27;latitude&#x27;, &#x27;housing_median_age&#x27;, &#x27;total_rooms&#x27;, &#x27;total_bedrooms&#x27;, &#x27;population&#x27;, &#x27;households&#x27;, &#x27;median_income&#x27;]</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-13\" type=\"checkbox\" ><label for=\"sk-estimator-id-13\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer(strategy=&#x27;median&#x27;)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-14\" type=\"checkbox\" ><label for=\"sk-estimator-id-14\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">CombinedAttributesAdder</label><div class=\"sk-toggleable__content\"><pre>CombinedAttributesAdder()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-15\" type=\"checkbox\" ><label for=\"sk-estimator-id-15\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">StandardScaler</label><div class=\"sk-toggleable__content\"><pre>StandardScaler()</pre></div></div></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-16\" type=\"checkbox\" ><label for=\"sk-estimator-id-16\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">cat</label><div class=\"sk-toggleable__content\"><pre>[&#x27;ocean_proximity&#x27;]</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-17\" type=\"checkbox\" ><label for=\"sk-estimator-id-17\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">OneHotEncoder</label><div class=\"sk-toggleable__content\"><pre>OneHotEncoder()</pre></div></div></div></div></div></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-18\" type=\"checkbox\" ><label for=\"sk-estimator-id-18\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">TopFeatureSelector</label><div class=\"sk-toggleable__content\"><pre>TopFeatureSelector(feature_importances=array([5.89419698e-02, 5.63495205e-02, 4.42741728e-02, 1.32544942e-02,\n",
       "       1.25134341e-02, 1.20148068e-02, 1.22129089e-02, 4.73380378e-01,\n",
       "       2.74675612e-02, 1.22755489e-01, 2.27348164e-02, 1.19768261e-03,\n",
       "       1.39886532e-01, 7.05372062e-05, 8.00138228e-04, 2.14555876e-03]),\n",
       "                   k=5)</pre></div></div></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-19\" type=\"checkbox\" ><label for=\"sk-estimator-id-19\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SVR</label><div class=\"sk-toggleable__content\"><pre>SVR(C=100.0, gamma=10.0, kernel=&#x27;linear&#x27;)</pre></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "Pipeline(steps=[('feature_selection',\n",
       "                 Pipeline(steps=[('full_pipeline',\n",
       "                                  ColumnTransformer(transformers=[('num',\n",
       "                                                                   Pipeline(steps=[('imputer',\n",
       "                                                                                    SimpleImputer(strategy='median')),\n",
       "                                                                                   ('attribs_adder',\n",
       "                                                                                    CombinedAttributesAdder()),\n",
       "                                                                                   ('std_scaler',\n",
       "                                                                                    StandardScaler())]),\n",
       "                                                                   ['longitude',\n",
       "                                                                    'latitude',\n",
       "                                                                    'housing_median_age',\n",
       "                                                                    'total_rooms',\n",
       "                                                                    'total_bedrooms',\n",
       "                                                                    'population',\n",
       "                                                                    'households',...\n",
       "                                  TopFeatureSelector(feature_importances=array([5.89419698e-02, 5.63495205e-02, 4.42741728e-02, 1.32544942e-02,\n",
       "       1.25134341e-02, 1.20148068e-02, 1.22129089e-02, 4.73380378e-01,\n",
       "       2.74675612e-02, 1.22755489e-01, 2.27348164e-02, 1.19768261e-03,\n",
       "       1.39886532e-01, 7.05372062e-05, 8.00138228e-04, 2.14555876e-03]),\n",
       "                                                     k=5))])),\n",
       "                ('Predictor', SVR(C=100.0, gamma=10.0, kernel='linear'))])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model= SVR(**random_search.best_params_)\n",
    "final_pipeline=Pipeline([(\"feature_selection\", create_and_select_k_best_features_pipeline),(\"Predictor\",model)]) \n",
    "final_pipeline.fit(housing, housing_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can now test it on some examples of the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predictions:\t [ 87683.01343346 291758.15018057 116104.29524525 148069.54376255]\n",
      "Labels:\t\t [72100.0, 279600.0, 82700.0, 112500.0]\n"
     ]
    }
   ],
   "source": [
    "some_data = housing.iloc[:4]\n",
    "some_labels = housing_labels.iloc[:4]\n",
    "\n",
    "print(\"Predictions:\\t\", final_pipeline.predict(some_data))\n",
    "print(\"Labels:\\t\\t\", list(some_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}