project_2_recommender_and_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "alike-morgan",
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from IPython.display import Markdown, display, HTML\n",
    "from collections import defaultdict\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from livelossplot import PlotLosses\n",
    "\n",
    "# Fix the dying kernel problem (only a problem in some installations - you can remove it, if it works without it)\n",
    "import os\n",
    "os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "blessed-knitting",
   "metadata": {},
   "source": [
    "# Load the dataset for recommenders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "victorian-bottom",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = os.path.join(\"data\", \"hotel_data\")\n",
    "\n",
    "interactions_df = pd.read_csv(os.path.join(data_path, \"hotel_data_interactions_df.csv\"), index_col=0)\n",
    "\n",
    "base_item_features = ['term', 'length_of_stay_bucket', 'rate_plan', 'room_segment', 'n_people_bucket', 'weekend_stay']\n",
    "\n",
    "column_values_dict = {\n",
    "    'term': ['WinterVacation', 'Easter', 'OffSeason', 'HighSeason', 'LowSeason', 'MayLongWeekend', 'NewYear', 'Christmas'],\n",
    "    'length_of_stay_bucket': ['[0-1]', '[2-3]', '[4-7]', '[8-inf]'],\n",
    "    'rate_plan': ['Standard', 'Nonref'],\n",
    "    'room_segment': ['[0-160]', '[160-260]', '[260-360]', '[360-500]', '[500-900]'],\n",
    "    'n_people_bucket': ['[1-1]', '[2-2]', '[3-4]', '[5-inf]'],\n",
    "    'weekend_stay': ['True', 'False']\n",
    "}\n",
    "\n",
    "interactions_df.loc[:, 'term'] = pd.Categorical(\n",
    "    interactions_df['term'], categories=column_values_dict['term'])\n",
    "interactions_df.loc[:, 'length_of_stay_bucket'] = pd.Categorical(\n",
    "    interactions_df['length_of_stay_bucket'], categories=column_values_dict['length_of_stay_bucket'])\n",
    "interactions_df.loc[:, 'rate_plan'] = pd.Categorical(\n",
    "    interactions_df['rate_plan'], categories=column_values_dict['rate_plan'])\n",
    "interactions_df.loc[:, 'room_segment'] = pd.Categorical(\n",
    "    interactions_df['room_segment'], categories=column_values_dict['room_segment'])\n",
    "interactions_df.loc[:, 'n_people_bucket'] = pd.Categorical(\n",
    "    interactions_df['n_people_bucket'], categories=column_values_dict['n_people_bucket'])\n",
    "interactions_df.loc[:, 'weekend_stay'] = interactions_df['weekend_stay'].astype('str')\n",
    "interactions_df.loc[:, 'weekend_stay'] = pd.Categorical(\n",
    "    interactions_df['weekend_stay'], categories=column_values_dict['weekend_stay'])\n",
    "\n",
    "display(HTML(interactions_df.head(15).to_html()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "realistic-third",
   "metadata": {},
   "source": [
    "# (Optional) Prepare numerical user features\n",
    "\n",
    "The method below is left here for convenience if you want to experiment with content-based user features as an input for your neural network."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "variable-jaguar",
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_users_df(interactions_df):\n",
    "    ########################\n",
    "    # Write your code here #\n",
    "    ########################\n",
    "    \n",
    "    return users_df, user_features\n",
    "    \n",
    "\n",
    "users_df, user_features = prepare_users_df(interactions_df)\n",
    "\n",
    "print(user_features)\n",
    "\n",
    "display(HTML(users_df.loc[users_df['user_id'].isin([706, 1736, 7779, 96, 1, 50, 115])].head(15).to_html()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "amino-keyboard",
   "metadata": {},
   "source": [
    "# (Optional) Prepare numerical item features\n",
    "\n",
    "The method below is left here for convenience if you want to experiment with content-based item features as an input for your neural network."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "formal-munich",
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_items_df(interactions_df):\n",
    "    ########################\n",
    "    # Write your code here #\n",
    "    ########################\n",
    "    \n",
    "    return items_df, item_features\n",
    "\n",
    "\n",
    "items_df, item_features = prepare_items_df(interactions_df)\n",
    "\n",
    "print(item_features)\n",
    "\n",
    "display(HTML(items_df.loc[items_df['item_id'].isin([0, 1, 2, 3, 4, 5, 6])].head(15).to_html()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "figured-imaging",
   "metadata": {},
   "source": [
    "# Neural network recommender\n",
    "\n",
    "<span style=\"color:red\"><font size=\"4\">**Task:**</font></span><br> \n",
    "Code a recommender based on a neural network model. You are free to choose any network architecture you find appropriate. The network can use the interaction vectors for users and items, embeddings of users and items, as well as user and item features (you can use the features you developed in the first project).\n",
    "\n",
    "Remember to keep control over randomness - in the init method add the seed as a parameter and initialize the random seed generator with that seed (both for numpy and pytorch):\n",
    "\n",
    "```python\n",
    "self.seed = seed\n",
    "self.rng = np.random.RandomState(seed=seed)\n",
    "```\n",
    "in the network model:\n",
    "```python\n",
    "self.seed = torch.manual_seed(seed)\n",
    "```\n",
    "\n",
    "You are encouraged to experiment with:\n",
    "  - the number of layers in the network, the number of neurons and different activation functions,\n",
    "  - different optimizers and their parameters,\n",
    "  - batch size and the number of epochs,\n",
    "  - embedding layers,\n",
    "  - content-based features of both users and items."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "unlike-recipient",
   "metadata": {},
   "outputs": [],
   "source": [
    "from recommenders.recommender import Recommender\n",
    "\n",
    "\n",
    "class NNRecommender(Recommender):\n",
    "    \"\"\"\n",
    "    Linear recommender class based on user and item features.\n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self, seed=6789, n_neg_per_pos=5):\n",
    "        \"\"\"\n",
    "        Initialize recommender params and variables.\n",
    "        \"\"\"\n",
    "        self.model = None\n",
    "        self.n_neg_per_pos = n_neg_per_pos\n",
    "        \n",
    "        self.recommender_df = pd.DataFrame(columns=['user_id', 'item_id', 'score'])\n",
    "        self.users_df = None\n",
    "        self.user_features = None\n",
    "        \n",
    "        self.seed = seed\n",
    "        self.rng = np.random.RandomState(seed=seed)\n",
    "        \n",
    "        # You can add more arguments if needed\n",
    "        ########################\n",
    "        # Write your code here #\n",
    "        ########################\n",
    "    \n",
    "    def fit(self, interactions_df, users_df, items_df):\n",
    "        \"\"\"\n",
    "        Training of the recommender.\n",
    "        \n",
    "        :param pd.DataFrame interactions_df: DataFrame with recorded interactions between users and items \n",
    "            defined by user_id, item_id and features of the interaction.\n",
    "        :param pd.DataFrame users_df: DataFrame with users and their features defined by user_id and the user feature columns.\n",
    "        :param pd.DataFrame items_df: DataFrame with items and their features defined by item_id and the item feature columns.\n",
    "        \"\"\"\n",
    "        \n",
    "        interactions_df = interactions_df.copy()\n",
    "        \n",
    "        # Prepare users_df and items_df \n",
    "        # (optional - use only if you want to train a hybrid model with content-based features)\n",
    "        \n",
    "        users_df, user_features = prepare_users_df(interactions_df)\n",
    "        \n",
    "        self.users_df = users_df\n",
    "        self.user_features = user_features\n",
    "        \n",
    "        items_df, item_features = prepare_items_df(interactions_df)\n",
    "        items_df = items_df.loc[:, ['item_id'] + item_features]\n",
    "        \n",
    "        # Generate negative interactions\n",
    "        \n",
    "        ########################\n",
    "        # Write your code here #\n",
    "        ########################\n",
    "        \n",
    "        negative_interactions = []\n",
    "        \n",
    "        interactions_df = pd.concat(\n",
    "            [interactions_df, pd.DataFrame(negative_interactions, columns=['user_id', 'item_id', 'interacted'])])\n",
    "        \n",
    "        # Merge user and item features\n",
    "        # (optional - use only if you want to train a hybrid model with content-based features)\n",
    "        \n",
    "        interactions_df = pd.merge(interactions_df, users_df, on=['user_id'])\n",
    "        interactions_df = pd.merge(interactions_df, items_df, on=['item_id'])\n",
    "\n",
    "        # Initialize the neural network model\n",
    "        \n",
    "        ########################\n",
    "        # Write your code here #\n",
    "        ########################\n",
    "        \n",
    "        # Train the model using an optimizer\n",
    "        \n",
    "        ########################\n",
    "        # Write your code here #\n",
    "        ########################\n",
    "    \n",
    "    def recommend(self, users_df, items_df, n_recommendations=1):\n",
    "        \"\"\"\n",
    "        Serving of recommendations. Scores items in items_df for each user in users_df and returns \n",
    "        top n_recommendations for each user.\n",
    "        \n",
    "        :param pd.DataFrame users_df: DataFrame with users and their features for which recommendations should be generated.\n",
    "        :param pd.DataFrame items_df: DataFrame with items and their features which should be scored.\n",
    "        :param int n_recommendations: Number of recommendations to be returned for each user.\n",
    "        :return: DataFrame with user_id, item_id and score as columns returning n_recommendations top recommendations \n",
    "            for each user.\n",
    "        :rtype: pd.DataFrame\n",
    "        \"\"\"\n",
    "        \n",
    "        # Clean previous recommendations (iloc could be used alternatively)\n",
    "        self.recommender_df = self.recommender_df[:0]\n",
    "        \n",
    "        # Prepare users_df and items_df\n",
    "        # (optional - use only if you want to train a hybrid model with content-based features)\n",
    "        \n",
    "        users_df = users_df.loc[:, 'user_id']\n",
    "        users_df = pd.merge(users_df, self.users_df, on=['user_id'], how='left').fillna(0)\n",
    "        \n",
    "        items_df, item_features = prepare_items_df(items_df)\n",
    "        items_df = items_df.loc[:, ['item_id'] + item_features]\n",
    "        \n",
    "        # Score the items\n",
    "    \n",
    "        recommendations = pd.DataFrame(columns=['user_id', 'item_id', 'score'])\n",
    "        \n",
    "        for ix, user in users_df.iterrows():\n",
    "            \n",
    "            # Calculate the score for the user and every item in items_df\n",
    "            ########################\n",
    "            # Write your code here #\n",
    "            ########################\n",
    "            \n",
    "            scores = []\n",
    "\n",
    "            chosen_ids = np.argsort(-scores)[:n_recommendations]\n",
    "            \n",
    "            recommendations = []\n",
    "            for item_id in chosen_ids:\n",
    "                recommendations.append(\n",
    "                    {\n",
    "                        'user_id': user['user_id'],\n",
    "                        'item_id': item_id,\n",
    "                        'score': scores[item_id]\n",
    "                    }\n",
    "                )\n",
    "            \n",
    "            user_recommendations = pd.DataFrame(recommendations)\n",
    "\n",
    "            self.recommender_df = pd.concat([self.recommender_df, user_recommendations])\n",
    "\n",
    "        return self.recommender_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "copyrighted-relative",
   "metadata": {},
   "source": [
    "# Quick test of the recommender"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "greatest-canon",
   "metadata": {},
   "outputs": [],
   "source": [
    "items_df = interactions_df.loc[:, ['item_id'] + base_item_features].drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "initial-capital",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fit method\n",
    "nn_recommender = NNRecommender()\n",
    "nn_recommender.fit(interactions_df, None, None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "digital-consolidation",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Recommender method\n",
    "\n",
    "recommendations = nn_recommender.recommend(pd.DataFrame([[1], [2], [3], [4], [5]], columns=['user_id']), items_df, 10)\n",
    "\n",
    "recommendations = pd.merge(recommendations, items_df, on='item_id', how='left')\n",
    "display(HTML(recommendations.to_html()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "advanced-eleven",
   "metadata": {},
   "source": [
    "# Tuning method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "strange-alaska",
   "metadata": {},
   "outputs": [],
   "source": [
    "from evaluation_and_testing.testing import evaluate_train_test_split_implicit\n",
    "\n",
    "seed = 6789"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "stable-theta",
   "metadata": {},
   "outputs": [],
   "source": [
    "from hyperopt import hp, fmin, tpe, Trials\n",
    "import traceback\n",
    "\n",
    "def tune_recommender(recommender_class, interactions_df, items_df, \n",
    "                     param_space, max_evals=1, show_progressbar=True, seed=6789):\n",
    "    # Split into train_validation and test sets\n",
    "\n",
    "    shuffle = np.arange(len(interactions_df))\n",
    "    rng = np.random.RandomState(seed=seed)\n",
    "    rng.shuffle(shuffle)\n",
    "    shuffle = list(shuffle)\n",
    "\n",
    "    train_test_split = 0.8\n",
    "    split_index = int(len(interactions_df) * train_test_split)\n",
    "\n",
    "    train_validation = interactions_df.iloc[shuffle[:split_index]]\n",
    "    test = interactions_df.iloc[shuffle[split_index:]]\n",
    "\n",
    "    # Tune\n",
    "\n",
    "    def loss(tuned_params):\n",
    "        recommender = recommender_class(seed=seed, **tuned_params)\n",
    "        hr1, hr3, hr5, hr10, ndcg1, ndcg3, ndcg5, ndcg10 = evaluate_train_test_split_implicit(\n",
    "            recommender, train_validation, items_df, seed=seed)\n",
    "        return -hr10\n",
    "\n",
    "    n_tries = 1\n",
    "    succeded = False\n",
    "    try_id = 0\n",
    "    while not succeded and try_id < n_tries:\n",
    "        try:\n",
    "            trials = Trials()\n",
    "            best_param_set = fmin(loss, space=param_space, algo=tpe.suggest, \n",
    "                                  max_evals=max_evals, show_progressbar=show_progressbar, trials=trials, verbose=True)\n",
    "            succeded = True\n",
    "        except:\n",
    "            traceback.print_exc()\n",
    "            try_id += 1\n",
    "            \n",
    "    if not succeded:\n",
    "        return None\n",
    "        \n",
    "    # Validate\n",
    "    \n",
    "    recommender = recommender_class(seed=seed, **best_param_set)\n",
    "\n",
    "    results = [[recommender_class.__name__] + list(evaluate_train_test_split_implicit(\n",
    "        recommender, {'train': train_validation, 'test': test}, items_df, seed=seed))]\n",
    "\n",
    "    results = pd.DataFrame(results, \n",
    "                           columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "    display(HTML(results.to_html()))\n",
    "    \n",
    "    return best_param_set"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "reliable-switzerland",
   "metadata": {},
   "source": [
    "## Tuning of the recommender\n",
    "\n",
    "<span style=\"color:red\"><font size=\"4\">**Task:**</font></span><br> \n",
    "Tune your model using the code below. You only need to put the class name of your recommender and choose an appropriate parameter space."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "obvious-astrology",
   "metadata": {},
   "outputs": [],
   "source": [
    "param_space = {\n",
    "    'n_neg_per_pos': hp.quniform('n_neg_per_pos', 1, 10, 1),\n",
    "    ########################\n",
    "    # Write your code here #\n",
    "    ########################\n",
    "}\n",
    "\n",
    "best_param_set = tune_recommender(NNRecommender, interactions_df, items_df,\n",
    "                                  param_space, max_evals=10, show_progressbar=True, seed=seed)\n",
    "\n",
    "print(\"Best parameters:\")\n",
    "print(best_param_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "accredited-strap",
   "metadata": {},
   "source": [
    "# Final evaluation\n",
    "\n",
    "<span style=\"color:red\"><font size=\"4\">**Task:**</font></span><br> \n",
    "Run the final evaluation of your recommender and present its results against the Amazon and Netflix recommenders' results. You just need to give the class name of your recommender and its tuned parameters below.\n",
    "\n",
    "It's optional, but for better effect you can include here the results from all recommenders created during in this class."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "given-homework",
   "metadata": {},
   "outputs": [],
   "source": [
    "nn_recommender = NNRecommender(n_neg_per_pos=1)  # Initialize your recommender here\n",
    "\n",
    "# Give the name of your recommender in the line below\n",
    "nn_tts_results = [['NNRecommender'] + list(evaluate_train_test_split_implicit(\n",
    "    nn_recommender, interactions_df, items_df))]\n",
    "\n",
    "nn_tts_results = pd.DataFrame(\n",
    "    nn_tts_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "display(HTML(nn_tts_results.to_html()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "suited-nomination",
   "metadata": {},
   "outputs": [],
   "source": [
    "from recommenders.amazon_recommender import AmazonRecommender\n",
    "\n",
    "amazon_recommender = AmazonRecommender()\n",
    "\n",
    "amazon_tts_results = [['AmazonRecommender'] + list(evaluate_train_test_split_implicit(\n",
    "    amazon_recommender, interactions_df, items_df))]\n",
    "\n",
    "amazon_tts_results = pd.DataFrame(\n",
    "    amazon_tts_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "display(HTML(amazon_tts_results.to_html()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "conservative-remedy",
   "metadata": {},
   "outputs": [],
   "source": [
    "from recommenders.netflix_recommender import NetflixRecommender\n",
    "\n",
    "netflix_recommender = NetflixRecommender(embedding_dim=8, n_epochs=200, print_type='live')\n",
    "\n",
    "netflix_tts_results = [['NetflixRecommender'] + list(evaluate_train_test_split_implicit(\n",
    "    netflix_recommender, interactions_df, items_df))]\n",
    "\n",
    "netflix_tts_results = pd.DataFrame(\n",
    "    netflix_tts_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "display(HTML(netflix_tts_results.to_html()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "moderate-printing",
   "metadata": {},
   "outputs": [],
   "source": [
    "tts_results = pd.concat([nn_tts_results, amazon_tts_results, netflix_tts_results]).reset_index(drop=True)\n",
    "display(HTML(tts_results.to_html()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "uniform-vegetable",
   "metadata": {},
   "source": [
    "# Summary\n",
    "\n",
    "<span style=\"color:red\"><font size=\"4\">**Task:**</font></span><br> \n",
    "Write a summary of your experiments. What worked well and what did not? What are your thoughts how could you possibly further improve the model?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "declared-howard",
   "metadata": {},
   "outputs": [],
   "source": [
    "###########################\n",
    "# Write your summary here #\n",
    "###########################"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}