class_05_amazon_recommender.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "verified-accommodation",
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from IPython.display import Markdown, display, HTML\n",
    "from collections import defaultdict\n",
    "from sklearn.model_selection import KFold\n",
    "import scipy.special as scisp\n",
    "\n",
    "# Fix the dying kernel problem (only a problem in some installations - you can remove it, if it works without it)\n",
    "import os\n",
    "os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "educated-tourist",
   "metadata": {},
   "source": [
    "# Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "looking-feeling",
   "metadata": {},
   "outputs": [],
   "source": [
    "ml_ratings_df = pd.read_csv(os.path.join(\"data\", \"movielens_small\", \"ratings.csv\")).rename(columns={'userId': 'user_id', 'movieId': 'item_id'})\n",
    "ml_movies_df = pd.read_csv(os.path.join(\"data\", \"movielens_small\", \"movies.csv\")).rename(columns={'movieId': 'item_id'})\n",
    "ml_df = pd.merge(ml_ratings_df, ml_movies_df, on='item_id')\n",
    "\n",
    "display(ml_movies_df.head(10))\n",
    "\n",
    "# Filter the data to reduce the number of movies\n",
    "seed = 6789\n",
    "rng = np.random.RandomState(seed=seed)\n",
    "left_ids = rng.choice(ml_movies_df['item_id'], size=100, replace=False)\n",
    "\n",
    "ml_ratings_df = ml_ratings_df.loc[ml_ratings_df['item_id'].isin(left_ids)]\n",
    "ml_movies_df = ml_movies_df.loc[ml_movies_df['item_id'].isin(left_ids)]\n",
    "ml_df = ml_df.loc[ml_df['item_id'].isin(left_ids)]\n",
    "\n",
    "print(\"Number of interactions left: {}\".format(len(ml_ratings_df)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "protecting-recognition",
   "metadata": {},
   "source": [
    "# Inner workings of the Amazon recommender fit method"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "plastic-brooklyn",
   "metadata": {},
   "source": [
    "## Shift item ids and user ids so that they are consecutive"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "optical-westminster",
   "metadata": {},
   "source": [
    "**Task 1.** Create a mapping from item ids in ml_ratings_df DataFrame into consecutive natural numbers starting from 0. Example:\n",
    "\n",
    "    {780: 0, 1500: 1, 3479: 2, 171: 3, 1914: 4, 4896: 5, ...}\n",
    "\n",
    "Name this mapping item_id_mapping. Create also a reverse mapping to this one. Name it item_id_reverse_mapping.\n",
    "\n",
    "Create analogous mappings for user ids. Name them user_id_mapping and user_id_reverse_mapping, respectively.\n",
    "\n",
    "Copy ml_ratings_df into interactions_df and apply the mappings to the user_id and item_id columns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "valuable-modem",
   "metadata": {},
   "outputs": [],
   "source": [
    "interactions_df = ml_ratings_df.copy()\n",
    "\n",
    "# Write your code here\n",
    "\n",
    "print(\"Item mapping\")\n",
    "print(item_id_mapping)\n",
    "print()\n",
    "\n",
    "print(\"Item reverse mapping\")\n",
    "print(item_id_reverse_mapping)\n",
    "print()\n",
    "\n",
    "print(\"User mapping\")\n",
    "print(user_id_mapping)\n",
    "print()\n",
    "\n",
    "print(\"User reverse mapping\")\n",
    "print(user_id_reverse_mapping)\n",
    "print()\n",
    "\n",
    "display(interactions_df.head(10))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "basic-meeting",
   "metadata": {},
   "source": [
    "## Get the number of items and users"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "close-massachusetts",
   "metadata": {},
   "outputs": [],
   "source": [
    "n_items = np.max(interactions_df['item_id']) + 1\n",
    "n_users = np.max(interactions_df['user_id']) + 1\n",
    "\n",
    "print(\"n_items={}\\nn_users={}\".format(n_items, n_users))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "permanent-corrections",
   "metadata": {},
   "source": [
    "## Get the maximal number of interactions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "equivalent-cassette",
   "metadata": {},
   "source": [
    "**Task 2.** Calculate the maximal number of interactions for a single user and set the max_interactions variable to this value."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "peripheral-natural",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write your code here\n",
    "\n",
    "print(\"max_interactions={}\".format(max_interactions))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "basic-production",
   "metadata": {},
   "source": [
    "## Calculate P_Y's"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "physical-preparation",
   "metadata": {},
   "source": [
    "**Task 3.** For every movie calculate the prior probability of interaction (the number of users who rated a given movie divided by the number of all users) and put those probabilities in a dictionary as follows:\n",
    "\n",
    "    {0: 0.17264957264957265, 1: 0.05042735042735043, 2: 0.015384615384615385, ...}\n",
    "    \n",
    "Set the result to p_y and print it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "concrete-transparency",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write your code here\n",
    "\n",
    "print(p_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "consolidated-constant",
   "metadata": {},
   "source": [
    "## For every X calculate the E[Y $\\cap$ X]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "electoral-cylinder",
   "metadata": {},
   "source": [
    "**Task 4.** Calculate $E_{XY}$ for all pairs of items as described in the Amazon paper (see lecture 5 one note). To do that first calculate powers of $P(Y)$ up to $k$ equal to $max\\_interactions$. Then calculate $\\alpha_k$ for every item and every $k$ between 1 and $max\\_interactions$. Finally, calculate $E_{XY}$ from those values.\n",
    "\n",
    "Print the submatrix of $E_{XY}$ of the first ten rows and ten columns."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "legislative-documentation",
   "metadata": {},
   "source": [
    "Note that for large datasets $E_{XY}$ are never calculated all at once, but only the needed ones are calculated on the fly from the powers of $P(Y)$ and $\\alpha_k$'s. But for smaller datasets it is more efficient to calculate all $E_{XY}$ once, sacrificing memory for a speed up in processing time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "alive-cameroon",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write your code here\n",
    "e_xy = np.zeros(shape=(n_items, n_items))\n",
    "e_xy[:][:] = -1e100\n",
    "    \n",
    "# Write your code here\n",
    "\n",
    "print(\"E[Y|X]\")\n",
    "print(np.around(e_xy[:10, :10], 3))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "acknowledged-threshold",
   "metadata": {},
   "source": [
    "## Get the user-item interaction matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "extraordinary-mexico",
   "metadata": {},
   "outputs": [],
   "source": [
    "# mapping to int is necessary because of how iterrows works\n",
    "r = np.zeros(shape=(n_users, n_items))\n",
    "for idx, interaction in interactions_df.iterrows():\n",
    "    r[int(interaction['user_id'])][int(interaction['item_id'])] = 1\n",
    "    \n",
    "print(r[:10, :10])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "lovely-password",
   "metadata": {},
   "source": [
    "## Calculate the number of users who bought both X and Y"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "waiting-mainstream",
   "metadata": {},
   "source": [
    "**Task 5.** Calculate the number of users who bought both X and Y ($N_{XY}$). Use the interaction matrix and matrix multiplication to achieve that. Print the submatrix of $N_{XY}$ of the first ten rows and ten columns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "rubber-detector",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write your code here\n",
    "\n",
    "print(n_xy[:10, :10])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "distinguished-consequence",
   "metadata": {},
   "source": [
    "## Calculate the scores"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "still-audience",
   "metadata": {},
   "source": [
    "**Task 6.** Calculate the \"Chi-squared\" scores for all pairs of items as described in the Amazon paper (see lecture 5 one note). Print the submatrix of the first ten rows and ten columns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "pointed-deputy",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write your code here\n",
    "\n",
    "print(np.around(scores[:10, :10], 3))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "endangered-stomach",
   "metadata": {},
   "source": [
    "## Final comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "prepared-fraction",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"E[Y|X]\")\n",
    "print(np.around(e_xy[:10, :10], 3))\n",
    "print()\n",
    "\n",
    "print(\"N(X, Y)\")\n",
    "print(n_xy[:10, :10])\n",
    "print()\n",
    "\n",
    "print(\"Scores\")\n",
    "print(np.around(scores[:10, :10], 3))\n",
    "print()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "distant-archive",
   "metadata": {},
   "source": [
    "# Inner workings of the Amazon recommender recommend method"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "incident-circulation",
   "metadata": {},
   "source": [
    "**Task 7.** Using the scores for all pairs of items generate recommendations for the user with original user_id=1. To do that you have to take all movies this user has already rated ($X$) and for every movie in the dataset ($Y$) sum up all the scores $score\\_{XY}$. Then you have to return items $Y$ with the highest score. Do not recommend movies already rated.\n",
    "\n",
    "Print ten first recommendations in the following form:\n",
    "\n",
    "Recommendation: 1, Honeymoon in Vegas (1992), 5.387478215471393\n",
    "\n",
    "using code in the form:\n",
    "\n",
    "    print(\"Recommendation: {}, {}, {}\".format(user_id, movie_title, score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aerial-shipping",
   "metadata": {},
   "outputs": [],
   "source": [
    "user_id = 1\n",
    "should_recommend_already_bought = False\n",
    "n_recommendations = 10\n",
    "\n",
    "# Write your code here"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "opponent-prediction",
   "metadata": {},
   "source": [
    "# Amazon recommender"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fancy-return",
   "metadata": {},
   "outputs": [],
   "source": [
    "from recommenders.recommender import Recommender\n",
    "\n",
    "class AmazonRecommender(Recommender):\n",
    "    \"\"\"\n",
    "    Basic item-to-item collaborative filtering algorithm used in Amazon.com as described in:\n",
    "    - Linden G., Smith B., York Y., Amazon.com Recommendations. Item-to-Item Collaborative Filtering,\n",
    "        IEEE Internet Computing, 2003,\n",
    "    - Smith B., Linden G., Two Decades of Recommender Systems at Amazon.com, IEEE Internet Computing, 2017.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.recommender_df = pd.DataFrame(columns=['user_id', 'item_id', 'score'])\n",
    "        self.interactions_df = None\n",
    "        self.item_id_mapping = None\n",
    "        self.user_id_mapping = None\n",
    "        self.item_id_reverse_mapping = None\n",
    "        self.user_id_reverse_mapping = None\n",
    "        self.e_xy = None\n",
    "        self.n_xy = None\n",
    "        self.scores = None\n",
    "        self.most_popular_items = None\n",
    "        self.should_recommend_already_bought = False\n",
    "\n",
    "    def initialize(self, **params):\n",
    "        if 'should_recommend_already_bought' in params:\n",
    "            self.should_recommend_already_bought = params['should_recommend_already_bought']\n",
    "\n",
    "    def fit(self, interactions_df, users_df, items_df):\n",
    "        \"\"\"\n",
    "        Training of the recommender.\n",
    "\n",
    "        :param pd.DataFrame interactions_df: DataFrame with recorded interactions between users and items\n",
    "            defined by user_id, item_id and features of the interaction.\n",
    "        :param pd.DataFrame users_df: DataFrame with users and their features defined by\n",
    "            user_id and the user feature columns.\n",
    "        :param pd.DataFrame items_df: DataFrame with items and their features defined\n",
    "            by item_id and the item feature columns.\n",
    "        \"\"\"\n",
    "\n",
    "        # Shift item ids and user ids so that they are consecutive\n",
    "\n",
    "        unique_item_ids = interactions_df['item_id'].unique()\n",
    "        self.item_id_mapping = dict(zip(unique_item_ids, list(range(len(unique_item_ids)))))\n",
    "        self.item_id_reverse_mapping = dict(zip(list(range(len(unique_item_ids))), unique_item_ids))\n",
    "        unique_user_ids = interactions_df['user_id'].unique()\n",
    "        self.user_id_mapping = dict(zip(unique_user_ids, list(range(len(unique_user_ids)))))\n",
    "        self.user_id_reverse_mapping = dict(zip(list(range(len(unique_user_ids))), unique_user_ids))\n",
    "        \n",
    "        interactions_df = interactions_df.copy()\n",
    "        interactions_df.replace({'item_id': self.item_id_mapping, 'user_id': self.user_id_mapping}, inplace=True)\n",
    "\n",
    "        # Get the number of items and users\n",
    "\n",
    "        self.interactions_df = interactions_df\n",
    "        n_items = np.max(interactions_df['item_id']) + 1\n",
    "        n_users = np.max(interactions_df['user_id']) + 1\n",
    "\n",
    "        # Get maximal number of interactions\n",
    "\n",
    "        n_user_interactions = interactions_df[['user_id', 'item_id']].groupby(\"user_id\").count()\n",
    "        # Unnecessary, but added for readability\n",
    "        n_user_interactions = n_user_interactions.rename(columns={'item_id': 'n_items'})\n",
    "        max_interactions = n_user_interactions['n_items'].max()\n",
    "\n",
    "        # Calculate P_Y's\n",
    "\n",
    "        n_interactions = len(interactions_df)\n",
    "        p_y = interactions_df[['item_id', 'user_id']].groupby(\"item_id\").count().reset_index()\n",
    "        p_y = p_y.rename(columns={'user_id': 'P_Y'})\n",
    "        p_y.loc[:, 'P_Y'] = p_y['P_Y'] / n_interactions\n",
    "        p_y = dict(zip(p_y['item_id'], p_y['P_Y']))\n",
    "\n",
    "        # Get the series of all items\n",
    "\n",
    "        # items = list(range(n_items))\n",
    "        items = interactions_df['item_id'].unique()\n",
    "\n",
    "        # For every X calculate the E[Y|X]\n",
    "\n",
    "        e_xy = np.zeros(shape=(n_items, n_items))\n",
    "        e_xy[:][:] = -1e100\n",
    "\n",
    "        p_y_powers = {}\n",
    "        for y in items:\n",
    "            p_y_powers[y] = np.array([p_y[y]**k for k in range(1, max_interactions + 1)])\n",
    "\n",
    "        for x in items:\n",
    "            # Get users who bought X\n",
    "            c_x = interactions_df.loc[interactions_df['item_id'] == x]['user_id'].unique()\n",
    "\n",
    "            # Get users who bought only X\n",
    "            c_only_x = interactions_df.loc[interactions_df['item_id'] != x]['user_id'].unique()\n",
    "            c_only_x = list(set(c_x.tolist()) - set(c_only_x.tolist()))\n",
    "\n",
    "            # Calculate the number of non-X interactions for each user who bought X\n",
    "            n_non_x_interactions = interactions_df.loc[interactions_df['item_id'] != x, ['user_id', 'item_id']]\n",
    "            n_non_x_interactions = n_non_x_interactions.groupby(\"user_id\").count()\n",
    "            # Unnecessary, but added for readability\n",
    "            n_non_x_interactions = n_non_x_interactions.rename(columns={'item_id': 'n_items'})\n",
    "\n",
    "            # Include users with zero non-X interactions\n",
    "            zero_non_x_interactions = pd.DataFrame([[0]]*len(c_only_x), columns=[\"n_items\"], index=c_only_x)\n",
    "            n_non_x_interactions = pd.concat([n_non_x_interactions, zero_non_x_interactions])\n",
    "\n",
    "            c_non_x = n_non_x_interactions.index.unique()\n",
    "            c_x_and_non_x = list(set.intersection(set(c_x.tolist()), set(c_non_x.tolist())))\n",
    "            n_non_x_interactions = n_non_x_interactions.loc[c_x_and_non_x]\n",
    "\n",
    "            # Calculate the expected numbers of Y products bought by clients who bought X\n",
    "            alpha_k = np.array([np.sum([(-1)**(k + 1) * scisp.binom(abs_c, k)\n",
    "                                        for abs_c in n_non_x_interactions[\"n_items\"]])\n",
    "                                for k in range(1, max_interactions + 1)])\n",
    "\n",
    "            for y in items:  # Optimize to use only those Y's which have at least one client who bought both X and Y\n",
    "                if y != x:\n",
    "                    e_xy[x][y] = np.sum(alpha_k * p_y_powers[y])\n",
    "                else:\n",
    "                    e_xy[x][y] = n_users * p_y[x]\n",
    "\n",
    "        self.e_xy = e_xy\n",
    "\n",
    "        # Calculate the number of users who bought both X and Y\n",
    "\n",
    "        # Get the user-item interaction matrix (mapping to int is necessary because of how iterrows works)\n",
    "        r = np.zeros(shape=(n_users, n_items))\n",
    "        for idx, interaction in interactions_df.iterrows():\n",
    "            r[int(interaction['user_id'])][int(interaction['item_id'])] = 1\n",
    "\n",
    "        # Get the number of users who bought both X and Y\n",
    "\n",
    "        n_xy = np.matmul(r.T, r)\n",
    "\n",
    "        self.n_xy = n_xy\n",
    "        \n",
    "        # Calculate the scores\n",
    "\n",
    "        self.scores = np.divide(n_xy - e_xy, np.sqrt(e_xy), out=np.zeros_like(n_xy), where=e_xy != 0)\n",
    "        \n",
    "        # Find the most popular items for the cold start problem\n",
    "        \n",
    "        offers_count = interactions_df.loc[:, ['item_id', 'user_id']].groupby(by='item_id').count()\n",
    "        offers_count = offers_count.sort_values('user_id', ascending=False)\n",
    "        self.most_popular_items = offers_count.index\n",
    "\n",
    "    def recommend(self, users_df, items_df, n_recommendations=1):\n",
    "        \"\"\"\n",
    "        Serving of recommendations. Scores items in items_df for each user in users_df and returns\n",
    "        top n_recommendations for each user.\n",
    "\n",
    "        :param pd.DataFrame users_df: DataFrame with users and their features for which\n",
    "            recommendations should be generated.\n",
    "        :param pd.DataFrame items_df: DataFrame with items and their features which should be scored.\n",
    "        :param int n_recommendations: Number of recommendations to be returned for each user.\n",
    "        :return: DataFrame with user_id, item_id and score as columns returning n_recommendations top recommendations\n",
    "            for each user.\n",
    "        :rtype: pd.DataFrame\n",
    "        \"\"\"\n",
    "\n",
    "        # Clean previous recommendations (iloc could be used alternatively)\n",
    "        self.recommender_df = self.recommender_df[:0]\n",
    "        \n",
    "        # Handle users not in the training data\n",
    "\n",
    "        # Map item ids\n",
    "        \n",
    "        items_df = items_df.copy()\n",
    "        items_df.replace({'item_id': self.item_id_mapping}, inplace=True)\n",
    "\n",
    "        # Generate recommendations\n",
    "\n",
    "        for idx, user in users_df.iterrows():\n",
    "            recommendations = []\n",
    "            \n",
    "            user_id = user['user_id']\n",
    "            \n",
    "            if user_id in self.user_id_mapping:\n",
    "                mapped_user_id = self.user_id_mapping[user_id]\n",
    "            \n",
    "                x_list = self.interactions_df.loc[self.interactions_df['user_id'] == mapped_user_id]['item_id'].tolist()\n",
    "                final_scores = np.sum(self.scores[x_list], axis=0)\n",
    "\n",
    "                # Choose n recommendations based on highest scores\n",
    "                if not self.should_recommend_already_bought:\n",
    "                    final_scores[x_list] = -1e100\n",
    "\n",
    "                chosen_ids = np.argsort(-final_scores)[:n_recommendations]\n",
    "\n",
    "                for item_id in chosen_ids:\n",
    "                    recommendations.append(\n",
    "                        {\n",
    "                            'user_id': self.user_id_reverse_mapping[mapped_user_id],\n",
    "                            'item_id': self.item_id_reverse_mapping[item_id],\n",
    "                            'score': final_scores[item_id]\n",
    "                        }\n",
    "                    )\n",
    "            else:  # For new users recommend most popular items\n",
    "                for i in range(n_recommendations):\n",
    "                    recommendations.append(\n",
    "                        {\n",
    "                            'user_id': user['user_id'],\n",
    "                            'item_id': self.item_id_reverse_mapping[self.most_popular_items[i]],\n",
    "                            'score': 1.0\n",
    "                        }\n",
    "                    )\n",
    "\n",
    "            user_recommendations = pd.DataFrame(recommendations)\n",
    "\n",
    "            self.recommender_df = pd.concat([self.recommender_df, user_recommendations])\n",
    "\n",
    "        return self.recommender_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "nonprofit-roads",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Quick test of the recommender\n",
    "\n",
    "amazon_recommender = AmazonRecommender()\n",
    "amazon_recommender.fit(ml_ratings_df, None, ml_movies_df)\n",
    "recommendations = amazon_recommender.recommend(pd.DataFrame([[1], [4], [6]], columns=['user_id']), ml_movies_df, 10)\n",
    "\n",
    "recommendations = pd.merge(recommendations, ml_movies_df, on='item_id', how='left')\n",
    "print(\"Recommendations\")\n",
    "display(HTML(recommendations.to_html()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "framed-negative",
   "metadata": {},
   "source": [
    "# Training-test split evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "romantic-music",
   "metadata": {},
   "outputs": [],
   "source": [
    "from evaluation_and_testing.testing import evaluate_train_test_split_implicit\n",
    "\n",
    "amazon_recommender = AmazonRecommender()\n",
    "\n",
    "amazon_tts_results = [['AmazonRecommender'] + list(evaluate_train_test_split_implicit(\n",
    "    amazon_recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df))]\n",
    "\n",
    "amazon_tts_results = pd.DataFrame(\n",
    "    amazon_tts_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "display(HTML(amazon_tts_results.to_html()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "saving-harrison",
   "metadata": {},
   "outputs": [],
   "source": [
    "from recommenders.tfidf_recommender import TFIDFRecommender\n",
    "\n",
    "tfidf_recommender = TFIDFRecommender()\n",
    "\n",
    "tfidf_tts_results = [['TFIDFRecommender'] + list(evaluate_train_test_split_implicit(\n",
    "    tfidf_recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df))]\n",
    "\n",
    "tfidf_tts_results = pd.DataFrame(\n",
    "    tfidf_tts_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "display(HTML(tfidf_tts_results.to_html()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "random-source",
   "metadata": {},
   "outputs": [],
   "source": [
    "tts_results = pd.concat([amazon_tts_results, tfidf_tts_results]).reset_index(drop=True)\n",
    "display(HTML(tts_results.to_html()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "continued-harassment",
   "metadata": {},
   "source": [
    "# Leave-one-out evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "prerequisite-lounge",
   "metadata": {},
   "outputs": [],
   "source": [
    "from evaluation_and_testing.testing import evaluate_leave_one_out_implicit\n",
    "\n",
    "amazon_recommender = AmazonRecommender()\n",
    "\n",
    "amazon_loo_results = [['AmazonRecommender'] + list(evaluate_leave_one_out_implicit(\n",
    "    amazon_recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df, max_evals=300, seed=6789))]\n",
    "\n",
    "amazon_loo_results = pd.DataFrame(\n",
    "    amazon_loo_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "display(HTML(amazon_loo_results.to_html()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "behind-cambodia",
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_recommender = TFIDFRecommender()\n",
    "\n",
    "tfidf_loo_results = [['TFIDFRecommender'] + list(evaluate_leave_one_out_implicit(\n",
    "    tfidf_recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df, max_evals=300, seed=6789))]\n",
    "\n",
    "tfidf_loo_results = pd.DataFrame(\n",
    "    tfidf_loo_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "display(HTML(tfidf_loo_results.to_html()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "lightweight-password",
   "metadata": {},
   "outputs": [],
   "source": [
    "loo_results = pd.concat([amazon_loo_results, tfidf_loo_results]).reset_index(drop=True)\n",
    "display(HTML(loo_results.to_html()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "mediterranean-residence",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}