lecture_03_evaluation_and_testing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "exciting-specific",
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from IPython.display import Markdown, display, HTML\n",
    "from collections import defaultdict\n",
    "from sklearn.model_selection import KFold\n",
    "import time\n",
    "\n",
    "# Fix the dying kernel problem (only a problem in some installations - you can remove it, if it works without it)\n",
    "import os\n",
    "os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "placed-afghanistan",
   "metadata": {},
   "source": [
    "# Load the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "electoral-indication",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>Adventure|Animation|Children|Comedy|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Jumanji (1995)</td>\n",
       "      <td>Adventure|Children|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Grumpier Old Men (1995)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Waiting to Exhale (1995)</td>\n",
       "      <td>Comedy|Drama|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Father of the Bride Part II (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>Heat (1995)</td>\n",
       "      <td>Action|Crime|Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>Sabrina (1995)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>Tom and Huck (1995)</td>\n",
       "      <td>Adventure|Children</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>Sudden Death (1995)</td>\n",
       "      <td>Action</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>GoldenEye (1995)</td>\n",
       "      <td>Action|Adventure|Thriller</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of chosen interactions: 9692\n"
     ]
    }
   ],
   "source": [
    "ml_ratings_df = pd.read_csv(os.path.join(\"data\", \"movielens_small\", \"ratings.csv\")).rename(columns={'userId': 'user_id', 'movieId': 'item_id'})\n",
    "ml_movies_df = pd.read_csv(os.path.join(\"data\", \"movielens_small\", \"movies.csv\")).rename(columns={'movieId': 'item_id'})\n",
    "ml_df = pd.merge(ml_ratings_df, ml_movies_df, on='item_id')\n",
    "ml_df.head(10)\n",
    "\n",
    "display(HTML(ml_movies_df.head(10).to_html()))\n",
    "\n",
    "# Filter the data to reduce the number of movies\n",
    "seed = 6789\n",
    "rng = np.random.RandomState(seed=seed)\n",
    "left_ids = rng.choice(ml_movies_df['item_id'], size=1000, replace=False)\n",
    "\n",
    "ml_ratings_df = ml_ratings_df.loc[ml_ratings_df['item_id'].isin(left_ids)]\n",
    "ml_movies_df = ml_movies_df.loc[ml_movies_df['item_id'].isin(left_ids)]\n",
    "ml_df = ml_df.loc[ml_df['item_id'].isin(left_ids)]\n",
    "\n",
    "print(\"Number of chosen interactions: {}\".format(len(ml_ratings_df)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "effective-renaissance",
   "metadata": {},
   "source": [
    "# Recommender class\n",
    "\n",
    "Remark: Docstrings written in reStructuredText (reST) used by Sphinx to automatically generate code documentation. It is also used by default by PyCharm (type triple quotes after defining a class or a method and hit enter)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "cooperative-synthesis",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Recommender(object):\n",
    "    \"\"\"\n",
    "    Base recommender class.\n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self):\n",
    "        \"\"\"\n",
    "        Initialize base recommender params and variables.\n",
    "        \n",
    "        :param int seed: Seed for the random number generator.\n",
    "        \"\"\"\n",
    "        pass\n",
    "    \n",
    "    def fit(self, interactions_df, users_df, items_df):\n",
    "        \"\"\"\n",
    "        Training of the recommender.\n",
    "        \n",
    "        :param pd.DataFrame interactions_df: DataFrame with recorded interactions between users and items \n",
    "            defined by user_id, item_id and features of the interaction.\n",
    "        :param pd.DataFrame users_df: DataFrame with users and their features defined by user_id and the user feature columns.\n",
    "        :param pd.DataFrame items_df: DataFrame with items and their features defined by item_id and the item feature columns.\n",
    "        \"\"\"\n",
    "        pass\n",
    "    \n",
    "    def recommend(self, users_df, items_df, n_recommendations=1):\n",
    "        \"\"\"\n",
    "        Serving of recommendations. Scores items in items_df for each user in users_df and returns \n",
    "        top n_recommendations for each user.\n",
    "        \n",
    "        :param pd.DataFrame users_df: DataFrame with users and their features for which recommendations should be generated.\n",
    "        :param pd.DataFrame items_df: DataFrame with items and their features which should be scored.\n",
    "        :param int n_recommendations: Number of recommendations to be returned for each user.\n",
    "        :return: DataFrame with user_id, item_id and score as columns returning n_recommendations top recommendations \n",
    "            for each user.\n",
    "        :rtype: pd.DataFrame\n",
    "        \"\"\"\n",
    "        \n",
    "        recommendations = pd.DataFrame(columns=['user_id', 'item_id', 'score'])\n",
    "        \n",
    "        for ix, user in users_df.iterrows():\n",
    "            user_recommendations = pd.DataFrame({'user_id': user['user_id'],\n",
    "                                                 'item_id': [-1] * n_recommendations,\n",
    "                                                 'score': [3.0] * n_recommendations})\n",
    "\n",
    "            recommendations = pd.concat([recommendations, user_recommendations])\n",
    "\n",
    "        return recommendations"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cleared-warehouse",
   "metadata": {},
   "source": [
    "# Evaluation measures"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "overall-perspective",
   "metadata": {},
   "source": [
    "## Explicit feedback - ratings"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "unlike-brush",
   "metadata": {},
   "source": [
    "### MAE - Mean Absolute error\n",
    "\n",
    "<center>\n",
    "$$\n",
    "    MAE = \\frac{\\sum_{i}^N |\\hat{r}_i - r_i|}{N}\n",
    "$$\n",
    "</center>\n",
    "\n",
    "where $\\hat{r}_i$ are the predicted ratings and $r_i$ are the real ratings and $N$ is the number of items in the test set.\n",
    "\n",
    "    + Easily interpretable.\n",
    "    - Non-differentiable which makes it unsuitable for gradient descent.\n",
    "    - The scale of errors dependent on data which reduces comparability between different datasets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "solved-economy",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAE = 0.016\n",
      "MAE = 0.340\n",
      "MAE = 1.940\n",
      "MAE = 1.060\n",
      "MAE = 0.940\n"
     ]
    }
   ],
   "source": [
    "def mae(r_pred, r_real):\n",
    "    return np.sum(np.abs(r_pred - r_real)) / len(r_pred)\n",
    "\n",
    "# Test\n",
    "\n",
    "# Very small differences\n",
    "print(\"MAE = {:.3f}\".format(mae(np.array([2.99, 1.98, 3.99, 4.97, 1.01]), np.array([3, 2, 4, 5, 1]))))\n",
    "# Small differences\n",
    "print(\"MAE = {:.3f}\".format(mae(np.array([2.8, 1.7, 3.8, 4.6, 1.6]), np.array([3, 2, 4, 5, 1]))))\n",
    "# Large differences\n",
    "print(\"MAE = {:.3f}\".format(mae(np.array([1.1, 4.2, 2.8, 3.2, 3.6]), np.array([3, 2, 4, 5, 1]))))\n",
    "# Medium differences with one large difference\n",
    "print(\"MAE = {:.3f}\".format(mae(np.array([2.1, 1.2, 3.8, 4.2, 3.6]), np.array([3, 2, 4, 5, 1]))))\n",
    "# Small differences with one very large difference\n",
    "print(\"MAE = {:.3f}\".format(mae(np.array([2.8, 1.7, 3.8, 4.6, 4.6]), np.array([3, 2, 4, 5, 1]))))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "tamil-anderson",
   "metadata": {},
   "source": [
    "### RMSE - Root Mean Squared Error\n",
    "\n",
    "<center>\n",
    "$$\n",
    "    RMSE = \\sqrt{\\frac{\\sum_{i}^N (\\hat{r}_i - r_i)^2}{N}}\n",
    "$$\n",
    "</center>\n",
    "\n",
    "where $\\hat{r}_i$ are the predicted ratings and $r_i$ are the real ratings and $N$ is the number of items in the test set.\n",
    "\n",
    "    + Very well-behaved analytically and therefore extensively used to train models, especially neural networks.\n",
    "    - The scale of errors dependent on data which reduced comparability between different datasets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "entitled-snake",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE = 0.018\n",
      "RMSE = 0.371\n",
      "RMSE = 1.994\n",
      "RMSE = 1.333\n",
      "RMSE = 1.630\n"
     ]
    }
   ],
   "source": [
    "def rmse(r_pred, r_real):\n",
    "    return np.sqrt(np.sum(np.power(r_pred - r_real, 2)) / len(r_pred))\n",
    "\n",
    "# Test\n",
    "\n",
    "# Very small differences\n",
    "print(\"RMSE = {:.3f}\".format(rmse(np.array([2.99, 1.98, 3.99, 4.97, 1.01]), np.array([3, 2, 4, 5, 1]))))\n",
    "# Small differences\n",
    "print(\"RMSE = {:.3f}\".format(rmse(np.array([2.8, 1.7, 3.8, 4.6, 1.6]), np.array([3, 2, 4, 5, 1]))))\n",
    "# Large differences\n",
    "print(\"RMSE = {:.3f}\".format(rmse(np.array([1.1, 4.2, 2.8, 3.2, 3.6]), np.array([3, 2, 4, 5, 1]))))\n",
    "# Medium differences with one large difference\n",
    "print(\"RMSE = {:.3f}\".format(rmse(np.array([2.1, 1.2, 3.8, 4.2, 3.6]), np.array([3, 2, 4, 5, 1]))))\n",
    "# Small differences with one very large difference\n",
    "print(\"RMSE = {:.3f}\".format(rmse(np.array([2.8, 1.7, 3.8, 4.6, 4.6]), np.array([3, 2, 4, 5, 1]))))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "unknown-arrival",
   "metadata": {},
   "source": [
    "### MRE - Mean Relative Error (more often called MAPE)\n",
    "\n",
    "<center>\n",
    "$$\n",
    "    MRE = \\frac{1}{N} \\sum_{i}^N \\frac{|\\hat{r}_i - r_i|}{|r_i|}\n",
    "$$\n",
    "</center>\n",
    "\n",
    "where $\\hat{r}_i$ are the predicted ratings and $r_i$ are the real ratings and $N$ is the number of items in the test set.\n",
    "\n",
    "    + Easily interpretable (average percentage error) and with a meaning understandable for business.\n",
    "    - Blows up when there are values close to zero among the predicted values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "dying-course",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MRE = 0.006\n",
      "MRE = 0.189\n",
      "MRE = 0.999\n",
      "MRE = 0.702\n",
      "MRE = 0.789\n"
     ]
    }
   ],
   "source": [
    "def mre(r_pred, r_real):\n",
    "    return 1 / len(r_pred) * np.sum(np.abs(r_pred - r_real) / np.abs(r_real))\n",
    "\n",
    "# Test\n",
    "\n",
    "# Very small differences\n",
    "print(\"MRE = {:.3f}\".format(mre(np.array([2.99, 1.98, 3.99, 4.97, 1.01]), np.array([3, 2, 4, 5, 1]))))\n",
    "# Small differences\n",
    "print(\"MRE = {:.3f}\".format(mre(np.array([2.8, 1.7, 3.8, 4.6, 1.6]), np.array([3, 2, 4, 5, 1]))))\n",
    "# Large differences\n",
    "print(\"MRE = {:.3f}\".format(mre(np.array([1.1, 4.2, 2.8, 3.2, 3.6]), np.array([3, 2, 4, 5, 1]))))\n",
    "# Medium differences with one large difference\n",
    "print(\"MRE = {:.3f}\".format(mre(np.array([2.1, 1.2, 3.8, 4.2, 3.6]), np.array([3, 2, 4, 5, 1]))))\n",
    "# Small differences with one very large difference for a small base value\n",
    "print(\"MRE = {:.3f}\".format(mre(np.array([2.8, 1.7, 3.8, 4.6, 4.6]), np.array([3, 2, 4, 5, 1]))))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "imported-contribution",
   "metadata": {},
   "source": [
    "### TRE - Total Relative Error\n",
    "\n",
    "<center>\n",
    "$$\n",
    "    TRE = \\frac{\\sum_{i}^N |\\hat{r}_i - r_i|}{\\sum_{i}^N |r_i|}\n",
    "$$\n",
    "</center>\n",
    "\n",
    "where $\\hat{r}_i$ are the predicted ratings and $r_i$ are the real ratings and $N$ is the number of items in the test set.\n",
    "\n",
    "    + Easily interpretable (total percentage error) and with a meaning understandable for business.\n",
    "    + Reliable even for very small predicted values.\n",
    "    - Does not distinguish between a case when one prediction is very bad and other are very good and a case when all predictions are mediocre."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "premium-trouble",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TRE = 0.005\n",
      "TRE = 0.113\n",
      "TRE = 0.647\n",
      "TRE = 0.353\n",
      "TRE = 0.313\n"
     ]
    }
   ],
   "source": [
    "def tre(r_pred, r_real):\n",
    "    return np.sum(np.abs(r_pred - r_real)) / np.sum(np.abs(r_real))\n",
    "\n",
    "# Test\n",
    "\n",
    "# Very small differences\n",
    "print(\"TRE = {:.3f}\".format(tre(np.array([2.99, 1.98, 3.99, 4.97, 1.01]), np.array([3, 2, 4, 5, 1]))))\n",
    "# Small differences\n",
    "print(\"TRE = {:.3f}\".format(tre(np.array([2.8, 1.7, 3.8, 4.6, 1.6]), np.array([3, 2, 4, 5, 1]))))\n",
    "# Large differences\n",
    "print(\"TRE = {:.3f}\".format(tre(np.array([1.1, 4.2, 2.8, 3.2, 3.6]), np.array([3, 2, 4, 5, 1]))))\n",
    "# Medium differences with one large difference\n",
    "print(\"TRE = {:.3f}\".format(tre(np.array([2.1, 1.2, 3.8, 4.2, 3.6]), np.array([3, 2, 4, 5, 1]))))\n",
    "# Small differences with one very large difference for a small base value\n",
    "print(\"TRE = {:.3f}\".format(tre(np.array([2.8, 1.7, 3.8, 4.6, 4.6]), np.array([3, 2, 4, 5, 1]))))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "quantitative-navigation",
   "metadata": {},
   "source": [
    "## Implicit feedback - binary indicators of interactions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "obvious-egypt",
   "metadata": {},
   "source": [
    "### HR@n - Hit Ratio \n",
    "How many hits did we score in the first n recommendations.\n",
    "<br/>\n",
    "<br/>\n",
    "<center>\n",
    "$$\n",
    "    \\text{HR@}n = \\frac{\\sum_{u} \\sum_{i \\in I_u} r_{u, i} \\cdot 1_{\\hat{D}_n(u)}(i)}{M}\n",
    "$$\n",
    "</center>\n",
    "\n",
    "where:\n",
    "  * $r_{u, i}$ is $1$ if there was an interaction between user $u$ and item $i$ in the test set and $0$ otherwise, \n",
    "  * $\\hat{D}_n$ is the set of the first $n$ recommendations for user $u$, \n",
    "  * $1_{\\hat{D}_n}(i)$ is $1$ if and only if $i \\in \\hat{D}_n$, otherwise it's equal to $0$,\n",
    "  * $M$ is the number of users.\n",
    "\n",
    "\n",
    "    + Easily interpretable.\n",
    "    - Does not take the rank of each recommendation into account."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "found-amazon",
   "metadata": {},
   "outputs": [],
   "source": [
    "def hr(recommendations, real_interactions, n=1):\n",
    "    \"\"\"\n",
    "    Assumes recommendations are ordered by user_id and then by score.\n",
    "    \"\"\"\n",
    "    # Transform real_interactions to a dict for a large speed-up\n",
    "    rui = defaultdict(lambda: 0)\n",
    "    \n",
    "    for idx, row in real_interactions.iterrows():\n",
    "        rui[(row['user_id'], row['item_id'])] = 1\n",
    "        \n",
    "    hr = 0.0\n",
    "    \n",
    "    previous_user_id = -1\n",
    "    rank = 0\n",
    "    for idx, row in recommendations.iterrows():\n",
    "        if previous_user_id == row['user_id']:\n",
    "            rank += 1\n",
    "        else:\n",
    "            rank = 1\n",
    "            \n",
    "        if rank <= n:\n",
    "            hr += rui[(row['user_id'], row['item_id'])]\n",
    "        \n",
    "        previous_user_id = row['user_id']\n",
    "    \n",
    "    hr /= len(recommendations['user_id'].unique())\n",
    "    \n",
    "    return hr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "canadian-cycling",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>77</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  item_id\n",
       "0        1       45\n",
       "1        1       22\n",
       "2        1       77\n",
       "3        2       13\n",
       "4        2       77"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>45</td>\n",
       "      <td>0.90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>0.80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>0.71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>77</td>\n",
       "      <td>0.55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>0.52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>0.85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "      <td>0.69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2</td>\n",
       "      <td>25</td>\n",
       "      <td>0.64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>0.60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2</td>\n",
       "      <td>77</td>\n",
       "      <td>0.53</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  item_id  score\n",
       "0        1       45   0.90\n",
       "1        1       13   0.80\n",
       "2        1       22   0.71\n",
       "3        1       77   0.55\n",
       "4        1        9   0.52\n",
       "5        2       11   0.85\n",
       "6        2       13   0.69\n",
       "7        2       25   0.64\n",
       "8        2        6   0.60\n",
       "9        2       77   0.53"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HR@3 = 1.5000\n"
     ]
    }
   ],
   "source": [
    "# Case 1\n",
    "    \n",
    "real_interactions = pd.DataFrame(\n",
    "    [\n",
    "        [1, 45],\n",
    "        [1, 22],\n",
    "        [1, 77],\n",
    "        [2, 13],\n",
    "        [2, 77]\n",
    "        \n",
    "    ], columns=['user_id', 'item_id'])\n",
    "\n",
    "display(real_interactions)\n",
    "    \n",
    "recommendations = pd.DataFrame(\n",
    "    [\n",
    "        [1, 45, 0.9],\n",
    "        [1, 13, 0.8],\n",
    "        [1, 22, 0.71],\n",
    "        [1, 77, 0.55],\n",
    "        [1, 9, 0.52],\n",
    "        [2, 11, 0.85],\n",
    "        [2, 13, 0.69],\n",
    "        [2, 25, 0.64],\n",
    "        [2, 6, 0.60],\n",
    "        [2, 77, 0.53]\n",
    "        \n",
    "    ], columns=['user_id', 'item_id', 'score'])\n",
    "\n",
    "display(recommendations)\n",
    "    \n",
    "print(\"HR@3 = {:.4f}\".format(hr(recommendations, real_interactions, n=3)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "guided-jungle",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>77</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  item_id\n",
       "0        1       45\n",
       "1        1       22\n",
       "2        1       77\n",
       "3        2       13\n",
       "4        2       77"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>0.90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>45</td>\n",
       "      <td>0.80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>0.71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>77</td>\n",
       "      <td>0.55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>0.52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>0.85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "      <td>0.69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2</td>\n",
       "      <td>25</td>\n",
       "      <td>0.64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>0.60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2</td>\n",
       "      <td>77</td>\n",
       "      <td>0.53</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  item_id  score\n",
       "0        1       13   0.90\n",
       "1        1       45   0.80\n",
       "2        1       22   0.71\n",
       "3        1       77   0.55\n",
       "4        1        9   0.52\n",
       "5        2       11   0.85\n",
       "6        2       13   0.69\n",
       "7        2       25   0.64\n",
       "8        2        6   0.60\n",
       "9        2       77   0.53"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HR@3 = 1.5000\n"
     ]
    }
   ],
   "source": [
    "# Case 2\n",
    "    \n",
    "real_interactions = pd.DataFrame(\n",
    "    [\n",
    "        [1, 45],\n",
    "        [1, 22],\n",
    "        [1, 77],\n",
    "        [2, 13],\n",
    "        [2, 77]\n",
    "        \n",
    "    ], columns=['user_id', 'item_id'])\n",
    "\n",
    "display(real_interactions)\n",
    "    \n",
    "recommendations = pd.DataFrame(\n",
    "    [\n",
    "        [1, 13, 0.9],\n",
    "        [1, 45, 0.8],\n",
    "        [1, 22, 0.71],\n",
    "        [1, 77, 0.55],\n",
    "        [1, 9, 0.52],\n",
    "        [2, 11, 0.85],\n",
    "        [2, 13, 0.69],\n",
    "        [2, 25, 0.64],\n",
    "        [2, 6, 0.60],\n",
    "        [2, 77, 0.53]\n",
    "        \n",
    "    ], columns=['user_id', 'item_id', 'score'])\n",
    "\n",
    "display(recommendations)\n",
    "    \n",
    "print(\"HR@3 = {:.4f}\".format(hr(recommendations, real_interactions, n=3)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "behind-munich",
   "metadata": {},
   "source": [
    "### NDCG@n - Normalized Discounted Cumulative Gain\n",
    "\n",
    "How many hits did we score in the first n recommendations discounted by the position of each recommendation.\n",
    "<br/>\n",
    "<br/>\n",
    "<center>\n",
    "$$\n",
    "    \\text{NDCG@}n = \\frac{\\sum_{u} \\sum_{i \\in I_u} \\frac{r_{u, i}}{log\\left(1 + v_{\\hat{D}_n(u)}(i)\\right)}}{M}\n",
    "$$\n",
    "</center>\n",
    "\n",
    "where:\n",
    "  * $r_{u, i}$ is $1$ if there was an interaction between user $u$ and item $i$ in the test set and $0$ otherwise, \n",
    "  * $\\hat{D}_n(u)$ is the set of the first $n$ recommendations for user $u$, \n",
    "  * $v_{\\hat{D}_n(u)}(i)$ is the position of item $i$ in recommendations $\\hat{D}_n$,\n",
    "  * $M$ is the number of users.\n",
    "\n",
    "\n",
    "    - Takes the rank of each recommendation into account."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "floral-anatomy",
   "metadata": {},
   "outputs": [],
   "source": [
    "def ndcg(recommendations, real_interactions, n=1):\n",
    "    \"\"\"\n",
    "    Assumes recommendations are ordered by user_id and then by score.\n",
    "    \"\"\"\n",
    "    # Transform real_interactions to a dict for a large speed-up\n",
    "    rui = defaultdict(lambda: 0)\n",
    "    \n",
    "    for idx, row in real_interactions.iterrows():\n",
    "        rui[(row['user_id'], row['item_id'])] = 1\n",
    "        \n",
    "    ndcg = 0.0\n",
    "    \n",
    "    previous_user_id = -1\n",
    "    rank = 0\n",
    "    for idx, row in recommendations.iterrows():\n",
    "        if previous_user_id == row['user_id']:\n",
    "            rank += 1\n",
    "        else:\n",
    "            rank = 1\n",
    "            \n",
    "        if rank <= n:\n",
    "            ndcg += rui[(row['user_id'], row['item_id'])] / np.log2(1 + rank)\n",
    "        \n",
    "        previous_user_id = row['user_id']\n",
    "    \n",
    "    ndcg /= len(recommendations['user_id'].unique())\n",
    "    \n",
    "    return ndcg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "headed-vermont",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>77</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  item_id\n",
       "0        1       45\n",
       "1        1       22\n",
       "2        1       77\n",
       "3        2       13\n",
       "4        2       77"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>45</td>\n",
       "      <td>0.90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>0.80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>0.71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>77</td>\n",
       "      <td>0.55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>0.52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>0.85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "      <td>0.69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2</td>\n",
       "      <td>25</td>\n",
       "      <td>0.64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>0.60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2</td>\n",
       "      <td>77</td>\n",
       "      <td>0.53</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  item_id  score\n",
       "0        1       45   0.90\n",
       "1        1       13   0.80\n",
       "2        1       22   0.71\n",
       "3        1       77   0.55\n",
       "4        1        9   0.52\n",
       "5        2       11   0.85\n",
       "6        2       13   0.69\n",
       "7        2       25   0.64\n",
       "8        2        6   0.60\n",
       "9        2       77   0.53"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NDCG@3 = 1.0655\n"
     ]
    }
   ],
   "source": [
    "# Case 1\n",
    "    \n",
    "real_interactions = pd.DataFrame(\n",
    "    [\n",
    "        [1, 45],\n",
    "        [1, 22],\n",
    "        [1, 77],\n",
    "        [2, 13],\n",
    "        [2, 77]\n",
    "        \n",
    "    ], columns=['user_id', 'item_id'])\n",
    "\n",
    "display(real_interactions)\n",
    "    \n",
    "recommendations = pd.DataFrame(\n",
    "    [\n",
    "        [1, 45, 0.9],\n",
    "        [1, 13, 0.8],\n",
    "        [1, 22, 0.71],\n",
    "        [1, 77, 0.55],\n",
    "        [1, 9, 0.52],\n",
    "        [2, 11, 0.85],\n",
    "        [2, 13, 0.69],\n",
    "        [2, 25, 0.64],\n",
    "        [2, 6, 0.60],\n",
    "        [2, 77, 0.53]\n",
    "        \n",
    "    ], columns=['user_id', 'item_id', 'score'])\n",
    "\n",
    "display(recommendations)\n",
    "    \n",
    "print(\"NDCG@3 = {:.4f}\".format(ndcg(recommendations, real_interactions, n=3)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "compressed-giving",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>77</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  item_id\n",
       "0        1       45\n",
       "1        1       22\n",
       "2        1       77\n",
       "3        2       13\n",
       "4        2       77"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>0.90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>45</td>\n",
       "      <td>0.80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>0.71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>77</td>\n",
       "      <td>0.55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>0.52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>0.85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "      <td>0.69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2</td>\n",
       "      <td>25</td>\n",
       "      <td>0.64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>0.60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2</td>\n",
       "      <td>77</td>\n",
       "      <td>0.53</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  item_id  score\n",
       "0        1       13   0.90\n",
       "1        1       45   0.80\n",
       "2        1       22   0.71\n",
       "3        1       77   0.55\n",
       "4        1        9   0.52\n",
       "5        2       11   0.85\n",
       "6        2       13   0.69\n",
       "7        2       25   0.64\n",
       "8        2        6   0.60\n",
       "9        2       77   0.53"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NDCG@3 = 0.8809\n"
     ]
    }
   ],
   "source": [
    "# Case 2\n",
    "    \n",
    "real_interactions = pd.DataFrame(\n",
    "    [\n",
    "        [1, 45],\n",
    "        [1, 22],\n",
    "        [1, 77],\n",
    "        [2, 13],\n",
    "        [2, 77]\n",
    "        \n",
    "    ], columns=['user_id', 'item_id'])\n",
    "\n",
    "display(real_interactions)\n",
    "    \n",
    "recommendations = pd.DataFrame(\n",
    "    [\n",
    "        [1, 13, 0.9],\n",
    "        [1, 45, 0.8],\n",
    "        [1, 22, 0.71],\n",
    "        [1, 77, 0.55],\n",
    "        [1, 9, 0.52],\n",
    "        [2, 11, 0.85],\n",
    "        [2, 13, 0.69],\n",
    "        [2, 25, 0.64],\n",
    "        [2, 6, 0.60],\n",
    "        [2, 77, 0.53]\n",
    "        \n",
    "    ], columns=['user_id', 'item_id', 'score'])\n",
    "\n",
    "display(recommendations)\n",
    "    \n",
    "print(\"NDCG@3 = {:.4f}\".format(ndcg(recommendations, real_interactions, n=3)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "appointed-baltimore",
   "metadata": {},
   "source": [
    "# Testing routines (offline)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bizarre-elevation",
   "metadata": {},
   "source": [
    "## Train and test set split"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fatty-blackjack",
   "metadata": {},
   "source": [
    "### Explicit feedback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "negative-cigarette",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>MRE</th>\n",
       "      <th>TRE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BaseRecommender</td>\n",
       "      <td>1.170155</td>\n",
       "      <td>0.349264</td>\n",
       "      <td>0.271796</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def evaluate_train_test_split_explicit(recommender, interactions_df, items_df, seed=6789):\n",
    "    rng = np.random.RandomState(seed=seed)\n",
    "    \n",
    "    if type(interactions_df) == dict:\n",
    "        # If interactions_df is a dict with already split data, use the split\n",
    "        interactions_df_train = interactions_df['train']\n",
    "        interactions_df_test = interactions_df['test']\n",
    "    else:    \n",
    "        # Otherwise split the dataset into train and test\n",
    "\n",
    "        shuffle = np.arange(len(interactions_df))\n",
    "        rng.shuffle(shuffle)\n",
    "        shuffle = list(shuffle)\n",
    "\n",
    "        train_test_split = 0.8\n",
    "        split_index = int(len(interactions_df) * train_test_split)\n",
    "\n",
    "        interactions_df_train = interactions_df.iloc[shuffle[:split_index]]\n",
    "        interactions_df_test = interactions_df.iloc[shuffle[split_index:]]\n",
    "    \n",
    "    # Train the recommender\n",
    "    \n",
    "    recommender.fit(interactions_df_train, None, items_df)\n",
    "    \n",
    "    # Gather predictions\n",
    "    \n",
    "    r_pred = []\n",
    "    \n",
    "    for idx, row in interactions_df_test.iterrows():\n",
    "        users_df = pd.DataFrame([row['user_id']], columns=['user_id'])\n",
    "        eval_items_df = pd.DataFrame([row['item_id']], columns=['item_id'])\n",
    "        eval_items_df = pd.merge(eval_items_df, items_df, on='item_id')\n",
    "        recommendations = recommender.recommend(users_df, eval_items_df, n_recommendations=1)\n",
    "        \n",
    "        r_pred.append(recommendations.iloc[0]['score'])\n",
    "    \n",
    "    # Gather real ratings\n",
    "    \n",
    "    r_real = np.array(interactions_df_test['rating'].tolist())\n",
    "    \n",
    "    # Return evaluation metrics\n",
    "    \n",
    "    return rmse(r_pred, r_real), mre(r_pred, r_real), tre(r_pred, r_real)\n",
    "\n",
    "recommender = Recommender()\n",
    "\n",
    "results = [['BaseRecommender'] + list(evaluate_train_test_split_explicit(\n",
    "    recommender, ml_ratings_df.loc[:, ['user_id', 'item_id', 'rating']], ml_movies_df))]\n",
    "\n",
    "results = pd.DataFrame(results, \n",
    "                       columns=['Recommender', 'RMSE', 'MRE', 'TRE'])\n",
    "\n",
    "display(HTML(results.to_html()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "naval-croatia",
   "metadata": {},
   "source": [
    "### Implicit feedback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "considerable-sunrise",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BaseRecommender</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def evaluate_train_test_split_implicit(recommender, interactions_df, items_df, seed=6789):\n",
    "    rng = np.random.RandomState(seed=seed)\n",
    "    \n",
    "    if type(interactions_df) == 'dict':\n",
    "        # If interactions_df is a dict with already split data, use the split\n",
    "        interactions_df_train = interactions_df['train']\n",
    "        interactions_df_test = interactions_df['test']\n",
    "    else:    \n",
    "        # Otherwise split the dataset into train and test\n",
    "\n",
    "        shuffle = np.arange(len(interactions_df))\n",
    "        rng.shuffle(shuffle)\n",
    "        shuffle = list(shuffle)\n",
    "\n",
    "        train_test_split = 0.8\n",
    "        split_index = int(len(interactions_df) * train_test_split)\n",
    "\n",
    "        interactions_df_train = interactions_df.iloc[shuffle[:split_index]]\n",
    "        interactions_df_test = interactions_df.iloc[shuffle[split_index:]]\n",
    "    \n",
    "    hr_1 = []\n",
    "    hr_3 = []\n",
    "    hr_5 = []\n",
    "    hr_10 = []\n",
    "    ndcg_1 = []\n",
    "    ndcg_3 = []\n",
    "    ndcg_5 = []\n",
    "    ndcg_10 = []\n",
    "    \n",
    "    # Train the recommender\n",
    "    \n",
    "    recommender.fit(interactions_df_train, None, items_df)\n",
    "    \n",
    "    # Make recommendations for each user in the test set and calculate the metric \n",
    "    # against all items of that user in the test set\n",
    "    \n",
    "    test_user_interactions = interactions_df_test.groupby(by='user_id')\n",
    "    \n",
    "    for user_id, user_interactions in test_user_interactions:\n",
    "        \n",
    "        recommendations = recommender.recommend(pd.DataFrame([user_id], columns=['user_id']), \n",
    "                                                items_df, n_recommendations=10)\n",
    "        \n",
    "        hr_1.append(hr(recommendations, user_interactions, n=1))\n",
    "        hr_3.append(hr(recommendations, user_interactions, n=3))\n",
    "        hr_5.append(hr(recommendations, user_interactions, n=5))\n",
    "        hr_10.append(hr(recommendations, user_interactions, n=10))\n",
    "        ndcg_1.append(ndcg(recommendations, user_interactions, n=1))\n",
    "        ndcg_3.append(ndcg(recommendations, user_interactions, n=3))\n",
    "        ndcg_5.append(ndcg(recommendations, user_interactions, n=5))\n",
    "        ndcg_10.append(ndcg(recommendations, user_interactions, n=10))\n",
    "                \n",
    "    hr_1 = np.mean(hr_1)\n",
    "    hr_3 = np.mean(hr_3)\n",
    "    hr_5 = np.mean(hr_5)\n",
    "    hr_10 = np.mean(hr_10)\n",
    "    ndcg_1 = np.mean(ndcg_1)\n",
    "    ndcg_3 = np.mean(ndcg_3)\n",
    "    ndcg_5 = np.mean(ndcg_5)\n",
    "    ndcg_10 = np.mean(ndcg_10)\n",
    "    \n",
    "    return hr_1, hr_3, hr_5, hr_10, ndcg_1, ndcg_3, ndcg_5, ndcg_10\n",
    "\n",
    "recommender = Recommender()\n",
    "\n",
    "results = [['BaseRecommender'] + list(evaluate_train_test_split_implicit(\n",
    "    recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df))]\n",
    "\n",
    "results = pd.DataFrame(results, \n",
    "                       columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "display(HTML(results.to_html()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "muslim-tunisia",
   "metadata": {},
   "source": [
    "## Leave-one-out, leave-k-out, cross-validation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "adjusted-spirit",
   "metadata": {},
   "source": [
    "### Explicit feedback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "flexible-runner",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>MRE</th>\n",
       "      <th>TRE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BaseRecommender</td>\n",
       "      <td>1.161536</td>\n",
       "      <td>0.382929</td>\n",
       "      <td>0.267866</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def evaluate_leave_one_out_explicit(recommender, interactions_df, items_df, max_evals=300, seed=6789):\n",
    "    rng = np.random.RandomState(seed=seed)\n",
    "    \n",
    "    # Prepare splits of the datasets\n",
    "    kf = KFold(n_splits=len(interactions_df), random_state=rng, shuffle=True)\n",
    "    \n",
    "    # For each split of the dataset train the recommender, generate recommendations and evaluate\n",
    "    \n",
    "    r_pred = []\n",
    "    r_real = []\n",
    "    n_eval = 1\n",
    "    for train_index, test_index in kf.split(interactions_df.index):\n",
    "        interactions_df_train = interactions_df.loc[interactions_df.index[train_index]]\n",
    "        interactions_df_test = interactions_df.loc[interactions_df.index[test_index]]\n",
    "\n",
    "        recommender.fit(interactions_df_train, None, items_df)\n",
    "        recommendations = recommender.recommend(interactions_df_test.loc[:, ['user_id']], \n",
    "                                                items_df.loc[items_df['item_id'] == interactions_df_test.iloc[0]['item_id']])\n",
    "        \n",
    "        r_pred.append(recommendations.iloc[0]['score'])\n",
    "        r_real.append(interactions_df_test.iloc[0]['rating'])\n",
    "        \n",
    "        if n_eval == max_evals:\n",
    "            break\n",
    "        n_eval += 1\n",
    "        \n",
    "    r_pred = np.array(r_pred)\n",
    "    r_real = np.array(r_real)\n",
    "        \n",
    "    # Return evaluation metrics\n",
    "    \n",
    "    return rmse(r_pred, r_real), mre(r_pred, r_real), tre(r_pred, r_real)\n",
    "\n",
    "recommender = Recommender()\n",
    "\n",
    "results = [['BaseRecommender'] + list(evaluate_leave_one_out_explicit(\n",
    "    recommender, ml_ratings_df.loc[:, ['user_id', 'item_id', 'rating']], ml_movies_df))]\n",
    "\n",
    "results = pd.DataFrame(results, \n",
    "                       columns=['Recommender', 'RMSE', 'MRE', 'TRE'])\n",
    "\n",
    "display(HTML(results.to_html()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "engaged-lloyd",
   "metadata": {},
   "source": [
    "### Implicit feedback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "surrounded-newton",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BaseRecommender</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def evaluate_leave_one_out_implicit(recommender, interactions_df, items_df, max_evals=300, seed=6789):\n",
    "    rng = np.random.RandomState(seed=seed)\n",
    "    \n",
    "    # Prepare splits of the datasets\n",
    "    kf = KFold(n_splits=len(interactions_df), random_state=rng, shuffle=True)\n",
    "    \n",
    "    hr_1 = []\n",
    "    hr_3 = []\n",
    "    hr_5 = []\n",
    "    hr_10 = []\n",
    "    ndcg_1 = []\n",
    "    ndcg_3 = []\n",
    "    ndcg_5 = []\n",
    "    ndcg_10 = []\n",
    "    \n",
    "    # For each split of the dataset train the recommender, generate recommendations and evaluate\n",
    "    \n",
    "    n_eval = 1\n",
    "    for train_index, test_index in kf.split(interactions_df.index):\n",
    "        interactions_df_train = interactions_df.loc[interactions_df.index[train_index]]\n",
    "        interactions_df_test = interactions_df.loc[interactions_df.index[test_index]]\n",
    "                \n",
    "        recommender.fit(interactions_df_train, None, items_df)\n",
    "        recommendations = recommender.recommend(interactions_df_test.loc[:, ['user_id']], items_df, n_recommendations=10)\n",
    "        \n",
    "        hr_1.append(hr(recommendations, interactions_df_test, n=1))\n",
    "        hr_3.append(hr(recommendations, interactions_df_test, n=3))\n",
    "        hr_5.append(hr(recommendations, interactions_df_test, n=5))\n",
    "        hr_10.append(hr(recommendations, interactions_df_test, n=10))\n",
    "        ndcg_1.append(ndcg(recommendations, interactions_df_test, n=1))\n",
    "        ndcg_3.append(ndcg(recommendations, interactions_df_test, n=3))\n",
    "        ndcg_5.append(ndcg(recommendations, interactions_df_test, n=5))\n",
    "        ndcg_10.append(ndcg(recommendations, interactions_df_test, n=10))\n",
    "        \n",
    "        if n_eval == max_evals:\n",
    "            break\n",
    "        n_eval += 1\n",
    "        \n",
    "    hr_1 = np.mean(hr_1)\n",
    "    hr_3 = np.mean(hr_3)\n",
    "    hr_5 = np.mean(hr_5)\n",
    "    hr_10 = np.mean(hr_10)\n",
    "    ndcg_1 = np.mean(ndcg_1)\n",
    "    ndcg_3 = np.mean(ndcg_3)\n",
    "    ndcg_5 = np.mean(ndcg_5)\n",
    "    ndcg_10 = np.mean(ndcg_10)\n",
    "    \n",
    "    return hr_1, hr_3, hr_5, hr_10, ndcg_1, ndcg_3, ndcg_5, ndcg_10\n",
    "\n",
    "recommender = Recommender()\n",
    "\n",
    "results = [['BaseRecommender'] + list(evaluate_leave_one_out_implicit(\n",
    "    recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df))]\n",
    "\n",
    "results = pd.DataFrame(results, \n",
    "                       columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "display(HTML(results.to_html()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "optional-chain",
   "metadata": {},
   "source": [
    "# Examples of evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "electoral-synthesis",
   "metadata": {},
   "source": [
    "## Explicit feedback"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "usual-archives",
   "metadata": {},
   "source": [
    "### Train-test split test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "bound-offer",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>MRE</th>\n",
       "      <th>TRE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>HighestRatedRecommender</td>\n",
       "      <td>1.272994</td>\n",
       "      <td>0.481631</td>\n",
       "      <td>0.277884</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               Recommender      RMSE       MRE       TRE\n",
       "0  HighestRatedRecommender  1.272994  0.481631  0.277884"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>MRE</th>\n",
       "      <th>TRE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LinearRegressionRecommender</td>\n",
       "      <td>1.016594</td>\n",
       "      <td>0.348461</td>\n",
       "      <td>0.230529</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Recommender      RMSE       MRE       TRE\n",
       "0  LinearRegressionRecommender  1.016594  0.348461  0.230529"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>MRE</th>\n",
       "      <th>TRE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SVRRecommender</td>\n",
       "      <td>0.998426</td>\n",
       "      <td>0.344447</td>\n",
       "      <td>0.2216</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Recommender      RMSE       MRE     TRE\n",
       "0  SVRRecommender  0.998426  0.344447  0.2216"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>MRE</th>\n",
       "      <th>TRE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>HighestRatedRecommender</td>\n",
       "      <td>1.272994</td>\n",
       "      <td>0.481631</td>\n",
       "      <td>0.277884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>LinearRegressionRecommender</td>\n",
       "      <td>1.016594</td>\n",
       "      <td>0.348461</td>\n",
       "      <td>0.230529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>SVRRecommender</td>\n",
       "      <td>0.998426</td>\n",
       "      <td>0.344447</td>\n",
       "      <td>0.221600</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Recommender      RMSE       MRE       TRE\n",
       "0      HighestRatedRecommender  1.272994  0.481631  0.277884\n",
       "1  LinearRegressionRecommender  1.016594  0.348461  0.230529\n",
       "2               SVRRecommender  0.998426  0.344447  0.221600"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total evaluation time: 130.65900230407715\n"
     ]
    }
   ],
   "source": [
    "from recommenders.basic_recommenders import MostPopularRecommender\n",
    "from recommenders.basic_recommenders import HighestRatedRecommender\n",
    "from recommenders.basic_content_based_recommenders import LinearRegressionRecommender\n",
    "from recommenders.basic_content_based_recommenders import SVRRecommender\n",
    "from recommenders.nearest_neighbors_recommender import ItemBasedCosineNearestNeighborsRecommender\n",
    "\n",
    "highest_rated_recommender = HighestRatedRecommender()\n",
    "lr_recommender = LinearRegressionRecommender()\n",
    "svr_recommender = SVRRecommender()\n",
    "\n",
    "recommenders = [highest_rated_recommender, lr_recommender, svr_recommender]\n",
    "\n",
    "all_results = []\n",
    "\n",
    "t0 = time.time()\n",
    "\n",
    "for recommender in recommenders:\n",
    "    results = [[type(recommender).__name__] + list(evaluate_train_test_split_explicit(\n",
    "        recommender, ml_ratings_df, ml_movies_df))]\n",
    "\n",
    "    results = pd.DataFrame(results, \n",
    "                       columns=['Recommender', 'RMSE', 'MRE', 'TRE'])\n",
    "    all_results.append(results)\n",
    "\n",
    "    display(results)\n",
    "    \n",
    "all_results = pd.concat(all_results).reset_index(drop=True)\n",
    "display(all_results)\n",
    "\n",
    "print('Total evaluation time: {}'.format(time.time() - t0))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "immediate-warehouse",
   "metadata": {},
   "source": [
    "### Leave-one-out test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "national-sight",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>MRE</th>\n",
       "      <th>TRE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>HighestRatedRecommender</td>\n",
       "      <td>1.380808</td>\n",
       "      <td>0.579605</td>\n",
       "      <td>0.313605</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               Recommender      RMSE       MRE       TRE\n",
       "0  HighestRatedRecommender  1.380808  0.579605  0.313605"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>MRE</th>\n",
       "      <th>TRE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LinearRegressionRecommender</td>\n",
       "      <td>1.045415</td>\n",
       "      <td>0.401399</td>\n",
       "      <td>0.242301</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Recommender      RMSE       MRE       TRE\n",
       "0  LinearRegressionRecommender  1.045415  0.401399  0.242301"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>MRE</th>\n",
       "      <th>TRE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SVRRecommender</td>\n",
       "      <td>1.030074</td>\n",
       "      <td>0.399686</td>\n",
       "      <td>0.235447</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Recommender      RMSE       MRE       TRE\n",
       "0  SVRRecommender  1.030074  0.399686  0.235447"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>MRE</th>\n",
       "      <th>TRE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>HighestRatedRecommender</td>\n",
       "      <td>1.380808</td>\n",
       "      <td>0.579605</td>\n",
       "      <td>0.313605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>LinearRegressionRecommender</td>\n",
       "      <td>1.045415</td>\n",
       "      <td>0.401399</td>\n",
       "      <td>0.242301</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>SVRRecommender</td>\n",
       "      <td>1.030074</td>\n",
       "      <td>0.399686</td>\n",
       "      <td>0.235447</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Recommender      RMSE       MRE       TRE\n",
       "0      HighestRatedRecommender  1.380808  0.579605  0.313605\n",
       "1  LinearRegressionRecommender  1.045415  0.401399  0.242301\n",
       "2               SVRRecommender  1.030074  0.399686  0.235447"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total evaluation time: 1840.7159786224365\n"
     ]
    }
   ],
   "source": [
    "from recommenders.basic_recommenders import MostPopularRecommender\n",
    "from recommenders.basic_recommenders import HighestRatedRecommender\n",
    "from recommenders.basic_content_based_recommenders import LinearRegressionRecommender\n",
    "from recommenders.basic_content_based_recommenders import SVRRecommender\n",
    "from recommenders.nearest_neighbors_recommender import ItemBasedCosineNearestNeighborsRecommender\n",
    "\n",
    "highest_rated_recommender = HighestRatedRecommender()\n",
    "lr_recommender = LinearRegressionRecommender()\n",
    "svr_recommender = SVRRecommender()\n",
    "\n",
    "recommenders = [highest_rated_recommender, lr_recommender, svr_recommender]\n",
    "\n",
    "all_results = []\n",
    "\n",
    "t0 = time.time()\n",
    "\n",
    "for recommender in recommenders:\n",
    "    results = [[type(recommender).__name__] + list(evaluate_leave_one_out_explicit(\n",
    "        recommender, ml_ratings_df, ml_movies_df))]\n",
    "\n",
    "    results = pd.DataFrame(results, \n",
    "                       columns=['Recommender', 'RMSE', 'MRE', 'TRE'])\n",
    "    all_results.append(results)\n",
    "\n",
    "    display(results)\n",
    "    \n",
    "all_results = pd.concat(all_results).reset_index(drop=True)\n",
    "display(all_results)\n",
    "\n",
    "print('Total evaluation time: {}'.format(time.time() - t0))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "distant-plaintiff",
   "metadata": {},
   "source": [
    "## Implicit feedback"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "sealed-liberty",
   "metadata": {},
   "source": [
    "### Train-test split test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "identified-destination",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>MostPopularRecommender</td>\n",
       "      <td>0.076063</td>\n",
       "      <td>0.284116</td>\n",
       "      <td>0.440716</td>\n",
       "      <td>0.680089</td>\n",
       "      <td>0.076063</td>\n",
       "      <td>0.192392</td>\n",
       "      <td>0.256306</td>\n",
       "      <td>0.332798</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Recommender      HR@1      HR@3      HR@5     HR@10    NDCG@1  \\\n",
       "0  MostPopularRecommender  0.076063  0.284116  0.440716  0.680089  0.076063   \n",
       "\n",
       "     NDCG@3    NDCG@5   NDCG@10  \n",
       "0  0.192392  0.256306  0.332798  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>HighestRatedRecommender</td>\n",
       "      <td>0.114094</td>\n",
       "      <td>0.239374</td>\n",
       "      <td>0.331096</td>\n",
       "      <td>0.568233</td>\n",
       "      <td>0.114094</td>\n",
       "      <td>0.186693</td>\n",
       "      <td>0.224823</td>\n",
       "      <td>0.302199</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               Recommender      HR@1      HR@3      HR@5     HR@10    NDCG@1  \\\n",
       "0  HighestRatedRecommender  0.114094  0.239374  0.331096  0.568233  0.114094   \n",
       "\n",
       "     NDCG@3    NDCG@5   NDCG@10  \n",
       "0  0.186693  0.224823  0.302199  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LinearRegressionRecommender</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006711</td>\n",
       "      <td>0.01566</td>\n",
       "      <td>0.080537</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.003942</td>\n",
       "      <td>0.007501</td>\n",
       "      <td>0.028281</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Recommender  HR@1      HR@3     HR@5     HR@10  NDCG@1  \\\n",
       "0  LinearRegressionRecommender   0.0  0.006711  0.01566  0.080537     0.0   \n",
       "\n",
       "     NDCG@3    NDCG@5   NDCG@10  \n",
       "0  0.003942  0.007501  0.028281  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SVRRecommender</td>\n",
       "      <td>0.006711</td>\n",
       "      <td>0.011186</td>\n",
       "      <td>0.013423</td>\n",
       "      <td>0.040268</td>\n",
       "      <td>0.006711</td>\n",
       "      <td>0.009241</td>\n",
       "      <td>0.010107</td>\n",
       "      <td>0.018459</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Recommender      HR@1      HR@3      HR@5     HR@10    NDCG@1    NDCG@3  \\\n",
       "0  SVRRecommender  0.006711  0.011186  0.013423  0.040268  0.006711  0.009241   \n",
       "\n",
       "     NDCG@5   NDCG@10  \n",
       "0  0.010107  0.018459  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ItemBasedCosineNearestNeighborsRecommender</td>\n",
       "      <td>0.14094</td>\n",
       "      <td>0.404922</td>\n",
       "      <td>0.610738</td>\n",
       "      <td>0.955257</td>\n",
       "      <td>0.14094</td>\n",
       "      <td>0.291677</td>\n",
       "      <td>0.376199</td>\n",
       "      <td>0.488492</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  Recommender     HR@1      HR@3      HR@5  \\\n",
       "0  ItemBasedCosineNearestNeighborsRecommender  0.14094  0.404922  0.610738   \n",
       "\n",
       "      HR@10   NDCG@1    NDCG@3    NDCG@5   NDCG@10  \n",
       "0  0.955257  0.14094  0.291677  0.376199  0.488492  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>MostPopularRecommender</td>\n",
       "      <td>0.076063</td>\n",
       "      <td>0.284116</td>\n",
       "      <td>0.440716</td>\n",
       "      <td>0.680089</td>\n",
       "      <td>0.076063</td>\n",
       "      <td>0.192392</td>\n",
       "      <td>0.256306</td>\n",
       "      <td>0.332798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>HighestRatedRecommender</td>\n",
       "      <td>0.114094</td>\n",
       "      <td>0.239374</td>\n",
       "      <td>0.331096</td>\n",
       "      <td>0.568233</td>\n",
       "      <td>0.114094</td>\n",
       "      <td>0.186693</td>\n",
       "      <td>0.224823</td>\n",
       "      <td>0.302199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LinearRegressionRecommender</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.006711</td>\n",
       "      <td>0.015660</td>\n",
       "      <td>0.080537</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.003942</td>\n",
       "      <td>0.007501</td>\n",
       "      <td>0.028281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>SVRRecommender</td>\n",
       "      <td>0.006711</td>\n",
       "      <td>0.011186</td>\n",
       "      <td>0.013423</td>\n",
       "      <td>0.040268</td>\n",
       "      <td>0.006711</td>\n",
       "      <td>0.009241</td>\n",
       "      <td>0.010107</td>\n",
       "      <td>0.018459</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ItemBasedCosineNearestNeighborsRecommender</td>\n",
       "      <td>0.140940</td>\n",
       "      <td>0.404922</td>\n",
       "      <td>0.610738</td>\n",
       "      <td>0.955257</td>\n",
       "      <td>0.140940</td>\n",
       "      <td>0.291677</td>\n",
       "      <td>0.376199</td>\n",
       "      <td>0.488492</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  Recommender      HR@1      HR@3      HR@5  \\\n",
       "0                      MostPopularRecommender  0.076063  0.284116  0.440716   \n",
       "1                     HighestRatedRecommender  0.114094  0.239374  0.331096   \n",
       "2                 LinearRegressionRecommender  0.000000  0.006711  0.015660   \n",
       "3                              SVRRecommender  0.006711  0.011186  0.013423   \n",
       "4  ItemBasedCosineNearestNeighborsRecommender  0.140940  0.404922  0.610738   \n",
       "\n",
       "      HR@10    NDCG@1    NDCG@3    NDCG@5   NDCG@10  \n",
       "0  0.680089  0.076063  0.192392  0.256306  0.332798  \n",
       "1  0.568233  0.114094  0.186693  0.224823  0.302199  \n",
       "2  0.080537  0.000000  0.003942  0.007501  0.028281  \n",
       "3  0.040268  0.006711  0.009241  0.010107  0.018459  \n",
       "4  0.955257  0.140940  0.291677  0.376199  0.488492  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total evaluation time: 569.9556179046631\n"
     ]
    }
   ],
   "source": [
    "from recommenders.basic_recommenders import MostPopularRecommender\n",
    "from recommenders.basic_recommenders import HighestRatedRecommender\n",
    "from recommenders.basic_content_based_recommenders import LinearRegressionRecommender\n",
    "from recommenders.basic_content_based_recommenders import SVRRecommender\n",
    "from recommenders.nearest_neighbors_recommender import ItemBasedCosineNearestNeighborsRecommender\n",
    "\n",
    "most_popular_recommender = MostPopularRecommender()\n",
    "highest_rated_recommender = HighestRatedRecommender()\n",
    "lr_recommender = LinearRegressionRecommender()\n",
    "svr_recommender = SVRRecommender()\n",
    "ibcnn_recommender = ItemBasedCosineNearestNeighborsRecommender(n_neighbors=30)\n",
    "\n",
    "recommenders = [most_popular_recommender, highest_rated_recommender, lr_recommender, svr_recommender, ibcnn_recommender]\n",
    "\n",
    "all_results = []\n",
    "\n",
    "t0 = time.time()\n",
    "\n",
    "for recommender in recommenders:\n",
    "    results = [[type(recommender).__name__] + list(evaluate_train_test_split_implicit(\n",
    "        recommender, ml_ratings_df, ml_movies_df))]\n",
    "\n",
    "    results = pd.DataFrame(results, \n",
    "                           columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "    all_results.append(results)\n",
    "\n",
    "    display(results)\n",
    "    \n",
    "all_results = pd.concat(all_results).reset_index(drop=True)\n",
    "display(all_results)\n",
    "\n",
    "print('Total evaluation time: {}'.format(time.time() - t0))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "creative-beast",
   "metadata": {},
   "source": [
    "### Leave-one-out test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "qualified-westminster",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>MostPopularRecommender</td>\n",
       "      <td>0.026667</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.163333</td>\n",
       "      <td>0.026667</td>\n",
       "      <td>0.040079</td>\n",
       "      <td>0.053997</td>\n",
       "      <td>0.079749</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Recommender      HR@1  HR@3      HR@5     HR@10    NDCG@1  \\\n",
       "0  MostPopularRecommender  0.026667  0.05  0.083333  0.163333  0.026667   \n",
       "\n",
       "     NDCG@3    NDCG@5   NDCG@10  \n",
       "0  0.040079  0.053997  0.079749  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>HighestRatedRecommender</td>\n",
       "      <td>0.013333</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.073333</td>\n",
       "      <td>0.113333</td>\n",
       "      <td>0.013333</td>\n",
       "      <td>0.02754</td>\n",
       "      <td>0.041603</td>\n",
       "      <td>0.054395</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               Recommender      HR@1  HR@3      HR@5     HR@10    NDCG@1  \\\n",
       "0  HighestRatedRecommender  0.013333  0.04  0.073333  0.113333  0.013333   \n",
       "\n",
       "    NDCG@3    NDCG@5   NDCG@10  \n",
       "0  0.02754  0.041603  0.054395  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LinearRegressionRecommender</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.003154</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Recommender  HR@1  HR@3  HR@5  HR@10  NDCG@1  NDCG@3  \\\n",
       "0  LinearRegressionRecommender   0.0   0.0   0.0   0.01     0.0     0.0   \n",
       "\n",
       "   NDCG@5   NDCG@10  \n",
       "0     0.0  0.003154  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SVRRecommender</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.003333</td>\n",
       "      <td>0.013333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00129</td>\n",
       "      <td>0.004515</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Recommender  HR@1  HR@3      HR@5     HR@10  NDCG@1  NDCG@3   NDCG@5  \\\n",
       "0  SVRRecommender   0.0   0.0  0.003333  0.013333     0.0     0.0  0.00129   \n",
       "\n",
       "    NDCG@10  \n",
       "0  0.004515  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ItemBasedCosineNearestNeighborsRecommender</td>\n",
       "      <td>0.076667</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.233333</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.076667</td>\n",
       "      <td>0.128134</td>\n",
       "      <td>0.154096</td>\n",
       "      <td>0.186049</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  Recommender      HR@1  HR@3      HR@5  \\\n",
       "0  ItemBasedCosineNearestNeighborsRecommender  0.076667  0.17  0.233333   \n",
       "\n",
       "      HR@10    NDCG@1    NDCG@3    NDCG@5   NDCG@10  \n",
       "0  0.333333  0.076667  0.128134  0.154096  0.186049  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>MostPopularRecommender</td>\n",
       "      <td>0.026667</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.163333</td>\n",
       "      <td>0.026667</td>\n",
       "      <td>0.040079</td>\n",
       "      <td>0.053997</td>\n",
       "      <td>0.079749</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>HighestRatedRecommender</td>\n",
       "      <td>0.013333</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.073333</td>\n",
       "      <td>0.113333</td>\n",
       "      <td>0.013333</td>\n",
       "      <td>0.027540</td>\n",
       "      <td>0.041603</td>\n",
       "      <td>0.054395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LinearRegressionRecommender</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.010000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.003154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>SVRRecommender</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.003333</td>\n",
       "      <td>0.013333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001290</td>\n",
       "      <td>0.004515</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ItemBasedCosineNearestNeighborsRecommender</td>\n",
       "      <td>0.076667</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.233333</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.076667</td>\n",
       "      <td>0.128134</td>\n",
       "      <td>0.154096</td>\n",
       "      <td>0.186049</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  Recommender      HR@1  HR@3      HR@5  \\\n",
       "0                      MostPopularRecommender  0.026667  0.05  0.083333   \n",
       "1                     HighestRatedRecommender  0.013333  0.04  0.073333   \n",
       "2                 LinearRegressionRecommender  0.000000  0.00  0.000000   \n",
       "3                              SVRRecommender  0.000000  0.00  0.003333   \n",
       "4  ItemBasedCosineNearestNeighborsRecommender  0.076667  0.17  0.233333   \n",
       "\n",
       "      HR@10    NDCG@1    NDCG@3    NDCG@5   NDCG@10  \n",
       "0  0.163333  0.026667  0.040079  0.053997  0.079749  \n",
       "1  0.113333  0.013333  0.027540  0.041603  0.054395  \n",
       "2  0.010000  0.000000  0.000000  0.000000  0.003154  \n",
       "3  0.013333  0.000000  0.000000  0.001290  0.004515  \n",
       "4  0.333333  0.076667  0.128134  0.154096  0.186049  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total evaluation time: 2517.236170053482\n"
     ]
    }
   ],
   "source": [
    "from recommenders.basic_recommenders import MostPopularRecommender\n",
    "from recommenders.basic_recommenders import HighestRatedRecommender\n",
    "from recommenders.basic_content_based_recommenders import LinearRegressionRecommender\n",
    "from recommenders.basic_content_based_recommenders import SVRRecommender\n",
    "from recommenders.nearest_neighbors_recommender import ItemBasedCosineNearestNeighborsRecommender\n",
    "\n",
    "most_popular_recommender = MostPopularRecommender()\n",
    "highest_rated_recommender = HighestRatedRecommender()\n",
    "lr_recommender = LinearRegressionRecommender()\n",
    "svr_recommender = SVRRecommender()\n",
    "ibcnn_recommender = ItemBasedCosineNearestNeighborsRecommender(n_neighbors=30)\n",
    "\n",
    "recommenders = [most_popular_recommender, highest_rated_recommender, lr_recommender, svr_recommender, ibcnn_recommender]\n",
    "\n",
    "all_results = []\n",
    "\n",
    "t0 = time.time()\n",
    "\n",
    "for recommender in recommenders:\n",
    "    results = [[type(recommender).__name__] + list(evaluate_leave_one_out_implicit(\n",
    "        recommender, ml_ratings_df, ml_movies_df))]\n",
    "\n",
    "    results = pd.DataFrame(results, \n",
    "                           columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "    all_results.append(results)\n",
    "\n",
    "    display(results)\n",
    "    \n",
    "all_results = pd.concat(all_results).reset_index(drop=True)\n",
    "display(all_results)\n",
    "\n",
    "print('Total evaluation time: {}'.format(time.time() - t0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "acute-armenia",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}