bmp25_+_lightfm_with_features_60epochs.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "bmp25 + lightfm_with_features_60epochs.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.1"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/gazon1/Recco-challenge/blob/master/bmp25_%2B_lightfm_with_features_60epochs.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cfucylfCunZK",
        "colab_type": "text"
      },
      "source": [
        "# Rekko challenge 2019\n",
        "\n",
        "```\n",
        "                           /$$$$$$$  /$$$$$$$$ /$$   /$$ /$$   /$$  /$$$$$$ \n",
        "                          | $$__  $$| $$_____/| $$  /$$/| $$  /$$/ /$$__  $$\n",
        "                          | $$  \\ $$| $$      | $$ /$$/ | $$ /$$/ | $$  \\ $$\n",
        "                          | $$$$$$$/| $$$$$   | $$$$$/  | $$$$$/  | $$  | $$\n",
        "                          | $$__  $$| $$__/   | $$  $$  | $$  $$  | $$  | $$\n",
        "                          | $$  \\ $$| $$      | $$\\  $$ | $$\\  $$ | $$  | $$\n",
        "                          | $$  | $$| $$$$$$$$| $$ \\  $$| $$ \\  $$|  $$$$$$/\n",
        "                          |__/  |__/|________/|__/  \\__/|__/  \\__/ \\______/ \n",
        "                                                                            \n",
        "```"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z5JoPw4QunZM",
        "colab_type": "text"
      },
      "source": [
        "Добро пожаловать на соревнование по машинному обучению от онлайн-кинотеатра [Okko](http://okko.tv) Rekko Challenge 2019.\n",
        "\n",
        "В этом ноутбуке мы покажем вам пример простого но полного решения, от загрузки данных до формирования ответа. Для работы нам понадобятся библиотеки `pandas`, `numpy`, `scipy`, `implicit`, `pprint`, `tqdm`. Установить их в вашем рабочем окружении можно следующей командой.\n",
        "```\n",
        "pip install pandas numpy scipy implicit pprint tqdm\n",
        "```"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BRFWg_QZunZN",
        "colab_type": "code",
        "outputId": "d530fbe4-69d8-4db6-b63f-825849cf8807",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "import os\n",
        "import json\n",
        "import pandas as pd\n",
        "\n",
        "import numpy as np\n",
        "\n",
        "\n",
        "import tqdm\n",
        "import scipy.sparse as sp\n",
        "from sklearn.metrics import average_precision_score as score\n",
        "import time\n",
        "from matplotlib import pyplot as plt\n",
        "\n",
        "from sklearn import preprocessing\n",
        "from itertools import chain\n",
        "\n",
        "from keras.models import model_from_json\n",
        "import keras\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.layers import Input, Embedding, SpatialDropout1D, Dropout, BatchNormalization, Dense,LSTM, CuDNNGRU\n",
        "from keras.models import Sequential\n",
        "from keras.layers import RepeatVector\n",
        "from keras.layers import TimeDistributed\n",
        "from keras.utils import plot_model\n",
        "from keras.models import Model\n",
        "from keras import optimizers\n",
        "\n",
        "from tensorflow.python.client import device_lib\n",
        "\n",
        "\n",
        "from keras import backend as K\n",
        "K.tensorflow_backend._get_available_gpus()\n",
        "\n",
        "# Let's set random seed\n",
        "import tensorflow as tf\n",
        "tf.set_random_seed(42)\n",
        "np.random.seed(42)\n",
        "\n",
        "from sklearn.preprocessing import LabelEncoder, MultiLabelBinarizer, StandardScaler\n",
        "import itertools\n",
        "from pprint import pprint\n",
        "\n",
        "DATA_PATH = ''\n",
        "%matplotlib inline\n",
        "\n",
        "# For Google Colab only:\n",
        "import sys\n",
        "sys.path.append('/content/gdrive/My Drive/okko competition')\n",
        "#from reco_utils.recommender.sar.sar_singlenode import SARSingleNode\n",
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')\n",
        "\n",
        "DATA_PATH = 'gdrive/My Drive/okko competition/data'\n",
        "CODE_PATH = 'gdrive/My Drive/okko competition/code'\n",
        "\n",
        "with open(os.path.join(DATA_PATH, 'test_users.json'), 'r') as f:\n",
        "    test_users = set(json.load(f)['users'])\n",
        "    \n",
        "with open(os.path.join(DATA_PATH, 'catalogue.json'), 'r') as f:\n",
        "    catalogue = json.load(f)\n",
        "    \n",
        "catalogue = {int(k): v for k, v in catalogue.items()}\n",
        "catalogue = pd.DataFrame.from_dict(catalogue,'index').reset_index()\n",
        "# av_hot - нужно для валидации. Эта колонка равна 1, если фильм или сериал может быть потребленным\n",
        "catalogue['av_hot'] = catalogue['availability'].apply(lambda x: 1 if x == [] or 'subscription' in x else 0)\n",
        "catalogue = catalogue.rename(index=str, columns={'index':'element_uid'})\n",
        "# catalogue.element_uid = catalogue.element_uid.astype('category')\n",
        "# catalogue['categ_id'] = catalogue.element_uid.cat.codes + 1"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sNk9yU2FvIbG",
        "colab_type": "code",
        "outputId": "8c5a246b-cdd3-448f-fe7d-fbe9f426da36",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 208
        }
      },
      "source": [
        "transactions = pd.read_csv(\n",
        "        os.path.join(DATA_PATH, 'transactions.csv'),\n",
        "        dtype={\n",
        "            'element_uid': np.uint16,\n",
        "            'user_uid': np.uint32,\n",
        "            'consumption_mode': 'category',\n",
        "            'ts': np.float64,\n",
        "            'watched_time': np.uint64,\n",
        "            'device_type': np.uint8,\n",
        "            'device_manufacturer': np.uint8\n",
        "        }\n",
        "    )\n",
        "\n",
        "ratings = pd.read_csv(\n",
        "        os.path.join(DATA_PATH, 'ratings.csv'),\n",
        "        dtype={\n",
        "            'element_uid': np.uint16,\n",
        "            'user_uid': np.uint32,\n",
        "            'ts': np.float64,\n",
        "            'rating': np.uint8\n",
        "        }\n",
        "    )\n",
        "\n",
        "bookmarks = pd.read_csv(\n",
        "        os.path.join(DATA_PATH, 'bookmarks.csv'),\n",
        "        dtype={\n",
        "            'element_uid': np.uint16,\n",
        "            'user_uid': np.uint32,\n",
        "            'ts': np.float64\n",
        "        }\n",
        "    )\n",
        "\n",
        "bookmarks['rating'] = 5\n",
        "_bookmarks = bookmarks[['rating', 'user_uid', 'element_uid','ts']]\n",
        "_ratings = ratings\n",
        "#_ratings['rating'] = 1\n",
        "\n",
        "_transaction = transactions[['element_uid', 'user_uid','ts','watched_time','consumption_mode']]\n",
        "#_transaction = _transaction[(_transaction['watched_time']>700)]\n",
        "_transaction['rating']=2\n",
        "_transaction.loc[(_transaction['consumption_mode']=='P')&(_transaction['watched_time']>1000),'rating'] = 8\n",
        "_transaction.loc[(_transaction['consumption_mode']=='R')&(_transaction['watched_time']>1000),'rating'] = 7\n",
        "_transaction.loc[(_transaction['consumption_mode']=='S')&(_transaction['watched_time']>1000),'rating'] = 6\n",
        "_transaction.loc[(_transaction['consumption_mode']=='P')&(_transaction['watched_time']<1000),'rating'] = 5\n",
        "\n",
        "all_ratings = pd.concat([ _transaction[['user_uid', 'element_uid', 'rating', 'ts']],_bookmarks[['user_uid', 'element_uid', 'rating', 'ts']]])\n",
        "all_ratings = pd.concat([all_ratings, _ratings[['user_uid', 'element_uid', 'rating','ts']]])\n",
        "all_ratings = all_ratings.drop_duplicates(['user_uid','element_uid'],keep='first')\n",
        "all_ratings = all_ratings.sort_values(by=['ts'],ascending=False)\n",
        "all_ratings= all_ratings.rename(index=str,columns={'element_uid':'itemID','user_uid':'userID','ts':'timestamp'})"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:40: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
            "/usr/local/lib/python3.6/dist-packages/pandas/core/indexing.py:543: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
            "  self.obj[item] = s\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q9tGSbMHUn7v",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "t=bookmarks.groupby('element_uid').agg({'ts':'count'}).reset_index()\n",
        "t['element_uid'] = t['element_uid'].astype('int64')\n",
        "t = t.rename(index=str, columns={'ts':'count of bookmarks'})\n",
        "catalogue = catalogue.merge(t,how='left', on='element_uid')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dM0gIafsQg9H",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "t=all_ratings.groupby('itemID').agg({'rating':'mean'}).reset_index()\n",
        "t = t.rename(index=str, columns={'rating':'average rating'})\n",
        "catalogue = catalogue.merge(t,how='left', left_on = 'element_uid', right_on='itemID')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2PZyxL5JQ-WR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "t=all_ratings.groupby('itemID').agg({'rating':'max'}).reset_index()\n",
        "t = t.rename(index=str, columns={'rating':'max rating'})\n",
        "catalogue = catalogue.merge(t,how='left', left_on = 'element_uid', right_on='itemID')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f3wEEToMSfGA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "t=all_ratings.groupby('itemID').agg({'rating':'min'}).reset_index()\n",
        "t = t.rename(index=str, columns={'rating':'min rating'})\n",
        "catalogue = catalogue.merge(t,how='left', left_on = 'element_uid', right_on='itemID')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y2xTel0CUWsT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "t=all_ratings.groupby('itemID').agg({'rating':'median'}).reset_index()\n",
        "t = t.rename(index=str, columns={'rating':'median rating'})\n",
        "catalogue = catalogue.merge(t,how='left', left_on = 'element_uid', right_on='itemID')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NOhHaftVUvh2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "t=all_ratings.groupby('itemID').agg({'rating':'count'}).reset_index()\n",
        "t = t.rename(index=str, columns={'rating':'count of ratings'})\n",
        "catalogue = catalogue.merge(t,how='left', left_on = 'element_uid', right_on='itemID')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vti7S4NVSiFJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "t = transactions.loc[transactions.consumption_mode == 'S'].groupby('element_uid').agg({'ts':'count'}).reset_index()\n",
        "t = t.rename(index=str, columns={'ts':'S count of transactions'})\n",
        "catalogue = catalogue.merge(t,how='left', left_on = 'element_uid', right_on='element_uid')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "00ccN51vTQ3t",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "t = transactions.loc[transactions.consumption_mode == 'P'].groupby('element_uid').agg({'ts':'count'}).reset_index()\n",
        "t = t.rename(index=str, columns={'ts':'P count of transactions'})\n",
        "catalogue = catalogue.merge(t,how='left', left_on = 'element_uid', right_on='element_uid')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2VYq_FtBTQzH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "t = transactions.loc[transactions.consumption_mode == 'R'].groupby('element_uid').agg({'ts':'count'}).reset_index()\n",
        "t = t.rename(index=str, columns={'ts':'R count of transactions'})\n",
        "catalogue = catalogue.merge(t,how='left', left_on = 'element_uid', right_on='element_uid')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gmYMjNz4TvOw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "t = transactions.groupby('element_uid').agg({'watched_time':'sum'}).reset_index()\n",
        "t = t.rename(index=str, columns={'watched_time': 'number of views'})\n",
        "catalogue = catalogue.merge(t,how='left', left_on = 'element_uid', right_on='element_uid')\n",
        "catalogue['number of views'] = catalogue['number of views'] / catalogue['duration']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jLFsaQ1oiaZm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "t = transactions.groupby('element_uid').agg({'watched_time':'mean'}).reset_index()\n",
        "t = t.rename(index=str, columns={'watched_time': 'mean number of views'})\n",
        "catalogue = catalogue.merge(t,how='left', left_on = 'element_uid', right_on='element_uid')\n",
        "catalogue['mean number of views'] = catalogue['mean number of views'] / catalogue['duration']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tNvUMk1YSMh0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "catalogue = catalogue.fillna(0).replace([np.inf, -np.inf], 0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bgUuPXzYunZS",
        "colab_type": "text"
      },
      "source": [
        "### Загрузка данных"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ulP9_F1hunZT",
        "colab_type": "text"
      },
      "source": [
        "Замените `DATA_PATH` на путь к данным, которые вы скачали со страницы соревнования."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gaz3EywcDQe8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "all_ratings.itemID = all_ratings.itemID.astype('category')\n",
        "all_ratings['categ_id'] = all_ratings.itemID.cat.codes + 1\n",
        "cat_to_element_uid = dict(zip(\n",
        "        range(1, len(all_ratings.itemID.cat.categories) + 1),\n",
        "        all_ratings.itemID.cat.categories))\n",
        "\n",
        "all_ratings.itemID = all_ratings.itemID.astype('category')\n",
        "all_ratings['categ_id'] = all_ratings.itemID.cat.codes + 1\n",
        "cat_to_element_uid = dict(zip(\n",
        "        range(1, len(all_ratings.itemID.cat.categories) + 1),\n",
        "        all_ratings.itemID.cat.categories\n",
        "    ))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R4ROKUGRxmgh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "for attr in [7,10,14,15,16,17,18,19,20,21,25,31,32,42,43,52,54,83,270]:\n",
        "  \n",
        "\n",
        "  def func(row,attr=attr):\n",
        "    if attr in row['attributes']:\n",
        "      return 1\n",
        "    else:\n",
        "      return 0\n",
        "\n",
        "  catalogue['is'+str(attr)] = catalogue.apply(func,axis=1)\n",
        "catalogue['ismovie'] = 0\n",
        "catalogue.loc[catalogue['type']=='movie','ismovie'] = 1\n",
        "catalogue['ismulti'] = 0\n",
        "catalogue.loc[catalogue['type']=='multipart_movie','ismulti'] = 1\n",
        "catalogue['isseries'] = 0\n",
        "catalogue.loc[catalogue['type']=='series','isseries'] = 1\n",
        "\n",
        "def av_func_p(row):\n",
        "  if 'purchase' in row['availability']:\n",
        "    return 1\n",
        "  else:\n",
        "    return 0\n",
        "def av_func_s(row):\n",
        "  if 'subscription' in row['availability']:\n",
        "    return 1\n",
        "  else:\n",
        "    return 0\n",
        "def av_func_r(row):\n",
        "  if 'rent' in row['availability']:\n",
        "    return 1\n",
        "  else:\n",
        "    return 0\n",
        "  \n",
        "def av_func_n(row):\n",
        "  if len(row['availability']) == 0:\n",
        "    return 1\n",
        "  else:\n",
        "    return 0\n",
        "catalogue['purch'] = catalogue.apply(av_func_p,axis=1)\n",
        "catalogue['subs'] = catalogue.apply(av_func_s,axis=1)\n",
        "catalogue['rent'] = catalogue.apply(av_func_r,axis=1)\n",
        "catalogue['not_known'] = catalogue.apply(av_func_n,axis=1)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BPNFFuZ1Ft-X",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "item_features = catalogue[catalogue['element_uid'].isin(all_ratings['itemID'].values)]\n",
        "item_features = item_features.drop(['type','availability','element_uid','attributes'],axis=1).values\n",
        "scaler = StandardScaler()\n",
        "item_features = scaler.fit_transform(item_features)\n",
        "\n",
        "item_features = np.hstack((np.identity(item_features.shape[0]),item_features))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eiZFI-loGI9_",
        "colab_type": "code",
        "outputId": "68c1eda4-7861-47fb-f9fe-5368be7b9ebc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "item_features.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(9580, 9629)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S3Edv96gvOKo",
        "colab_type": "code",
        "outputId": "36336fbf-4451-4196-c97e-badb946601a9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "from collections import defaultdict\n",
        "\n",
        "filtered_elements = defaultdict(set)\n",
        "\n",
        "for user_uid, element_uid in tqdm.tqdm(transactions.loc[:, ['user_uid', 'element_uid']].values):\n",
        "    if user_uid not in test_users:\n",
        "        continue\n",
        "    filtered_elements[user_uid].add(element_uid)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 9643012/9643012 [02:19<00:00, 69058.82it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L0zx419iunZX",
        "colab_type": "text"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KNoxDmlRunaX",
        "colab_type": "text"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k69v33OTunaY",
        "colab_type": "text"
      },
      "source": [
        "Для примера мы воспользуемся методом K ближайших соседей, реализованным в библиотеке `implicit`. В качестве данных используем только информацию о рейтингах.\n",
        "\n",
        "Необходимо построить разреженную матрицу, где строкам будут соответствовать элементы, столбцам — пользователи, а на пересечении пользователя и элемента будет находиться количественная оценка степени их взаимодействия, если таковое имело место.\n",
        "\n",
        "Не забудем добавить `1` к рейтингу, чтобы избежать деления на ноль во время вычисления `tf-idf`."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "STLPGJYUunaZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "all_ratings['userID'] = all_ratings['userID'].astype('category')\n",
        "all_ratings['itemID'] = all_ratings['itemID'].astype('category')\n",
        "\n",
        "ratings_matrix = sp.coo_matrix(\n",
        "    (all_ratings['rating'].astype(np.float32) + 1,\n",
        "        (\n",
        "            all_ratings['itemID'].cat.codes.copy(),\n",
        "            all_ratings['userID'].cat.codes.copy()\n",
        "        )\n",
        "    )\n",
        ")\n",
        "\n",
        "ratings_matrix = ratings_matrix.tocsr()\n",
        "ratings_matrix_T = ratings_matrix.T.tocsr()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dm4tP1Ybunac",
        "colab_type": "code",
        "outputId": "5b4cd119-7b4f-4f8a-d404-cf797562d045",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "sparsity = ratings_matrix.nnz / (ratings_matrix.shape[0] * ratings_matrix.shape[1])\n",
        "print('Sparsity: %.6f' % sparsity)\n",
        "ratings_matrix.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sparsity: 0.002172\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(9580, 500000)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IqIpvnyTunag",
        "colab_type": "text"
      },
      "source": [
        "Обучить модель крайне просто."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JoTAmIH1unap",
        "colab_type": "text"
      },
      "source": [
        "Отображения из оригинальной категории во внутреннюю пригодится нам в дальнейшем."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MWbQyBlF1Bay",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# основная функция для валидации модели\n",
        "def validation(dict_):\n",
        "  d1 = {}\n",
        "  with open(DATA_PATH+'answer_50.json', 'r') as f:\n",
        "    d1 = json.load(f)\n",
        "  av = []\n",
        "\n",
        "\n",
        "  for key in d1.keys():\n",
        "      if key in dict_.keys():\n",
        "          ar1 = np.array(d1[str(key)])\n",
        "         # print(ar1)\n",
        "          ar2 = np.array(dict_[str(key)])\n",
        "          inter = np.intersect1d(ar1,ar2)\n",
        "          av.append(len(inter))\n",
        "          to_add = []\n",
        "          for el in ar1:\n",
        "              if el  in inter:\n",
        "                  to_add.append(el)\n",
        "          for el in ar1:\n",
        "              if el  not in to_add:\n",
        "                  to_add.append(el)\n",
        "\n",
        "\n",
        "          to_add = np.array(to_add)\n",
        "  return np.mean(av),np.var(av)\n",
        "\n",
        "def shuffle(dict_):\n",
        "  d = {}\n",
        "  d1 = {}\n",
        "  with open(DATA_PATH+'my_best.json', 'r') as f:\n",
        "    d1 = json.load(f)\n",
        "  \n",
        "  for key in dict_.keys():\n",
        "    if key in d1.keys():\n",
        "      ar1 = np.array(dict_[key])\n",
        "       # print(ar1)\n",
        "      ar2 = np.array(d1[key])\n",
        "      inter = np.intersect1d(ar1,ar2)\n",
        "\n",
        "      to_add = []\n",
        "      for el in ar1:\n",
        "        if el  in inter:\n",
        "          to_add.append(el)\n",
        "      for el in ar1:\n",
        "        if el  not in to_add:\n",
        "          to_add.append(el)\n",
        "        \n",
        "        \n",
        "      to_add = np.array(to_add)\n",
        "        \n",
        "      d[key] = list(map(int,list(to_add)))\n",
        "\n",
        "    else:\n",
        "      d[key] = dict_[key]\n",
        "  for key in d1.keys():\n",
        "    if key not in d.keys():\n",
        "      d[key] = d1[key]\n",
        "      \n",
        "  return d\n",
        "       \n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G5VawlNh34tH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from lightfm import LightFM\n",
        "from lightfm.evaluation import precision_at_k\n",
        "from lightfm.evaluation import auc_score\n",
        "from scipy import sparse\n",
        "from lightfm.datasets import fetch_movielens"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_alk_erN4UYI",
        "colab_type": "code",
        "outputId": "d717c482-d600-4a5a-8bc5-eb9780ef13f1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 156
        }
      },
      "source": [
        "#%%time\n",
        "model = LightFM(loss='warp',no_components=20, learning_rate=0.01)\n",
        "#model.fit(ratings_matrix_T, epochs=10, num_threads=4,verbose=True)\n",
        "\n",
        "model.fit(ratings_matrix_T, item_features=sparse.csr_matrix(item_features), epochs=60, num_threads=4,verbose=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 0\n",
            "Epoch 1\n",
            "Epoch 2\n",
            "Epoch 3\n",
            "Epoch 4\n",
            "Epoch 5\n",
            "Epoch 6\n",
            "Epoch 7\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qu-xSgOGBR9z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "d1 = {}\n",
        "t = sparse.csr_matrix(item_features)\n",
        "for user in tqdm.tqdm(list(test_users)):\n",
        "   # print(user)\n",
        "  try:\n",
        "    cats = list(np.argsort(-model.predict(user_uid_to_cat[int(user)],[i for i in element_uid_to_cat.values()],\\\n",
        "                                           item_features=t,\n",
        "                                          num_threads=30  )))[:80]\n",
        "\n",
        "    d1[str(user)] = [int(all_ratings['itemID'].cat.categories[i]) for i in cats if i not in filtered_elements_cat[user]]\n",
        "      #print(d[str(user)])\n",
        "    d1[str(user)] = d1[str(user)][:20]\n",
        "  except:\n",
        "    pass\n",
        "print(validation(d1))\n",
        "\n",
        "to_send = shuffle(d1)\n",
        "    \n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JzWcFHhSVrtk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open(os.path.join(DATA_PATH, 'answer_40_epochs_lightfm_with_features.json'), 'w') as f:\n",
        "    json.dump(to_send, f)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "354vG3MFxxaq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "np.argsort(-model.predict(user_uid_to_cat[13],[i for i in element_uid_to_cat.values()],\\\n",
        "                                           item_features=t,num_threads=30  ))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EWRbaaHBOU2L",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}