LSTM_from_kaggle_baseline_Oleg.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "LSTM_from_kaggle_baseline_Oleg.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/gazon1/Recco-challenge/blob/master/LSTM_from_kaggle_baseline_Oleg.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "gly_24bqgYEB"
      },
      "source": [
        "from sklearn.metrics import # Rekko challenge 2019\n",
        "\n",
        "```\n",
        "                           /$$$$$$$  /$$$$$$$$ /$$   /$$ /$$   /$$  /$$$$$$ \n",
        "                          | $$__  $$| $$_____/| $$  /$$/| $$  /$$/ /$$__  $$\n",
        "                          | $$  \\ $$| $$      | $$ /$$/ | $$ /$$/ | $$  \\ $$\n",
        "                          | $$$$$$$/| $$$$$   | $$$$$/  | $$$$$/  | $$  | $$\n",
        "                          | $$__  $$| $$__/   | $$  $$  | $$  $$  | $$  | $$\n",
        "                          | $$  \\ $$| $$      | $$\\  $$ | $$\\  $$ | $$  | $$\n",
        "                          | $$  | $$| $$$$$$$$| $$ \\  $$| $$ \\  $$|  $$$$$$/\n",
        "                          |__/  |__/|________/|__/  \\__/|__/  \\__/ \\______/ \n",
        "                                                                            \n",
        "```"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "P5c3Bz9BgYEI"
      },
      "source": [
        "Добро пожаловать на соревнование по машинному обучению от онлайн-кинотеатра [Okko](http://okko.tv) Rekko Challenge 2019.\n",
        "\n",
        "В этом ноутбуке мы покажем вам пример простого но полного решения, от загрузки данных до формирования ответа. Для работы нам понадобятся библиотеки `pandas`, `numpy`, `scipy`, `implicit`, `pprint`, `tqdm`. Установить их в вашем рабочем окружении можно следующей командой.\n",
        "```\n",
        "pip install pandas numpy scipy implicit pprint tqdm\n",
        "```"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "L9ZqWVchgYEN",
        "outputId": "ba19d8a6-91f5-4f43-cce9-8e441164501d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import os\n",
        "import json\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "\n",
        "import tqdm\n",
        "import scipy.sparse as sp\n",
        "from sklearn.metrics import average_precision_score as score\n",
        "import time\n",
        "from matplotlib import pyplot as plt\n",
        "\n",
        "import itertools\n",
        "from pprint import pprint\n",
        "\n",
        "DATA_PATH = ''\n",
        "%matplotlib inline\n",
        "\n",
        "# For Google Colab only:\n",
        "import sys\n",
        "sys.path.append('/content/gdrive/My Drive/okko competition')\n",
        "#from reco_utils.recommender.sar.sar_singlenode import SARSingleNode\n",
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')\n",
        "\n",
        "DATA_PATH = 'gdrive/My Drive/okko competition/data'\n",
        "CODE_PATH = 'gdrive/My Drive/okko competition/code'"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eZLrpj1FHc4K",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Hyperparametrs:\n",
        "\n",
        "WATCHED_TIME = 600 # выкидываем передачи, просмотренные меньше WATCHED_TIME. ЧТобы ничего не выкидывать - ставим 0\n",
        "MIN_RATING = 3 # ниже этого рейтинга все выкидываем\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "Vsbu4E36gYEf"
      },
      "source": [
        "### Загрузка данных"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "9-21CbKCgYFB"
      },
      "source": [
        "`catalogue.json` содержит анонимизированную метаинформацию о доступных в сервисе фильмах и сериалах."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "rjmKosCfgYFD",
        "colab": {}
      },
      "source": [
        "with open(os.path.join(DATA_PATH, 'catalogue.json'), 'r') as f:\n",
        "    catalogue = json.load(f)\n",
        "    \n",
        "catalogue = {int(k): v for k, v in catalogue.items()}\n",
        "catalogue = pd.DataFrame.from_dict(catalogue,'index').reset_index()\n",
        "# av_hot - нужно для валидации. Эта колонка равна 1, если фильм или сериал может быть потребленным\n",
        "catalogue['av_hot'] = catalogue['availability'].apply(lambda x: 1 if x == [] or 'subscription' in x else 0)\n",
        "catalogue = catalogue.rename(index=str, columns={'index':'element_uid'})"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BMG24yhF3yYA",
        "colab_type": "code",
        "outputId": "d28ee50d-4974-4d00-af19-2617c09feeb8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 375
        }
      },
      "source": [
        "catalogue.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>element_uid</th>\n",
              "      <th>type</th>\n",
              "      <th>availability</th>\n",
              "      <th>duration</th>\n",
              "      <th>feature_1</th>\n",
              "      <th>feature_2</th>\n",
              "      <th>feature_3</th>\n",
              "      <th>feature_4</th>\n",
              "      <th>feature_5</th>\n",
              "      <th>attributes</th>\n",
              "      <th>av_hot</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>movie</td>\n",
              "      <td>[]</td>\n",
              "      <td>80</td>\n",
              "      <td>2.912198e+07</td>\n",
              "      <td>0.575260</td>\n",
              "      <td>0</td>\n",
              "      <td>1.128332</td>\n",
              "      <td>0.654707</td>\n",
              "      <td>[31115, 6713, 10906, 31116, 31117, 270, 24431,...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>movie</td>\n",
              "      <td>[purchase, rent]</td>\n",
              "      <td>120</td>\n",
              "      <td>6.610431e+06</td>\n",
              "      <td>0.773224</td>\n",
              "      <td>3</td>\n",
              "      <td>1.112014</td>\n",
              "      <td>0.654707</td>\n",
              "      <td>[2786, 385, 2799, 3730, 886, 7, 11700, 42, 20,...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>movie</td>\n",
              "      <td>[]</td>\n",
              "      <td>80</td>\n",
              "      <td>1.315874e+07</td>\n",
              "      <td>0.699502</td>\n",
              "      <td>0</td>\n",
              "      <td>1.110127</td>\n",
              "      <td>0.680410</td>\n",
              "      <td>[31442, 31443, 31444, 31445, 113, 31446, 42, 3...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>series</td>\n",
              "      <td>[]</td>\n",
              "      <td>20</td>\n",
              "      <td>4.157712e+07</td>\n",
              "      <td>0.702981</td>\n",
              "      <td>0</td>\n",
              "      <td>1.141929</td>\n",
              "      <td>0.654707</td>\n",
              "      <td>[34361, 34362, 23033, 14887, 270, 20089, 43, 25]</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>movie</td>\n",
              "      <td>[purchase, rent, subscription]</td>\n",
              "      <td>70</td>\n",
              "      <td>3.999579e+07</td>\n",
              "      <td>0.626596</td>\n",
              "      <td>8</td>\n",
              "      <td>1.130076</td>\n",
              "      <td>0.592716</td>\n",
              "      <td>[26732, 26733, 26734, 9367, 7792, 336, 26735, ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   element_uid    type                    availability  duration  \\\n",
              "0            0   movie                              []        80   \n",
              "1            1   movie                [purchase, rent]       120   \n",
              "2            2   movie                              []        80   \n",
              "3            3  series                              []        20   \n",
              "4            4   movie  [purchase, rent, subscription]        70   \n",
              "\n",
              "      feature_1  feature_2  feature_3  feature_4  feature_5  \\\n",
              "0  2.912198e+07   0.575260          0   1.128332   0.654707   \n",
              "1  6.610431e+06   0.773224          3   1.112014   0.654707   \n",
              "2  1.315874e+07   0.699502          0   1.110127   0.680410   \n",
              "3  4.157712e+07   0.702981          0   1.141929   0.654707   \n",
              "4  3.999579e+07   0.626596          8   1.130076   0.592716   \n",
              "\n",
              "                                          attributes  av_hot  \n",
              "0  [31115, 6713, 10906, 31116, 31117, 270, 24431,...       1  \n",
              "1  [2786, 385, 2799, 3730, 886, 7, 11700, 42, 20,...       0  \n",
              "2  [31442, 31443, 31444, 31445, 113, 31446, 42, 3...       1  \n",
              "3   [34361, 34362, 23033, 14887, 270, 20089, 43, 25]       1  \n",
              "4  [26732, 26733, 26734, 9367, 7792, 336, 26735, ...       1  "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "-M9hcZGfgYF4"
      },
      "source": [
        " - `attributes` — мешок атрибутов\n",
        " - `availability` — доступность (может содержать значения `purchase`, `rent` и `subscription`)\n",
        " - `duration` — длительность в минутах, округлённая до десятков (продолжительность серии для сериалов и многосерийных фильмов)\n",
        " - `feature_1..5` — пять анонимизированных вещественных и порядковых признаков\n",
        " - `type` — принимает значения `movie`, `multipart_movie` или `series`"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "5N0G4hz_gYGB"
      },
      "source": [
        "`test_users.json` содержит список пользователей, для которых необходимо построить предсказание"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "yht-nWQ7gYGG",
        "outputId": "e0db8ac3-4e1a-4d4e-8758-670fd2019ee2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "with open(os.path.join(DATA_PATH, 'test_users.json'), 'r') as f:\n",
        "    test_users = set(json.load(f)['users'])\n",
        "len(test_users)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "50000"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "I-OYZb6TgYGl"
      },
      "source": [
        "`transactions.csv` — список всех транзакций за определённый период времени"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "x2FKH3PKgYGp",
        "outputId": "a8332e2f-85a8-43db-debc-f4533299a909",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "%%time\n",
        "transactions = pd.read_csv(\n",
        "    os.path.join(DATA_PATH, 'transactions.csv'),\n",
        "    dtype={\n",
        "        'element_uid': np.uint16,\n",
        "        'user_uid': np.uint32,\n",
        "        'consumption_mode': 'category',\n",
        "        'ts': np.float64,\n",
        "        'watched_time': np.uint64,\n",
        "        'device_type': np.uint8,\n",
        "        'device_manufacturer': np.uint8\n",
        "    }\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 7.71 s, sys: 582 ms, total: 8.29 s\n",
            "Wall time: 10.6 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "2ZiS_BRmgYHN",
        "outputId": "3a195de9-0122-408c-c466-55f75c9c4606",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "%%time\n",
        "ratings = pd.read_csv(\n",
        "    os.path.join(DATA_PATH, 'ratings.csv'),\n",
        "    dtype={\n",
        "        'element_uid': np.uint16,\n",
        "        'user_uid': np.uint32,\n",
        "        'ts': np.float64,\n",
        "        'rating': np.uint8\n",
        "    }\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 251 ms, sys: 14 ms, total: 265 ms\n",
            "Wall time: 599 ms\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "OdaGsulxgYH2",
        "outputId": "10e2b4e2-cfc0-4491-83f7-7641226e5cc7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "%%time\n",
        "bookmarks = pd.read_csv(\n",
        "    os.path.join(DATA_PATH, 'bookmarks.csv'),\n",
        "    dtype={\n",
        "        'element_uid': np.uint16,\n",
        "        'user_uid': np.uint32,\n",
        "        'ts': np.float64\n",
        "    }\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 458 ms, sys: 12.6 ms, total: 471 ms\n",
            "Wall time: 1.6 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "YtypT4T-gYIn",
        "outputId": "4a832343-0800-4e70-dc85-e311630cfbf2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        }
      },
      "source": [
        "bookmarks['rating'] = 1\n",
        "_bookmarks = bookmarks[['rating', 'user_uid', 'element_uid','ts']]\n",
        "\n",
        "_ratings = ratings[ratings['rating']>5]\n",
        "_ratings['rating'] = 1\n",
        "\n",
        "_transaction = transactions[['element_uid', 'user_uid','ts','watched_time']]\n",
        "_transaction = _transaction[(_transaction['watched_time']>700)]\n",
        "_transaction['rating'] = 1"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
            "  \"\"\"\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Iq4iedsaIjez",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "all_ratings = pd.concat([ _transaction[['user_uid', 'element_uid', 'rating', 'ts']],_bookmarks[['user_uid', 'element_uid', 'rating', 'ts']]])\n",
        "all_ratings = pd.concat([all_ratings, _ratings[['user_uid', 'element_uid', 'rating','ts']]])\n",
        "all_ratings = all_ratings.drop_duplicates(['user_uid','element_uid'],keep='first')\n",
        "all_ratings = all_ratings.sort_values(by=['ts'])\n",
        "all_ratings= all_ratings.rename(index=str,columns={'element_uid':'itemID','user_uid':'userID','ts':'timestamp'})"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CDOngt5TNKfh",
        "colab_type": "code",
        "outputId": "e432a8f6-0e4c-4151-dd3d-144e9795462b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 141
        }
      },
      "source": [
        "\n",
        "all_ratings.head(3)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>userID</th>\n",
              "      <th>itemID</th>\n",
              "      <th>rating</th>\n",
              "      <th>timestamp</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>9643011</th>\n",
              "      <td>283774</td>\n",
              "      <td>6189</td>\n",
              "      <td>1</td>\n",
              "      <td>4.173063e+07</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9643010</th>\n",
              "      <td>59148</td>\n",
              "      <td>6099</td>\n",
              "      <td>1</td>\n",
              "      <td>4.173063e+07</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9643009</th>\n",
              "      <td>50431</td>\n",
              "      <td>8888</td>\n",
              "      <td>1</td>\n",
              "      <td>4.173063e+07</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "         userID  itemID  rating     timestamp\n",
              "9643011  283774    6189       1  4.173063e+07\n",
              "9643010   59148    6099       1  4.173063e+07\n",
              "9643009   50431    8888       1  4.173063e+07"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P7jbtMzV3XJo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df = pd.merge(all_ratings, catalogue, how='left', left_on='itemID', right_on='element_uid')\n",
        "del all_ratings\n",
        "df['ismovie'] = 0\n",
        "df.loc[df['type']=='movie','ismovie'] = 1\n",
        "df['isseries'] = 0\n",
        "df.loc[df['type']=='series','isseries'] = 1\n",
        "df['ismulti'] = 0\n",
        "df.loc[df['type']=='multipart_movie','ismulti'] = 1\n",
        "\n",
        "\n",
        "def func(row):\n",
        "  return [row['itemID'],row['av_hot'],row['ismovie'],row['isseries'],row['ismulti']]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5tBXwYCr5ISz",
        "colab_type": "code",
        "outputId": "759cdfc9-9341-4f8d-86f5-c589026c6c54",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1122
        }
      },
      "source": [
        "%%time\n",
        "# it will take about 6 minutes\n",
        "df['gen'] = df.apply(func,axis=1)\n",
        "#df.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-25-0498c01c4b59>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mget_ipython\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_cell_magic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'time'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m''\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"# it will take about 6 minutes\\ndf['gen'] = df.apply(func,axis=1)\\n#df.shape\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36mrun_cell_magic\u001b[0;34m(self, magic_name, line, cell)\u001b[0m\n\u001b[1;32m   2115\u001b[0m             \u001b[0mmagic_arg_s\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvar_expand\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstack_depth\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2116\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuiltin_trap\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2117\u001b[0;31m                 \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmagic_arg_s\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2118\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2119\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m</usr/local/lib/python3.6/dist-packages/decorator.py:decorator-gen-60>\u001b[0m in \u001b[0;36mtime\u001b[0;34m(self, line, cell, local_ns)\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/IPython/core/magic.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(f, *a, **k)\u001b[0m\n\u001b[1;32m    186\u001b[0m     \u001b[0;31m# but it's overkill for just that one bit of state.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    187\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mmagic_deco\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 188\u001b[0;31m         \u001b[0mcall\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    189\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    190\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcallable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/IPython/core/magics/execution.py\u001b[0m in \u001b[0;36mtime\u001b[0;34m(self, line, cell, local_ns)\u001b[0m\n\u001b[1;32m   1191\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1192\u001b[0m             \u001b[0mst\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclock2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1193\u001b[0;31m             \u001b[0mexec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mglob\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlocal_ns\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1194\u001b[0m             \u001b[0mend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclock2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1195\u001b[0m             \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<timed exec>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36mapply\u001b[0;34m(self, func, axis, broadcast, raw, reduce, args, **kwds)\u001b[0m\n\u001b[1;32m   4875\u001b[0m                         \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4876\u001b[0m                         \u001b[0mreduce\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreduce\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 4877\u001b[0;31m                         ignore_failures=ignore_failures)\n\u001b[0m\u001b[1;32m   4878\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4879\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_apply_broadcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m_apply_standard\u001b[0;34m(self, func, axis, ignore_failures, reduce)\u001b[0m\n\u001b[1;32m   4931\u001b[0m                     \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_agg_axis\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4932\u001b[0m                     result = lib.reduce(values, func, axis=axis, dummy=dummy,\n\u001b[0;32m-> 4933\u001b[0;31m                                         labels=labels)\n\u001b[0m\u001b[1;32m   4934\u001b[0m                     \u001b[0;32mreturn\u001b[0m \u001b[0mSeries\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4935\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32mpandas/_libs/src/reduce.pyx\u001b[0m in \u001b[0;36mpandas._libs.lib.reduce\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;32mpandas/_libs/src/reduce.pyx\u001b[0m in \u001b[0;36mpandas._libs.lib.Reducer.get_result\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;32m<ipython-input-24-39fc2a6f6460>\u001b[0m in \u001b[0;36mfunc\u001b[0;34m(row)\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrow\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mrow\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'itemID'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mrow\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'av_hot'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mrow\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'ismovie'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mrow\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'isseries'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mrow\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'ismulti'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/core/series.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m    621\u001b[0m         \u001b[0mkey\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_apply_if_callable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    622\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 623\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_value\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    624\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    625\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_scalar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "DpRmLjBRgYI7"
      },
      "source": [
        "### Решение"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O8AfzgnoKUFB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df.itemID = df.itemID.astype('category')\n",
        "df['categ_id'] = df.itemID.cat.codes + 1\n",
        "cat_to_element_uid = dict(zip(\n",
        "    range(1, len(df.itemID.cat.categories) + 1),\n",
        "    df.itemID.cat.categories\n",
        "))\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QNv6IJEyPR0G",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df.to_csv(DATA_PATH+'gen.csv')\n",
        "df = df[['userID','itemID','categ_id','gen']]\n",
        "df.to_csv(DATA_PATH+'df.csv')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tsVSaD93DJEV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#del all_ratings\n",
        "\n",
        "df = pd.read_csv(\n",
        "    os.path.join(DATA_PATH+ 'df.csv')\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UKyvJs40ikvt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def func(x):\n",
        "  return x[1:-1].split(',')\n",
        "df['gen'] = df['gen'].apply(func)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6JtPGT84KZc3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%time\n",
        "import tqdm\n",
        "tqdm.tqdm.pandas()\n",
        "sequences = df.groupby('userID').agg({'gen': sum})['gen'].progress_apply(list)\n",
        "# it will take about 2 mins\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NC9f0Dc7OeEf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#type(sequences)\n",
        "seqq = [sequences.values[0:,][i] for i in range(sequences.shape[0])]\n",
        "#del sequences"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1PB3QIc7KkW_",
        "colab_type": "code",
        "outputId": "0c01b9b9-b61d-4054-ddc4-201470867712",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        }
      },
      "source": [
        "#sequences2use = sequences[len(sequences) >= 20]\n",
        "#sequences.values[0:,][1]\n",
        "print(len(seqq))\n",
        "print(seqq[1])\n",
        "seqq = np.array(seqq)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "496046\n",
            "['1653', ' 0', ' 1', ' 0', ' 0', '4120', ' 0', ' 1', ' 0', ' 0', '2245', ' 1', ' 1', ' 0', ' 0', '8237', ' 1', ' 1', ' 0', ' 0', '10084', ' 1', ' 1', ' 0', ' 0', '6976', ' 1', ' 1', ' 0', ' 0', '6131', ' 1', ' 1', ' 0', ' 0', '5445', ' 1', ' 1', ' 0', ' 0', '6915', ' 1', ' 1', ' 0', ' 0', '434', ' 1', ' 1', ' 0', ' 0', '6794', ' 1', ' 1', ' 0', ' 0', '7449', ' 1', ' 1', ' 0', ' 0', '8085', ' 1', ' 1', ' 0', ' 0', '4728', ' 0', ' 1', ' 0', ' 0', '1844', ' 1', ' 1', ' 0', ' 0', '7642', ' 1', ' 1', ' 0', ' 0', '9412', ' 1', ' 1', ' 0', ' 0', '9932', ' 0', ' 1', ' 0', ' 0']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8HYrk_M_KnZH",
        "colab_type": "code",
        "outputId": "38a12c83-7e4f-488a-c109-e686a422e452",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "maxlen = 18*5 # Length of sequences in X\n",
        "X = []\n",
        "y = []\n",
        "\n",
        "def slice_sequence(seq, num_slices):\n",
        "  n = len(seq)\n",
        "  if n > maxlen:\n",
        "    X.append(seq[:maxlen-1])\n",
        "    y.append(seq[maxlen])\n",
        " # if n == maxlen:\n",
        " #   X.append(seq[:maxlen-5])\n",
        " #   y.append(seq[maxlen-5])\n",
        " #   if seq[maxlen-5] == 0:\n",
        " #     print(seq)\n",
        "\n",
        "  \n",
        "        \n",
        "for seq in tqdm.tqdm(seqq):\n",
        "    if len(seq) <= 5*5:\n",
        "        slice_sequence(seq, 2)\n",
        "    elif len(seq) <= 6*5:\n",
        "        slice_sequence(seq, 2)\n",
        "    elif len(seq) <= 8*5:\n",
        "        slice_sequence(seq, 2)\n",
        "    elif len(seq) <= 12*5:\n",
        "        slice_sequence(seq, 2)\n",
        "    elif len(seq) <= 16:\n",
        "        slice_sequence(seq, 2)\n",
        "    elif len(seq) <= 20:\n",
        "        slice_sequence(seq, 2)\n",
        "    elif len(seq) <= 26:\n",
        "        slice_sequence(seq, 2)\n",
        "    else:\n",
        "        slice_sequence(seq, 2)\n",
        "        \n",
        "X = np.ravel(X).reshape((-1, 18, 5))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 496046/496046 [00:01<00:00, 269585.27it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cHoQ_wRTRznC",
        "colab_type": "code",
        "outputId": "5e2bfbcf-7938-4f12-b845-14ee5c476d2a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "maxlen = 18 # Length of sequences in X\n",
        "X = []\n",
        "y = []\n",
        "\n",
        "def slice_sequence(seq, num_slices):\n",
        "    for i in range(1, num_slices):\n",
        "        X.append(seq[-(i+maxlen): -i])\n",
        "        y.append(seq[-i])\n",
        "        \n",
        "        \n",
        "sequences2use = sequences[sequences.apply(len) >= 5]\n",
        "for seq in tqdm.tqdm(sequences2use):\n",
        "    if len(seq) <= 5:\n",
        "        slice_sequence(seq, 2)\n",
        "    elif len(seq) <= 6:\n",
        "        slice_sequence(seq, 3)\n",
        "    elif len(seq) <= 8:\n",
        "        slice_sequence(seq, 4)\n",
        "    elif len(seq) <= 12:\n",
        "        slice_sequence(seq, 6)\n",
        "    elif len(seq) <= 16:\n",
        "        slice_sequence(seq, 8)\n",
        "    elif len(seq) <= 20:\n",
        "        slice_sequence(seq, 11)\n",
        "    elif len(seq) <= 26:\n",
        "        slice_sequence(seq, 16)\n",
        "    else:\n",
        "        slice_sequence(seq, 23)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 496046/496046 [00:26<00:00, 19073.78it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wswdzBYXKrDk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "lens = [len(x) for x in X]\n",
        "max(lens), min(lens), np.mean(lens), np.median(lens)\n",
        "max_features = df.categ_id.unique().size + 1\n",
        "#del df"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CW09FyDFKvcW",
        "colab_type": "code",
        "outputId": "20a36f4a-a231-4600-8728-f934f34104d1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "from keras.preprocessing.sequence import pad_sequences\n",
        "\n",
        "# We should pad our sequences with 0 values, so they all will have the same length\n",
        "X = pad_sequences(X, maxlen=maxlen,padding='post',truncating='post',value=0)\n",
        "y = np.array(y)\n",
        "X.shape, y.shape\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((8433040, 18), (8433040,))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 256
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cJmoNPO5Kx4A",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from keras.layers import Input, Embedding, SpatialDropout1D, Dropout, BatchNormalization, Dense,LSTM, CuDNNGRU#, CuDNNLSTM, CuDNNGRU\n",
        "from keras.models import Model\n",
        "from keras import optimizers\n",
        "\n",
        "# Let's set random seed\n",
        "import tensorflow as tf\n",
        "tf.set_random_seed(42)\n",
        "np.random.seed(42)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H1v42gD8Kz0l",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "max_features_all = df.categ_id.unique().size + 1\n",
        "maxlen=18\n",
        "embed_size = 512\n",
        "\n",
        "DEBUG = True\n",
        "\n",
        "def lstm129(num_features, seq_len):\n",
        "    inp = Input(shape=(seq_len,num_features))\n",
        "    x = inp\n",
        "    if DEBUG:\n",
        "      print('input:', x.shape)\n",
        "#     x = Embedding(max_features_all + 2, 256)(inp)\n",
        "#     x = SpatialDropout1D(0.05)(x)\n",
        "    if DEBUG:\n",
        "      print('after Spatial dropout:', x.shape)\n",
        "    x = BatchNormalization(epsilon=1e-6,weights=None)(x)\n",
        "    x = CuDNNGRU(256,return_sequences=False)(x)\n",
        "    if DEBUG:\n",
        "      print('after CuDNNGRU:', x.shape)\n",
        "#     x = Dropout(0.02)(x)\n",
        "    outp = Dense(max_features_all, activation=\"softmax\")(x)\n",
        "    if DEBUG:\n",
        "      print('after Dense:', outp.shape)\n",
        "    model = Model(inputs=inp, outputs=outp)\n",
        "    model.compile(loss='sparse_categorical_crossentropy', optimizer=optimizers.SGD(lr=0.01, \\\n",
        "                                                                    momentum=0.0, decay=0.0, nesterov=True, clipnorm=1.),\n",
        "                  metrics=['sparse_categorical_accuracy'])\n",
        "    return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EoXKM1tkEsrM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "embed_size = 128\n",
        "\n",
        "def old_lstm129():\n",
        "    inp = Input(shape=(maxlen,))\n",
        "    x = Embedding(max_features_all, embed_size)(inp)\n",
        "    x = SpatialDropout1D(0.05)(x)\n",
        "    x = CuDNNGRU(128,return_sequences=False)(x)\n",
        "    x = Dropout(0.02)(x)\n",
        "    outp = Dense(max_features, activation=\"softmax\")(x)\n",
        "    model = Model(inputs=inp, outputs=outp)\n",
        "    model.compile(loss='sparse_categorical_crossentropy', optimizer='rmsprop',\n",
        "                  metrics=['sparse_categorical_accuracy'])\n",
        "    return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s75UL6Cdi1m6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn import preprocessing\n",
        "le = preprocessing.LabelEncoder()\n",
        "\n",
        "\n",
        "_X = np.where(np.ravel(X) == 0, np.full(np.ravel(X).shape, -1), np.ravel(X)).reshape(X.shape)\n",
        "_X = np.ravel(_X)\n",
        "le = preprocessing.LabelEncoder()\n",
        "le = preprocessing.LabelEncoder()\n",
        "all_movies = list(set(y).union(set(_X[0::5])))\n",
        "le.fit(all_movies)\n",
        "\n",
        "#теперь в _y будут числа от 0 до кол-во фильмов (max_features)\n",
        "_y = le.transform(y)\n",
        "\n",
        "_X[0::5] = le.transform(_X[0::5])\n",
        "_X = _X.reshape((-1,18,5))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FjAqDExHHrKE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model = old_lstm129()\n",
        "model.fit(_X[:, :, 0], _y, batch_size=2048, epochs=10, verbose=True, validation_split=0.01, shuffle=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wcvXC6_8bH_d",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# раскаменти, чтобы обучить модель с фичами вместо embeding слоя\n",
        "\n",
        "# start = 0\n",
        "# end = _X.shape[0]\n",
        "# DEBUG = False\n",
        "# model = lstm129(5, 18)\n",
        "\n",
        "# model.fit(_X[start:end,:18, :], _y[start:end], epochs=10, batch_size=2048)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1ShyDogma_nE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from keras.models import model_from_json\n",
        "import keras\n",
        "def save_model(model, prefix=''):\n",
        "    model_json = model.to_json()\n",
        "    with open(os.path.join(CODE_PATH, prefix + \"model.json\"), \"w\") as json_file:\n",
        "        json_file.write(model_json)\n",
        "    # serialize weights to HDF5\n",
        "    model.save_weights(os.path.join(CODE_PATH, prefix + \"model.h5\"))\n",
        "    print(\"Saved model to disk\")\n",
        "    \n",
        "def load_model():\n",
        "    json_file = open(os.path.join(CODE_PATH, 'model.json'), 'r')\n",
        "    loaded_model_json = json_file.read()\n",
        "    json_file.close()\n",
        "    loaded_model = model_from_json(loaded_model_json)\n",
        "    # load weights into new model\n",
        "    loaded_model.load_weights(os.path.join(CODE_PATH, \"model.h5\"))\n",
        "    return loaded_model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QdUMk30hfDdT",
        "colab_type": "code",
        "outputId": "efd419fa-bd4b-4a9c-c7e6-2581bb675d3f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 555
        }
      },
      "source": [
        "from tensorflow.python.client import device_lib\n",
        "print(device_lib.list_local_devices())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[name: \"/device:CPU:0\"\n",
            "device_type: \"CPU\"\n",
            "memory_limit: 268435456\n",
            "locality {\n",
            "}\n",
            "incarnation: 14146287626785672023\n",
            ", name: \"/device:XLA_CPU:0\"\n",
            "device_type: \"XLA_CPU\"\n",
            "memory_limit: 17179869184\n",
            "locality {\n",
            "}\n",
            "incarnation: 4273390814707937227\n",
            "physical_device_desc: \"device: XLA_CPU device\"\n",
            ", name: \"/device:XLA_GPU:0\"\n",
            "device_type: \"XLA_GPU\"\n",
            "memory_limit: 17179869184\n",
            "locality {\n",
            "}\n",
            "incarnation: 10310275435120479926\n",
            "physical_device_desc: \"device: XLA_GPU device\"\n",
            ", name: \"/device:GPU:0\"\n",
            "device_type: \"GPU\"\n",
            "memory_limit: 11276946637\n",
            "locality {\n",
            "  bus_id: 1\n",
            "  links {\n",
            "  }\n",
            "}\n",
            "incarnation: 1650850629698700918\n",
            "physical_device_desc: \"device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7\"\n",
            "]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Lybm8r1bfMRz",
        "colab_type": "code",
        "outputId": "e612fed7-abd2-454d-860a-380e20539026",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "from keras import backend as K\n",
        "K.tensorflow_backend._get_available_gpus()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['/job:localhost/replica:0/task:0/device:GPU:0']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kREpG_TULRON",
        "colab_type": "code",
        "outputId": "c1df1c76-c423-4833-8278-238476089dd0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 237
        }
      },
      "source": [
        "#model = lstm129()\n",
        "#model.load_weights('model128.json')\n",
        "\n",
        "sequences_test = seqq.apply(lambda x: x[-maxlen:])\n",
        "sequences_test = seqq.apply(lambda x: [0 for i in range(maxlen - len(x))] + x)\n",
        "test_users_in_sequences = sorted(set(sequences_test.index) & set(test_users))\n",
        "X_test = np.array(sequences_test[test_users_in_sequences].tolist())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "error",
          "ename": "AttributeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-246-c4c9dbf62bd6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0msequences_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mseqq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mmaxlen\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0msequences_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mseqq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmaxlen\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mtest_users_in_sequences\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msorted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msequences_test\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m&\u001b[0m \u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_users\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mX_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msequences_test\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtest_users_in_sequences\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtolist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mAttributeError\u001b[0m: 'numpy.ndarray' object has no attribute 'apply'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RvlAPBxXLU3O",
        "colab_type": "code",
        "outputId": "38b70fff-8329-423c-be9e-4026f5a94c04",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "%%time\n",
        "from itertools import chain\n",
        "batch_size = 2048\n",
        "n_batches = int(X_test.shape[0]/batch_size) + 1\n",
        "preds = []\n",
        "\n",
        "for batch_ind in tqdm.tqdm(range(n_batches)):\n",
        "    batch = X_test[batch_ind*batch_size: (batch_ind + 1)*batch_size]\n",
        "    curr_preds = model.predict(batch)\n",
        "    curr_preds = np.argsort(-curr_preds)[:, :20]\n",
        "    curr_preds = [[cat_to_element_uid[x] for x in row] for row in curr_preds]\n",
        "    preds.append([' '.join(map(lambda x: str(x), row)) for row in curr_preds])\n",
        "    \n",
        "preds = list(chain(*preds))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 25/25 [01:00<00:00,  2.01s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 1min 11s, sys: 3.76 s, total: 1min 14s\n",
            "Wall time: 1min\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WKVov1PsLXZW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "top_20_videos = [2714, 747, 2639, 3783, 2245, 6127, 10061, 3916,\n",
        "                 8771, 3336, 7079, 3567, 6195, 4141, 1016,\n",
        "                 7185, 1004, 9467, 72, 1521]\n",
        "\n",
        "sam_sub = dict()\n",
        "for el in test_users:\n",
        "    sam_sub[el] = top_20_videos"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m7v5ookQLZMy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "keys = [int(x) for x in sam_sub.keys()]\n",
        "df = pd.DataFrame(keys,columns=['userID'])\n",
        "df['itemID'] = df.apply(lambda x: sam_sub[x[0]],axis=1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d05lh-SPLc5E",
        "colab_type": "code",
        "outputId": "d3feb6e9-78d3-45d6-b2e5-7b95126d9403",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104
        }
      },
      "source": [
        "df.index = df.userID\n",
        "df.itemID[test_users_in_sequences] = preds"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
            "  \n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U68MFoYRLBIs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ans = {}\n",
        "for el, el1 in df.values:\n",
        "    try:\n",
        "        el1 = el1.split(' ')\n",
        "        for i in range(len(el1)):\n",
        "            el1[i] = int(el1[i])\n",
        "        ans[str(el)]= el1\n",
        "    except:\n",
        "        ans[str(el)] = el1\n",
        "    \n",
        "with open(os.path.join(DATA_PATH, 'my_best.json'), 'w') as f:\n",
        "    json.dump(ans, f)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zOoquR7XMlsj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}