S2SChatbot.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 111,
     "resources": {
      "http://localhost:8080/nbextensions/google.colab/files.js": {
       "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=",
       "headers": [
        [
         "content-type",
         "application/javascript"
        ]
       ],
       "ok": true,
       "status": 200,
       "status_text": ""
      }
     }
    },
    "colab_type": "code",
    "id": "nQsRpTjJGfNA",
    "outputId": "b72528fc-86d9-4b2b-c2c2-bb04a3b30f80"
   },
   "outputs": [],
   "source": [
    "from google.colab import files\n",
    "files.upload()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "EaTPCNIoQl5Z",
    "outputId": "3da84714-c86f-4825-f3eb-608c0a783545"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import collections\n",
    "\n",
    "import numpy as np\n",
    "import pickle\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Model\n",
    "from keras.layers import GRU, Input, Dense, TimeDistributed, Activation, RepeatVector, Bidirectional\n",
    "from keras.layers import Embedding, CuDNNLSTM, GlobalMaxPooling1D, GlobalAveragePooling1D, CuDNNGRU\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.optimizers import Adam\n",
    "from keras.losses import sparse_categorical_crossentropy\n",
    "from keras.utils.vis_utils import plot_model\n",
    "from keras.callbacks import ModelCheckpoint, EarlyStopping, TensorBoard\n",
    "from nltk.translate.bleu_score import sentence_bleu, corpus_bleu\n",
    "import helper\n",
    "import json\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm\n",
    "tqdm.pandas()\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ztAV9J1pQwIq"
   },
   "outputs": [],
   "source": [
    "# df = pd.read_csv('scrap_context.csv')\n",
    "# df_tp = pd.read_csv('reddit.txt', sep=\"_eou_\",error_bad_lines=False, header=['title','response'] )\n",
    "# df_tp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "MBH7cUSSQyaA"
   },
   "outputs": [],
   "source": [
    "# df.shape\n",
    "# df_tp.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gUFEEoQAvfYh"
   },
   "outputs": [],
   "source": [
    "# df_tp.iloc[0]['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "rDDtubAGvNh5"
   },
   "outputs": [],
   "source": [
    "# df = pd.DataFrame(columns=['title','response'])\n",
    "# df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qCv0i2ym7TZq"
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame(columns=['title','response'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "uotiKW4Y7TTg"
   },
   "outputs": [],
   "source": [
    "# df_1 = pd.read_csv('general2.txt', sep='_eou_')\n",
    "# df_1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xdpGLQUruzOL"
   },
   "outputs": [],
   "source": [
    "# df = pd.DataFrame(columns=['title','response'])\n",
    "# for i in tqdm(range(1,len(df_tp)-1)):\n",
    "#   if 1 <= len(df_tp.iloc[i]['text'].split()) <= 7:\n",
    "#     if 1 <= len(df_tp.iloc[i+1]['text'].split()) <= 7:\n",
    "#       title = df_tp.iloc[i]['text']\n",
    "#       response = df_tp.iloc[i+1]['text']\n",
    "#       df = df.append({'title':title, 'response':response}, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "KrQ7wBAIv_vc"
   },
   "outputs": [],
   "source": [
    "# df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0pJFg-LCwnH_"
   },
   "outputs": [],
   "source": [
    "# df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Yv5LsxaqyJFm"
   },
   "outputs": [],
   "source": [
    "# df_gen = pd.read_csv('general.csv', names=['title','response'])\n",
    "# df_gen.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "R5CFoRp4zbud"
   },
   "outputs": [],
   "source": [
    "# df_gen.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "ZqXY_9UuL9is",
    "outputId": "40e81d0d-0527-4d39-f110-c3698f3654d0"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(500, 2)"
      ]
     },
     "execution_count": 5,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df_gen2 = pd.read_csv('general2.csv', names=['title','response'])\n",
    "# df_gen2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2iHF6BYvMMqF"
   },
   "outputs": [],
   "source": [
    "# df_gen2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tTNsCn5HNIuI"
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('data.csv', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 204
    },
    "colab_type": "code",
    "id": "2JXDSVBlNRqu",
    "outputId": "e827b1c0-fb68-4614-d489-f8b53816bf7c"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>response</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>hi what is your name?</td>\n",
       "      <td>hi this is Sofia.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>nice to meet you!</td>\n",
       "      <td>nice to meet you too!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>which programming language do you use?</td>\n",
       "      <td>i like python.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>where do you live?</td>\n",
       "      <td>i live in gwalior.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>what is your major?</td>\n",
       "      <td>i study software engineering.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    title                       response\n",
       "0                   hi what is your name?              hi this is Sofia.\n",
       "1                       nice to meet you!          nice to meet you too!\n",
       "2  which programming language do you use?                 i like python.\n",
       "3                      where do you live?             i live in gwalior.\n",
       "4                     what is your major?  i study software engineering."
      ]
     },
     "execution_count": 6,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "V_RzOYyNyPQq",
    "outputId": "57364ecc-4e8c-47e7-f3e7-c893014b004a"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 199/199 [00:00<00:00, 385.37it/s]\n"
     ]
    }
   ],
   "source": [
    "# Append the data of both the csv files to one dataframe.\n",
    "# for i in tqdm(range(0,len(df_gen)-1)):\n",
    "#   title = df_gen.iloc[i]['title']\n",
    "#   response = df_gen.iloc[i]['response']\n",
    "#   df = df.append({'title':title, 'response':response}, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "QPNsPvvsMi9-",
    "outputId": "2bf18646-a4fa-4651-e868-70a6d21ac9b0"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 499/499 [00:01<00:00, 381.37it/s]\n"
     ]
    }
   ],
   "source": [
    "# for i in tqdm(range(0,len(df_gen2)-1)):\n",
    "#   title = df_gen2.iloc[i]['title']\n",
    "#   response = df_gen2.iloc[i]['response']\n",
    "#   df = df.append({'title':title, 'response':response}, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 204
    },
    "colab_type": "code",
    "id": "JEwMZ3Oyyc6-",
    "outputId": "489309ab-d247-4f0d-a515-68df765c5ded"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>response</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>hi what is your name?</td>\n",
       "      <td>hi this is Sofia.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>nice to meet you!</td>\n",
       "      <td>nice to meet you too!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>which programming language do you use?</td>\n",
       "      <td>i like python.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>where do you live?</td>\n",
       "      <td>i live in gwalior.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>what is your major?</td>\n",
       "      <td>i study software engineering.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    title                       response\n",
       "0                   hi what is your name?              hi this is Sofia.\n",
       "1                       nice to meet you!          nice to meet you too!\n",
       "2  which programming language do you use?                 i like python.\n",
       "3                      where do you live?             i live in gwalior.\n",
       "4                     what is your major?  i study software engineering."
      ]
     },
     "execution_count": 7,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "OVEIKcFDQ0cd",
    "outputId": "c2f79ac2-e6c0-403b-96b6-5ec6d489aa87"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(698, 2)"
      ]
     },
     "execution_count": 8,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df = df[(df['title'] != '[deleted]') & (df['response'] != '[deleted]')]\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5ylGOvQluepD"
   },
   "outputs": [],
   "source": [
    "# from google.colab import files\n",
    "\n",
    "# df.to_csv('data.csv')\n",
    "# files.download('data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "G2g_gH-tQ2G2"
   },
   "outputs": [],
   "source": [
    "contraction_mapping = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\", 'u.s':'america', 'e.g':'for example'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "GeSUuqPQQ30U"
   },
   "outputs": [],
   "source": [
    "def clean_contractions(text, mapping):\n",
    "    specials = [\"’\", \"‘\", \"´\", \"`\"]\n",
    "    for s in specials:\n",
    "        text = text.replace(s, \"'\")\n",
    "    text = ' '.join([mapping[t] if t in mapping else t for t in text.split(\" \")])\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BWRUC_6BQ5Rm"
   },
   "outputs": [],
   "source": [
    "df['title'] = df['title'].apply(lambda x: x.lower())\n",
    "df['response'] = df['response'].apply(lambda x: x.lower())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "ij6hWJCtQ63i",
    "outputId": "476cd4b3-2b6c-48f4-9e9d-294a6d61b78e"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 698/698 [00:00<00:00, 77684.66it/s]\n",
      "100%|██████████| 698/698 [00:00<00:00, 133932.21it/s]\n"
     ]
    }
   ],
   "source": [
    "df['title'] = df['title'].progress_apply(lambda x: clean_contractions(x, contraction_mapping))\n",
    "df['response'] = df['response'].progress_apply(lambda x: clean_contractions(x, contraction_mapping))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OUd52BjLQ8TB"
   },
   "outputs": [],
   "source": [
    "punct = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n",
    " '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n",
    " '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n",
    " '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n",
    " '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "eyaujG8EQ9-z"
   },
   "outputs": [],
   "source": [
    "punct_mapping = {\"‘\": \"'\", \"₹\": \"e\", \"´\": \"'\", \"°\": \"\", \"€\": \"e\", \"™\": \"tm\", \"√\": \" sqrt \", \"×\": \"x\", \"²\": \"2\", \"—\": \"-\", \"–\": \"-\", \"’\": \"'\", \"_\": \"-\", \"`\": \"'\", '“': '\"', '”': '\"', '“': '\"', \"£\": \"e\", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', '!':' '}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "XJ_cm4cDQ_jX"
   },
   "outputs": [],
   "source": [
    "def clean_special_chars(text, punct, mapping):\n",
    "    for p in mapping:\n",
    "        text = text.replace(p, mapping[p])\n",
    "    \n",
    "    for p in punct:\n",
    "        text = text.replace(p, f' {p} ')\n",
    "    \n",
    "    specials = {'\\u200b': ' ', '…': ' ... ', '\\ufeff': '', 'करना': '', 'है': ''}  \n",
    "    for s in specials:\n",
    "        text = text.replace(s, specials[s])\n",
    "    \n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "Be9UNfglRBBX",
    "outputId": "14936661-1098-4365-d8f1-30ec041fa0e4"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 698/698 [00:00<00:00, 19249.54it/s]\n",
      "100%|██████████| 698/698 [00:00<00:00, 21776.60it/s]\n"
     ]
    }
   ],
   "source": [
    "df['title'] = df['title'].progress_apply(lambda x: clean_special_chars(x, punct, punct_mapping))\n",
    "df['response'] = df['response'].progress_apply(lambda x: clean_special_chars(x, punct, punct_mapping))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "J_nCqAN3RCbw"
   },
   "outputs": [],
   "source": [
    "mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \"whst\": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7ckSRGECRD1Y"
   },
   "outputs": [],
   "source": [
    "def correct_spelling(x, dic):\n",
    "    for word in dic.keys():\n",
    "        x = x.replace(word, dic[word])\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "iE9wjCH0RFYb",
    "outputId": "0de8bcb7-e584-43be-ff99-ef8f866495b1"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 698/698 [00:00<00:00, 50433.67it/s]\n",
      "100%|██████████| 698/698 [00:00<00:00, 56053.61it/s]\n"
     ]
    }
   ],
   "source": [
    "df['title'] = df['title'].progress_apply(lambda x: correct_spelling(x, mispell_dict))\n",
    "df['response'] = df['response'].progress_apply(lambda x: correct_spelling(x, mispell_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "PMO5O3JuRG_C",
    "outputId": "6bfe1d78-83a8-4bf6-87a1-2b0b87b0dc34"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 698/698 [00:00<00:00, 118532.09it/s]\n",
      "100%|██████████| 698/698 [00:00<00:00, 115161.05it/s]\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "# text = re.sub(r'^https?:\\/\\/.*[\\r\\n]*', '', text, flags=re.MULTILINE)\n",
    "df['title'] = df['title'].progress_apply(lambda x: re.sub(r'^https?:\\/\\/.*[\\r\\n]*', '', x, flags=re.MULTILINE))\n",
    "df['response'] = df['response'].progress_apply(lambda x: re.sub(r'^https?:\\/\\/.*[\\r\\n]*', '', x, flags=re.MULTILINE))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "8oz6M3ikRIaa",
    "outputId": "7af41b35-2848-4106-e08c-15c238ac5eab"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 698/698 [00:00<00:00, 138258.52it/s]\n",
      "100%|██████████| 698/698 [00:00<00:00, 175170.48it/s]\n"
     ]
    }
   ],
   "source": [
    "df['title'] = df['title'].progress_apply(lambda x: re.sub(r'\\/r\\/[a-z]+ ', '', x, flags=re.MULTILINE))\n",
    "df['response'] = df['response'].progress_apply(lambda x: re.sub(r'\\/r\\/[a-z]+ ', '', x, flags=re.MULTILINE))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Hnrq0CrrRKmL"
   },
   "outputs": [],
   "source": [
    "def preprocess_sentence(w):\n",
    "\n",
    "    # creating a space between a word and the punctuation following it\n",
    "    # eg: \"he is a boy.\" => \"he is a boy .\"\n",
    "    # Reference:- https://stackoverflow.com/questions/3645931/python-padding-punctuation-with-white-spaces-keeping-punctuation\n",
    "#     w = re.sub(r\"([?.!,¿])\", r\" \\1 \", w)\n",
    "#     w = re.sub(r'[\" \"]+', \" \", w)\n",
    "\n",
    "    # replacing everything with space except (a-z, A-Z, \".\", \"?\", \"!\", \",\")\n",
    "    w = re.sub(r\"[^a-zA-Z?.!¿]+\", \" \", w)\n",
    "\n",
    "    w = w.rstrip().strip()\n",
    "    \n",
    "    # adding a start and an end token to the sentence\n",
    "    # so that the model know when to start and stop predicting.\n",
    "#     w = '<start> ' + w + ' <end>'\n",
    "    \n",
    "    return w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "t7raZWv8RMAl",
    "outputId": "cd1d3b74-1de6-478b-8bfc-d23e6948c956"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 698/698 [00:00<00:00, 89061.33it/s]\n",
      "100%|██████████| 698/698 [00:00<00:00, 127078.05it/s]\n"
     ]
    }
   ],
   "source": [
    "df['title'] = df['title'].progress_apply(lambda x: preprocess_sentence(x))\n",
    "df['response'] = df['response'].progress_apply(lambda x: preprocess_sentence(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "e2fjQGRXRNfn",
    "outputId": "9e6b24f3-2e86-4ec8-fb58-7beef24cb472"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TensorFlow Version: 1.14.0\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.python.layers.core import Dense\n",
    "from tensorflow.python.ops.rnn_cell_impl import _zero_state_tensors\n",
    "from tensorflow.contrib.seq2seq import AttentionWrapper as attention_wrapper\n",
    "from tensorflow.contrib.seq2seq import BeamSearchDecoder as beam_search_decoder\n",
    "print('TensorFlow Version: {}'.format(tf.__version__))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7sM0U7xgVC1T"
   },
   "outputs": [],
   "source": [
    "# title list\n",
    "titles=df['title'].values\n",
    "import string\n",
    "title=[]\n",
    "responses=df['response'].values\n",
    "response=[]\n",
    "for i in range(len(titles)):\n",
    "  if 1 <= len(titles[i].split()) <= 15:\n",
    "    if 1 <= len(responses[i].split()) <= 15:\n",
    "      title.append( titles[i].lower().strip() )\n",
    "      response.append( responses[i].lower().strip() )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "qqK66YFK7V9C",
    "outputId": "701c0ec5-82b5-482b-d572-d22a23881178"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Title samples : 698\n",
      "Response samples : 698\n"
     ]
    }
   ],
   "source": [
    "print('Title samples :', len(title))\n",
    "print('Response samples :', len(response))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 102
    },
    "colab_type": "code",
    "id": "BJwFU7lwRRa1",
    "outputId": "28c0cf45-a730-4ee5-8623-80a89383b2ac"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hi what is your name?\n",
      "([240, 23, 13, 21, 238, 11, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 6)\n",
      "hi this is chinmay.\n",
      "([0, 240, 51, 13, 274, 7, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 6)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"_START_ am of . people ' _END_ _PAD_ _PAD_ _PAD_ _PAD_\""
      ]
     },
     "execution_count": 27,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "import pprint\n",
    "from collections import Counter\n",
    "\n",
    "\n",
    "# maximum length of input and target sentences including paddings\n",
    "enc_sentence_length = 17\n",
    "dec_sentence_length = 17\n",
    "\n",
    "# input_batches = [\n",
    "#     ['hi what is your name?', 'nice to meet you!'],\n",
    "#     ['which programming language do you use?', 'see you later.'],\n",
    "#     ['where do you live?', 'what is your major?'],\n",
    "#     ['what do you want to drink?', 'what is your favorite beer?']]\n",
    "\n",
    "# target_batches = [\n",
    "#     ['hi this is chinmay.', 'nice to meet you too!'],\n",
    "#     ['i like python.', 'bye bye.'],\n",
    "#     ['i live in gwalior, madhya pradesh.', 'i study software engineering.'],\n",
    "#     ['beer please!', 'leffe brown!']]\n",
    "\n",
    "\n",
    "all_input_sentences = []\n",
    "# for input_batch in input_batches:\n",
    "#     all_input_sentences.extend(input_batches)\n",
    "    \n",
    "all_target_sentences = []\n",
    "# for target_batch in target_batches:\n",
    "#     all_target_sentences.extend(target_batches)\n",
    "    \n",
    "for t in range(len(title)-1):\n",
    "  all_input_sentences.extend([[title[t],title[t+1]]])\n",
    "  \n",
    "for r in range(len(response)-1):\n",
    "  all_target_sentences.extend([[response[r],response[r+1]]])\n",
    "    \n",
    "def tokenizer(sentence):\n",
    "    tokens = re.findall(r\"[\\w]+|[^\\s\\w]\", str(sentence))\n",
    "    return tokens\n",
    "\n",
    "_START_ = \"_START_\"\n",
    "_PAD_ = \"_PAD_\"\n",
    "_END_ = \"_END_\"\n",
    "\n",
    "def build_vocab(sentences, max_vocab_size=None):\n",
    "    word_counter = Counter()\n",
    "    vocab = dict()\n",
    "    reverse_vocab = dict()\n",
    "    \n",
    "    for sentence in sentences:\n",
    "        tokens = tokenizer(sentence)\n",
    "        word_counter.update(tokens)\n",
    "        \n",
    "    if max_vocab_size is None:\n",
    "        max_vocab_size = len(word_counter)\n",
    "    \n",
    "    vocab[_START_] = 0\n",
    "    vocab[_PAD_] = 1\n",
    "    vocab[_END_] = 2\n",
    "    vocab_idx = 3\n",
    "    for key, value in word_counter.most_common(max_vocab_size):\n",
    "            vocab[key] = vocab_idx\n",
    "            vocab_idx += 1\n",
    "            \n",
    "    for key, value in vocab.items():\n",
    "        reverse_vocab[value] = key\n",
    "            \n",
    "    return vocab, reverse_vocab, max_vocab_size\n",
    "\n",
    "vocab,reverse_vocab,max_vocab_size = build_vocab(all_input_sentences+all_target_sentences)\n",
    "\n",
    "def token2idx(word, vocab):\n",
    "    return vocab[word]\n",
    "\n",
    "def sent2idx(sent, vocab=vocab, max_sentence_length=enc_sentence_length, is_target=False):\n",
    "    tokens = tokenizer(sent)\n",
    "    current_length = len(tokens)\n",
    "    pad_length = max_sentence_length - current_length\n",
    "    if is_target:\n",
    "        return [0] + [token2idx(token, vocab) for token in tokens] + [2] + [1] * (pad_length-1), current_length + 1\n",
    "    else:\n",
    "        return [token2idx(token, vocab) for token in tokens] + [1] * pad_length, current_length\n",
    "\n",
    "def idx2token(idx, reverse_vocab):\n",
    "    return reverse_vocab[idx]\n",
    "\n",
    "def idx2sent(indices, reverse_vocab=reverse_vocab):\n",
    "    return \" \".join([idx2token(idx, reverse_vocab) for idx in indices])\n",
    "\n",
    "# Enc Example\n",
    "print('hi what is your name?')\n",
    "print(sent2idx('hi what is your name?'))\n",
    "\n",
    "# Dec Example\n",
    "print('hi this is chinmay.')\n",
    "print(sent2idx('hi this is chinmay.', max_sentence_length=dec_sentence_length, is_target=True))\n",
    "\n",
    "idx2sent([0, 16, 41, 7, 36, 3, 2, 1, 1, 1, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-4sxpY9hNA3s"
   },
   "outputs": [],
   "source": [
    "# f = open(\"senttoindex.pkl\",\"wb\")\n",
    "# pickle.dump(vocab,f)\n",
    "# f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "H_e0fV9aNUdW"
   },
   "outputs": [],
   "source": [
    "# f = open(\"indextosent.pkl\",\"wb\")\n",
    "# pickle.dump(reverse_vocab,f)\n",
    "# f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "vOugU2viN22z"
   },
   "outputs": [],
   "source": [
    "# json.load( open( \"idx2sent.json\" ) )\n",
    "# json.dump( reverse_vocab, open( \"idxtosent.json\", 'w' ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LPLVD80eN2v-"
   },
   "outputs": [],
   "source": [
    "# json.dump( vocab, open( \"senttoindex.json\", 'w' ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "xawGAKmCdndi",
    "outputId": "215826c9-7437-44ee-ea27-04082f397e1d"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "697"
      ]
     },
     "execution_count": 32,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(all_input_sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yIpm45eXWiNP"
   },
   "outputs": [],
   "source": [
    "def get_optimizer(opt):\n",
    "    if opt == \"adam\":\n",
    "        optfn = tf.train.AdamOptimizer\n",
    "    elif opt == \"sgd\":\n",
    "        optfn = tf.train.GradientDescentOptimizer\n",
    "    else:\n",
    "        assert(False)\n",
    "    return optfn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "suobBVkDYbFg"
   },
   "outputs": [],
   "source": [
    "def single_rnn_cell(cell_name,dim_size, train_phase = True, keep_prob = 0.75):\n",
    "    if cell_name == \"gru\":\n",
    "        cell = tf.contrib.rnn.GRUCell(dim_size)\n",
    "    elif cell_name == \"lstm\":\n",
    "        cell = tf.contrib.rnn.LSTMCell(dim_size)\n",
    "    else:\n",
    "        cell = tf.contrib.rnn.BasicRNNCell(dim_size)\n",
    "    if train_phase and keep_prob < 1.0:\n",
    "        cell = tf.contrib.rnn.DropoutWrapper(\n",
    "              cell=cell,\n",
    "              input_keep_prob=keep_prob,\n",
    "              output_keep_prob=keep_prob)\n",
    "    return cell\n",
    "\n",
    "def multi_rnn_cell(cell_name,dim_size,num_layers = 1, train_phase = True, keep_prob=0.75):\n",
    "    cells = []\n",
    "    for _ in range(num_layers):\n",
    "        cell = single_rnn_cell(cell_name,dim_size, train_phase, keep_prob)\n",
    "        cells.append(cell)\n",
    "    \n",
    "    if len(cells) > 1:\n",
    "        final_cell = tf.contrib.rnn.MultiRNNCell(cells=cells)\n",
    "    else:\n",
    "        final_cell = cells[0]\n",
    "    return final_cell"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "H2M9Kfq8ZC0a"
   },
   "outputs": [],
   "source": [
    "class BasicS2SModel(object):\n",
    "    def __init__(self, vocab, batch_size = 2, dim_size=128, rnn_cell = 'gru', num_layers=2, max_gradient_norm=5.0, atten_size=30, \n",
    "                 learning_rate=0.001, learning_rate_decay_factor=0.98, dropout=0.2,max_inference_lenght=10,\n",
    "                 max_source_len = 10, max_target_len = 10,beam_size =3, optimizer=\"adam\", mode ='train',\n",
    "                 use_beam_search = False):\n",
    "        assert mode in ['train', 'inference']\n",
    "        self.start_token = vocab.get(_START_)\n",
    "        self.end_token = vocab.get(_END_)\n",
    "        self.train_phase = True if mode == 'train' else False\n",
    "        self.cell_name = rnn_cell\n",
    "        self.dim_size = dim_size\n",
    "        self.vocab_size = len(vocab)\n",
    "        self.num_layers = num_layers\n",
    "        self.keep_prob_config = 1.0 - dropout\n",
    "        self.atten_size = atten_size\n",
    "        \n",
    "        # decoder\n",
    "        self.max_inference_lenght = max_inference_lenght\n",
    "        \n",
    "        # beam search\n",
    "        self.beam_size = beam_size\n",
    "        self.beam_search = use_beam_search\n",
    "        \n",
    "        # learning\n",
    "        self.learning_rate = tf.Variable(float(learning_rate), trainable=False)\n",
    "        self.learning_rate_decay_op = self.learning_rate.assign(self.learning_rate * learning_rate_decay_factor)\n",
    "        self.global_step = tf.Variable(0, trainable=False)\n",
    "        \n",
    "        # if we use beam search decoder, we need to specify the batch size and max source len\n",
    "        if self.beam_search:\n",
    "            self.batch_size = batch_size\n",
    "            self.source_tokens = tf.placeholder(tf.int32, shape=[batch_size, max_source_len])\n",
    "            self.source_length = tf.placeholder(tf.int32, shape=[batch_size,])\n",
    "        else:\n",
    "            self.source_tokens = tf.placeholder(tf.int32,shape=[None,None])\n",
    "            self.source_length = tf.placeholder(tf.int32,shape=[None,])\n",
    "            \n",
    "        if self.train_phase:\n",
    "            self.target_tokens = tf.placeholder(tf.int32, shape=[None, None])\n",
    "            self.target_length = tf.placeholder(tf.int32, shape=[None,])\n",
    "         \n",
    "        with tf.variable_scope(\"S2S\",initializer = tf.uniform_unit_scaling_initializer(1.0)):\n",
    "            self.setup_embeddings()\n",
    "            #self.setup_encoder()\n",
    "            self.setup_bidirection_encoder()\n",
    "            self.setup_attention_decoder()\n",
    "                \n",
    "        if self.train_phase:\n",
    "            opt = get_optimizer(optimizer)(self.learning_rate)\n",
    "            params = tf.trainable_variables()\n",
    "            gradients = tf.gradients(self.losses, params)\n",
    "            clipped_gradients, _ = tf.clip_by_global_norm(gradients, max_gradient_norm)\n",
    "            self.gradient_norm = tf.global_norm(gradients)\n",
    "            self.param_norm = tf.global_norm(params)\n",
    "            self.updates = opt.apply_gradients(zip(clipped_gradients, params), global_step=self.global_step)\n",
    "    \n",
    "    def setup_embeddings(self):\n",
    "        with tf.variable_scope(\"Embeddings\"):\n",
    "            with tf.device('/cpu:0'):\n",
    "                self.enc_emd = tf.get_variable(\"encode_embedding\", [self.vocab_size, self.dim_size])\n",
    "                self.dec_emd = tf.get_variable(\"decode_embedding\", [self.vocab_size, self.dim_size])\n",
    "                self.encoder_inputs = tf.nn.embedding_lookup(self.enc_emd, self.source_tokens)\n",
    "                if self.train_phase:\n",
    "                    self.decoder_inputs = tf.nn.embedding_lookup(self.dec_emd, self.target_tokens)\n",
    "    \n",
    "    def setup_encoder(self):\n",
    "        cell = multi_rnn_cell(self.cell_name,self.dim_size, self.num_layers, self.train_phase,self.keep_prob_config)\n",
    "        outputs,state = tf.nn.dynamic_rnn(cell,inputs=self.encoder_inputs,sequence_length=self.source_length,dtype=tf.float32)\n",
    "        self.encode_output = outputs\n",
    "        self.encode_state = state\n",
    "        # using the state of last layer of rnn as initial state\n",
    "        self.decode_initial_state = self.encode_state[-1]\n",
    "        \n",
    "    def setup_bidirection_encoder(self):\n",
    "        fw_cell = single_rnn_cell('gru',self.dim_size, train_phase=self.train_phase, keep_prob=self.keep_prob_config)\n",
    "        bw_cell = single_rnn_cell('gru',self.dim_size, train_phase=self.train_phase, keep_prob=self.keep_prob_config)\n",
    "        \n",
    "        with tf.variable_scope(\"Encoder\"):\n",
    "            outputs,states = tf.nn.bidirectional_dynamic_rnn(\n",
    "                cell_fw = fw_cell,\n",
    "                cell_bw = bw_cell,\n",
    "                dtype = tf.float32,\n",
    "                sequence_length = self.source_length,\n",
    "                inputs = self.encoder_inputs\n",
    "                )\n",
    "            outputs_concat = tf.concat(outputs, 2)\n",
    "        self.encode_output = outputs_concat\n",
    "        self.encode_state = states\n",
    "        \n",
    "        # use Dense layer to convert bi-direction state to decoder inital state\n",
    "        convert_layer = Dense(self.dim_size,dtype=tf.float32,name=\"bi_convert\")\n",
    "        self.decode_initial_state = convert_layer(tf.concat(self.encode_state,axis=1))\n",
    "        \n",
    "    def setup_training_decoder_layer(self):\n",
    "        max_dec_len = tf.reduce_max(self.target_length, name='max_dec_len')\n",
    "        training_helper = tf.contrib.seq2seq.TrainingHelper(self.decoder_inputs,self.target_length,name=\"training_helper\")\n",
    "        training_decoder = tf.contrib.seq2seq.BasicDecoder(\n",
    "            cell = self.dec_cell,\n",
    "            helper = training_helper,\n",
    "            initial_state = self.initial_state,\n",
    "            output_layer = self.output_layer\n",
    "        )\n",
    "        train_dec_outputs, train_dec_last_state,_ = tf.contrib.seq2seq.dynamic_decode(\n",
    "            training_decoder,\n",
    "            output_time_major=False,\n",
    "            impute_finished=True,\n",
    "            maximum_iterations=max_dec_len)\n",
    "        \n",
    "        # logits: [batch_size x max_dec_len x vocab_size]\n",
    "        logits = tf.identity(train_dec_outputs.rnn_output, name='logits')\n",
    "\n",
    "        # targets: [batch_size x max_dec_len x vocab_size]\n",
    "        targets = tf.slice(self.target_tokens, [0, 0], [-1, max_dec_len], 'targets')\n",
    "\n",
    "        masks = tf.sequence_mask(self.target_length,max_dec_len,dtype=tf.float32,name=\"mask\")\n",
    "        self.losses = tf.contrib.seq2seq.sequence_loss(logits=logits,targets=targets,weights=masks,name=\"losses\")\n",
    "        \n",
    "        # prediction sample for validation\n",
    "        self.valid_predictions = tf.identity(train_dec_outputs.sample_id, name='valid_preds')\n",
    "    \n",
    "    def setup_inference_decoder_layer(self):\n",
    "        start_tokens = tf.tile(tf.constant([self.start_token],dtype=tf.int32),[self.batch_size])\n",
    "        inference_helper = tf.contrib.seq2seq.GreedyEmbeddingHelper(\n",
    "            embedding=self.dec_emd,\n",
    "            start_tokens=start_tokens,\n",
    "            end_token=self.end_token)\n",
    "        \n",
    "        inference_decoder = tf.contrib.seq2seq.BasicDecoder(\n",
    "            cell = self.dec_cell,\n",
    "            helper = inference_helper, \n",
    "            initial_state=self.initial_state,\n",
    "            output_layer=self.output_layer)\n",
    "        \n",
    "        infer_dec_outputs, infer_dec_last_state,_ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                    inference_decoder,\n",
    "                    output_time_major=False,\n",
    "                    impute_finished=True,\n",
    "                    maximum_iterations=self.max_inference_lenght)\n",
    "        # [batch_size x dec_sentence_length], tf.int32\n",
    "        self.predictions = tf.identity(infer_dec_outputs.sample_id, name='predictions')\n",
    "            \n",
    "    def setup_beam_search_decoder_layer(self):\n",
    "        start_tokens = tf.tile(tf.constant([self.start_token],dtype=tf.int32),[self.batch_size])\n",
    "        bsd = tf.contrib.seq2seq.BeamSearchDecoder(\n",
    "                    cell=self.dec_cell,\n",
    "                    embedding=self.dec_emd,\n",
    "                    start_tokens= start_tokens,\n",
    "                    end_token=self.end_token,\n",
    "                    initial_state=self.initial_state,\n",
    "                    beam_width=self.beam_size,\n",
    "                    output_layer=self.output_layer,\n",
    "                    length_penalty_weight=0.0)\n",
    "        # final_outputs are instances of FinalBeamSearchDecoderOutput\n",
    "        final_outputs, final_state, final_sequence_lengths = tf.contrib.seq2seq.dynamic_decode(\n",
    "            bsd, \n",
    "            output_time_major=False,\n",
    "           # impute_finished=True,\n",
    "            maximum_iterations=self.max_inference_lenght\n",
    "        )\n",
    "        beam_predictions = final_outputs.predicted_ids\n",
    "        self.beam_predictions = tf.transpose(beam_predictions,perm=[0,2,1])\n",
    "        self.beam_prob = final_outputs.beam_search_decoder_output.scores\n",
    "        self.beam_ids = final_outputs.beam_search_decoder_output.predicted_ids\n",
    "        \n",
    "    def setup_attention_decoder(self):\n",
    "        #dec_cell = multi_rnn_cell('gru',self.dim_size,num_layers=self.num_layers, train_phase = self.train_phase, keep_prob=self.keep_prob_config)\n",
    "        dec_cell = [single_rnn_cell(self.cell_name,self.dim_size,self.train_phase,self.keep_prob_config) for i in range(self.num_layers)]\n",
    "        if self.beam_search:\n",
    "            memory = tf.contrib.seq2seq.tile_batch(self.encode_output,multiplier = self.beam_size)\n",
    "            memory_sequence_length = tf.contrib.seq2seq.tile_batch(self.source_length,multiplier = self.beam_size)\n",
    "        else:\n",
    "            memory = self.encode_output\n",
    "            memory_sequence_length = self.source_length\n",
    "            \n",
    "        attn_mech = tf.contrib.seq2seq.BahdanauAttention(\n",
    "            num_units = self.atten_size,\n",
    "            memory = memory,\n",
    "            memory_sequence_length = memory_sequence_length,\n",
    "            name = \"BahdanauAttention\"\n",
    "        )\n",
    "        dec_cell[0] = tf.contrib.seq2seq.AttentionWrapper(\n",
    "            cell=dec_cell[0],\n",
    "            attention_mechanism=attn_mech,\n",
    "            attention_layer_size=self.atten_size)\n",
    "        \n",
    "        if self.beam_search:\n",
    "            tile_state = tf.contrib.seq2seq.tile_batch(self.decode_initial_state,self.beam_size)\n",
    "            initial_state = [tile_state for i in range(self.num_layers)]\n",
    "            cell_state = dec_cell[0].zero_state(dtype=tf.float32,batch_size=self.batch_size*self.beam_size)\n",
    "            initial_state[0] = cell_state.clone(cell_state=initial_state[0])\n",
    "            self.initial_state = tuple(initial_state)\n",
    "        else:\n",
    "            # we use dynamic batch size\n",
    "            self.batch_size = tf.shape(self.encoder_inputs)[0]\n",
    "            initial_state = [self.decode_initial_state for i in range(self.num_layers)]\n",
    "            cell_state = dec_cell[0].zero_state(dtype=tf.float32, batch_size = self.batch_size)\n",
    "            initial_state[0] = cell_state.clone(cell_state=initial_state[0])\n",
    "            self.initial_state = tuple(initial_state)\n",
    "            \n",
    "        print(self.initial_state)\n",
    "        self.dec_cell = tf.contrib.rnn.MultiRNNCell(dec_cell)\n",
    "        self.output_layer = Dense(self.vocab_size,kernel_initializer = tf.truncated_normal_initializer(mean = 0.0, stddev=0.1))\n",
    "        if self.train_phase:\n",
    "            self.setup_training_decoder_layer()\n",
    "        else:\n",
    "            if self.beam_search:\n",
    "                self.setup_beam_search_decoder_layer()\n",
    "            else:\n",
    "                self.setup_inference_decoder_layer()\n",
    "    \n",
    "    def train_one_step(self,sess,encode_input,encode_len,decode_input,decode_len):\n",
    "        feed_dict = {}\n",
    "        feed_dict[self.source_tokens] = encode_input\n",
    "        feed_dict[self.source_length] = encode_len\n",
    "        feed_dict[self.target_tokens] = decode_input\n",
    "        feed_dict[self.target_length] = decode_len\n",
    "        valid_predictions,loss,_ = sess.run([self.valid_predictions,self.losses,self.updates],feed_dict=feed_dict)\n",
    "        return valid_predictions,loss\n",
    "    \n",
    "    def inference(self,sess,encode_input,encode_len):\n",
    "        feed_dict = {}\n",
    "        feed_dict[self.source_tokens] = encode_input\n",
    "        feed_dict[self.source_length] = encode_len\n",
    "        if self.beam_search:\n",
    "            predictions,probs,ids = sess.run([self.beam_predictions,self.beam_prob,self.beam_ids],feed_dict=feed_dict)\n",
    "            return predictions,ids\n",
    "        else:\n",
    "            predictions = sess.run([self.predictions],feed_dict=feed_dict)\n",
    "            return predictions\n",
    "    \n",
    "    def save_model(self,sess,checkpoint_dir):\n",
    "        writer = tf.summary.FileWriter(checkpoint_dir, sess.graph)\n",
    "        saver = tf.train.Saver(tf.global_variables())\n",
    "        saver.save(sess,checkpoint_dir + \"model.ckpt\",global_step=self.global_step)\n",
    "        \n",
    "    def restore_model(self,sess,checkpoint_dir):\n",
    "        saver = tf.train.Saver(tf.global_variables())\n",
    "        saver.restore(sess,tf.train.latest_checkpoint(checkpoint_dir))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 581
    },
    "colab_type": "code",
    "id": "SBNlzSI3ZKAl",
    "outputId": "714fc148-1fd0-4584-866e-36563a47b1c2"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0815 10:04:41.689500 139967090608000 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/util/deprecation.py:507: UniformUnitScaling.__init__ (from tensorflow.python.ops.init_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.initializers.variance_scaling instead with distribution=uniform to get equivalent behavior.\n",
      "W0815 10:04:41.720247 139967090608000 deprecation.py:323] From <ipython-input-34-e72657281cdd>:3: GRUCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.GRUCell, and will be replaced by that in Tensorflow 2.0.\n",
      "W0815 10:04:41.730442 139967090608000 deprecation.py:323] From <ipython-input-35-5597b33b1874>:84: bidirectional_dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.Bidirectional(keras.layers.RNN(cell))`, which is equivalent to this API\n",
      "W0815 10:04:41.731960 139967090608000 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/rnn.py:464: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n",
      "W0815 10:04:41.879231 139967090608000 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/rnn_cell_impl.py:564: calling Constant.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "W0815 10:04:41.896697 139967090608000 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/rnn_cell_impl.py:574: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "W0815 10:04:42.141234 139967090608000 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/rnn.py:244: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "W0815 10:04:42.842146 139967090608000 deprecation.py:323] From <ipython-input-35-5597b33b1874>:201: MultiRNNCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.StackedRNNCells, and will be replaced by that in Tensorflow 2.0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(AttentionWrapperState(cell_state=<tf.Tensor 'S2S/bi_convert/BiasAdd/Identity:0' shape=(?, 128) dtype=float32>, attention=<tf.Tensor 'S2S/AttentionWrapperZeroState/zeros_2/Identity:0' shape=(?, 30) dtype=float32>, time=<tf.Tensor 'S2S/AttentionWrapperZeroState/zeros_1:0' shape=() dtype=int32>, alignments=<tf.Tensor 'S2S/AttentionWrapperZeroState/zeros/Identity:0' shape=(?, ?) dtype=float32>, alignment_history=(), attention_state=<tf.Tensor 'S2S/AttentionWrapperZeroState/zeros_3/Identity:0' shape=(?, ?) dtype=float32>), <tf.Tensor 'S2S/bi_convert/BiasAdd:0' shape=(?, 128) dtype=float32>)\n",
      "(AttentionWrapperState(cell_state=<tf.Tensor 'S2S/bi_convert/BiasAdd/Identity:0' shape=(?, 128) dtype=float32>, attention=<tf.Tensor 'S2S/AttentionWrapperZeroState/zeros_2/Identity:0' shape=(?, 30) dtype=float32>, time=<tf.Tensor 'S2S/AttentionWrapperZeroState/zeros_1:0' shape=() dtype=int32>, alignments=<tf.Tensor 'S2S/AttentionWrapperZeroState/zeros/Identity:0' shape=(?, ?) dtype=float32>, alignment_history=(), attention_state=<tf.Tensor 'S2S/AttentionWrapperZeroState/zeros_3/Identity:0' shape=(?, ?) dtype=float32>), <tf.Tensor 'S2S/bi_convert/BiasAdd:0' shape=(?, 128) dtype=float32>)\n",
      "beam\n",
      "(AttentionWrapperState(cell_state=<tf.Tensor 'S2S/tile_batch_2/Reshape:0' shape=(6, 128) dtype=float32>, attention=<tf.Tensor 'S2S/AttentionWrapperZeroState/zeros_2:0' shape=(6, 30) dtype=float32>, time=<tf.Tensor 'S2S/AttentionWrapperZeroState/zeros_1:0' shape=() dtype=int32>, alignments=<tf.Tensor 'S2S/AttentionWrapperZeroState/zeros:0' shape=(6, 10) dtype=float32>, alignment_history=(), attention_state=<tf.Tensor 'S2S/AttentionWrapperZeroState/zeros_3:0' shape=(6, 10) dtype=float32>), <tf.Tensor 'S2S/tile_batch_2/Reshape:0' shape=(6, 128) dtype=float32>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0815 10:04:49.003839 139967090608000 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py:985: to_int64 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "model = BasicS2SModel(vocab=vocab,mode=\"train\")\n",
    "\n",
    "tf.reset_default_graph()\n",
    "model = BasicS2SModel(vocab=vocab,mode=\"inference\")\n",
    "\n",
    "print(\"beam\")\n",
    "tf.reset_default_graph()\n",
    "model = BasicS2SModel(vocab=vocab,mode=\"inference\",use_beam_search=True,beam_size=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 54
    },
    "colab_type": "code",
    "id": "rnSQ6JqENVLH",
    "outputId": "bf5f7166-6132-45ac-efae-57406c338d08"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(AttentionWrapperState(cell_state=<tf.Tensor 'S2S/bi_convert/BiasAdd/Identity:0' shape=(?, 128) dtype=float32>, attention=<tf.Tensor 'S2S/AttentionWrapperZeroState/zeros_2/Identity:0' shape=(?, 30) dtype=float32>, time=<tf.Tensor 'S2S/AttentionWrapperZeroState/zeros_1:0' shape=() dtype=int32>, alignments=<tf.Tensor 'S2S/AttentionWrapperZeroState/zeros/Identity:0' shape=(?, ?) dtype=float32>, alignment_history=(), attention_state=<tf.Tensor 'S2S/AttentionWrapperZeroState/zeros_3/Identity:0' shape=(?, ?) dtype=float32>), <tf.Tensor 'S2S/bi_convert/BiasAdd:0' shape=(?, 128) dtype=float32>, <tf.Tensor 'S2S/bi_convert/BiasAdd:0' shape=(?, 128) dtype=float32>)\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "model = BasicS2SModel(vocab=vocab,num_layers=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "QAX_tYC9ZNQI",
    "outputId": "3ef82156-faae-42dd-b8f8-86bc9361efe7"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start to run\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:50, 13.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0\n",
      "epoch loss:  3144.470274090767\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move to _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ move on and speak\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.79it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1\n",
      "epoch loss:  2273.845334172249\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ move strong happened speak\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2\n",
      "epoch loss:  1896.8293200917542\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ what strong ? speak\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 18.06it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3\n",
      "epoch loss:  1617.2040611784905\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ move happened time speak\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.71it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4\n",
      "epoch loss:  1374.858656448312\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ move with to flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:48, 17.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 5\n",
      "epoch loss:  1154.6074039274827\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ move with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 6\n",
      "epoch loss:  956.7287890203297\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.65it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 7\n",
      "epoch loss:  789.2109757526778\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ move something people speak\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 8\n",
      "epoch loss:  601.1568242823705\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ move with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 9\n",
      "epoch loss:  482.58003842667677\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ move with to flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 10\n",
      "epoch loss:  373.79689831720316\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go yourself the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 11\n",
      "epoch loss:  284.1963379048757\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 12\n",
      "epoch loss:  227.10404265476973\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ believe with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 13\n",
      "epoch loss:  200.12864326828276\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 14\n",
      "epoch loss:  173.52273639576742\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.66it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 15\n",
      "epoch loss:  150.6462235505751\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with your flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 16\n",
      "epoch loss:  144.47801132559107\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with to flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 17\n",
      "epoch loss:  105.45508008652541\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 18.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 18\n",
      "epoch loss:  112.56182409183862\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ believe with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 19\n",
      "epoch loss:  98.54453961904801\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.60it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 20\n",
      "epoch loss:  86.32281836366747\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 21\n",
      "epoch loss:  60.990382402008436\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 18.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 22\n",
      "epoch loss:  50.154526189719036\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with your flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 23\n",
      "epoch loss:  51.99370015353543\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with a flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 14.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 24\n",
      "epoch loss:  39.68923259639996\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.70it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 25\n",
      "epoch loss:  33.62043382642696\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 18.48it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 26\n",
      "epoch loss:  36.129153820842475\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.79it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 27\n",
      "epoch loss:  32.350799475328586\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 28\n",
      "epoch loss:  40.69672221219116\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 29\n",
      "epoch loss:  39.35348985359906\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with people flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:48, 17.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 30\n",
      "epoch loss:  35.150225956445865\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:48, 17.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 31\n",
      "epoch loss:  28.273298315597458\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 32\n",
      "epoch loss:  26.20029483348833\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 33\n",
      "epoch loss:  19.02314428720456\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.67it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 34\n",
      "epoch loss:  24.25677172352789\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 35\n",
      "epoch loss:  30.760705134951877\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 36\n",
      "epoch loss:  31.69596145423634\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 14.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 37\n",
      "epoch loss:  30.200108511491635\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 38\n",
      "epoch loss:  21.34951921167101\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 39\n",
      "epoch loss:  17.418128152125178\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.65it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 40\n",
      "epoch loss:  18.626276441032246\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.82it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 41\n",
      "epoch loss:  20.610405946404455\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 18.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 42\n",
      "epoch loss:  24.78026511040366\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 43\n",
      "epoch loss:  17.70977259735281\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 44\n",
      "epoch loss:  17.040365970186087\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 18.06it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 45\n",
      "epoch loss:  18.111616802769632\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with your flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 46\n",
      "epoch loss:  22.976563000205942\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 17.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 47\n",
      "epoch loss:  25.000062897961698\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 14.60it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 48\n",
      "epoch loss:  29.74657986289921\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with the flow\n",
      "Target: go with the flow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [00:47, 18.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 49\n",
      "epoch loss:  24.772872344739795\n",
      "Input: my heart is broken\n",
      "Prediction: _START_ move on _START_ _START_\n",
      "Target: move on\n",
      "Input: i broke her heart\n",
      "Prediction: _START_ go with the flow\n",
      "Target: go with the flow\n"
     ]
    }
   ],
   "source": [
    "\n",
    "saver = tf.train.Saver()\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    # saver.restore(sess,tf.train.latest_checkpoint('./'))\n",
    "    \n",
    "    print(\"start to run\")\n",
    "    loss_history = []\n",
    "    for epoch in range(50):\n",
    "        all_preds = []\n",
    "        epoch_loss = 0\n",
    "#         i=0\n",
    "        for input_batch, target_batch in tqdm(zip(all_input_sentences, all_target_sentences)):\n",
    "            input_batch_tokens = []\n",
    "            target_batch_tokens = []\n",
    "            enc_sentence_lengths = []\n",
    "            dec_sentence_lengths = []\n",
    "\n",
    "            for input_sent in input_batch:\n",
    "                tokens, sent_len = sent2idx(input_sent)\n",
    "                input_batch_tokens.append(tokens)\n",
    "                enc_sentence_lengths.append(sent_len)\n",
    "\n",
    "            for target_sent in target_batch:\n",
    "                tokens, sent_len = sent2idx(target_sent,\n",
    "                             vocab=vocab,\n",
    "                             max_sentence_length=dec_sentence_length,\n",
    "                             is_target=True)\n",
    "                target_batch_tokens.append(tokens)\n",
    "                dec_sentence_lengths.append(sent_len)\n",
    "#             i = i+1\n",
    "#             print(i)\n",
    "            batch_preds, batch_loss = model.train_one_step(sess,input_batch_tokens,enc_sentence_lengths,target_batch_tokens,dec_sentence_lengths)\n",
    "            epoch_loss += batch_loss\n",
    "            all_preds.append(batch_preds)\n",
    "        loss_history.append(epoch_loss)\n",
    "        if epoch % 1 == 0:\n",
    "            print('Epoch', epoch)\n",
    "            print('epoch loss: ', epoch_loss )\n",
    "            for input_sent, target_sent, pred in zip(input_batch, target_batch, batch_preds):\n",
    "                print('Input:', input_sent)\n",
    "                print('Prediction:', idx2sent(pred, reverse_vocab=reverse_vocab))\n",
    "                print('Target:', target_sent)\n",
    "\n",
    "        \n",
    "        writer = tf.summary.FileWriter('./checkpoints/', sess.graph)\n",
    "        \n",
    "        saver.save(sess,\"./checkpoints/model.ckpt\")\n",
    "    # ,global_step=model.global_step\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-PcxR4I54iR7"
   },
   "outputs": [],
   "source": [
    "# from google.colab import files\n",
    "# files.download(\"/content/file.zip\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "XhtosVjmZrB6",
    "outputId": "3b4b4fc6-77f2-451d-c220-d768119b1d53"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(AttentionWrapperState(cell_state=<tf.Tensor 'S2S/bi_convert/BiasAdd/Identity:0' shape=(?, 128) dtype=float32>, attention=<tf.Tensor 'S2S/AttentionWrapperZeroState/zeros_2/Identity:0' shape=(?, 30) dtype=float32>, time=<tf.Tensor 'S2S/AttentionWrapperZeroState/zeros_1:0' shape=() dtype=int32>, alignments=<tf.Tensor 'S2S/AttentionWrapperZeroState/zeros/Identity:0' shape=(?, ?) dtype=float32>, alignment_history=(), attention_state=<tf.Tensor 'S2S/AttentionWrapperZeroState/zeros_3/Identity:0' shape=(?, ?) dtype=float32>), <tf.Tensor 'S2S/bi_convert/BiasAdd:0' shape=(?, 128) dtype=float32>, <tf.Tensor 'S2S/bi_convert/BiasAdd:0' shape=(?, 128) dtype=float32>)\n",
      "Input: hi what is your name ?\n",
      "[  0 240   1   1   1   1   1   1   1   2]\n",
      "Prediction: _START_ hi _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: hi this is sofia .\n",
      "Input: nice to meet you\n",
      "[  0 241  97 356 300 821   1   1   1   2]\n",
      "Prediction: _START_ nice something delicious birthday books _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: nice to meet you too\n",
      "Input: nice to meet you\n",
      "[  0 241  97 356 300 821   1   1   1   2]\n",
      "Prediction: _START_ nice something delicious birthday books _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: nice to meet you too\n",
      "Input: which programming language do you use ?\n",
      "[  0 179  55 784   1   1   1   1   1   2]\n",
      "Prediction: _START_ these like python _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: i like python .\n",
      "Input: which programming language do you use ?\n",
      "[  0 179  55 784   1   1   1   1   1   2]\n",
      "Prediction: _START_ these like python _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: i like python .\n",
      "Input: where do you live ?\n",
      "[ 0 44 90  1  1  1  1  1  1  2]\n",
      "Prediction: _START_ take live _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: i live in gwalior .\n",
      "Input: where do you live ?\n",
      "[ 0 44 90  1  1  1  1  1  1  2]\n",
      "Prediction: _START_ take live _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: i live in gwalior .\n",
      "Input: what is your major ?\n",
      "[  0 465 790 786 821   1   1   1   1   2]\n",
      "Prediction: _START_ software engineer engineering books _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: i study software engineering .\n",
      "Input: what is your major ?\n",
      "[  0 465 790 786 821   1   1   1   1   2]\n",
      "Prediction: _START_ software engineer engineering books _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: i study software engineering .\n",
      "Input: what do you want to drink ?\n",
      "[  0 373 221   1   1   1   1   1   1   2]\n",
      "Prediction: _START_ beer please _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: beer please\n",
      "Input: what do you want to drink ?\n",
      "[  0 373 221   1   1   1   1   1   1   2]\n",
      "Prediction: _START_ beer please _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: beer please\n",
      "Input: what is your favorite beer ?\n",
      "[  0 787 788   1   1   1   1   1   1   2]\n",
      "Prediction: _START_ leffe brown _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: leffe brown\n",
      "Input: what is your favorite beer ?\n",
      "[  0 787 788   1   1   1   1   1   1   2]\n",
      "Prediction: _START_ leffe brown _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: leffe brown\n",
      "Input: see you later .\n",
      "[  0 252   1   1   1   1   1   1   1   2]\n",
      "Prediction: _START_ bye _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: bye bye .\n",
      "Input: see you later .\n",
      "[  0 252   1   1   1   1   1   1   1   2]\n",
      "Prediction: _START_ bye _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: bye bye .\n",
      "Input: thank you\n",
      "[  0 789   1   1   1   1   1   1   1   2]\n",
      "Prediction: _START_ welcome _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: welcome\n",
      "Input: thank you\n",
      "[  0 789   1   1   1   1   1   1   1   2]\n",
      "Prediction: _START_ welcome _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: welcome\n",
      "Input: what is your occupation ?\n",
      "[  0 465 790   1   1   1   1   1   1   2]\n",
      "Prediction: _START_ software engineer _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: software engineer\n",
      "Input: what is your occupation ?\n",
      "[  0 465 790   1   1   1   1   1   1   2]\n",
      "Prediction: _START_ software engineer _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: software engineer\n",
      "Input: who is your father ?\n",
      "[  0 274   1   1   1   1   1   1   1   2]\n",
      "Prediction: _START_ chinmay _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: chinmay\n",
      "Input: who is your father ?\n",
      "[  0 274   1   1   1   1   1   1   1   2]\n",
      "Prediction: _START_ chinmay _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: chinmay\n",
      "Input: what can you do ?\n",
      "[  0 792 466 275 468   1   1   1   1   2]\n",
      "Prediction: _START_ chatting watching here video _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: i am here to help you .\n",
      "Input: what can you do ?\n",
      "[  0 792 466 275 468   1   1   1   1   2]\n",
      "Prediction: _START_ chatting watching here video _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: i am here to help you .\n",
      "Input: i am depressed\n",
      "[  0  48 116  22  87   1   1   1   1   2]\n",
      "Prediction: _START_ get happened be positive _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: why ?\n",
      "Input: i am depressed\n",
      "[  0  48 116  22  87   1   1   1   1   2]\n",
      "Prediction: _START_ get happened be positive _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: why ?\n",
      "Input: my boss is harassing me\n",
      "[  0 344 791   1   1   1   1   1   1   2]\n",
      "Prediction: _START_ okay gotcha _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: okay gotcha\n",
      "Input: my boss is harassing me\n",
      "[  0 344 791   1   1   1   1   1   1   2]\n",
      "Prediction: _START_ okay gotcha _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: okay gotcha\n",
      "Input: what should i do ?\n",
      "[ 0 89 46  1  1  1  1  1  1  2]\n",
      "Prediction: _START_ go with _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: try watching recommended video on this site\n",
      "Input: what should i do ?\n",
      "[ 0 89 46  1  1  1  1  1  1  2]\n",
      "Prediction: _START_ go with _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: try watching recommended video on this site\n",
      "Input: what can i do ?\n",
      "[ 0 89 46  1  1  1  1  1  1  2]\n",
      "Prediction: _START_ go with _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: try watching recommended video on this site\n",
      "Input: what can i do ?\n",
      "[ 0 89 46  1  1  1  1  1  1  2]\n",
      "Prediction: _START_ go with _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: try watching recommended video on this site\n",
      "Input: i want to kill myself\n",
      "[  0 114 195 370 195  47  32 195  57   2]\n",
      "Prediction: _START_ keep fighting cheerful fighting hard life fighting time _END_\n",
      "Target: do not life is beautiful .\n",
      "Input: i want to kill myself\n",
      "[  0 114 195 370 195  47  32 195  57   2]\n",
      "Prediction: _START_ keep fighting cheerful fighting hard life fighting time _END_\n",
      "Target: do not life is beautiful .\n",
      "Input: my friend fooled me\n",
      "[ 0 14  1  1  1  1  1  1  1  2]\n",
      "Prediction: _START_ do _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: do not be sad .\n",
      "Input: my friend fooled me\n",
      "[ 0 14  1  1  1  1  1  1  1  2]\n",
      "Prediction: _START_ do _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: do not be sad .\n",
      "Input: my girlfriend fooled me\n",
      "[ 0 14  1  1  1  1  1  1  1  2]\n",
      "Prediction: _START_ do _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: do not be sad .\n",
      "Input: my girlfriend fooled me\n",
      "[ 0 14  1  1  1  1  1  1  1  2]\n",
      "Prediction: _START_ do _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: do not be sad .\n",
      "Input: my gf fooled me\n",
      "[ 0 14  1  1  1  1  1  1  1  2]\n",
      "Prediction: _START_ do _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: do not be sad . it is okay\n",
      "Input: my gf fooled me\n",
      "[ 0 14  1  1  1  1  1  1  1  2]\n",
      "Prediction: _START_ do _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: do not be sad . it is okay\n",
      "Input: my gf backstabbed me\n",
      "[  0 344 248   1   1   1   1   1   1   2]\n",
      "Prediction: _START_ okay understand _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: i understand\n",
      "Input: my gf backstabbed me\n",
      "[  0 344 248   1   1   1   1   1   1   2]\n",
      "Prediction: _START_ okay understand _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: i understand\n",
      "Input: my gf betrayed me\n",
      "[ 0 14  1  1  1  1  1  1  1  2]\n",
      "Prediction: _START_ do _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: do not worry focus on yourself\n",
      "Input: my gf betrayed me\n",
      "[ 0 14  1  1  1  1  1  1  1  2]\n",
      "Prediction: _START_ do _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: do not worry focus on yourself\n",
      "Input: my girlfriend betrayed me\n",
      "[ 0 14  1  1  1  1  1  1  1  2]\n",
      "Prediction: _START_ do _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: do not worry focus on yourself\n",
      "Input: my girlfriend betrayed me\n",
      "[ 0 14  1  1  1  1  1  1  1  2]\n",
      "Prediction: _START_ do _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: do not worry focus on yourself\n",
      "Input: i do not want to live anymore\n",
      "[  0 265 478 385 821   1   1   1   1   2]\n",
      "Prediction: _START_ stay meditation read books _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: stay positive\n",
      "Input: i do not want to live anymore\n",
      "[  0 265 478 385 821   1   1   1   1   2]\n",
      "Prediction: _START_ stay meditation read books _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: stay positive\n",
      "Input: i like to swim\n",
      "[ 0 15  1  1  1  1  1  1  1  2]\n",
      "Prediction: _START_ me _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: me too\n",
      "Input: i like to swim\n",
      "[ 0 15  1  1  1  1  1  1  1  2]\n",
      "Prediction: _START_ me _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: me too\n",
      "Input: i like to jog\n",
      "[ 0 15  1  1  1  1  1  1  1  2]\n",
      "Prediction: _START_ me _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: me too\n",
      "Input: i like to jog\n",
      "[ 0 15  1  1  1  1  1  1  1  2]\n",
      "Prediction: _START_ me _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: me too\n",
      "Input: i like to drink coffee\n",
      "[ 0 15  1  1  1  1  1  1  1  2]\n",
      "Prediction: _START_ me _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: me too\n",
      "Input: i like to drink coffee\n",
      "[ 0 15  1  1  1  1  1  1  1  2]\n",
      "Prediction: _START_ me _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: me too\n",
      "Input: i have no friends\n",
      "[  0 110 506  95 851  95 218  57 125   2]\n",
      "Prediction: _START_ tell approaching new company new relations time them _END_\n",
      "Target: i am your friend\n",
      "Input: i have no friends\n",
      "[  0 110 506  95 851  95 218  57 125   2]\n",
      "Prediction: _START_ tell approaching new company new relations time them _END_\n",
      "Target: i am your friend\n",
      "Input: i do not have friends\n",
      "[  0 110 275  95 211  57  29  21   1   2]\n",
      "Prediction: _START_ tell here new much time on your _PAD_ _END_\n",
      "Target: tell me i am your friend\n",
      "Input: i do not have friends\n",
      "[  0 110 275  95 211  57  29  21   1   2]\n",
      "Prediction: _START_ tell here new much time on your _PAD_ _END_\n",
      "Target: tell me i am your friend\n",
      "Input: i have lot of work\n",
      "[  0  89 470 265   1   1   1   1   1   2]\n",
      "Prediction: _START_ go worries stay _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: no worries stay calm\n",
      "Input: i have lot of work\n",
      "[  0  89 470 265   1   1   1   1   1   2]\n",
      "Prediction: _START_ go worries stay _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: no worries stay calm\n",
      "Input: i have lot to do\n",
      "[  0  89 470 265  47 267   1   1   1   2]\n",
      "Prediction: _START_ go worries stay hard calm _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: no worries stay calm\n",
      "Input: i have lot to do\n",
      "[  0  89 470 265  47 267   1   1   1   2]\n",
      "Prediction: _START_ go worries stay hard calm _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: no worries stay calm\n",
      "Input: i have never heard about this\n",
      "[  0 209 116  87  29  87   1   1   1   2]\n",
      "Prediction: _START_ seriously happened positive on positive _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: seriously ?\n",
      "Input: i have never heard about this\n",
      "[  0 209 116  87  29  87   1   1   1   2]\n",
      "Prediction: _START_ seriously happened positive on positive _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: seriously ?\n",
      "Input: hey\n",
      "[  0 250  84 497 821   1   1   1   1   2]\n",
      "Prediction: _START_ hello day hara books _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: hey\n",
      "Input: hey\n",
      "[  0 250  84 497 821   1   1   1   1   2]\n",
      "Prediction: _START_ hello day hara books _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: hey\n",
      "Input: what is up ?\n",
      "[  0 190  29  22  21   1   1   1   1   2]\n",
      "Prediction: _START_ nothing on be your _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: nothing much\n",
      "Input: what is up ?\n",
      "[  0 190  29  22  21   1   1   1   1   2]\n",
      "Prediction: _START_ nothing on be your _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: nothing much\n",
      "Input: what are you doing ?\n",
      "[  0 792   1   1   1   1   1   1   1   2]\n",
      "Prediction: _START_ chatting _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: chatting with you\n",
      "Input: what are you doing ?\n",
      "[  0 792   1   1   1   1   1   1   1   2]\n",
      "Prediction: _START_ chatting _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: chatting with you\n",
      "Input: what will you do years after now ?\n",
      "[  0  44 171  73   1   1   1   1   1   2]\n",
      "Prediction: _START_ take everyone help _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: help everyone\n",
      "Input: what will you do years after now ?\n",
      "[  0  44 171  73   1   1   1   1   1   2]\n",
      "Prediction: _START_ take everyone help _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: help everyone\n",
      "Input: today is my birthday\n",
      "[  0 155 300   1   1   1   1   1   1   2]\n",
      "Prediction: _START_ happy birthday _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: happy birthday\n",
      "Input: today is my birthday\n",
      "[  0 155 300   1   1   1   1   1   1   2]\n",
      "Prediction: _START_ happy birthday _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: happy birthday\n",
      "Input: nobody wished me\n",
      "[0 8 1 1 1 1 1 1 1 2]\n",
      "Prediction: _START_ i _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: i will wish you happy birthday .\n",
      "Input: nobody wished me\n",
      "[0 8 1 1 1 1 1 1 1 2]\n",
      "Prediction: _START_ i _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: i will wish you happy birthday .\n",
      "Input: am i bad boy ?\n",
      "[  0 198 116  42  30   1   1   1   1   2]\n",
      "Prediction: _START_ sometimes happened good have _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: not at all\n",
      "Input: am i bad boy ?\n",
      "[  0 198 116  42  30   1   1   1   1   2]\n",
      "Prediction: _START_ sometimes happened good have _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: not at all\n",
      "Input: do you like me ?\n",
      "[ 0 58  1  1  1  1  1  1  1  2]\n",
      "Prediction: _START_ yes _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: yes totally\n",
      "Input: do you like me ?\n",
      "[ 0 58  1  1  1  1  1  1  1  2]\n",
      "Prediction: _START_ yes _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: yes totally\n",
      "Input: do you love me ?\n",
      "[ 0 58  1  1  1  1  1  1  1  2]\n",
      "Prediction: _START_ yes _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _PAD_ _END_\n",
      "Target: yes totally\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "with tf.Session() as sess:\n",
    "    model = BasicS2SModel(vocab=vocab,mode=\"inference\",use_beam_search=False,num_layers=3)\n",
    "    saver = tf.train.Saver(tf.global_variables())\n",
    "    saver.restore(sess,tf.train.latest_checkpoint('./checkpoints'))\n",
    "    \n",
    "    \n",
    "    for input_batch, target_batch in zip(all_input_sentences[:40], all_target_sentences[:40]):\n",
    "      batch_preds = []\n",
    "      batch_tokens = []\n",
    "      batch_sent_lens = []\n",
    "      for input_sent in input_batch:\n",
    "          tokens, sent_len = sent2idx(input_sent)\n",
    "          batch_tokens.append(tokens)\n",
    "          batch_sent_lens.append(sent_len)\n",
    "\n",
    "      batch_preds = model.inference(sess,batch_tokens,batch_sent_lens)\n",
    "      # print(batch_preds)\n",
    "      \n",
    "      # ans = []\n",
    "      for preds in batch_preds[0]:\n",
    "      #     # print(preds)\n",
    "      #     ans1 = []\n",
    "          for i in range(len(preds)-1):\n",
    "            if preds[i] == preds[i+1]:\n",
    "      #         if preds[i] not in ans1:\n",
    "      #           ans1.append(preds[i])\n",
    "      #           if i == len(preds)-1:\n",
    "      #             ans1.append(preds[i+1])\n",
    "              preds[i+1:9] = 1\n",
    "              preds[9] = 2\n",
    "      #     ans.append(ans1)\n",
    "      # batch_preds[0]\n",
    "      for input_sent, target_sent, pred in zip(input_batch, target_batch, batch_preds[0]):\n",
    "          print('Input:', input_sent)\n",
    "          print(pred)\n",
    "          print('Prediction:', idx2sent(pred, reverse_vocab=reverse_vocab))\n",
    "          print('Target:', target_sent)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "mdFeSsApuGKc"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Chatbot5.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}