Lecture-5/Lecture-5.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Introduction to deep learning\n",
    "\n",
    "## Using keras package:\n",
    "\n",
    "Open a shell and install Keras.\n",
    "\n",
    "For anaconda: conda install keras\n",
    "\n",
    "For standard python: pip install keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import keras"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Loading IMDB data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv(\"../Lecture-4/labeledTrainData.tsv\", header=0, delimiter=\"\\t\", quoting=0)\n",
    "\n",
    "#First shuffle data\n",
    "df = df.sample(frac=1, random_state= 100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creating the training and the testing data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20000,) (5000,)\n"
     ]
    }
   ],
   "source": [
    "X_train_text = df['review'][:20000]\n",
    "y_train = df['sentiment'][:20000]\n",
    "#y_train = keras.utils.to_categorical(y_train)\n",
    "X_valid_text = df['review'][20000:]\n",
    "y_valid = df['sentiment'][20000:]\n",
    "\n",
    "print(y_train.shape, y_valid.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Vecotizing the data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20000, 16414)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "\n",
    "#vect = CountVectorizer(binary = False, min_df = 10)\n",
    "vect = TfidfVectorizer(min_df = 10)\n",
    "\n",
    "X_train = vect.fit_transform(X_train_text)\n",
    "print(X_train.shape)\n",
    "X_valid = vect.transform(X_valid_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training logistic regression as the baseline:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8944\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics  import accuracy_score\n",
    "\n",
    "LR = LogisticRegression(C = 4.1)\n",
    "LR.fit(X_train, y_train)\n",
    "p_valid = LR.predict(X_valid)\n",
    "print(accuracy_score(y_valid, p_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating 2 layers MLP:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_9 (Dense)              (None, 32)                525280    \n",
      "_________________________________________________________________\n",
      "dense_10 (Dense)             (None, 1)                 33        \n",
      "=================================================================\n",
      "Total params: 525,313\n",
      "Trainable params: 525,313\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Activation\n",
    "\n",
    "model = Sequential([\n",
    "    Dense(32, activation = 'relu', input_shape=(X_train.shape[1],)),\n",
    "    Dense(1, activation = 'sigmoid'),\n",
    "])\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compile the model using SGD with learning rate as .1:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# For a binary classification problem\n",
    "opt = keras.optimizers.SGD(lr=0.4)\n",
    "model.compile(optimizer=opt, loss='binary_crossentropy', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training the model with batch_size = 1024 for 5 epochs:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 20000 samples, validate on 5000 samples\n",
      "Epoch 1/5\n",
      "20000/20000 [==============================] - 5s 240us/step - loss: 0.6910 - acc: 0.5825 - val_loss: 0.6888 - val_acc: 0.5108\n",
      "Epoch 2/5\n",
      "20000/20000 [==============================] - 4s 221us/step - loss: 0.6857 - acc: 0.6540 - val_loss: 0.6829 - val_acc: 0.6782\n",
      "Epoch 3/5\n",
      "20000/20000 [==============================] - 4s 222us/step - loss: 0.6794 - acc: 0.7284 - val_loss: 0.6764 - val_acc: 0.7444\n",
      "Epoch 4/5\n",
      "20000/20000 [==============================] - 4s 222us/step - loss: 0.6722 - acc: 0.7499 - val_loss: 0.6691 - val_acc: 0.7220\n",
      "Epoch 5/5\n",
      "20000/20000 [==============================] - 4s 221us/step - loss: 0.6639 - acc: 0.7612 - val_loss: 0.6602 - val_acc: 0.7702\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x2429e48cc50>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train, y_train, validation_data = (X_valid, y_valid), epochs=5, batch_size=1024)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training using Adam optimizer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 20000 samples, validate on 5000 samples\n",
      "Epoch 1/5\n",
      "20000/20000 [==============================] - 5s 272us/step - loss: 0.3450 - acc: 0.8387 - val_loss: 0.2733 - val_acc: 0.8814\n",
      "Epoch 2/5\n",
      "20000/20000 [==============================] - 5s 246us/step - loss: 0.0947 - acc: 0.9664 - val_loss: 0.3711 - val_acc: 0.8788\n",
      "Epoch 3/5\n",
      "20000/20000 [==============================] - 5s 248us/step - loss: 0.0228 - acc: 0.9925 - val_loss: 0.5679 - val_acc: 0.8716\n",
      "Epoch 4/5\n",
      "20000/20000 [==============================] - 5s 248us/step - loss: 0.0072 - acc: 0.9979 - val_loss: 0.7174 - val_acc: 0.8684\n",
      "Epoch 5/5\n",
      "20000/20000 [==============================] - 5s 248us/step - loss: 0.0024 - acc: 0.9991 - val_loss: 0.7840 - val_acc: 0.8712\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x242a231b7b8>"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = Sequential([\n",
    "    Dense(32, activation = 'relu', input_shape=(X_train.shape[1],)),\n",
    "    Dense(1, activation = 'sigmoid'),\n",
    "])\n",
    "\n",
    "# For a binary classification problem\n",
    "opt = keras.optimizers.Adam(lr=0.1)\n",
    "model.compile(optimizer=opt, loss='binary_crossentropy', metrics=['accuracy'])\n",
    "model.fit(X_train, y_train, validation_data = (X_valid, y_valid), epochs=5, batch_size=512)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using tanh as activation function:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 20000 samples, validate on 5000 samples\n",
      "Epoch 1/5\n",
      "20000/20000 [==============================] - 5s 249us/step - loss: 0.4023 - acc: 0.8050 - val_loss: 0.2753 - val_acc: 0.8862\n",
      "Epoch 2/5\n",
      "20000/20000 [==============================] - 5s 232us/step - loss: 0.1281 - acc: 0.9571 - val_loss: 0.3590 - val_acc: 0.8824\n",
      "Epoch 3/5\n",
      "20000/20000 [==============================] - 5s 229us/step - loss: 0.0511 - acc: 0.9859 - val_loss: 0.4396 - val_acc: 0.8726\n",
      "Epoch 4/5\n",
      "20000/20000 [==============================] - 5s 230us/step - loss: 0.0198 - acc: 0.9955 - val_loss: 0.5849 - val_acc: 0.8692\n",
      "Epoch 5/5\n",
      "20000/20000 [==============================] - 5s 228us/step - loss: 0.0065 - acc: 0.9990 - val_loss: 0.7085 - val_acc: 0.8658\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x242a3c8ed68>"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = Sequential([\n",
    "    Dense(32, activation = 'tanh', input_shape=(X_train.shape[1],)),\n",
    "    Dense(1, activation = 'sigmoid'),\n",
    "])\n",
    "\n",
    "# For a binary classification problem\n",
    "opt = keras.optimizers.Adam(lr=0.1)\n",
    "model.compile(optimizer=opt, loss='binary_crossentropy', metrics=['accuracy'])\n",
    "model.fit(X_train, y_train, validation_data = (X_valid, y_valid), epochs=5, batch_size=1024)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Adding dropout layer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 20000 samples, validate on 5000 samples\n",
      "Epoch 1/5\n",
      "20000/20000 [==============================] - 5s 268us/step - loss: 0.5915 - acc: 0.6317 - val_loss: 0.3724 - val_acc: 0.8700\n",
      "Epoch 2/5\n",
      "20000/20000 [==============================] - 5s 238us/step - loss: 0.5124 - acc: 0.6984 - val_loss: 0.3414 - val_acc: 0.8862\n",
      "Epoch 3/5\n",
      "20000/20000 [==============================] - 5s 239us/step - loss: 0.4876 - acc: 0.7081 - val_loss: 0.3273 - val_acc: 0.8844\n",
      "Epoch 4/5\n",
      "20000/20000 [==============================] - 5s 233us/step - loss: 0.4671 - acc: 0.7250 - val_loss: 0.3041 - val_acc: 0.8908\n",
      "Epoch 5/5\n",
      "20000/20000 [==============================] - 5s 233us/step - loss: 0.4458 - acc: 0.7444 - val_loss: 0.3070 - val_acc: 0.8928\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x242c69f9a20>"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from keras.layers import Dropout\n",
    "\n",
    "model = Sequential([\n",
    "    Dense(32, activation = 'relu', input_shape=(X_train.shape[1],)),\n",
    "    Dropout(.9),\n",
    "    Dense(1, activation = 'sigmoid'),\n",
    "])\n",
    "\n",
    "# For a binary classification problem\n",
    "opt = keras.optimizers.Adam(lr=0.1)\n",
    "model.compile(optimizer=opt, loss='binary_crossentropy', metrics=['accuracy'])\n",
    "model.fit(X_train, y_train, validation_data = (X_valid, y_valid), epochs=5, batch_size=1024)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Increasing number of cells:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 20000 samples, validate on 5000 samples\n",
      "Epoch 1/5\n",
      "20000/20000 [==============================] - 9s 439us/step - loss: 0.6746 - acc: 0.6258 - val_loss: 0.3693 - val_acc: 0.8668\n",
      "Epoch 2/5\n",
      "20000/20000 [==============================] - 8s 418us/step - loss: 0.4805 - acc: 0.7622 - val_loss: 0.3076 - val_acc: 0.8848\n",
      "Epoch 3/5\n",
      "20000/20000 [==============================] - 9s 431us/step - loss: 0.4242 - acc: 0.8007 - val_loss: 0.2852 - val_acc: 0.8836\n",
      "Epoch 4/5\n",
      "20000/20000 [==============================] - 8s 422us/step - loss: 0.3906 - acc: 0.8191 - val_loss: 0.2744 - val_acc: 0.8914\n",
      "Epoch 5/5\n",
      "20000/20000 [==============================] - 8s 408us/step - loss: 0.3685 - acc: 0.8332 - val_loss: 0.2723 - val_acc: 0.8942\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x242ab8c2ba8>"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = Sequential([\n",
    "    Dense(128, activation = 'relu', input_shape=(X_train.shape[1],)),\n",
    "    Dropout(.9),\n",
    "    Dense(1, activation = 'sigmoid'),\n",
    "])\n",
    "\n",
    "# For a binary classification problem\n",
    "opt = keras.optimizers.Adam(lr=0.1)\n",
    "model.compile(optimizer=opt, loss='binary_crossentropy', metrics=['accuracy'])\n",
    "model.fit(X_train, y_train, validation_data = (X_valid, y_valid), epochs=5, batch_size=1024)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 20000 samples, validate on 5000 samples\n",
      "Epoch 1/5\n",
      "20000/20000 [==============================] - 7s 354us/step - loss: 0.4359 - acc: 0.7941 - val_loss: 0.2819 - val_acc: 0.8820\n",
      "Epoch 2/5\n",
      "20000/20000 [==============================] - 6s 293us/step - loss: 0.1856 - acc: 0.9317 - val_loss: 0.2515 - val_acc: 0.8942\n",
      "Epoch 3/5\n",
      "20000/20000 [==============================] - 6s 292us/step - loss: 0.1165 - acc: 0.9621 - val_loss: 0.2674 - val_acc: 0.8878\n",
      "Epoch 4/5\n",
      "20000/20000 [==============================] - 6s 302us/step - loss: 0.0769 - acc: 0.9803 - val_loss: 0.2877 - val_acc: 0.8852\n",
      "Epoch 5/5\n",
      "20000/20000 [==============================] - 6s 303us/step - loss: 0.0518 - acc: 0.9900 - val_loss: 0.3131 - val_acc: 0.8846\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x242bb386978>"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from keras.layers import BatchNormalization\n",
    "\n",
    "model = Sequential([\n",
    "    Dense(64, activation = 'relu', input_shape=(X_train.shape[1],)),\n",
    "    BatchNormalization(),\n",
    "    Dropout(.5),\n",
    "    Dense(1, activation = 'sigmoid'),\n",
    "])\n",
    "\n",
    "# For a binary classification problem\n",
    "opt = keras.optimizers.Adam(lr=0.001)\n",
    "model.compile(optimizer=opt, loss='binary_crossentropy', metrics=['accuracy'])\n",
    "model.fit(X_train, y_train, validation_data = (X_valid, y_valid), epochs=5, batch_size=1024)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Adding more layers:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 20000 samples, validate on 5000 samples\n",
      "Epoch 1/5\n",
      "20000/20000 [==============================] - 6s 324us/step - loss: 0.6146 - acc: 0.6835 - val_loss: 0.3312 - val_acc: 0.8666\n",
      "Epoch 2/5\n",
      "20000/20000 [==============================] - 6s 288us/step - loss: 0.3971 - acc: 0.8337 - val_loss: 0.2905 - val_acc: 0.8860\n",
      "Epoch 3/5\n",
      "20000/20000 [==============================] - 6s 289us/step - loss: 0.3575 - acc: 0.8543 - val_loss: 0.2911 - val_acc: 0.8862\n",
      "Epoch 4/5\n",
      "20000/20000 [==============================] - 6s 289us/step - loss: 0.3264 - acc: 0.8658 - val_loss: 0.2741 - val_acc: 0.8860\n",
      "Epoch 5/5\n",
      "20000/20000 [==============================] - 6s 289us/step - loss: 0.3061 - acc: 0.8765 - val_loss: 0.2786 - val_acc: 0.8878\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x24294d13ac8>"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = Sequential([\n",
    "    Dense(64, activation = 'relu', input_shape=(X_train.shape[1],)),\n",
    "    Dropout(.8),\n",
    "    Dense(8, activation = 'tanh'),\n",
    "    Dense(1, activation = 'sigmoid'),\n",
    "])\n",
    "\n",
    "# For a binary classification problem\n",
    "opt = keras.optimizers.Adam(lr=0.1)\n",
    "model.compile(optimizer=opt, loss='binary_crossentropy', metrics=['accuracy'])\n",
    "model.fit(X_train, y_train, validation_data = (X_valid, y_valid), epochs=5, batch_size=1024)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Getting word vector:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16414, 32)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word2vec = model.get_weights()[0]\n",
    "word2vec.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compute the distance between words using cosine distance:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from scipy.spatial.distance import cdist\n",
    "dist = cdist(word2vec, word2vec, 'cosine')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Print closest and farthest words:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "love ['favorite' 'enjoyed' 'best' ..., 'laughable' 'stupid' 'worst']\n",
      "amazing ['wonderful' 'favorite' 'delight' ..., 'unfortunately' 'bad' 'ridiculous']\n",
      "bad ['stupid' 'worst' 'worse' ..., 'well' 'favorite' 'great']\n",
      "horrible ['worse' 'waste' 'awful' ..., 'episodes' 'freedom' 'great']\n"
     ]
    }
   ],
   "source": [
    "vocab = np.array(vect.get_feature_names())\n",
    "\n",
    "words = ['love', 'amazing', 'bad', 'horrible']\n",
    "for word in words:\n",
    "    A = vocab[dist[vect.vocabulary_[word]].argsort()[1:]]\n",
    "    print(word, A)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multi-class\n",
    "\n",
    "Loading 20 news groups:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_20newsgroups\n",
    "\n",
    "cats = ['sci.space', 'talk.politics.mideast', 'comp.graphics']\n",
    "newsgroups_train = fetch_20newsgroups(subset='train', categories=cats, remove=('headers', 'footers', 'quotes'), shuffle = True)\n",
    "train_data = newsgroups_train.data\n",
    "\n",
    "newsgroups_valid = fetch_20newsgroups(subset='test', categories=cats, remove=('headers', 'footers', 'quotes'), shuffle = True)\n",
    "valid_data = newsgroups_valid.data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using count vectorizer to create features:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1741, 5899) (1159, 5899)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "count = CountVectorizer(binary = False, min_df = 5)\n",
    "\n",
    "X_train = count.fit_transform(train_data)\n",
    "X_valid = count.transform(valid_data)\n",
    "print(X_train.shape, X_valid.shape)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For categorical classification, the target (y) must be converted to one-hot-encoder:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1741, 3) (1159, 3)\n"
     ]
    }
   ],
   "source": [
    "y_train = keras.utils.to_categorical(newsgroups_train.target,)\n",
    "y_valid = keras.utils.to_categorical(newsgroups_valid.target)\n",
    "print(y_train.shape, y_valid.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creating the MLP with 3 sofmax outputs and start training:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 1741 samples, validate on 1159 samples\n",
      "Epoch 1/10\n",
      "1741/1741 [==============================] - 1s 774us/step - loss: 1.0465 - acc: 0.6037 - val_loss: 0.5887 - val_acc: 0.8464\n",
      "Epoch 2/10\n",
      "1741/1741 [==============================] - 1s 297us/step - loss: 0.3490 - acc: 0.9018 - val_loss: 0.3505 - val_acc: 0.8766\n",
      "Epoch 3/10\n",
      "1741/1741 [==============================] - 1s 298us/step - loss: 0.1845 - acc: 0.9437 - val_loss: 0.3658 - val_acc: 0.8861\n",
      "Epoch 4/10\n",
      "1741/1741 [==============================] - 1s 302us/step - loss: 0.1155 - acc: 0.9598 - val_loss: 0.3788 - val_acc: 0.8861\n",
      "Epoch 5/10\n",
      "1741/1741 [==============================] - 1s 302us/step - loss: 0.0998 - acc: 0.9696 - val_loss: 0.3537 - val_acc: 0.8896\n",
      "Epoch 6/10\n",
      "1741/1741 [==============================] - 1s 303us/step - loss: 0.0815 - acc: 0.9719 - val_loss: 0.3928 - val_acc: 0.8921\n",
      "Epoch 7/10\n",
      "1741/1741 [==============================] - 1s 294us/step - loss: 0.0691 - acc: 0.9770 - val_loss: 0.4095 - val_acc: 0.8913\n",
      "Epoch 8/10\n",
      "1741/1741 [==============================] - 1s 302us/step - loss: 0.0501 - acc: 0.9770 - val_loss: 0.4028 - val_acc: 0.8921\n",
      "Epoch 9/10\n",
      "1741/1741 [==============================] - 1s 299us/step - loss: 0.0522 - acc: 0.9776 - val_loss: 0.4202 - val_acc: 0.8913\n",
      "Epoch 10/10\n",
      "1741/1741 [==============================] - 1s 299us/step - loss: 0.0437 - acc: 0.9816 - val_loss: 0.4352 - val_acc: 0.8904\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x242c9a50f28>"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = Sequential([\n",
    "    Dense(128, activation = 'relu', input_shape=(X_train.shape[1],)),\n",
    "    Dropout(.5),\n",
    "    Dense(3, activation = 'softmax'),\n",
    "])\n",
    "\n",
    "# For a binary classification problem\n",
    "opt = keras.optimizers.Adam(lr=0.01)\n",
    "model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "model.fit(X_train, y_train, validation_data = (X_valid, y_valid), epochs=10, batch_size=256)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Saving and loading model weights:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.save('model.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Loading weights:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_46 (Dense)             (None, 128)               755200    \n",
      "_________________________________________________________________\n",
      "dropout_14 (Dropout)         (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_47 (Dense)             (None, 3)                 387       \n",
      "=================================================================\n",
      "Total params: 755,587\n",
      "Trainable params: 755,587\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from keras.models import load_model\n",
    "\n",
    "model = load_model('model.h5')\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Auto saving the model based on best validation accuracy:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 1741 samples, validate on 1159 samples\n",
      "Epoch 1/10\n",
      "1741/1741 [==============================] - 2s 920us/step - loss: 1.1874 - acc: 0.5560 - val_loss: 0.5403 - val_acc: 0.7869\n",
      "Epoch 2/10\n",
      "1741/1741 [==============================] - 1s 349us/step - loss: 0.3037 - acc: 0.8983 - val_loss: 0.3897 - val_acc: 0.8723\n",
      "Epoch 3/10\n",
      "1741/1741 [==============================] - 1s 385us/step - loss: 0.1736 - acc: 0.9495 - val_loss: 0.3566 - val_acc: 0.8801\n",
      "Epoch 4/10\n",
      "1741/1741 [==============================] - 1s 344us/step - loss: 0.1145 - acc: 0.9650 - val_loss: 0.3624 - val_acc: 0.8896\n",
      "Epoch 5/10\n",
      "1741/1741 [==============================] - 1s 294us/step - loss: 0.0887 - acc: 0.9724 - val_loss: 0.4244 - val_acc: 0.8792\n",
      "Epoch 6/10\n",
      "1741/1741 [==============================] - 0s 262us/step - loss: 0.0610 - acc: 0.9782 - val_loss: 0.4428 - val_acc: 0.8870\n",
      "Epoch 7/10\n",
      "1741/1741 [==============================] - 0s 275us/step - loss: 0.0526 - acc: 0.9793 - val_loss: 0.3949 - val_acc: 0.8861\n",
      "Epoch 8/10\n",
      "1741/1741 [==============================] - 0s 274us/step - loss: 0.0434 - acc: 0.9828 - val_loss: 0.3894 - val_acc: 0.8852\n",
      "Epoch 9/10\n",
      "1741/1741 [==============================] - 0s 274us/step - loss: 0.0402 - acc: 0.9822 - val_loss: 0.4008 - val_acc: 0.8861\n",
      "Epoch 10/10\n",
      "1741/1741 [==============================] - 0s 269us/step - loss: 0.0375 - acc: 0.9845 - val_loss: 0.4214 - val_acc: 0.8852\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x242cd7ef1d0>"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = Sequential([\n",
    "    Dense(128, activation = 'relu', input_shape=(X_train.shape[1],)),\n",
    "    Dropout(.2),\n",
    "    Dense(3, activation = 'softmax'),\n",
    "])\n",
    "\n",
    "# For a binary classification problem\n",
    "opt = keras.optimizers.Adam(lr=0.01)\n",
    "model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "checkpoint = keras.callbacks.ModelCheckpoint('model.h5', monitor='val_acc', save_best_only=True, verbose = False)\n",
    "\n",
    "model.fit(X_train, y_train, validation_data = (X_valid, y_valid), epochs=10, batch_size=256, callbacks = [checkpoint])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}