CIFAR10ConvolutionalNetwork-Library.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "%reset -f\n",
    "import tensorflow as tf\n",
    "import matplotlib.pylab as plt\n",
    "import numpy as np\n",
    "import pdb\n",
    "from time import time\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from tensorflow.contrib.layers.python.layers import fully_connected, convolution2d, max_pool2d, flatten"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Why doesn't the network perform as well as the examples online? One reason could be the we perform no image preprocessing, like stretching, whitening, rotating and flipping. Another could be that we are implementing the layer operations manually and are not using existing libraries. Let's see if that makes a difference."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We define the Chunk and CIPHAR classes just as before."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class Chunk():\n",
    "    def __init__(self,chunk_filename):\n",
    "        self.chunk = self.unpickle(chunk_filename)\n",
    "        self.images = np.array([self.format_image(image) for image in self.chunk['data']])\n",
    "        self.labels = np.array([self.one_hot(label) for label in self.chunk['labels']])\n",
    "        self.idx = 0\n",
    "        \n",
    "    def one_hot(self,key):\n",
    "        vector = np.zeros(10, dtype=int)\n",
    "        vector[key] = int(1)\n",
    "        return vector\n",
    "                       \n",
    "        \n",
    "    def unpickle(self,chunk):\n",
    "        import cPickle\n",
    "        fo = open(chunk, 'rb')\n",
    "        dictionary = cPickle.load(fo)\n",
    "        fo.close()\n",
    "        return dictionary\n",
    "    def format_image(self,vector):\n",
    "        return vector.reshape(-1,32,32).transpose(1,2,0)\n",
    "    \n",
    "    def next_batch(self,number):\n",
    "        if self.idx < len(self.labels):\n",
    "            batch_images = self.images[self.idx : self.idx + number]\n",
    "            batch_labels = self.labels[self.idx : self.idx + number]\n",
    "            self.idx += number\n",
    "            return batch_images, batch_labels\n",
    "        else:\n",
    "            raise Exception('Reached end of chunk')\n",
    "            \n",
    "            \n",
    "\n",
    "class CIPHAR10():\n",
    "    def __init__(self,test_filepath,train_filepaths):\n",
    "        self.test_data = self.load_data(test_filepath)\n",
    "        self.train_data = [self.load_data(chunk) for chunk in train_filepaths]\n",
    "        \n",
    "    def load_data(self,filepath):\n",
    "        return Chunk(filepath)\n",
    "\n",
    "train_filepaths = ['./data/cifar-10-batches-py/data_batch_{}'.format(i) for i in range(1,6)]\n",
    "test_filepaths = './data/cifar-10-batches-py/test_batch'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def lrn(x):\n",
    "    return tf.nn.lrn(x, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now however we are using layer from the library `tensorflow.contrib.layers.python.layers`. These are reliable implementations that have been communtity tested."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph() \n",
    "X = tf.placeholder(tf.float32, shape = [None, 32, 32, 3])\n",
    "Y = tf.placeholder(tf.float32, shape = [None, 10])\n",
    "dropout_prob = tf.placeholder(tf.float32)\n",
    "\n",
    "conv1 = convolution2d(X, 128, [3,3])\n",
    "pool1 = max_pool2d(conv1, (3,3), (2,2), padding='SAME')\n",
    "lrn1 = lrn(pool1)\n",
    "lrn1 = tf.nn.dropout(lrn1, dropout_prob)\n",
    "conv2 = convolution2d(pool1, 256, [3,3])\n",
    "pool2 = max_pool2d(conv2, (3,3), (2,2), padding='SAME')\n",
    "lrn2 = lrn(pool2)\n",
    "lrn2 = tf.nn.dropout(lrn2, dropout_prob)\n",
    "conv3 = convolution2d(lrn2, 512, [3,3])\n",
    "pool3 = max_pool2d(conv3, (3,3), (2,2), padding='SAME')\n",
    "lrn3 = lrn(pool3)\n",
    "lrn3 = tf.nn.dropout(lrn3, dropout_prob)\n",
    "\n",
    "\n",
    "flat1 = flatten(lrn3)\n",
    "fc1 = fully_connected(flat1, 2000)\n",
    "fc1 = tf.nn.dropout(fc1, dropout_prob)\n",
    "fc2 = fully_connected(fc1, 1000)\n",
    "fc2 = tf.nn.dropout(fc2, dropout_prob)\n",
    "\n",
    "\n",
    "y = fully_connected(fc2, 10, None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cost = tf.reduce_sum(tf.nn.softmax_cross_entropy_with_logits(y,Y))\n",
    "optimiser = tf.train.AdamOptimizer(0.0005).minimize(cost)\n",
    "results = tf.cast(tf.equal(tf.arg_max(y,1), tf.arg_max(Y,1)), tf.float32)\n",
    "count = tf.reduce_sum(results)\n",
    "accuracy = tf.reduce_mean(results)\n",
    "init = tf.initialize_all_variables()\n",
    "\n",
    "batch_size = 500\n",
    "n_batches = 20\n",
    "epochs = 50\n",
    "n_random_select = 200\n",
    "test_batch_size = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processed epoch 0 in 23.3828620911s, at 2138.31821807 images/s\n",
      "Loss: 21.570230484. Accuracy on total test set: 33.27\n",
      "processed epoch 1 in 22.395179987s, at 2232.62327113 images/s\n",
      "Loss: 17.7248802185. Accuracy on total test set: 37.49\n",
      "processed epoch 2 in 22.3864250183s, at 2233.49641397 images/s\n",
      "Loss: 16.4685516357. Accuracy on total test set: 43.65\n",
      "processed epoch 3 in 22.3926529884s, at 2232.87522143 images/s\n",
      "Loss: 16.5044059753. Accuracy on total test set: 47.23\n",
      "processed epoch 4 in 22.3886089325s, at 2233.27854583 images/s\n",
      "Loss: 15.5775403976. Accuracy on total test set: 52.84\n",
      "processed epoch 5 in 22.411823988s, at 2230.96522741 images/s\n",
      "Loss: 13.1671161652. Accuracy on total test set: 53.5\n",
      "processed epoch 6 in 22.4050939083s, at 2231.63536848 images/s\n",
      "Loss: 13.0299911499. Accuracy on total test set: 58.05\n",
      "processed epoch 7 in 22.4167859554s, at 2230.47140207 images/s\n",
      "Loss: 12.226805687. Accuracy on total test set: 62.5\n",
      "processed epoch 8 in 22.4067959785s, at 2231.46584848 images/s\n",
      "Loss: 13.3273267746. Accuracy on total test set: 60.3\n",
      "processed epoch 9 in 22.4060049057s, at 2231.54463325 images/s\n",
      "Loss: 11.7454185486. Accuracy on total test set: 59.96\n",
      "processed epoch 10 in 22.4124679565s, at 2230.90112597 images/s\n",
      "Loss: 12.1382265091. Accuracy on total test set: 65.6\n",
      "processed epoch 11 in 22.3895580769s, at 2233.18387207 images/s\n",
      "Loss: 10.4416236877. Accuracy on total test set: 65.78\n",
      "processed epoch 12 in 22.4197018147s, at 2230.18131166 images/s\n",
      "Loss: 11.7010660172. Accuracy on total test set: 65.46\n",
      "processed epoch 13 in 22.3960888386s, at 2232.53266945 images/s\n",
      "Loss: 8.83016967773. Accuracy on total test set: 68.36\n",
      "processed epoch 14 in 22.41507411s, at 2230.64174379 images/s\n",
      "Loss: 10.1679077148. Accuracy on total test set: 70.67\n",
      "processed epoch 15 in 22.4199390411s, at 2230.157714 images/s\n",
      "Loss: 10.6100492477. Accuracy on total test set: 68.79\n",
      "processed epoch 16 in 22.4128050804s, at 2230.8675697 images/s\n",
      "Loss: 7.74372577667. Accuracy on total test set: 72.21\n",
      "processed epoch 17 in 22.4113061428s, at 2231.01677704 images/s\n",
      "Loss: 8.35270023346. Accuracy on total test set: 70.91\n",
      "processed epoch 18 in 22.3981771469s, at 2232.32451784 images/s\n",
      "Loss: 8.20824623108. Accuracy on total test set: 70.4\n",
      "processed epoch 19 in 22.4130029678s, at 2230.84787307 images/s\n",
      "Loss: 8.33349609375. Accuracy on total test set: 73.29\n",
      "processed epoch 20 in 22.4093770981s, at 2231.2088275 images/s\n",
      "Loss: 8.16432189941. Accuracy on total test set: 73.37\n",
      "processed epoch 21 in 22.4067268372s, at 2231.47273421 images/s\n",
      "Loss: 7.00791501999. Accuracy on total test set: 75.43\n",
      "processed epoch 22 in 22.4031040668s, at 2231.83358211 images/s\n",
      "Loss: 7.84757852554. Accuracy on total test set: 74.98\n",
      "processed epoch 23 in 22.4181239605s, at 2230.33827844 images/s\n",
      "Loss: 7.00962162018. Accuracy on total test set: 74.49\n",
      "processed epoch 24 in 22.403678894s, at 2231.77631837 images/s\n",
      "Loss: 6.59524917603. Accuracy on total test set: 76.74\n",
      "processed epoch 25 in 22.4074599743s, at 2231.3997239 images/s\n",
      "Loss: 6.80548858643. Accuracy on total test set: 75.76\n",
      "processed epoch 26 in 22.4214789867s, at 2230.00454295 images/s\n",
      "Loss: 7.04673719406. Accuracy on total test set: 76.94\n",
      "processed epoch 27 in 22.3807618618s, at 2234.06157077 images/s\n",
      "Loss: 6.50636863708. Accuracy on total test set: 77.76\n",
      "processed epoch 28 in 22.4026148319s, at 2231.88232156 images/s\n",
      "Loss: 8.7635269165. Accuracy on total test set: 76.79\n",
      "processed epoch 29 in 22.4016940594s, at 2231.97405819 images/s\n",
      "Loss: 5.72778892517. Accuracy on total test set: 78.89\n",
      "processed epoch 30 in 22.4043869972s, at 2231.70578183 images/s\n",
      "Loss: 8.01434135437. Accuracy on total test set: 77.15\n",
      "processed epoch 31 in 22.4113008976s, at 2231.01729919 images/s\n",
      "Loss: 4.51046228409. Accuracy on total test set: 78.08\n",
      "processed epoch 32 in 22.3950500488s, at 2232.63622501 images/s\n",
      "Loss: 5.84792613983. Accuracy on total test set: 78.17\n",
      "processed epoch 33 in 22.3936579227s, at 2232.77501927 images/s\n",
      "Loss: 4.9031920433. Accuracy on total test set: 79.31\n",
      "processed epoch 34 in 22.3952970505s, at 2232.61160088 images/s\n",
      "Loss: 4.42173576355. Accuracy on total test set: 78.63\n",
      "processed epoch 35 in 22.3892641068s, at 2233.21319368 images/s\n",
      "Loss: 5.22908115387. Accuracy on total test set: 78.4\n",
      "processed epoch 36 in 22.3741848469s, at 2234.71828548 images/s\n",
      "Loss: 6.89174461365. Accuracy on total test set: 80.02\n",
      "processed epoch 37 in 22.42030406s, at 2230.12140541 images/s\n",
      "Loss: 5.19748067856. Accuracy on total test set: 79.86\n",
      "processed epoch 38 in 22.4131100178s, at 2230.83721805 images/s\n",
      "Loss: 5.19211912155. Accuracy on total test set: 80.05\n",
      "processed epoch 39 in 22.4114699364s, at 2231.00047172 images/s\n",
      "Loss: 4.13872718811. Accuracy on total test set: 80.67\n",
      "processed epoch 40 in 22.4180760384s, at 2230.34304614 images/s\n",
      "Loss: 7.05006122589. Accuracy on total test set: 80.26\n",
      "processed epoch 41 in 22.4128119946s, at 2230.8668815 images/s\n",
      "Loss: 6.13943433762. Accuracy on total test set: 80.89\n",
      "processed epoch 42 in 22.4198760986s, at 2230.16397504 images/s\n",
      "Loss: 6.18099880219. Accuracy on total test set: 80.09\n",
      "processed epoch 43 in 22.4357020855s, at 2228.59083302 images/s\n",
      "Loss: 6.03069782257. Accuracy on total test set: 81.2\n",
      "processed epoch 44 in 22.4155409336s, at 2230.59528869 images/s\n",
      "Loss: 4.10362768173. Accuracy on total test set: 79.85\n",
      "processed epoch 45 in 22.4154810905s, at 2230.60124376 images/s\n",
      "Loss: 5.77927207947. Accuracy on total test set: 80.93\n",
      "processed epoch 46 in 22.4137120247s, at 2230.77730029 images/s\n",
      "Loss: 4.67257499695. Accuracy on total test set: 80.92\n",
      "processed epoch 47 in 22.4046261311s, at 2231.68196191 images/s\n",
      "Loss: 6.03124713898. Accuracy on total test set: 80.83\n",
      "processed epoch 48 in 22.401473999s, at 2231.99598393 images/s\n",
      "Loss: 4.83545684814. Accuracy on total test set: 82.0\n",
      "processed epoch 49 in 22.3991661072s, at 2232.22595702 images/s\n",
      "Loss: 4.19632673264. Accuracy on total test set: 81.42\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    ciphar10 = CIPHAR10(test_filepaths,train_filepaths)\n",
    "\n",
    "    for ep in range(epochs):\n",
    "        t = time()\n",
    "        for chunk in ciphar10.train_data:\n",
    "            for b in range(n_batches):\n",
    "                b_images, b_labels = chunk.next_batch(batch_size)\n",
    "                _, c = sess.run([optimiser, cost], feed_dict={X:b_images, Y:b_labels, dropout_prob:0.5})\n",
    "            chunk.idx = 0\n",
    "        epoch_time = time() - t\n",
    "        print \"processed epoch {} in {}s, at {} images/s\".format(ep, epoch_time, 50000 / epoch_time)\n",
    "        \n",
    "#         if ep % 10 == 0:\n",
    "        total_count = 0\n",
    "        for i in range(1000):\n",
    "            t_batch = ciphar10.test_data.images[test_batch_size*i:test_batch_size*i + test_batch_size]\n",
    "            t_labels = ciphar10.test_data.labels[test_batch_size*i:test_batch_size*i + test_batch_size]\n",
    "            cst, cnt, acc = sess.run([cost,count,accuracy], feed_dict={X: t_batch, Y: t_labels, dropout_prob:1})\n",
    "            total_count += cnt\n",
    "        print \"Loss: {}. Accuracy on total test set: {}\".format(cst,total_count/100)\n",
    "            \n",
    "\n",
    "#         if ep % 10 == 0:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "We see that we reach a maximum of 80% and then starts overfitting, even with dropout layers inbetween each compute layers. Not quite as high as the 86% that is known to be possible with these sorts of networks. These sorts of results can be achieved with clever image preprocessing."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}