Diabetic_keras_Subset_callbacks_rescale_class_weights_Accuracy_80.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Convolutional Neural Network (CNN) implementation for Diabetic Retinopathy Detection \n",
    "https://www.kaggle.com/c/diabetic-retinopathy-detection/data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import keras \n",
    "\n",
    "from tqdm import tqdm\n",
    "import os\n",
    "from sklearn.model_selection import train_test_split\n",
    "from cv2 import cv2\n",
    "from PIL import Image\n",
    "import tensorflow as tf\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "from keras.layers import Dense, Dropout, Flatten, Input, ZeroPadding2D\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img\n",
    "from keras.preprocessing import image\n",
    "from keras.utils import plot_model\n",
    "from keras.models import Model\n",
    "from keras.layers.convolutional import Conv2D\n",
    "from keras.layers.pooling import MaxPooling2D\n",
    "from numpy import array \n",
    "from keras import regularizers\n",
    "from keras import optimizers\n",
    "from keras.models import load_model\n",
    "from keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\n",
    "from PIL import ImageFile\n",
    "ImageFile.LOAD_TRUNCATED_IMAGES = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "sess = tf.Session(config=config)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I have already downloaded the dataset from Kaggle.\n",
    "I have noted all the steps to download the database in a seperate file.\n",
    "Below I am loading a CSV file containing training labels "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_csv = pd.read_csv('/storage/trainLabels.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since these labels dont have .jpeg extension, lets add it to the dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_csv['image'] = df_csv['image'].astype(str) + '.jpeg'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are 35125 images in the training set, 'level' is the column indicating the labels for its respective images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image</th>\n",
       "      <th>level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>35121</th>\n",
       "      <td>44347_right.jpeg</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35122</th>\n",
       "      <td>44348_left.jpeg</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35123</th>\n",
       "      <td>44348_right.jpeg</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35124</th>\n",
       "      <td>44349_left.jpeg</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35125</th>\n",
       "      <td>44349_right.jpeg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  image  level\n",
       "35121  44347_right.jpeg      0\n",
       "35122   44348_left.jpeg      0\n",
       "35123  44348_right.jpeg      0\n",
       "35124   44349_left.jpeg      0\n",
       "35125  44349_right.jpeg      1"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_csv.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It is important to remove all imgaes of size 0 KB, below command will print all files of size zero. Replace print with delete to delete those images. Dont forget to cd into train directory "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!find /storage/train/ -size 0 -print"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Make a list of all images after deleating size zero images."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!ls -1 >> /storage/name.txt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read all the content into a dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_txt = pd.read_table('/storage/name.txt',  header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_txt.columns = ['name']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>31974</th>\n",
       "      <td>6483_left.jpeg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31975</th>\n",
       "      <td>6483_right.jpeg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31976</th>\n",
       "      <td>6484_left.jpeg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31977</th>\n",
       "      <td>6484_right.jpeg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31978</th>\n",
       "      <td>6488_left.jpeg</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  name\n",
       "31974   6483_left.jpeg\n",
       "31975  6483_right.jpeg\n",
       "31976   6484_left.jpeg\n",
       "31977  6484_right.jpeg\n",
       "31978   6488_left.jpeg"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_txt.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remove the image entries from df_csv which we just deleted from \\train folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df_csv[df_csv.image.isin(df_txt.name.values)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image</th>\n",
       "      <th>level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>35121</th>\n",
       "      <td>44347_right.jpeg</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35122</th>\n",
       "      <td>44348_left.jpeg</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35123</th>\n",
       "      <td>44348_right.jpeg</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35124</th>\n",
       "      <td>44349_left.jpeg</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35125</th>\n",
       "      <td>44349_right.jpeg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  image  level\n",
       "35121  44347_right.jpeg      0\n",
       "35122   44348_left.jpeg      0\n",
       "35123  44348_right.jpeg      0\n",
       "35124   44349_left.jpeg      0\n",
       "35125  44349_right.jpeg      1"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Only 31979 images left after removing unwanted images."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(31979, 2)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "23542"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df[df['level'] == 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2203"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df[df['level'] == 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4796"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df[df['level'] == 2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "795"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df[df['level'] == 3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "643"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df[df['level'] == 4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_0 = df[df['level'] == 0]\n",
    "df_1_4 = df[df['level'] != 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "23542"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8437"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_1_4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4708"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_0 = df_0.sample(frac = 0.2)\n",
    "len(df_0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13145"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_0_4 = pd.concat([df_0, df_1_4])\n",
    "len(df_0_4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image</th>\n",
       "      <th>level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1743</th>\n",
       "      <td>2181_right.jpeg</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35082</th>\n",
       "      <td>44284_left.jpeg</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34594</th>\n",
       "      <td>43728_left.jpeg</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2210</th>\n",
       "      <td>2736_left.jpeg</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1065</th>\n",
       "      <td>1316_right.jpeg</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 image  level\n",
       "1743   2181_right.jpeg      0\n",
       "35082  44284_left.jpeg      0\n",
       "34594  43728_left.jpeg      0\n",
       "2210    2736_left.jpeg      0\n",
       "1065   1316_right.jpeg      0"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_0_4.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_main = df_0_4.sample(frac=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image</th>\n",
       "      <th>level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4851</th>\n",
       "      <td>6093_right.jpeg</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9731</th>\n",
       "      <td>12257_right.jpeg</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18554</th>\n",
       "      <td>23394_left.jpeg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28780</th>\n",
       "      <td>36427_left.jpeg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9209</th>\n",
       "      <td>11574_right.jpeg</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  image  level\n",
       "4851    6093_right.jpeg      2\n",
       "9731   12257_right.jpeg      2\n",
       "18554   23394_left.jpeg      1\n",
       "28780   36427_left.jpeg      1\n",
       "9209   11574_right.jpeg      0"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_main.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13145, 2)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_main.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are 5 types of labels 0/1/2/3/4, they are distinguished as below\n",
    "NDPR - Non Proliferative Diabetic Retinopathy\n",
    "\n",
    "Class\t Name\n",
    "0\t    Normal\t\n",
    "1\t    Mild NPDR\n",
    "2\t    Moderate NPDR\n",
    "3\t    Severe NPDR\t\n",
    "4\t    PDR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Its important to split the whole dataset into training and validation dataset apart from testing dataset which we have seperately."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "train, valid = train_test_split(df_main, test_size=0.1, random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11830, 2)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels_count = dict()\n",
    "for img_class in range(5):    \n",
    "    labels_count[img_class] = len(train[train['level'] == img_class])\n",
    "\n",
    "total_count = sum(labels_count.values())\n",
    "class_weights = {cls: total_count / count for cls, count in \n",
    "                 enumerate(labels_count.values())}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_values([4226, 1996, 4323, 715, 570])\n"
     ]
    }
   ],
   "source": [
    "print (labels_count.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11830"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1315, 2)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>29507</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16917</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31921</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29320</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4874</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       0  1  2  3  4\n",
       "29507  0  0  1  0  0\n",
       "16917  1  0  0  0  0\n",
       "31921  0  0  0  0  1\n",
       "29320  0  0  0  0  1\n",
       "4874   1  0  0  0  0"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_series = pd.Series(train['level'])\n",
    "train_OH = pd.get_dummies(train_series)\n",
    "train_OH.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image</th>\n",
       "      <th>level</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9568</th>\n",
       "      <td>12056_left.jpeg</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4953</th>\n",
       "      <td>6234_right.jpeg</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4657</th>\n",
       "      <td>5845_right.jpeg</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12650</th>\n",
       "      <td>15868_left.jpeg</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18301</th>\n",
       "      <td>23050_right.jpeg</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  image  level  0  1  2  3  4\n",
       "9568    12056_left.jpeg      0  1  0  0  0  0\n",
       "4953    6234_right.jpeg      0  1  0  0  0  0\n",
       "4657    5845_right.jpeg      0  1  0  0  0  0\n",
       "12650   15868_left.jpeg      3  0  0  0  1  0\n",
       "18301  23050_right.jpeg      1  0  1  0  0  0"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train = pd.concat([train, train_OH], axis= 1)\n",
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image</th>\n",
       "      <th>Normal</th>\n",
       "      <th>Mild NPDR</th>\n",
       "      <th>Moderate NPDR</th>\n",
       "      <th>Severe NPDR</th>\n",
       "      <th>PDR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9568</th>\n",
       "      <td>12056_left.jpeg</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4953</th>\n",
       "      <td>6234_right.jpeg</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4657</th>\n",
       "      <td>5845_right.jpeg</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12650</th>\n",
       "      <td>15868_left.jpeg</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18301</th>\n",
       "      <td>23050_right.jpeg</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  image  Normal  Mild NPDR  Moderate NPDR  Severe NPDR  PDR\n",
       "9568    12056_left.jpeg       1          0              0            0    0\n",
       "4953    6234_right.jpeg       1          0              0            0    0\n",
       "4657    5845_right.jpeg       1          0              0            0    0\n",
       "12650   15868_left.jpeg       0          0              0            1    0\n",
       "18301  23050_right.jpeg       0          1              0            0    0"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.rename(columns={0: 'Normal', 1: 'Mild NPDR', 2: 'Moderate NPDR', 3: 'Severe NPDR', 4: 'PDR'}, inplace=True)\n",
    "df_train = df_train.drop('level', axis=1)\n",
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>13737</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8094</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12128</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24373</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12021</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       0  1  2  3  4\n",
       "13737  1  0  0  0  0\n",
       "8094   0  0  0  1  0\n",
       "12128  1  0  0  0  0\n",
       "24373  1  0  0  0  0\n",
       "12021  0  0  1  0  0"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_series = pd.Series(valid['level'])\n",
    "valid_OH = pd.get_dummies(valid_series)\n",
    "valid_OH.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image</th>\n",
       "      <th>level</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4201</th>\n",
       "      <td>5274_right.jpeg</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24853</th>\n",
       "      <td>31435_right.jpeg</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12864</th>\n",
       "      <td>16154_left.jpeg</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29694</th>\n",
       "      <td>37582_left.jpeg</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19917</th>\n",
       "      <td>25119_right.jpeg</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  image  level  0  1  2  3  4\n",
       "4201    5274_right.jpeg      1  0  1  0  0  0\n",
       "24853  31435_right.jpeg      1  0  1  0  0  0\n",
       "12864   16154_left.jpeg      2  0  0  1  0  0\n",
       "29694   37582_left.jpeg      2  0  0  1  0  0\n",
       "19917  25119_right.jpeg      3  0  0  0  1  0"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_valid = pd.concat([valid, valid_OH], axis= 1)\n",
    "df_valid.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image</th>\n",
       "      <th>Normal</th>\n",
       "      <th>Mild NPDR</th>\n",
       "      <th>Moderate NPDR</th>\n",
       "      <th>Severe NPDR</th>\n",
       "      <th>PDR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4201</th>\n",
       "      <td>5274_right.jpeg</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24853</th>\n",
       "      <td>31435_right.jpeg</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12864</th>\n",
       "      <td>16154_left.jpeg</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29694</th>\n",
       "      <td>37582_left.jpeg</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19917</th>\n",
       "      <td>25119_right.jpeg</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  image  Normal  Mild NPDR  Moderate NPDR  Severe NPDR  PDR\n",
       "4201    5274_right.jpeg       0          1              0            0    0\n",
       "24853  31435_right.jpeg       0          1              0            0    0\n",
       "12864   16154_left.jpeg       0          0              1            0    0\n",
       "29694   37582_left.jpeg       0          0              1            0    0\n",
       "19917  25119_right.jpeg       0          0              0            1    0"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_valid.rename(columns={0: 'Normal', 1: 'Mild NPDR', 2: 'Moderate NPDR', 3: 'Severe NPDR', 4: 'PDR'}, inplace=True)\n",
    "df_valid = df_valid.drop('level', axis=1)\n",
    "df_valid.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Image data generator can be used to pre processes images before training. \n",
    "This function can be used to perform image augmentation, chnage brightness, horizontal/vertical flip, image roration, etc.\n",
    "https://keras.io/preprocessing/image/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "TrainData = ImageDataGenerator(\n",
    "    width_shift_range= 0.2, height_shift_range= 0.2,\n",
    "    rotation_range= 90, rescale = 1/255,\n",
    "    horizontal_flip= True, vertical_flip=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "ValidData = ImageDataGenerator(\n",
    "    horizontal_flip=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "flow_from_dataframe function can be used to read labels from dataframe and images from /storage/train/ directory. Image and level are the column names of df_train dataframe.   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = ['Normal', 'Mild NPDR', 'Moderate NPDR', 'Severe NPDR', 'PDR']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 11830 images.\n"
     ]
    }
   ],
   "source": [
    "train_generator = TrainData.flow_from_dataframe(\n",
    "            dataframe=df_train, directory=\"/storage/train/\", \n",
    "            x_col=\"image\", y_col= columns, \n",
    "            class_mode=\"other\", target_size=(256,256), \n",
    "            batch_size= 32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 1315 images.\n"
     ]
    }
   ],
   "source": [
    "valid_generator = ValidData.flow_from_dataframe(\n",
    "            dataframe=df_valid, directory=\"/storage/train/\", \n",
    "            x_col=\"image\", y_col= columns, \n",
    "            class_mode=\"other\", target_size=(256,256), \n",
    "            batch_size=32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we will define the model >>\n",
    "Models in Keras/TF come in 2 forms - Sequential (model = Sequential()) or using Funtional API\n",
    "Below code is using Funtional API which is usually used for complex models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "visible = Input(shape=(256,256,3))\n",
    "conv1 = Conv2D(16, kernel_size=(3,3), activation='relu', strides=(1, 1))(visible)\n",
    "conv2 = Conv2D(16, kernel_size=(3,3), activation='relu', strides=(1, 1))(conv1)\n",
    "bat1 = BatchNormalization()(conv2)\n",
    "zero1 = ZeroPadding2D(padding=(1, 1))(bat1)\n",
    "\n",
    "conv3 = Conv2D(32, kernel_size=(3,3), activation='relu', padding='valid', kernel_regularizer=regularizers.l2(0.05))(zero1)\n",
    "conv4 = Conv2D(32, kernel_size=(3,3), activation='relu', padding='valid', kernel_regularizer=regularizers.l2(0.05))(conv3)\n",
    "bat2 = BatchNormalization()(conv4)\n",
    "\n",
    "conv5 = Conv2D(64, kernel_size=(3,3), activation='relu',strides=(1, 1), padding='valid')(bat2)\n",
    "conv6 = Conv2D(64, kernel_size=(3,3), activation='relu',strides=(1, 1), padding='valid')(conv5)\n",
    "bat3 = BatchNormalization()(conv6)\n",
    "pool1 = MaxPooling2D(pool_size=(2, 2))(bat3)\n",
    "zero2 = ZeroPadding2D(padding=(1, 1))(pool1)\n",
    "\n",
    "conv7 = Conv2D(128, kernel_size=(3,3), activation='relu', padding='valid', kernel_regularizer=regularizers.l2(0.01))(zero2)\n",
    "conv8 = Conv2D(128, kernel_size=(2,2), activation='relu', strides=(1, 1), padding='valid')(conv7)\n",
    "bat4 = BatchNormalization()(conv8)\n",
    "\n",
    "conv9 = Conv2D(64, kernel_size=(3,3), activation='relu', padding='valid', kernel_regularizer=regularizers.l2(0.02))(bat4)\n",
    "conv10 = Conv2D(64, kernel_size=(3,3), activation='relu', padding='valid', kernel_regularizer=regularizers.l2(0.02))(conv9)\n",
    "bat5 = BatchNormalization()(conv10)\n",
    "\n",
    "conv11 = Conv2D(64, kernel_size=(3,3), activation='relu', strides=(1, 1))(bat5)\n",
    "conv12 = Conv2D(64, kernel_size=(3,3), activation='relu', strides=(1, 1))(conv11)\n",
    "bat6 = BatchNormalization()(conv12)\n",
    "pool2 = MaxPooling2D(pool_size=(2, 2))(bat6)\n",
    "\n",
    "conv13 = Conv2D(64, kernel_size=(3,3), activation='relu', padding='valid', kernel_regularizer=regularizers.l2(0.02))(pool2)\n",
    "conv14 = Conv2D(64, kernel_size=(3,3), activation='relu', padding='valid', kernel_regularizer=regularizers.l2(0.02))(conv13)\n",
    "bat7 = BatchNormalization()(conv14)\n",
    "\n",
    "conv15 = Conv2D(128, kernel_size=(3,3), activation='relu', padding='valid', kernel_regularizer=regularizers.l2(0.05))(bat7)\n",
    "conv16 = Conv2D(128, kernel_size=(2,2), activation='relu', strides=(1, 1), padding='valid')(conv15)\n",
    "bat8 = BatchNormalization()(conv16)\n",
    "\n",
    "flat = Flatten()(bat8)\n",
    "hidden1 = Dense(32, activation='relu')(flat)\n",
    "drop1 = Dropout(0.3)(hidden1)\n",
    "\n",
    "hidden2 = Dense(32, activation='relu')(drop1)\n",
    "drop2 = Dropout(0.2)(hidden2)\n",
    "\n",
    "output = Dense(5, activation='sigmoid')(drop2)\n",
    "model = Model(inputs=visible, outputs=output)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are variety of optimizers available >> https://keras.io/optimizers/\n",
    "More about loss functions >> https://keras.io/losses/\n",
    "Metrics >> https://keras.io/metrics/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "#opt = optimizers.adam(lr=0.01, decay=0.0)\n",
    "opt = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)\n",
    "#opt = optimizers.RMSprop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer= opt, \n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "Callbacks=[EarlyStopping(patience=3, restore_best_weights=True), \n",
    "           ReduceLROnPlateau(patience=2), \n",
    "           ModelCheckpoint(filepath='ImageDataGen_Size256_oneHOT_ClassWeights_Callbacks_SGD_L2.h5', save_best_only=True)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_3 (InputLayer)         (None, 256, 256, 3)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_33 (Conv2D)           (None, 254, 254, 16)      448       \n",
      "_________________________________________________________________\n",
      "conv2d_34 (Conv2D)           (None, 252, 252, 16)      2320      \n",
      "_________________________________________________________________\n",
      "batch_normalization_17 (Batc (None, 252, 252, 16)      64        \n",
      "_________________________________________________________________\n",
      "zero_padding2d_5 (ZeroPaddin (None, 254, 254, 16)      0         \n",
      "_________________________________________________________________\n",
      "conv2d_35 (Conv2D)           (None, 252, 252, 32)      4640      \n",
      "_________________________________________________________________\n",
      "conv2d_36 (Conv2D)           (None, 250, 250, 32)      9248      \n",
      "_________________________________________________________________\n",
      "batch_normalization_18 (Batc (None, 250, 250, 32)      128       \n",
      "_________________________________________________________________\n",
      "conv2d_37 (Conv2D)           (None, 248, 248, 64)      18496     \n",
      "_________________________________________________________________\n",
      "conv2d_38 (Conv2D)           (None, 246, 246, 64)      36928     \n",
      "_________________________________________________________________\n",
      "batch_normalization_19 (Batc (None, 246, 246, 64)      256       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_5 (MaxPooling2 (None, 123, 123, 64)      0         \n",
      "_________________________________________________________________\n",
      "zero_padding2d_6 (ZeroPaddin (None, 125, 125, 64)      0         \n",
      "_________________________________________________________________\n",
      "conv2d_39 (Conv2D)           (None, 123, 123, 128)     73856     \n",
      "_________________________________________________________________\n",
      "conv2d_40 (Conv2D)           (None, 122, 122, 128)     65664     \n",
      "_________________________________________________________________\n",
      "batch_normalization_20 (Batc (None, 122, 122, 128)     512       \n",
      "_________________________________________________________________\n",
      "conv2d_41 (Conv2D)           (None, 120, 120, 64)      73792     \n",
      "_________________________________________________________________\n",
      "conv2d_42 (Conv2D)           (None, 118, 118, 64)      36928     \n",
      "_________________________________________________________________\n",
      "batch_normalization_21 (Batc (None, 118, 118, 64)      256       \n",
      "_________________________________________________________________\n",
      "conv2d_43 (Conv2D)           (None, 116, 116, 64)      36928     \n",
      "_________________________________________________________________\n",
      "conv2d_44 (Conv2D)           (None, 114, 114, 64)      36928     \n",
      "_________________________________________________________________\n",
      "batch_normalization_22 (Batc (None, 114, 114, 64)      256       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_6 (MaxPooling2 (None, 57, 57, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_45 (Conv2D)           (None, 55, 55, 64)        36928     \n",
      "_________________________________________________________________\n",
      "conv2d_46 (Conv2D)           (None, 53, 53, 64)        36928     \n",
      "_________________________________________________________________\n",
      "batch_normalization_23 (Batc (None, 53, 53, 64)        256       \n",
      "_________________________________________________________________\n",
      "conv2d_47 (Conv2D)           (None, 51, 51, 128)       73856     \n",
      "_________________________________________________________________\n",
      "conv2d_48 (Conv2D)           (None, 50, 50, 128)       65664     \n",
      "_________________________________________________________________\n",
      "batch_normalization_24 (Batc (None, 50, 50, 128)       512       \n",
      "_________________________________________________________________\n",
      "flatten_3 (Flatten)          (None, 320000)            0         \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 32)                10240032  \n",
      "_________________________________________________________________\n",
      "dropout_5 (Dropout)          (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 32)                1056      \n",
      "_________________________________________________________________\n",
      "dropout_6 (Dropout)          (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 5)                 165       \n",
      "=================================================================\n",
      "Total params: 10,853,045\n",
      "Trainable params: 10,851,925\n",
      "Non-trainable params: 1,120\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "370/370 [==============================] - 1849s 5s/step - loss: 29.1990 - acc: 0.7959 - val_loss: 10.7663 - val_acc: 0.8000\n",
      "Epoch 2/5\n",
      "370/370 [==============================] - 2584s 7s/step - loss: 17.5359 - acc: 0.7999 - val_loss: 3.8036 - val_acc: 0.8000\n",
      "Epoch 3/5\n",
      "370/370 [==============================] - 2398s 6s/step - loss: 16.2506 - acc: 0.8000 - val_loss: 3.2556 - val_acc: 0.8000\n",
      "Epoch 4/5\n",
      "370/370 [==============================] - 2064s 6s/step - loss: 20347.9284 - acc: 0.7999 - val_loss: 321.0431 - val_acc: 0.8000\n",
      "Epoch 5/5\n",
      "370/370 [==============================] - 2482s 7s/step - loss: 88.7576 - acc: 0.8000 - val_loss: 12.5082 - val_acc: 0.8000\n"
     ]
    }
   ],
   "source": [
    "with tf.device('/device:GPU:0'): \n",
    "    m1= model.fit_generator(\n",
    "            train_generator,\n",
    "            epochs=5,\n",
    "            validation_data=valid_generator,\n",
    "            class_weight = class_weights,\n",
    "            callbacks= Callbacks,\n",
    "            verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "#save data before one hot encoding\n",
    "train.to_csv(r'train.csv', header=None, index=None, sep=' ', mode='a')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_train = pd.read_csv('train.csv', names=['image', 'level'], sep=' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 11830 images.\n"
     ]
    }
   ],
   "source": [
    "#Use full training data\n",
    "LoadData = ImageDataGenerator(\n",
    "    horizontal_flip=True)\n",
    "\n",
    "hard_data_gen = LoadData.flow_from_dataframe(\n",
    "            dataframe= df_train, directory=\"/storage/train/\", \n",
    "            x_col=\"image\", y_col= None, shuffle=False,\n",
    "            class_mode= None, target_size=(256,256), \n",
    "            batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "370/370 [==============================] - 3233s 9s/step\n"
     ]
    }
   ],
   "source": [
    "pre_model = load_model('ImageDataGen_Size256_oneHOT_ClassWeights_Callbacks_SGD_L2.h5')\n",
    "\n",
    "hard_data_gen.reset()\n",
    "pred = pre_model.predict_generator(hard_data_gen, use_multiprocessing= True, verbose=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "predicted_class_indices = np.argmax(pred,axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11830,)"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predicted_class_indices.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = pd.DataFrame({'predictions':predicted_class_indices})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred.to_csv(r'pred.csv', header=None, index=None, sep=' ', mode='a')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "####Try with wrongly predicted images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "images_trained = pd.read_csv('train.csv', names=['image', 'level'], sep=' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "images_predicted = pd.read_csv('pred.csv', names=['pre'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "Compare = pd.concat([images_trained, images_predicted], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11830, 3)"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Compare.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_filtered = Compare[Compare['level'] != Compare['pre']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_filtered.to_csv(r'df_filtered.csv', sep=' ', mode='a')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lets plot training and validation loss and accuracy. Before usinng below line import plot_history() function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEWCAYAAACufwpNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8VNX9+P/Xm7Avsu9BAaFgwiZEQBAFREWwLIqAghO1lWq11tr2K/bX71fR1lpr1VL92KrFD4MsKohSRRERwQWFsBhZlEVQggECsggEsr1/f5ybOAlJCDAzd5K8nz7mMTNnztz7nmuY99xzzj1HVBVjjDEmHKr4HYAxxpiKw5KKMcaYsLGkYowxJmwsqRhjjAkbSyrGGGPCxpKKMcaYsLGkYowxJmwsqRgTISKyQ0SG+B2HMdFkScUYY0zYWFIxJspE5DYR2Soi34vIAhFp5ZWLiDwpIntF5LCIfCEiXbzXhonIRhH5QUR2icjv/P0UxhTPkooxUSQig4G/AGOBlsA3wBzv5SuBS4GfAPW9Ovu91/4D/EJV6wFdgPejGLYxZVbV7wCMqWQmANNUdQ2AiNwPHBCRtkA2UA/oDKxU1U0h78sGEkTkc1U9AByIatTGlJGdqRgTXa1wZycAqOoR3NlIa1V9H3gaeAbYKyLPicg5XtXrgGHANyKyTEQujnLcxpSJJRVjous74Lz8JyJSB2gM7AJQ1amq2gtIwDWD/d4rX6WqI4FmwOvAK1GO25gysaRiTGRVE5Ga+TdgNnCLiPQQkRrAI8BnqrpDRC4SkT4iUg04ChwH8kSkuohMEJH6qpoNHAbyfPtExpTCkooxkbUQyAy5DQT+LzAPSAfOB8Z7dc8Bnsf1l3yDaxb7m/faTcAOETkM3I7rmzEm5ogt0mWMMSZc7EzFGGNM2FhSMcYYEzaWVIwxxoSNJRVjjDFhU+muqG/SpIm2bdvW7zCMMaZcWb169T5VbXqqepUuqbRt25aUlBS/wzDGmHJFRL45dS1r/jLGGBNGllSMMcaEjSUVY4wxYVPp+lSKk52dTVpaGsePH/c7FGMK1KxZk/j4eKpVq+Z3KMaUmSUVIC0tjXr16tG2bVtExO9wjEFV2b9/P2lpabRr187vcIwpM2v+Ao4fP07jxo0toZiYISI0btzYzp5NuWNJxWMJxcQa+5s05ZElFWOMMWFjSSUG7N+/nx49etCjRw9atGhB69atC55nZWWVaRu33HILX331Val1nnnmGWbOnBmOkLnkkktYt25dWLZVVEpKCr/4xS8AOHDgAMOHD6d79+4kJiYSDAYL6t17770kJiZywQUX8Jvf/IaSlnF48skn6dSpEwkJCfzhD38A4MSJEyQnJ9O1a1d69OjB8uXLC+pPnjyZ+Ph4GjRoUGg7x48fZ8yYMXTo0IGLL76Yb7/9FoBgMFjw/6tHjx6ICOvXrwfg8ssv59ChQ+E7OAaA2bOhVSs4cMDvSMxJVDUiN6ANsBTYCGwAfu2VNwIWA1u8+4ZeuQBTga1AKtAzZFvJXv0tQHJIeS/gC+89U/HWhynt1qtXLy1q48aNJ5X55YEHHtC//e1vJ5Xn5eVpbm6uDxEVr3///rp27dqIbHvUqFG6fv16VVWdMmWK/uEPf1BV1d27d2uDBg00Oztbly1bpgMGDNDc3FzNzs7Wiy66SD/88MOTtvXuu+/qlVdeqcePH1dV1T179qiq6lNPPaU///nPVVU1PT1de/XqpXl5eaqq+sknn+jOnTu1fv36hbb1j3/8Q++8805VVZ0xY4beeOONJ+1vzZo12rFjx4LnL7zwgj766KNnfCxi6W8zllx8sSqoPvus35FUHkCKluG7P5JnKjnAb1U1AegL3CkiCcBkYImqdgSWeM8BrgY6erdJwLMAItIIeADoA/QGHhCRht57ngVuC3nf0Ah+nqjbunUrCQkJTJgwgcTERNLT05k0aRJJSUkkJiby0EMPFdTNP3PIycmhQYMGTJ48me7du3PxxRezd+9eAP74xz/y1FNPFdSfPHkyvXv3plOnTnzyyScAHD16lOuuu46EhATGjBlDUlLSKc9IXnrpJbp27UqXLl0KzgRycnK46aabCsqnTp0KuLOGhIQEunXrxsSJE0/a1qFDh/jqq69ITEwEXL/CDz/8AMCRI0do0qQJcXFxiAjHjx8nKyuLEydOkJOTQ7NmzU7a3rPPPsv9999PjRo1AArqbNy4kcGDBwPQokUL6tSpw9q1awG4+OKLadGixUnbeuONN0hOTgZg7NixLFq06KQ6s2fP5oYbbih4PnLkSGbNmlXq8TOnZ8sWWLHCPQ45cTUxImJDilU1HbdcKqr6g4hsAloDI3FLqgJMBz4A7vPKg15G/FREGohIS6/uYlX9HkBEFgNDReQD4BxV/dQrDwKjgLfPJu577oFwt+r06AHed/lp+/LLLwkGgyQlJQHw6KOP0qhRI3Jychg0aBBjxowhISGh0HsOHTrEZZddxqOPPsq9997LtGnTmDx58knbVlVWrlzJggULeOihh3jnnXf45z//SYsWLZg3bx6ff/45PXv2LDW+tLQ0/vjHP5KSkkL9+vUZMmQIb775Jk2bNmXfvn188cUXABw8eBCAxx57jG+++Ybq1asXlIVauXIlXbt2LXj+61//mmuuuYZWrVpx+PBh5s6di4gwYMAA+vXrR4sWLVBV7rnnHn7yk5+ctL3NmzfzwQcfcN9991GrVi3+/ve/06tXL7p3784bb7zB2LFj2bFjB2vXrmXnzp2lft5du3bRpk0bAKpXr06dOnU4ePBgQTOZqvLyyy8XSjZNmjThhx9+KFTPnJ0ZM6BKFbj7bvfvavNmKOZ/vfFJVPpURKQtcCHwGdDcSzgAu4Hm3uPWwM6Qt6V5ZaWVpxVTXtz+J4lIioikZGRknNVnibbzzz+/IKGA+yXcs2dPevbsyaZNm9i4ceNJ76lVqxZXX301AL169WLHjh3Fbvvaa689qc5HH33E+PFuyfT8fozSfPbZZwwePJgmTZpQrVo1brzxRpYvX06HDh346quvuPvuu1m0aBH169cHIDExkYkTJzJz5sxiL+pLT0+nadMfJ0JduHAhvXv3ZteuXaxevZo77riDI0eO8NVXX7Ft2zZ27drFzp07efvttwvOtkLl5ORw6NAhPvvsM/7yl78wbtw4AG677TaaN29Or169+N3vfke/fv2Ii4sr9bOeyieffEKjRo3o3LlzofKmTZuSnp5ewrvM6cjLc2cnV1wBv/+9Sy4zZvgdlQkV8YsfRaQuMA+4R1UPhw6TVFUVkeJ7V8NIVZ8DngNISkoqdX9nekYRKXXq1Cl4vGXLFv7xj3+wcuVKGjRowMSJE4u9jqF69eoFj+Pi4sjJySl22/lNQqXVOVONGzcmNTWVt99+m2eeeYZ58+bx3HPPsWjRIpYtW8aCBQt45JFHSE1NLfRlXqtWrUKf6cUXX+TBBx9EROjUqRNt2rRh8+bNLFq0iH79+hUcn6FDh/Lpp5/Sr1+/QnHEx8cXJM+LL76Y7OxsDhw4QMOGDfnHP/5RUK93797FnumEat26NTt37qRFixZkZWVx9OjRQmcfc+bMKdT0le/48ePUqlXrNI6eKcmHH8I338Ajj7iO+iuucElmyhSXYIz/Ivq/QUSq4RLKTFV9zSve4zVr4d3v9cp34Tr388V7ZaWVxxdTXmEdPnyYevXqcc4555Cenl5sm/7Z6t+/P6+88goAX3zxRbFnQqH69OnD0qVL2b9/Pzk5OcyZM4fLLruMjIwMVJXrr7+ehx56iDVr1pCbm0taWhqDBw/mscceY9++fRw7dqzQ9i644AK2bt1a8Pzcc89lyZIlgDuL2bp1K+3atePcc89l2bJl5OTkkJ2dzbJly7jgggtOim/UqFEsXboUgE2bNgHQsGFDjh49WrDvt99+m7p1654yqYwYMYLp06cD8Morr3DllVcWvJabm8vcuXMLzvJCy/ft28e5555b6rZN2QSDULcujBrlngcC8O23EDJ4z/gsYklF3CnJf4BNqvpEyEsLcKO58O7fCCkPiNMXOOQ1ky0CrhSRhl4H/ZXAIu+1wyLS19tXIGRbFVLPnj1JSEigc+fOBAIB+vfvH/Z9/OpXv2LXrl0kJCQwZcoUEhISCpquihMfH8/DDz/MwIED6dGjB3379mX48OHs3LmTSy+9lB49enDLLbfwyCOPkJOTw4033ki3bt3o2bMnv/vd76hXr16h7SUmJpKRkcHRo0cBePDBB1m2bBndunXjiiuu4PHHH6dhw4aMHz+eNm3a0K1bN7p3707v3r0LmvxuueWWgsEFt912G5s2baJLly5MnDixYEjy7t27ufDCC7ngggt44oknCpIFuKHKbdu25fDhw8THx/OnP/0JgEmTJpGenk6HDh14+umneeSRRwres3TpUs4///yTkseqVau45JJLqGI/o8/asWPw6qtw/fVQu7YrGzUK6tWDkP99xm9lGSJ2JjfgEkBxw4PXebdhQGPcqK8twHtAI/1xSPEzwDbcMOGkkG3dihs2vBW4JaQ8CVjvvedpKsCQYr9lZ2drZmamqqpu3rxZ27Ztq9nZ2VGN4bHHHtMXX3wxqvuMlF/+8pf6wQcfnPH77W/zR7NmuWHES5cWLr/1VtW6dVWPHPElrEqDMg4pjuTor4+8RFGcy4upr8CdJWxrGjCtmPIUoMtZhGmKOHLkCJdffjk5OTmoKv/+97+pWjW6847eddddvPbaa6euWA5ceOGFXHbZZX6HUSEEg3DuuXDppYXLAwGYNg1efx0mTPAnNvMjm6XYFNKgQQNWr17tawy1atViQgX5dvj5z3/udwgVQno6vPsu3H//yR3yAwZA27auCayC/NmUa9bQa4yJeTNnuuHEgcDJr1WpAjfdBO+9B7sq9FCd8sGSijEmpqm6s5C+fUu+yDEQcPVeeim6sZmTWVIxxsS0zz+H9euLP0vJ16ED9Ovn+l1KmFfURIklFWNMTAsGoVo18CZDKFFyMmzcCD53CVZ6llSMMTErJ8f1p/z0p9CoUel1x46FGjVskkm/WVKJAYMGDTrp6vinnnqKO+64o9T31a1bF4DvvvuOMWPGFFtn4MCBpKSklLqdp556qtCV7cOGDSt2ssfT9eCDD/L444+f9XaKk5mZyWWXXUZubi7gpmlp0KAB11xzTaF6EyZMoFOnTnTp0oVbb72V7OzsYrcXFxdXsB7KiBEjCsq3b99Onz596NChA+PGjStY3+bbb79l0KBBXHjhhXTr1o2FCxcCsHjxYnr16kXXrl3p1asX77//fsG2Vq9eTdeuXenQoQN33313wfovv/vd7wrVMz96913Yu7f0pq98DRrAyJEwaxaUcRkiEwlluZilIt1i8eLHf//733rzzTcXKuvTp48uW7as1PfVqVPnlNu+7LLLdNWqVaXWOe+88zQjI+PUgZ6mktaGCYenn35an3rqqYLn7733ni5YsECHDx9eqN5bb72leXl5mpeXp+PHj9f/+Z//KXZ7JR3L66+/XmfPnq2qqr/4xS8K3n/bbbcVPN6wYYOed955qurWU9m1a5eqqn7xxRfaqlWrgm1ddNFFumLFCs3Ly9OhQ4fqwoULVVV1x44desUVVxS7f7//Nv02bpxq48aqJ06Urf6bb7oLJF9/PbJxVUb4ffFjeXXPO/ewbnd4577v0aIHTw0teabKMWPG8Mc//pGsrCyqV6/Ojh07+O677xgwYABHjhxh5MiRHDhwgOzsbP70pz8xcuTIQu/fsWMH11xzDevXryczM5NbbrmFzz//nM6dO5OZmVlQ74477mDVqlVkZmYyZswYpkyZwtSpU/nuu+8YNGgQTZo0YenSpbRt25aUlBSaNGnCE088wbRp7rrTn//859xzzz3s2LGDq6++mksuuYRPPvmE1q1b88Ybb5Q6aeK6deu4/fbbOXbsGOeffz7Tpk2jYcOGTJ06lX/9619UrVqVhIQE5syZw7Jly/j1r38NuPVUli9fftJ0LjNnziy0Tsnll1/OBx98cNJ+hw0bVvC4d+/epKWlnVSnJKrK+++/X7Cf5ORkHnzwQe644w5EhMOHDwNuqYFWrVoB7mLHfImJiWRmZnLixAm+//57Dh8+TN++fQEIBAK8/vrrXH311Zx33nns37+f3bt3F7uOS2V18KC7oHHSJAiZI7VUV10FzZq50WJF/pmYKLHmrxjQqFEjevfuzdtvu6Vg5syZw9ixYxERatasyfz581mzZg1Lly7lt7/9bUGzSXGeffZZateuzaZNm5gyZUqhCxn//Oc/k5KSQmpqKsuWLSM1NZW7776bVq1asXTp0oKJF/OtXr2aF198kc8++4xPP/2U559/vmAhqy1btnDnnXeyYcMGGjRowLx580r9jIFAgL/+9a+kpqbStWtXpkyZArj1YdauXUtqair/+te/AHj88cd55plnWLduHR9++OFJySorK4uvv/6atm3blu0AA9nZ2cyYMYOhQ4tfx+348eMkJSXRt29fXn/9dcAt89ygQYOCGQXi4+PZ5V0I8eCDD/LSSy8RHx/PsGHD+Oc//3nSNufNm0fPnj2pUaMGu3btIj7+x/lPQ7cFbl63jz/+uMyfpzJ49VU4caJsTV/5qlZ1F0C++Sbs3x+52EzJ7EyliNLOKCLphhtuYM6cOYwcOZI5c+bwn//8B3C/lv/whz+wfPlyqlSpwq5du9izZ0+Jv2iXL1/O3XffDUC3bt3o1q1bwWuvvPIKzz33HDk5OaSnp7Nx48ZCrxf10UcfMXr06ILp5a+99lo+/PBDRowYQbt27ejRowdQ+pot4H7JHzx4sGC6kuTkZK6//vqCGCdMmMCoUaMY5U09279/f+69914mTJjAtddeW+jLGGDfvn2nveDVL3/5Sy699FIGDBhQ7OvffPMNrVu35uuvv2bw4MF07dq11Ik0Z8+ezc0338xvf/tbVqxYwU033cT69esLJo7csGED9913H++++26Z4mvWrBnffffdaX2mii4YhAsugF69Tu99ycnw5JMwZw7cWezETyaS7EwlRowcOZIlS5awZs0ajh07Ri/vX9LMmTPJyMhg9erVrFu3jubNmxe7hsqpbN++nccff5wlS5aQmprK8OHDz2g7+fLXYoGzW4/lrbfe4s4772TNmjVcdNFF5OTkMHnyZF544QUyMzPp378/X375ZaH3FF1z5VSmTJlCRkYGTzzxRIl1Wrd267u1b9+egQMHsnbtWho3bszBgwcLPltaWlpBvf/85z+MHTsWcOu0HD9+nH379hXUGz16NMFgkPPPP79g+6FNb6HbAltzpaht2+Cjj9xZipQ0g2AJuneHbt1sFJhfLKnEiLp16zJo0CBuvfXWQgs9HTp0iGbNmlGtWjWWLl3KN998U+p2Lr300oI+gPXr15Oamgq4tVjq1KlD/fr12bNnT0FTG0C9evUK1oEPNWDAAF5//XWOHTvG0aNHmT9/fom/9EtTv359GjZsyIcffgjAjBkzuOyyy8jLy2Pnzp0MGjSIv/71rxw6dIgjR46wbds2unbtyn333cdFF110UlJp2LAhubm5ZUosL7zwAosWLWL27NklTj9/4MABTpw4AbizoI8//piEhAREhEGDBjF37lwApk+fXtCfFbrOy6ZNmzh+/DhNmzbl4MGDDB8+nEcffbTQ0gQtW7bknHPO4dNPP0VVCQaDhfrGNm/eTJcuNjdqvpdecsnkTOfySk6GlSuhyJ+OiYay9OZXpFssjv7KN3/+fAV006ZNBWUZGRnat29f7dKli958883auXNn3b59u6r+OGJp+/btmpiYqKqqx44d03Hjxmnnzp119OjR2rt374LRX8nJydqxY0cdPHiwjh49umB6+alTp+pPfvITHThwoKoWHg3297//XRMTEzUxMVGffPLJk/anqvq3v/1NH3jggZM+T+jor7Vr12qfPn20a9euOnLkSP3+++81KytL+/fvr126dNHExET9y1/+oqqqd911lyYmJmrXrl11/Pjxevz48ZO2feutt+rixYsLnl9yySXapEkTrVmzprZu3VrfeecdVVWNi4vT9u3ba/fu3bV79+46ZcoUVVVdtWqV/uxnP1NV1Y8//li7dOmi3bp10y5duugLL7xQsN1t27bpRRddpOeff76OGTOmIJYNGzZov379tFu3btq9e3ddtGiRqqo+/PDDWrt27YL9de/eXffs2VOwz8TERG3fvr3eeeedmpeXp6qqWVlZ2rlz52KXGIiVv81oystTbd9e9fLLz3wb6emqcXGq998fvrgqO8o4+ku0lE7fiigpKUmLXrexadOmYlcNNLFrzZo1PPnkk8yoAAuU5w/EePjhh096rTL+bX78MVxyiRvBdTqd9EUNHw6pqbBjB4SsWG3OkIisVtWkU9Wz5i9TLvXs2ZNBgwYVXPxYnuXk5PDb3/7W7zBiRjAIderAtdee3XYCAUhLg2JGmpsIiuRywtNEZK+IrA8pe1lE1nm3HSKyzitvKyKZIa/9K+Q9vUTkCxHZKiJTvaWDEZFGIrJYRLZ49w3PJt7KdsZWEdx6663EVYCfoNdff32xo9kq499kZia8/DJcd51bi/5sjBgB9evbUsPRFskzlf8FCl0UoKrjVLWHqvYA5gGhy/tty39NVW8PKX8WuA3o6N3ytzkZWKKqHXHLE08+00Br1qzJ/v37K+U/YhObVJX9+/dTs2ZNv0OJqv/+Fw4dOrtmr3y1arn5wObNgyNHzn57pmwiuZzwchFpW9xr3tnGWGBwadsQkZbAOar6qfc8CIwC3gZGAgO9qtOBD4D7ziTW+Ph40tLSyMjIOJO3GxMRNWvWPOkanYouGIT4eBg4MDzbCwTg+eddYklODs82Ten8uvhxALBHVbeElLUTkbXAYeCPqvoh0BoInVcjzSsDaK6q6d7j3UDzknYmIpOASeCGghZVrVo12rVrd4YfxRgTDnv2wDvvwO9/H76O9f79oX17l6wsqUSHXx31NwCzQ56nA+eq6oXAvcAsETmnrBvzhruV2Halqs+papKqJjVt2vRMYzbGRNDs2ZCb65YGDhcRd7aydCl8+234tmtKFvWkIiJVgWuBl/PLVPWEqu73Hq8GtgE/AXYBoef/8V4ZwB6veSy/mWxv5KM3xkRKMAhJSZCQEN7t3nSTLTUcTX6cqQwBvlTVgmYtEWkqInHe4/a4DvmvveatwyLS1+uHCQBveG9bAOSf0CaHlBtjypkvvoC1a8PTQV9U+/YwYIAtNRwtkRxSPBtYAXQSkTQR+Zn30ngKN30BXAqkekOM5wK3q+r33mu/BF4AtuLOYPLnF3kUuEJEtuAS1aOR+izGmMgKBt0MwyEzFIVVcjJ89ZWbusVEll1Rb4zxVU4OtGkDffq49VMi4dAhaNECbr0VnnkmMvuo6OyKemNMubBkCezeHZmmr3z168Po0W4wgDd3qIkQSyrGGF8Fg9CwoZurK5ICAThwAN56K7L7qewsqRhjfHP4MMyfD+PHQ8gSPRExZAi0bGnTtkSaJRVjjG/mzXPzfUWy6Stf/lLDCxeCTZ4ROZZUjDG+CQahY0fXSR8NycluYMDsouNPTdhYUjHG+GLHDjctfXLy6S8ZfKa6dIELL7SlhiPJkooxxhf5V7hPnBjd/SYnw+rVsGFDdPdbWVhSMcZEnao7Wxg4EM47L7r7vuEG179iZyuRYUnFGBN1n30GW7ZEp4O+qGbN4Oqr3ZlSBVg4NOZYUjHGRF0w6BbRuu46f/YfCMB337kLL014WVIxxkTViRMwZ467wv2cMi9wEV7XXAMNGtg1K5FgScUYE1VvveWubPej6StfzZrugsv5890FmCZ8LKkYY6IqGHRXtg8Z4m8cgYC78HLuXH/jqGgsqRhjoiYjw52pTJwYviWDz1Tfvu7CSxsFFl6WVIwxUTNnjrui3c+mr3z5Sw0vWwbbt/sdTcVhScUYEzXBoLuivUsXvyNxbrrJ3dtSw+FjScUYExUbN0JKSmycpeQ77zx3AaYtNRw+kVxOeJqI7BWR9SFlD4rILhFZ592Ghbx2v4hsFZGvROSqkPKhXtlWEZkcUt5ORD7zyl8WkeqR+izGmLM3Y4brR4nUksFnKjkZtm6FFSv8jqRiiOSZyv8CQ4spf1JVe3i3hQAikoBbuz7Re8//iEiciMQBzwBXAwnADV5dgL962+oAHAB+FsHPYow5C7m5rolp6FBo3tzvaAq77jqoXds67MMlYklFVZcD35ex+khgjqqeUNXtwFagt3fbqqpfq2oWMAcYKSICDAbyBwNOB0aF9QMYY8Lmgw8gLS22mr7y1asH114LL78Mx4/7HU3550efyl0ikuo1jzX0yloDO0PqpHllJZU3Bg6qak6R8mKJyCQRSRGRlAxbnceYqJs+3a0TP2KE35EULxCAgwfhv//1O5LyL9pJ5VngfKAHkA78PRo7VdXnVDVJVZOaNm0ajV0aYzxHjrgVHseNc1eyx6LBg6F1a5u2JRyimlRUdY+q5qpqHvA8rnkLYBfQJqRqvFdWUvl+oIGIVC1SboyJMa+9BseOxWbTV764OHdB5jvvwJ49fkdTvkU1qYhIy5Cno4H8kWELgPEiUkNE2gEdgZXAKqCjN9KrOq4zf4GqKrAUGOO9Pxl4IxqfwRhzeoJBaN8e+vXzO5LSBQJuQMGsWX5HUr5FckjxbGAF0ElE0kTkZ8BjIvKFiKQCg4DfAKjqBuAVYCPwDnCnd0aTA9wFLAI2Aa94dQHuA+4Vka24Ppb/ROqzGGPOzM6d8P777gs7WksGn6mEBEhKslFgZ6vqqaucGVUtbjR6iV/8qvpn4M/FlC8EFhZT/jU/Np8ZY2LQzJnuosL8K9djXXIy/OpXkJoK3br5HU35ZFfUG2MiIn/J4Esucc1f5cH48VCtmp2tnA1LKsaYiEhJgU2b3K//8qJJExg+3F2omZNz6vrmZJZUjDEREQxCjRpw/fV+R3J6AgE3AmzxYr8jKZ8sqRhjwi4rC2bPhlGj3EWP5cmwYdCokV2zcqYsqRhjwu7tt2H//ti+NqUkNWq4SS9ff91dZW9OjyUVY0zYBYPQrBlceaXfkZyZQABOnIBXX/XML87nAAAfdklEQVQ7kvLHkooxJqy+/97NoTVhAlSN2EULkXXRRdC5s40COxOWVIwxYfXyy5CdXT6bvvLlLzX80UewbZvf0ZQvllSMMWE1fbq7cLBHD78jOTsTJ7rkMmOG35GUL5ZUjDFh89VX8Nln5fssJV+bNm724mAQ8vL8jqb8sKRijAmbGTOgShW48Ua/IwmP5GTYvh0+/tjvSMoPSyrGmLDIy3NJ5coroWXLU9cvD0aPhjp1rMP+dFhSMcaExfLl8O23FaPpK1/dujBmDLzyCmRm+h1N+WBJxRgTFsGgW+995Ei/IwmvQAAOH4Y3bMWmMrGkYow5a8eOuQsFr78eatf2O5rwGjjQddrbtC1lY0nFGHPW5s93a9FXpKavfFWquPVg3n0X0tP9jib2RXLlx2kisldE1oeU/U1EvhSRVBGZLyINvPK2IpIpIuu8279C3tPLWy1yq4hMFXHrx4lIIxFZLCJbvPuGkfosxpjSBYPQti0MGOB3JJERCLiBCDNn+h1J7Ivkmcr/AkOLlC0GuqhqN2AzcH/Ia9tUtYd3uz2k/FngNty69R1DtjkZWKKqHYEl3nNjTJTt2gXvved+zVepoG0fnTpBnz6uCUzV72hiW8T+BFR1OfB9kbJ3vXXnAT4F4kvbhoi0BM5R1U9VVYEgMMp7eSSQ38o5PaTcGBNFs2a5X/HlZcngM5WcDOvXw7p1fkcS2/z8XXEr8HbI83YislZElolI/kl0ayAtpE6aVwbQXFXzWzh3A81L2pGITBKRFBFJycjICFP4xhhV9+v94ouhY0e/o4msceOgenW7ZuVUfEkqIvL/ATlAfgtlOnCuql4I3AvMEpFzyro97yymxJNSVX1OVZNUNalp06ZnEbkxJtS6dbBhQ8XsoC+qUSP46U9dv0p2tt/RxK6oJxURuRm4BpjgJQNU9YSq7vcerwa2AT8BdlG4iSzeKwPY4zWP5TeT7Y3KBzDGFAgG3a/3sWP9jiQ6AgHIyIBFi/yOJHZFNamIyFDg/wAjVPVYSHlTEYnzHrfHdch/7TVvHRaRvt6orwCQfwnSAiDZe5wcUm6MiYLsbNef8tOful/xlcHVV0OTJnbNSmkiOaR4NrAC6CQiaSLyM+BpoB6wuMjQ4UuBVBFZB8wFblfV/E7+XwIvAFtxZzD5/TCPAleIyBZgiPfcGBMlixbB3r2uA7uyqFbNTZa5YAEcOOB3NLFJtJKNj0tKStKUlBS/wzCm3Bs7FpYuhe++c1+2lcXq1ZCUBM8+C7fffur6FYWIrFbVpFPVq6Cjyo0xkXTggPu1fuONlSuhAPTsCYmJNgqsJJZUjDGn7dVX4cSJyjHqq6j8pYZXrIDNm/2OJvZYUjHGnLZgEBIS3K/2ymjiRDd7gC01fDJLKsaY07Jtm1sJMRBwv9oro1atYMgQW2q4OJZUjDGnZcYMl0wmTPA7En8lJ7tFyZYv9zuS2FKmpCIi54tIDe/xQBG5O3+GYWNM5ZGX536dDxkC8aXO3FfxjRrlFiWza1YKK+uZyjwgV0Q6AM8BbYBZEYvKGBOTPv4Ytm+vnB30RdWu7RYlmzsXjh71O5rYUdakkufNLjwa+Keq/h5oGbmwjDGxKBiEOnVg9Gi/I4kNgYBbnGz+fL8jiR1lTSrZInIDbjqUN72ySjY63ZjKLTMTXnkFxoxxicW4RcnatrVrVkKVNancAlwM/FlVt4tIO8AG0xlTiSxYAIcPW9NXqPylht97zy1WZsqYVFR1o6reraqzvWV766nqXyMcmzEmhgSD0KYNDBzodySxJRBw68q89JLfkcSGso7++kBEzhGRRsAa4HkReSKyoRljYsXu3W4CyfyL/syPOnSAfv1c0q1kUykWq6x/HvVV9TBwLRBU1T64mYGNMZXArFmQm2tNXyVJToaNG91kk5VdWZNKVW8hrLH82FFvjKkkgkHo3Rs6d/Y7ktg0dizUqGEd9lD2pPIQsAjYpqqrvIW0tkQuLGNMrPj8c3ezs5SSNWgAI0e6M7qsLL+j8VdZO+pfVdVuqnqH9/xrVb0usqEZY2LBjBluevtx4/yOJLYFArB/P7z99qnrVmRl7aiPF5H5IrLXu80TkUo+SYMxFV9ODsycCcOHu2V0TcmuugqaNbNpW8ra/PUibk34Vt7tv15ZqURkmpeE1oeUNRKRxSKyxbtv6JWLiEwVka0ikioiPUPek+zV3yIiySHlvUTkC+89U7117I0xYfLee27klzV9nVrVqm6SzTffdGcslVVZk0pTVX1RVXO82/8CTcvwvv8FhhYpmwwsUdWOwBLvOcDVQEfvNgl4FlwSAh4A+gC9gQfyE5FX57aQ9xXdlzHmLASD0KgRDBvmdyTlQyAA2dkwZ47fkfinrEllv4hMFJE47zYROGUuVtXlwPdFikcC+SeI04FRIeVBdT4FGngjzq4CFqvq96p6AFgMDPVeO0dVP1VVBYIh2zLGnKVDh9ycVuPHu5FN5tR69IBu3Sr3KLCyJpVbccOJdwPpwBjg5jPcZ3NVTfce7waae49bAztD6qV5ZaWVpxVTfhIRmSQiKSKSkpGRcYZhG1O5zJ0Lx4+7azBM2QUCsHIlfPml35H4o6yjv75R1RGq2lRVm6nqKOCsR395ZxgRvwZVVZ9T1SRVTWratCytdsaYYBA6dYKLLvI7kvJlwgQ360BlPVs5mwkX7j3D9+3xmq7w7vd65btw67Tki/fKSiuPL6bcGHOWtm93KxpW5iWDz1SLFm4k2IwZbhaCyuZsksqZ/qktwE2hj3f/Rkh5wBsF1hc45DWTLQKuFJGGXgf9lcAi77XDItLXG/UVCNmWMeYs5E+OOHGiv3GUV8nJkJYGH3zgdyTRdzZJ5ZTNViIyG1gBdBKRNBH5GfAocIWIbMHNH/aoV30h8DWwFXge+CWAqn4PPAys8m4PeWV4dV7w3rMNqOSXHRlz9lRd082gQXDuuX5HUz6NGAH161fOa1ZES5lWU0R+oPjkIUAtVa0aqcAiJSkpSVNSUvwOw5iYtWKFm3X3xRfh5pv9jqb8mjTJXTi6Zw/Uret3NGdPRFaratKp6pV6pqKq9VT1nGJu9cpjQjHGnNr06W799etsIqazEgjAsWMwb57fkUSXrYxgjClw/Di8/DJcey3Uq+d3NOVb//7Qvn3lGwVmScUYU+DNN+HgQZuWJRxE3HFcuhS+/dbvaKLHkooxpkAwCK1aweDBfkdSMdx0U+VbatiSijEGgL173bTtEydCXJzf0VQM7dvDgAGVa6lhSyrGGMBNgpiT435dm/BJToavvnJTt1QGllSMMYD7Nd2zJ3Tp4nckFcuYMVCzZuXpsLekYoxhwwZYvdo66COhfn0YPRpmz4YTJ/yOJvIsqRhjCAbdIlM33OB3JBVTIAAHDsBbb/kdSeRZUjGmksvNdaOTrr7aLYdrwm/IEGjZsnJM22JJxZhK7v334bvvrOkrkvKXGl64ECr6kk6WVIyp5IJBaNAArrnG70gqtkDAja6bPdvvSCLLkooxldgPP8Brr8G4cW6Ekomcrl3hwgsr/igwSyrGVGKvveYmPbSmr+gIBNwouw0b/I4kciypGFOJBYNw/vlw8cV+R1I53Hijm62gIp+tWFIxppL69ls32aEtGRw9zZq5UXYvvVRxlxqOelIRkU4isi7kdlhE7hGRB0VkV0j5sJD33C8iW0XkKxG5KqR8qFe2VUQmR/uzGFOevfSSm4/KpmWJruRkN9puyRK/I4mMUld+jPjOReKAXUAf4BbgiKo+XqROAjAb6A20At4DfuK9vBm4AkjDLTV8g6puLG2ftvKjMS6ZXHABNG8Oy5b5HU3lcvy4u2Zl2DC3MmR5EZaVH6PgcmCbqn5TSp2RwBxVPaGq23Hr0ff2bltV9WtVzQLmeHWNMaewapWb5NA66KOvZk0YPx7mz4fDh/2OJvz8TirjcWch+e4SkVQRmSYiDb2y1sDOkDppXllJ5ScRkUkikiIiKRkV/cojY8ogGHRfbmPG+B1J5RQIQGYmzJ3rdyTh51tSEZHqwAjgVa/oWeB8oAeQDvw9XPtS1edUNUlVk5o2bRquzRpTLmVluQvwRo1ykx2a6OvbFzp2rJijwPw8U7kaWKOqewBUdY+q5qpqHvA8rnkLXJ9Lm5D3xXtlJZUbY0qxcCF8/701ffkpf6nhZctg+3a/owkvP5PKDYQ0fYlIy5DXRgPrvccLgPEiUkNE2gEdgZW4jvmOItLOO+sZ79U1xpRi+nTXQX/FFX5HUrnlj7qraEsN+5JURKQObtTWayHFj4nIFyKSCgwCfgOgqhuAV4CNwDvAnd4ZTQ5wF7AI2AS84tU1xpRg3z43/fqECW6SQ+Of886DgQMr3lLDvvxZqepRoHGRshJHy6vqn4E/F1O+EFgY9gCNqaBefhmys921EsZ/yclwyy2wYgX06+d3NOHh9+gvY0wUBYPQvTt06+Z3JAbguuugdu2K1WFvScWYSuLLL2HlSuugjyX16sG117ozyOPH/Y4mPCypGFNJzJgBVaq4SQ1N7AgE4OBB+O9//Y4kPCypGFMJ5OW5pHLVVdCihd/RmFCDB0Pr1hVnqWFLKsZUAh98ADt3WtNXLIqLg4kT4Z13YM8ev6M5e5ZUjKkEgkE45xwYabPjxaRAwE2FP2uW35GcPUsqxlRwR4+6OabGjoVatfyOxhQnIQGSkirGKDBLKsZUcPPnu8RiTV+xLRCAdesgNdXvSM6OJRVjKrhgENq1g/79/Y7ElOaGG9wsB+X9bMWSijEV2K5d8N57bp6pKvavPaY1aQLDh7u5wHJy/I7mzNmfmTEV2MyZtmRweZKc7EaALV7sdyRnzpKKMRWUqrv2oV8/6NDB72hMWQwbBo0ale9rViypGFNBrVkDGzdaB315UqOG61t5/XV3lX15ZEnFmAoqGHRfUmPH+h2JOR2BAJw4Aa++euq6sciSijEVUHa2u5BuxAho2NDvaMzpuOgi6Ny5/I4Cs6RiTAX0zjtuQS5r+ip/8pca/ugj2LbN72hOnyUVYyqgYBCaNnUTSJryZ+JEl1xmzPA7ktPnW1IRkR3e8sHrRCTFK2skIotFZIt339ArFxGZKiJbRSRVRHqGbCfZq79FRGw9O1PpHTgACxa4Ke6rVfM7GnMm2rRxsxcHg26G6fLE7zOVQaraQ1WTvOeTgSWq2hFY4j0HuBro6N0mAc+CS0LAA0AfoDfwQH4iMqayeuUVyMqypq/yLjkZtm+Hjz/2O5LT43dSKWokkD9CezowKqQ8qM6nQAMRaQlcBSxW1e9V9QCwGBga7aCNiSXTp0NiIlx4od+RmLMxejTUqVP+rlnxM6ko8K6IrBaRSV5Zc1VN9x7vBpp7j1sDO0Pem+aVlVReiIhMEpEUEUnJyMgI52cwJqZs2QIrVrizFBG/ozFno25dGDPGnXlmZvodTdn5mVQuUdWeuKatO0Xk0tAXVVVxieesqepzqpqkqklNmzYNxyaNiUn5SwZPnOh3JCYcAgH44Qd3MWR54VtSUdVd3v1eYD6uT2SP16yFd7/Xq74LaBPy9nivrKRyYyqd/CWDhwyBVq38jsaEw8CBrtO+PF2z4ktSEZE6IlIv/zFwJbAeWADkj+BKBt7wHi8AAt4osL7AIa+ZbBFwpYg09Dror/TKjKl0PvoIduywDvqKpEoVNxnou+9Cevqp68cCv85UmgMficjnwErgLVV9B3gUuEJEtgBDvOcAC4Gvga3A88AvAVT1e+BhYJV3e8grM6bSCQZdO/yoUaeua8qPQMCdhc6c6XckZSOu66LySEpK0pSUFL/DMCasMjOheXO47jp48UW/ozHh1revW70zNdW/ARgisjrk8o8SxdqQYmPMGXj9ddeha01fFVMgAOvXu+WGY50lFWMqgGAQzj0XLrvM70hMJIwf72ZHKA8d9pZUjCnn0tNdR64tGVxxNWoEP/2p61fJzvY7mtLZn6Ax5dysWa4j15YMrtiSkyEjAxbF+PhWSyrGlHPBIPTpA506+R2JiaShQ6FJk9iftsWSijHl2OefuxFB1kFf8VWv7maeXrDAzUQdqyypGFOOBYOuA3fcOL8jMdEQCLgZqF9+2e9ISmZJxZhyKifHddxecw00bux3NCYaevZ0M1DH8igwSyrGlFPvvgt79ljTV2WSv9TwihWwebPf0RTPkoox5VQw6M5Qhg3zOxITTRMnuqHjsbrUsCUVY8qhgwfdVfQ33OA6cE3l0aqVm4k6VpcatqRiTDk0dy6cOGFNX5VVcjJ8+y0sX+53JCezpGJMORQMQufOkHTK6f1MRTRqFNSrF5vXrFhSMaac+fpr+PBDWzK4MqtdG66/3p2xHj3qdzSFWVIxppyZMcMlkwkT/I7E+CkQgCNHYP58vyMpzJKKMeWIqmv6GjTIzUpsKq8BA6Bt29i7ZiXqSUVE2ojIUhHZKCIbROTXXvmDIrJLRNZ5t2Eh77lfRLaKyFciclVI+VCvbKuITI72ZzEm2j75xDV/WQe9yV9q+L33IC3N72h+5MeZSg7wW1VNAPoCd4pIgvfak6raw7stBPBeGw8kAkOB/xGROBGJA54BrgYSgBtCtmNMhRQMuvb0667zOxITCwIBd/YaS0sNRz2pqGq6qq7xHv8AbAJal/KWkcAcVT2hqttx69T39m5bVfVrVc0C5nh1jamQjh93cz5dd51bi96YDh2gXz/3YyNWVob3tU9FRNoCFwKfeUV3iUiqiEwTkYZeWWtgZ8jb0ryyksqL288kEUkRkZSMjIwwfgJjoue//4VDh6zpyxQWCMDGjbB6td+ROL4lFRGpC8wD7lHVw8CzwPlADyAd+Hu49qWqz6lqkqomNW3aNFybNSaqgkFo3dp10huTb+xYqFEjdjrsfUkqIlINl1BmquprAKq6R1VzVTUPeB7XvAWwC2gT8vZ4r6ykcmMqnL174e233bxPcXF+R2NiScOGMGKEWwE0K8vvaPwZ/SXAf4BNqvpESHnLkGqjgfXe4wXAeBGpISLtgI7ASmAV0FFE2olIdVxn/oJofAZjom3WLMjNtSWDTfGSk2H/fvfDw29Vfdhnf+Am4AsRWeeV/QE3eqsHoMAO4BcAqrpBRF4BNuJGjt2pqrkAInIXsAiIA6ap6oZofhBjoiUYhF693FoaxhR15ZXQrJmbtmWkz8OVop5UVPUjoLjJJRaW8p4/A38upnxhae8zpiL44gtYuxamTvU7EhOrqlVzMyw8/bQ7Y/Fz0Ta7ot6YGDdjBlStCuPH+x2JiWWBAGRnw5w5/sZhScWYGJabCy+95BbisoGLpjQ9ekC3bv6PArOkYkwMW7IE0tPt2hRTNoEArFwJX37pXwyWVIyJYcEgNGgA11zjdySmPJgwwc0J5ufZiiUVY2LU4cPw2muuL6VGDb+jMeVBixZw1VWuHy43158YLKkYE6PmzYPMTGv6MqcnOdnNWvzBB/7s35KKMTEqGISOHaFvX78jMeXJiBFQv75/Sw1bUjEmBn3zjfulaUsGm9NVq5abD2zePLcyZLRZUjEmBr30krufONHfOEz5FAjAsWMusUSbJRVjYkz+ksGXXeaWizXmdPXvD+3b+zMKzJKKMTFm5UrYvNk66M2ZE3F/P0uXwrffRnffllSMiTHTp0PNmjBmjN+RmPLsppvcWW9+U2q0WFIxJoacOOHmbho9Gs45x+9oTHnWvj0MGOB+pERzqWFLKsbEkLfeggMHrOnLhEcg4JpSV66M3j4tqRgTQ4JBaNkShgzxOxJTEVx/vWtKjWaHvR+LdBljirFvnztTueceN9W9cVSV7LxsjmYd5Wj20YL7rNwsalerTe1qtalTrQ51qtehdrXaVK1iBy9f/fowahTMng1PPBGd6X7s6BsTI+bMgZyc8tn0lad5HMs+xtGso+4+5Mu/pPv8+kezS693NOsouVr2iayqx1UvSDS1q9WmTvU6hR4XJKHiykIel/Ra9bjqETyS4Zec7P623noLrr028vsr90lFRIYC/8AtKfyCqj7qc0jGnJFg0K2J0bVrZLaflZtV6Iu6rF/+JyWAYu4zczJPKxZBCr7si943qd2k4Eu/aEIILaseV53MnMyTklTo5zqW82PZD1k/sPvI7oLX8+ufTsICqFqlaqlJqOBxGZNW0Xo14mogYZxGYcgQN9Hk9OmWVE5JROKAZ4ArgDRglYgsUNWN4d7Xp2uOsu/7HG+/3v6RHx+Le17odXG1QsuqhPyxiICIFJqGI/+PKbSsioR/G6HvKSn2omVVqhTeb9F9lFYWSRVhH998A6tS8vjL45nsPXqKX/WlfPmXlixy8nJOK6bqcdWL/dJvXLsx51Y798fyIl+WJSWL0Ptwf3GeKVUlKzfrpERTKDEVea1QvZzCZfuO7Tvpvdl52acVUxWpUraEdKozsZDHP72pNtP+VYfdexrSonlchI6mIxrNsWZhJiIXAw+q6lXe8/sBVPUvJb0nKSlJU1JSTntfzX4znIwGC8801MpBi/mSKK7MFK9K3mlVF6TYX/An3Z/Bl771TYRPdm522ZNUSfVKeW9WblaZY3nnpxu4qmfCGX0OEVmtqkmnqlfe/2paAztDnqcBfYpWEpFJwCTv6RER+eoM99cE2HeG742kGInrpB8oTUBjIK6TxMjxOslpxaUoR73/IqxCHK8oitm4hj6YeDZxnVeWSuU9qZSJqj4HPHe22xGRlLJk6mizuE6PxXV6LK7TU9njKu/XqewC2oQ8j/fKjDHG+KC8J5VVQEcRaSci1YHxwAKfYzLGmEqrXDd/qWqOiNwFLMINKZ6mqhsiuMuzbkKLEIvr9Fhcp8fiOj2VOq5yPfrLGGNMbCnvzV/GGGNiiCUVY4wxYWNJpQgRmSYie0VkfQmvi4hMFZGtIpIqIj1jJK6BInJIRNZ5t/8XpbjaiMhSEdkoIhtE5NfF1In6MStjXFE/ZiJSU0RWisjnXlxTiqlTQ0Re9o7XZyLSNkbiullEMkKO188jHVfIvuNEZK2IvFnMa1E/XmWMy5fjJSI7ROQLb58nXekd8X+Pqmq3kBtwKdATWF/C68OAt3Hzr/QFPouRuAYCb/pwvFoCPb3H9YDNQILfx6yMcUX9mHnHoK73uBrwGdC3SJ1fAv/yHo8HXo6RuG4Gno7235i373uBWcX9//LjeJUxLl+OF7ADaFLK6xH992hnKkWo6nLg+1KqjASC6nwKNBCRljEQly9UNV1V13iPfwA24WY6CBX1Y1bGuKLOOwZHvKfVvFvR0TIjgene47nA5RLhibLKGJcvRCQeGA68UEKVqB+vMsYVqyL679GSyukrbmoY37+sPBd7zRdvi0hitHfuNTtciPuVG8rXY1ZKXODDMfOaTNYBe4HFqlri8VLVHOAQ0DgG4gK4zmsymSsibYp5PRKeAv4PUNLkaL4crzLEBf4cLwXeFZHV4qaoKiqi/x4tqVQca4DzVLU78E/g9WjuXETqAvOAe1T1cDT3XZpTxOXLMVPVXFXtgZsBoreIdInGfk+lDHH9F2irqt2Axfx4dhAxInINsFdVV0d6X6ejjHFF/Xh5LlHVnsDVwJ0icmmU9gtYUjkTMTk1jKoezm++UNWFQDURaRKNfYtINdwX90xVfa2YKr4cs1PF5ecx8/Z5EFgKDC3yUsHxEpGqQH1gv99xqep+VT3hPX0B6BWFcPoDI0RkBzAHGCwiLxWp48fxOmVcPh0vVHWXd78XmA/0LlIlov8eLamcvgVAwBtB0Rc4pKrpfgclIi3y25FFpDfu/23Ev4i8ff4H2KSqT5RQLerHrCxx+XHMRKSpiDTwHtfCrQX0ZZFqC4Bk7/EY4H31elj9jKtIu/sIXD9VRKnq/aoar6ptcZ3w76vqxCLVon68yhKXH8dLROqISL38x8CVQNERoxH991iup2mJBBGZjRsV1ERE0oAHcJ2WqOq/gIW40RNbgWPALTES1xjgDhHJATKB8ZH+h+XpD9wEfOG1xwP8ATg3JDY/jllZ4vLjmLUEpotbYK4K8IqqvikiDwEpqroAlwxniMhW3OCM8RGOqaxx3S0iI4AcL66boxBXsWLgeJUlLj+OV3NgvvdbqSowS1XfEZHbITr/Hm2aFmOMMWFjzV/GGGPCxpKKMcaYsLGkYowxJmwsqRhjjAkbSyrGGGPCxpKKMWEgIrkhs9GuE5HJYdx2WylhdmpjYo1dp2JMeGR6U5wYU6nZmYoxEeStbfGYt77FShHp4JW3FZH3vckGl4jIuV55cxGZ701y+bmI9PM2FSciz4tb6+Rd76p3RORucWvGpIrIHJ8+pjEFLKkYEx61ijR/jQt57ZCqdgWexs1sC24Cy+neZIMzgale+VRgmTfJZU9gg1feEXhGVROBg8B1Xvlk4EJvO7dH6sMZU1Z2Rb0xYSAiR1S1bjHlO4DBqvq1N8HlblVtLCL7gJaqmu2Vp6tqExHJAOJDJiLMn7p/sap29J7fB1RT1T+JyDvAEdwMy6+HrIlijC/sTMWYyNMSHp+OEyGPc/mxP3Q48AzurGaVN0uvMb6xpGJM5I0LuV/hPf6EHyc+nAB86D1eAtwBBYtm1S9poyJSBWijqkuB+3BTvp90tmRMNNmvGmPCo1bIbMgA76hq/rDihiKSijvbuMEr+xXwooj8Hsjgx5lifw08JyI/w52R3AGUNC15HPCSl3gEmOqthWKMb6xPxZgI8vpUklR1n9+xGBMN1vxljDEmbOxMxRhjTNjYmYoxxpiwsaRijDEmbCypGGOMCRtLKsYYY8LGkooxxpiw+f8Bx+EOMG4zi9oAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f2729ec4198>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEWCAYAAABMoxE0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl4FFXW+PHvIayyy6qAgICyhohhUUE2I0ERXFDAhU3Hxw2UcWN8nddxG3Uc9/HHO9oBQRHEBXABBJHNQYSwBAEVUXEIRA0Bwm628/ujKm2HbJ10OpXlfJ6nn3RV3bp9upL06bq36l5RVYwxxpjiquJ1AMYYY8o3SyTGGGNCYonEGGNMSCyRGGOMCYklEmOMMSGxRGKMMSYklkiMMcaExBKJMQUQkZUiclBEangdizFllSUSY/IhIm2AfoACw0vxdauW1msZUxIskRiTv7HAOuANYFz2ShGpJSLPicjPIpIqIl+ISC13W18RWSsih0Rkj4iMd9evFJFbAuoYLyJfBCyriNwpIt8D37vrXnLrOCwiG0WkX0D5CBF5SER+EJEj7vZWIvKqiDwX+CZE5EMRmRKOA2QMWCIxpiBjgdnuY4iINHPX/xM4H7gQOB14AMgSkdbAYuAVoAkQBWwpwutdCfQGOrvLG9w6TgfeBt4VkZrutj8DY4DLgHrAROA4MBMYIyJVAESkMXCJu78xYWGJxJg8iEhfoDUwT1U3Aj8A17sf0BOBu1V1r6pmqupaVf0duB74TFXnqGq6qqaoalESyVOqekBVTwCo6ltuHRmq+hxQAzjXLXsL8LCqfqeOBLfseiAVGOyWGw2sVNVfQzwkxuTLEokxeRsHLFXV/e7y2+66xkBNnMRyqlb5rA/WnsAFEblPRL5xm88OAfXd1y/stWYCN7rPbwTeDCEmYwplnXrGnMLt77gOiBCRX9zVNYAGwBnASaAdkHDKrnuAXvlUeww4LWC5eR5l/ENxu/0hD+CcWWxX1SwROQhIwGu1A7blUc9bwDYR6Q50AhbkE5MxJcLOSIzJ7UogE6evIsp9dALW4PSbTAeeF5Ez3U7vC9zLg2cDl4jIdSJSVUQaiUiUW+cW4GoROU1E2gM3FxJDXSADSAaqisj/4vSFZPMBj4tIB3FEikgjAFVNxOlfeRN4P7upzJhwsURiTG7jgBmq+l9V/SX7AfwLuAGYCnyN82F9AHgGqKKq/8Xp/L7XXb8F6O7W+QKQBvyK0/Q0u5AYPgWWADuBn3HOggKbvp4H5gFLgcNAHFArYPtMoBvWrGVKgdjEVsZUPCJyMU4TV2u1f3ITZnZGYkwFIyLVgLsBnyURUxoskRhTgYhIJ+AQzkUBL3ocjqkkrGnLGGNMSOyMxBhjTEgqxX0kjRs31jZt2ngdhjHGlCsbN27cr6pNCitXKRJJmzZtiI+P9zoMY4wpV0Tk52DKWdOWMcaYkFgiMcYYExJLJMYYY0JiicQYY0xILJEYY4wJSVgTiYjEish3IrJLRKbmsf0sEVkhIptFZKuIXBaw7S/uft+JyJBg6zTGGFO6wpZIRCQCeBUYijMc9xgR6XxKsYdxZqA7D2cmt//n7tvZXe4CxAL/zx2uO5g6jTHGlKJw3kfSC9ilqj8CiMhcYASwI6CM8sccC/WBfe7zEcBcd/rSn0RkF39MGFRYnSXmniX3sOWXosyUaiqjrKw/HpmZOZcLe6hClSrBPSIicq8TKTw+U74F+7eU199e06woPpn0Io0bF/46oQhnImlBzvkTEoHep5T5G7BURCYBtYFLAvZdd8q+LdznhdUJgIjcCtwKcNZZZxU9elNuqBbtwzvHP13gvkVMAtmJwGvBJqIqVaDKqclI8k9SwTwqg+L8feX7d5fftnz+9kL9+9r9CyQnU64TSTDGAG+o6nMicgHwpoh0LYmKVfU14DWA6OjoYv06Xoy1wVOLQhV++w1OnvzjceJEzuWC1ge77tT1mZmhxV29OtSsCbVqOT9PfeS5vnbe64tUR03nwzgtLfRj4F93rGj7h/pBVaNG8d97cY5f1arw++8l87cV7PqMjNCOUdWqxTgOdUI7ZoHHqzTOWsOZSPYCrQKWW7rrAt2M0weCqn4pIjWBxoXsW1idxiN33w2vvFK8fQv7B6lXL7QPoPzW16jh/Tfr7JhKmyqkp5fsh/Cp61JS8i8X6heAoghMeHn9HTRqFJ7EV6OG82Fe0YXzLW4AOohIW5wP+9HA9aeU+S8wGHjDnUehJs4c1R8Cb4vI88CZQAdgPSBB1Gk8cOQITJ8Ol14Ko0cX7Z+xenVr6/eCiHPsq1d3EnVpy8goenJKTy/6h3tZ+LJQ0YUtkahqhojchTP3dAQwXVW3i8hjQLyqfogzt/XrIjIFp+N9vDuj23YRmYfTiZ4B3KmqmQB51Rmu92CCN28eHDsGf/sbXHCB19GY8qBqVahTx3mY8q1STGwVHR2tNvpveF1wAaSmwvbtdnZhTEUhIhtVNbqwcnbCZ0K2fTusWwe33GJJxJjKyBKJCVlcHFSrBjfd5HUkxhgvWCIxIfn9d5g1C668EpoUOo+aMaYiskRiQrJwoXOJ5803ex2JMcYrlkhMSHw+OOssuOSSwssaYyomSySm2Hbvhs8+g4kTnSE2jDGVkyUSU2wzZjg/J0zwNg5jjLcskZhiycz84052GxPTmMrNEokplqVLITHRuXfEGFO5WSIxxRIX5wxNPXy415EYY7xmicQU2W+/OZf9jhvnDPhnjKncLJGYIps1yxm51e4dMcaAJRJTRKrOvSMXXgidOnkdjTGmLKgEU66YkrR2LXz3nXPFljHGgJ2RmCLy+aBuXbj2Wq8jMcaUFZZITNBSU50JrMaMscmIjDF/sERigjZ3Lhw/bp3sxpicLJGYoPl80K0b9OzpdSTGmLLEEokJSkICxMfbLIjGmNwskZigxMVBjRpw441eR2KMKWsskZhCnTgBb74JV10Fp5/udTTGmLLGEokp1Pz5cOiQDdBojMmbJRJTqLg4aNsWBg70OhJjTFlkicQU6Icf4PPPnUt+q9hfizEmD/bRYAo0fbqTQMaP9zoSY0xZZYnE5Csjw5lOd+hQaNHC62iMMWWVJRKTryVLICnJOtmNMQWzRGLy5fNBs2Zw+eVeR2KMKcsskZg8JSXBxx87fSPVqnkdjTGmLLNEYvI0cyZkZsLEiV5HYowp6yyRmFxUnXtHLr4YzjnH62iMMWWdJRKTy+rVsGuXdbIbY4JjicTk4vNB/fpwzTVeR2KMKQ8skZgcDh6E996D66+H007zOhpjTHlgicTk8PbbcPKkNWsZY4JnicTkEBcH550HPXp4HYkxprywRGL8Nm2CzZvtbMQYUzSWSIyfzwc1azr9I8YYEyxLJAaA48dh9mwYORIaNPA6GmNMeRLWRCIisSLynYjsEpGpeWx/QUS2uI+dInIoYNszIrLNfYwKWD9IRDa562eKSNVwvofK4v334fBha9YyxhRd2BKJiEQArwJDgc7AGBHpHFhGVaeoapSqRgGvAB+4+14O9ACigN7AfSJST0SqADOB0araFfgZGBeu91CZ+HzQvr1zN7sxxhRFOM9IegG7VPVHVU0D5gIjCig/BpjjPu8MrFbVDFU9BmwFYoFGQJqq7nTLLQPstrkQ7dzp3M1+yy0g4nU0xpjyJpyJpAWwJ2A50V2Xi4i0BtoCn7urEoBYETlNRBoDA4FWwH6gqohEu+VGuuvzqvNWEYkXkfjk5OSQ30xFFhcHEREwzs7tjDHFUFY620cD76lqJoCqLgUWAWtxzlK+BDJVVd2yL4jIeuAIkJlXhar6mqpGq2p0kyZNSuM9lEvp6fDGGzBsGDRv7nU0xpjyKJyJZC85zxZauuvyMpo/mrUAUNUn3f6TGECAne76L1W1n6r2AlZnrzfF88kn8Ntv1slujCm+cCaSDUAHEWkrItVxksWHpxYSkY5AQ5yzjux1ESLSyH0eCUQCS93lpu7PGsCDwP+F8T1UeD4fnHkmxMZ6HYkxprwK26WzqpohIncBnwIRwHRV3S4ijwHxqpqdVEYDc91mq2zVgDXi9PweBm5U1Qx32/0iMgwnCU5T1c8xxZKYCIsXw9SpUNUuojbGFJPk/PyumKKjozU+Pt7rMMqcJ56Av/7VmXukXTuvozHGlDUislFVowsrV1Y6200py8qC6dNh0CBLIsaY0FgiqaRWrICffrJOdmNM6CyRVFI+HzRsCFdd5XUkxpjyzhJJJZSSAh98ADfe6Iz2a4wxobBEUgnNng1padasZYwpGZZIKhlVeP116NkTIiO9jsYYUxFYIqlkNmyAbdvsbMQYU3IskVQyPh+cdhqMHu11JMaYisISSSVy9CjMmQPXXQf16nkdjTGmorBEUom8+66TTKxZyxhTkiyRVCI+H3TsCBde6HUkxpiKxBJJJbFjB6xdCzffbLMgGmNKliWSSiIuzhnhd+xYryMxxlQ0lkgqgbQ0mDULRoyApk29jsYYU9FYIqkEPvwQ9u+3TnZjTHhYIqkEfD5o1QpiYryOxBhTEVkiqeB+/hmWLoUJEyAiwutojDEVkSWSCu6NN5yfEyZ4GoYxpgKzRFKBZWY6syDGxECbNl5HY4ypqCyRVGCffQb//a91shtjwssSSQXm80GjRjB8uNeRGGMqMkskFVRyMixc6NyAWKOG19EYYyoySyQV1JtvQnq6MySKMcaEkyWSCkjVada64ALo0sXraIwxFZ0lkgroyy/hm2/sbMQYUzoKTSQiMklEGpZGMKZk+HxQpw6MGuV1JMaYyiCYM5JmwAYRmScisSI2CHlZdvgwvPOOM5VunTpeR2OMqQwKTSSq+jDQAYgDxgPfi8jfRaRdmGMzxfDOO3D8uN07YowpPUH1kaiqAr+4jwygIfCeiPwjjLGZYvD5oGtX6NXL60iMMZVFMH0kd4vIRuAfwH+Abqp6O3A+cE2Y4zNF8PXXsH69zYJojCldVYMoczpwtar+HLhSVbNEZFh4wjLFERcH1avDjTd6HYkxpjIJpmlrMXAge0FE6olIbwBV/SZcgZmiOXnSuQnxqqugcWOvozHGVCbBJJJpwNGA5aPuOlOGLFgABw5YJ7sxpvQFk0jE7WwHnCYtgmsSM6XI53OGih80yOtIjDGVTTCJ5EcRmSwi1dzH3cCP4Q7MBO/HH2H5cpg4EarYWAXGmFIWzMfObcCFwF4gEegN3BrOoEzRzJjhJJDx472OxBhTGRXaRKWqvwGjSyEWUwwZGU4iiY2FVq28jsYYUxkVmkhEpCZwM9AFqJm9XlUnhjEuE6RPP4W9e+Hll72OxBhTWQXTtPUm0BwYAqwCWgJHwhmUCZ7PB02bwjC7o8cY45FgEkl7Vf0rcExVZwKX4/STFMod5PE7EdklIlPz2P6CiGxxHztF5FDAtmdEZJv7GBWwfrCIbHL3+UJE2gcTS0X0yy/w8ccwbpxzI6IxxnghmMt4092fh0SkK854W00L20lEIoBXgRicTvoNIvKhqu7ILqOqUwLKTwLOc59fDvQAooAawEoRWayqh3HuYRmhqt+IyB3AwziDSVY6s2Y5fSQ274gxxkvBnJG85s5H8jDwIbADeCaI/XoBu1T1R1VNA+YCIwooPwaY4z7vDKxW1QxVPQZsBWLdbQrUc5/XB/YFEUuFkz0LYr9+cO65XkdjjKnMCjwjEZEqwGFVPQisBs4uQt0tgD0By9mXDuf1Oq2BtsDn7qoE4BEReQ44DRiIk8AAbgEWicgJ4DDQJ586b8W9TPmss84qQtjlw5o18P338D//43UkxpjKrsAzEvcu9gdKIY7RwHuqmum+7lJgEbAW5yzlSyDTLTsFuExVWwIzgOfzqlBVX1PVaFWNbtKkSbjjL3VxcVCvHowc6XUkxpjKLpimrc9E5D4RaSUip2c/gthvLxB4Z0NLd11eRvNHsxYAqvqkqkapagwgwE4RaQJ0V9Wv3GLv4NwsWakcOgTvvgvXXw+1a3sdjTGmsgumsz37iqk7A9YphTdzbQA6iEhbnAQyGrj+1EIi0hFnoqwvA9ZFAA1UNUVEIoFIYKm7ub6InKOqO3E68ivdCMRz5sCJEzZAozGmbAjmzva2xalYVTNE5C7gUyACmK6q20XkMSBeVT90i44G5gYODAlUA9a408MfBm5U1QwAEfkT8L6IZAEHgUp3Y6TPB927Q48eXkdijDHB3dk+Nq/1qjqrsH1VdRFOX0fguv89Zflveex3EufKrbzqnA/ML+y1K6rNm2HTJnjlFZsF0RhTNgTTtNUz4HlNYDCwCSg0kZiSFxcHNWrADTd4HYkxxjiCadqaFLgsIg1w7gkxpezECXjrLedKrYYNvY7GGGMcxZm94hjOPR+mlL3/PqSm2p3sxpiyJZg+ko9wrtICJ/F0BuaFMyiTN58P2rWD/v29jsQYY/4QTB/JPwOeZwA/q2pimOIx+fj+e1i1Cv7+d5sF0RhTtgSTSP4LJLlXUiEitUSkjaruDmtkJofp0yEiwhnp1xhjypJgvtu+C2QFLGe660wpSU+HN96Ayy+HM8/0OhpjjMkpmERS1R29FwD3uc1+UYoWLXLmHrFOdmNMWRRMIkkWkeHZCyIyAtgfvpDMqeLi4Iwz4LLLvI7EGGNyC6aP5DZgtoj8y11OBPK8292UvL174ZNP4MEHoWowvy1jjCllwdyQ+APQR0TquMtHwx6V8Zs5E7KyYGKlG1HMGFNeFNq0JSJ/F5EGqnpUVY+KSEMReaI0gqvssrKcZq0BA6B9pZ2Z3hhT1gXTRzJUVQ9lL7izJVprfSlYuRJ+/NGGizfGlG3BJJIIEamRvSAitYAaBZQ3JSQuDho0gKuv9joSY4zJXzDdt7OB5SIyA2emwvHAzHAGZeDAAWdsrT/9CWrV8joaY4zJXzCd7c+ISAJwCc6YW58CrcMdWGU3ezb8/rvdO2KMKfuCHbXpV5wkci0wiEo4vW1pUnUGaDz/fIiK8joaY4wpWL5nJCJyDjDGfewH3gFEVQeWUmyV1saNsHUrTJvmdSTGGFO4gpq2vgXWAMNUdReAiEwplagqOZ/P6RcZM8brSIwxpnAFNW1dDSQBK0TkdREZjNPZbsLo2DF4+2247jqoX9/raIwxpnD5JhJVXaCqo4GOwArgHqCpiEwTkUtLK8DK5t134cgR62Q3xpQfhXa2q+oxVX1bVa8AWgKbgQfDHlklFRcH55wDfft6HYkxxgSnSHPtqepBVX1NVQeHK6DK7Ntv4YsvnDvZxRoRjTHlhE3aWobExTkj/I61sZWNMeWIJZIyIi3NGen3iiugWTOvozHGmOBZIikjPvoIkpNtgEZjTPljiaSMiIuDFi1gyBCvIzHGmKKxRFIG7NkDS5Y4k1dFRHgdjTHGFI0lkjJgxgxnfK0JE7yOxBhjis4SiceysmD6dLjkEmjb1utojDGm6CyReGz5cvj5Z+tkN8aUX5ZIPObzwemnw5VXeh2JMcYUjyUSD+3fD/PnOzcg1rDJi40x5ZQlEg+9+Sakp9sAjcaY8s0SiUdUnXtHeveGrl29jsYYY4rPEolHvvoKtm+3TnZjTPlnicQjPh/Urg2jRnkdiTHGhMYSiQeOHIG5c50kUreu19EYY0xoLJF44J13nCl1rVnLGFMRhDWRiEisiHwnIrtEZGoe218QkS3uY6eIHArY9oyIbHMfowLWrwnYZ5+ILAjnewiHuDjo3Bn69PE6EmOMCV3VcFUsIhHAq0AMkAhsEJEPVXVHdhlVnRJQfhJwnvv8cqAHEAXUAFaKyGJVPayq/QL2eR9YGK73EA7btsG6dfD88zYLojGmYgjnGUkvYJeq/qiqacBcYEQB5ccAc9znnYHVqpqhqseArUBsYGERqQcMAsrVGUlcHFSrBjfd5HUkxhhTMsKZSFoAewKWE911uYhIa6At8Lm7KgGIFZHTRKQxMBBodcpuVwLLVfVwPnXeKiLxIhKfnJwcwtsoOb//DrNmOcOhNG7sdTTGGFMyykpn+2jgPVXNBFDVpcAiYC3OWcqXQOYp+wSeweSiqq+parSqRjdp0iQ8URfRwoVw4IB1shtjKpZwJpK95DyLaOmuy8toTkkKqvqkqkapagwgwM7sbe5ZSi/gkxKNOMx8Pmjd2hky3hhjKopwJpINQAcRaSsi1XGSxYenFhKRjkBDnLOO7HURItLIfR4JRAJLA3YbCXysqifDGH+J+uknWLbMmQWxSlk5DzTGmBIQtqu2VDVDRO4CPgUigOmqul1EHgPiVTU7qYwG5qqqBuxeDVgjzmVNh4EbVTUjYPto4OlwxR4OM2Y4V2mNH+91JMYYU7Ik5+d3xRQdHa3x8fGevX5mJrRp4wzOuHixZ2EYY0yRiMhGVY0urJw1spSCpUshMdE62Y0xFZMlklLg80GTJnDFFV5HYowxJc8SSZj9+it8+KEzC2L16l5HY4wxJc8SSZjNmgUZGTYLojGm4rJEEkbZsyBedBF06uR1NMYYEx6WSMLoP/+B776zTnZjTMVmiSSMfD5n4qprr/U6EmOMCR9LJGGSmgrz5sGYMc6UusYYU1FZIgmTuXPhxAlr1jLGVHyWSMLE54PISIgu9J5QY4wp3yyRhMGWLRAf75yN2CyIxpiKzhJJGMTFQY0acMMNXkdijDHhZ4mkhJ04AW+9BVdfDaef7nU0xhgTfpZIStj8+XDokHWyG2MqD0skJczng7PPhgEDvI7EGGNKhyWSErRrF6xYYbMgGmMqF/u4K0HTpzsJxGZBNMZUJpZISkhGBrzxBlx2GbRo4XU0xhhTeiyRlJDFiyEpyTrZjTGVjyWSEuLzQbNmzhmJMcZUJlW9DqAiSEqCTz6B++6DatW8jsaUpPT0dBITEzl58qTXoRgTNjVr1qRly5ZUK+YHmCWSEjBzJmRm2iyIFVFiYiJ169alTZs2iI13YyogVSUlJYXExETatm1brDqsaStEqk6zVv/+0KGD19GYknby5EkaNWpkScRUWCJCo0aNQjrrtkQSolWr4IcfrJO9IrMkYiq6UP/GLZGEyOeD+vXhmmu8jsQYY7xhiSQEBw/Ce+85o/zWquV1NKYiSklJISoqiqioKJo3b06LFi38y2lpaUHVMWHCBL777rsCy7z66qvMnj27JEKuEP75z3/y9ttvA87vYPDgwXTo0IEhQ4aQmpqa5z5//vOf6dKlC506dWLKlCmoKgAbNmyga9eutG/fnilTpvjL51evqnLHHXfQvn17IiMj2bJli3+fuLg4OnToQIcOHXjrrbf86wcPHpxvXKVCVSv84/zzz9dw+Ne/VEF106awVG/KgB07dngdgt8jjzyizz77bK71WVlZmpmZ6UFE3kpPTw9LvWlpadqtWzfNyMhQVdUpU6b4j/vjjz+uDz30UK59Vq1apf369dPMzExNT0/Xnj176po1a1RVtUePHrp+/XrNysrSmJgYXbp0aYH1Lly4UIcNG6aqqmvWrNELL7xQVVWTk5O1bdu2evDgQd2/f7+2adNGDx06pKqqPp9Pn3766ZDed15/60C8BvEZa2ckxaQKr78OPXrAeed5HY0pDffc4wzGWZKPe+4pXiy7du2ic+fO3HDDDXTp0oWkpCRuvfVWoqOj6dKlC4899pi/bN++fdmyZQsZGRk0aNCAqVOn0r17dy644AJ+++03AB5++GFefPFFf/mpU6fSq1cvzj33XNauXQvAsWPHuOaaa+jcuTMjR44kOjo6x7flbI888gg9e/aka9eu3Hbbbf5v5jt37mTQoEF0796dHj16sHv3bgD+/ve/061bN7p3787//M//5IgZ4JdffqF9+/YA+Hw+rrzySgYOHMiQIUM4fPgwgwYNokePHkRGRvLxxx/745gxYwaRkZF0796dCRMmkJqaytlnn01GRgYABw8ezLGcbdmyZfTq1YuIiAgAFi5cyLhx4wAYN24cCxYsyPWeRYSTJ0+SlpbG77//TkZGBk2bNmXPnj2cPHmSnj17IiLcdNNN/v3zq3fhwoWMHTvWfxx++eUXkpOTWbx4MUOHDqVBgwY0atSIQYMGsXTpUgBGjBjhP4PygiWSYtq0CRIS7JJf451vv/2WKVOmsGPHDlq0aMHTTz9NfHw8CQkJLFu2jB07duTaJzU1lf79+5OQkMAFF1zA9OnT86xbVVm/fj3PPvusPym98sorNG/enB07dvDXv/6VzZs357nv3XffzYYNG/j6669JTU1lyZIlAIwZM4YpU6aQkJDA2rVradq0KR999BGLFy9m/fr1JCQkcO+99xb6vjdv3swHH3zA8uXLqVWrFgsWLGDTpk189tln/qajhIQEnnnmGVauXElCQgLPPfcc9evX56KLLvLHM2fOHK699lqqVs15F8R//vMfzj//fP9ySkoKTZo0AaBFixYkJSXliqlfv35ceOGFNG/enDPPPJMrrriCc845h71799KqVSt/uZYtW7J3794C681vn4Lqaty4MUeOHOHQoUOFHr9wsPtIisnng5o14frrvY7ElBb3C3uZ0a5dO6Kjo/3Lc+bMIS4ujoyMDPbt28eOHTvo3Llzjn1q1arF0KFDATj//PNZs2ZNnnVfffXV/jLZZw5ffPEFDz74IADdu3enS5cuee67fPlynn32WU6ePMn+/fs5//zz6dOnD/v37+eKK64AnBvgAD777DMmTpxILbeT8fQgZoO79NJLadiwIeAkvKlTp/LFF19QpUoV9uzZw/79+/n8888ZNWqUv77sn7fccgsvv/wyw4YNY8aMGbz55pu56k9KSuK8IjYzfPfdd/zwww/s3buXzMxMLrnkEoYMGUKVUhwGvEmTJiQlJdGgQYNSe81sdkZSDMePw9tvw7XXgge/M2MAqF27tv/5999/z0svvcTnn3/O1q1biY2NzfO+gOrVq/ufR0RE5GrWyVajRo1Cy+Tl+PHj3HXXXcyfP5+tW7cyceLEYt2fULVqVbKysgBy7R/4vmfNmkVqaiqbNm1iy5YtNG7cuMDX69+/Pzt37mTFihVUq1aNjh075ipTq1atHHU0atSI5ORkwDlbOOOMM3Lt88EHH3DhhRdSu3Zt6tWrR2xsLOvWraNFixbs2bPHXy4xMZEW7qiu+dWb3z4F1ZV9nGp5dNUZFe9YAAAU10lEQVSPJZJieO89OHzY7h0xZcfhw4epW7cu9erVIykpiU8//bTEX+Oiiy5i3rx5AHz99dd5Np2dOHGCKlWq+Jta3n//fQAaNmxIkyZN+OijjwDnQ+/48ePExMQwffp0Tpw4AcCBAwcAaNOmDRs3bgTgvffeyzem1NRUmjZtStWqVVm2bJm/qWfQoEG88847/vqyfwLceOON3HDDDUyYMCHPOjt16sSuXbv8y8OHD2fmzJkAzJw5kxEjRuTa56yzzmLVqlVkZGSQnp7OqlWr6NSpE61ataJGjRps2LABVeXNN9/0759fvcOHD2fWrFmAcxbYrFkzmjRpQmxsLIsXL+bQoUOkpKSwfPlyLr30UgAyMzPZv38/Z511Vr7HKpwskRSDz+fcxd6vn9eRGOPo0aMHnTt3pmPHjowdO5aLLrqoxF9j0qRJ7N27l86dO/Poo4/SuXNn6tevn6NMo0aNGDduHJ07d2bo0KH07t3bv2327Nk899xzREZG0rdvX5KTkxk2bBixsbFER0cTFRXFCy+8AMD999/PSy+9RI8ePTh48GC+Md10002sXbuWbt26MXfuXDq4w0t0796dBx54gIsvvpioqCjuv/9+/z433HADqampjBo1Ks86L7vsMlatWuVffuihh/jkk0/o0KEDq1ev9tf11VdfcdtttwEwevRoWrVq5e/c79Wrl78Jcdq0aYwfP5727dvTqVMn/4d/fvVeccUVtGjRgnbt2nH77bfz6quvAk7T1V/+8heio6Pp3bs3jz32mP/4b9iwgb59+5ZqU1oOwVzaVd4fJXn577ffOpf8hnilnSknytLlv15LT0/XEydOqKrqzp07tU2bNmG7BDec5syZo+PHjy+wzBVXXKE//PBDKUUUujvuuENXrlwZUh2hXP5rne1FNH06RESAe9WeMZXG0aNHGTx4MBkZGagq//73v3Nd8VTW3X777Xz22Wf+K7fy88wzz7Bv3z7OPvvsUoosNOeddx79+/f37PVF3Wu8K7Lo6GiNj48PuZ70dGjZEi68EObPL4HATJn3zTff0KlTJ6/DMCbs8vpbF5GNqhqdzy5+1kdSBB9/DL/9Zp3sxhgTyBJJEfh8cOaZMGSI15EYY0zZYYkkSImJsGQJTJgA5axZ2BhjwiqsiUREYkXkOxHZJSJT89j+gohscR87ReRQwLZnRGSb+xgVsF5E5Em3/DciMjmc7yHbG29AVhZMnFgar2aMMeVH2BKJiEQArwJDgc7AGBHJMV6Dqk5R1ShVjQJeAT5w970c6AFEAb2B+0SknrvbeKAV0FFVOwFzw/UesmVlQVwcDB4M5eQiDlNBDBw4MNfNhS+++CK33357gfvVqVMHgH379jFy5Mg8ywwYMIDCLkJ58cUXOX78uH/5sssu82w8p7Jm8+bN3OwOtqeqTJ482T/0+6ZNm/LcZ86cOXTr1o3IyEhiY2PZv38/4NwwGRMTQ4cOHYiJifHfO1NQvTNnzvQPKZ99YyPAxo0b6datG+3bt2fy5Mn+QTPvu+8+Pv/887Aci7DduwFcAHwasPwX4C8FlF8LxLjP7wf+GrAtDrjOfb4eaF+UWEK9j2TZMufekbffDqkaUw55fR/Jv//971z3PPTu3VtXrVpV4H61a9cutO7+/fvrhg0bCizTunVrTU5OLjzQMiqcQ+yPHDlSt2zZoqqqn3zyicbGxmpWVpZ++eWX2qtXr1zl09PTtUmTJv7jef/99+sjjzzif/7UU0+pqupTTz2lDzzwQIH1pqSkaNu2bTUlJUUPHDigbdu21QMHDqiqas+ePfXLL7/UrKwsjY2N1UWLFqmq6u7duzUmJibf91NW7yNpAewJWE7EObvIRURaA22B7HSZADwiIs8BpwEDgezxGNoBo0TkKiAZmKyq3+dR563ArUDIwwb4fNCwIVx1VUjVmHLuniX3sOWX3MOmhyKqeRQvxuY/GuTIkSN5+OGHSUtLo3r16uzevZt9+/bRr18/jh49yogRIzh48CDp6ek88cQTuYbv2L17N8OGDWPbtm2cOHGCCRMmkJCQQMeOHf3DkoBzf8WGDRs4ceIEI0eO5NFHH+Xll19m3759DBw4kMaNG7NixQratGlDfHw8jRs35vnnn/ePHnzLLbdwzz33sHv3boYOHUrfvn1Zu3YtLVq0YOHChbnGgProo4944oknSEtLo1GjRsyePZtmzZpx9OhRJk2aRHx8PCLCI488wjXXXMOSJUt46KGHyMzMpHHjxixfvpy//e1v1KlTh/vuuw+Arl27+oeRHzJkCL1792bjxo0sWrSIp59+Otf7A+eO8Lvvvptjx45Ro0YNli9fzuWXX87LL79MVFQU4Azl/uqrr9K9e3d//EeOHGHr1q3+ddlDv4sIffr04dChQyQlJeUYlyv7Q/fYsWM0atSIw4cP+4fHX7hwIStXrgScIeUHDBjAM888k2+9K1euJCYmxj8YZUxMDEuWLGHAgAEcPnyYPn36ADB27FgWLFjA0KFDad26NSkpKfzyyy80b9680L/Noigrne2jgfdUNRNAVZcCi3DOUuYAXwKZbtkawEl1rm1+HchzHGxVfU1Vo1U1Onuo5uJISXHuGbnpJme0X2NK0+mnn06vXr1YvHgxAHPnzuW6665DRKhZsybz589n06ZNrFixgnvvvdffjJGXadOmcdppp/HNN9/w6KOP+seyAnjyySeJj49n69atrFq1iq1btzJ58mTOPPNMVqxYwYoVK3LUtXHjRmbMmMFXX33FunXreP311/3Dyn///ffceeedbN++nQYNGvjH2wrUt29f1q1bx+bNmxk9ejT/+Mc/AHj88cepX78+X3/9NVu3bmXQoEEkJyfzpz/9iffff5+EhATefffdQo/b999/zx133MH27dtp3bp1nu8vLS2NUaNG8dJLL5GQkMBnn31GrVq1uPnmm3njjTcAZw6VkydP5kgiAPHx8XTt2tW/XNAQ79mqVavGtGnT6NatG2eeeSY7duzwN439+uuv/qTTvHlzfv311wLrLWh9y5Yt842jR48e/Oc//yn0+BVVOM9I9uL0ZWRr6a7Ly2jgzsAVqvok8CSAiLwN7HQ3JeL2pQDzgRklFG+e3noL0tJs3hFDgWcO4TRmzBjmzp3LiBEjmDt3LnFxcYDzDfehhx5i9erVVKlShb179/Lrr7/m+21z9erVTJ7sXJsSGRlJZGSkf9u8efN47bXXyMjIICkpiR07duTYfqovvviCq666yj8S79VXX82aNWsYPnw4bdu29X+bDxyGPlBiYiKjRo0iKSmJtLQ02rZtCzjDys+d+0e3Z8OGDfnoo4+4+OKL/WWCGWq+devW/m/l+b0/EeGMM86gZ8+eANSr53TDXnvttTz++OM8++yzTJ8+nfHjx+eqPykpiaJ+QU1PT2fatGls3ryZs88+m0mTJvHUU0/x8MMP5ygnIohIkeoOVtOmTdm3b1+J1xvOM5INQAcRaSsi1XGSxYenFhKRjkBDnLOO7HURItLIfR4JRAJL3c0LcJq6APrzR4IpcapOs1bPnlDA/5QxYTVixAiWL1/Opk2bOH78uH/SpdmzZ5OcnMzGjRvZsmULzZo1K9aQ7T/99BP//Oc/Wb58OVu3buXyyy8vVj3Zsoegh/yHoZ80aRJ33XUXX3/9Nf/+979DHmoecg43HzjUfFHf32mnnUZMTAwLFy5k3rx53HDDDbnKnDrUfGFDvAP+GR/btWuHiHDdddf5Z59s1qyZf2KrpKQkmjZtWmC9Ba1PTEzMN45wDTUftkSiqhnAXcCnwDfAPFXdLiKPicjwgKKjgbma85y8GrBGRHYArwE3uvUBPA1cIyJfA08BYbvPfP162LbN7mQ33qpTpw4DBw5k4sSJjBkzxr8+ewj1atWqsWLFCn7++ecC67n44ov907Fu27aNrVu3As4Q9LVr16Z+/fr8+uuv/mY0gLp163LkyJFcdfXr148FCxZw/Phxjh07xvz58+lXhOGwU1NT/R9wgVccxcTE+Ee7BWc63D59+rB69Wp++uknIOdQ89lXMW3atMm//VT5vb9zzz2XpKQkNmzYADj9HtlJ75ZbbmHy5Mn07NnTP4lWoLyGmp81axaqyrp166hfv36ueUtatGjBjh07/HOQLFu2zD8kSWFDyp9a75AhQ1i6dCkHDx7k4MGDLF26lCFDhnDGGWdQr1491q1bh6oya9asHP1mO3fuzNEkV2KC6ZEv74/iXrX1pz+pnnaaampqsXY3FYDXV21lmz9/vgL6zTff+NclJydrnz59tGvXrjp+/Hjt2LGj/vTTT6r6x1VbP/30k3bp0kVVVY8fP66jRo3Sjh076lVXXaW9evXyX7U1btw47dChgw4aNEivuuoqnTFjhqqqvvzyy3rOOefogAEDVDXnVVzPPfecdunSRbt06aIvvPBCrtdTVX322Wf9VyYFWrBggbZt21Z79Oih9913n/bv319VVY8cOaJjx47VLl26aGRkpL7//vuqqrpo0SKNiorSyMhIveSSS/zvJyYmRjt37qwTJkzwv/9TYyjo/a1fv1579+6tkZGR2rt3bz1y5Ih/n3PPPVcXL16c7++ka9euevjwYVV1rg6744479Oyzz9auXbvmuBque/fu/ufTpk3Tjh07ardu3XTYsGG6f/9+VVXdv3+/Dho0SNu3b6+DBw/WlJSUQuuNi4vTdu3aabt27XT69On+9Rs2bNAuXbro2WefrXfeeadmZWWpqmpaWpp27Ngx3xGbQ7lqywZtLMAzz8ChQ/DUU2EIypQLNmhj5bRv3z4GDBjAt99+m+8cHy+88AJ169bllnLSZJF9Ycbjjz+e53YbtDFMHnzQkogxlc2sWbPo3bs3Tz75ZIETRd1+++05+oPKuoyMDO69996w1G1nJMYUwM5ITGVhZyTGhFFl+LJlKrdQ/8YtkRhTgJo1a5KSkmLJxFRYqkpKSgo1Q7jj2gZEN6YALVu2JDEx0X/JpjEVUc2aNXPcEV9UlkiMKUC1atX8d1QbY/JmTVvGGGNCYonEGGNMSCyRGGOMCUmluI9ERJKBggciyl9jYH8JhlNSLK6isbiKxuIqmooaV2tVLXSY40qRSEIhIvHB3JBT2iyuorG4isbiKprKHpc1bRljjAmJJRJjjDEhsURSuNe8DiAfFlfRWFxFY3EVTaWOy/pIjDHGhMTOSIwxxoTEEokxxpiQWCIBRGS6iPwmItvy2S4i8rKI7BKRrSLSo4zENUBEUkVki/v431KKq5WIrBCRHSKyXUTuzqNMqR+zIOMq9WMmIjVFZL2IJLhxPZpHmRoi8o57vL4SkTZlJK7xIpIccLxKbTpAEYkQkc0i8nEe20r9eAUZlyfHS0R2i8jX7mvmmnwp7P+PwczHW9EfwMVAD2BbPtsvAxYDAvQBviojcQ0APvbgeJ0B9HCf1wV2Ap29PmZBxlXqx8w9BnXc59WAr4A+p5S5A/g/9/lo4J0yEtd44F+l/Tfmvvafgbfz+n15cbyCjMuT4wXsBhoXsD2s/492RgKo6mrgQAFFRgCz1LEOaCAiZ5SBuDyhqkmqusl9fgT4BmhxSrFSP2ZBxlXq3GNw1F2s5j5OvcplBDDTff4eMFhEpAzE5QkRaQlcDvjyKVLqxyvIuMqqsP4/WiIJTgtgT8ByImXgA8p1gds0sVhEupT2i7tNCufhfJsN5OkxKyAu8OCYuc0hW4DfgGWqmu/xUtUMIBVoVAbiArjGbQ55T0RahTsm14vAA0BWPts9OV5BxAXeHC8FlorIRhG5NY/tYf1/tERSvm3CGQunO/AKsKA0X1xE6gDvA/eo6uHSfO2CFBKXJ8dMVTNVNQpoCfQSka6l8bqFCSKuj4A2qhoJLOOPs4CwEZFhwG+qujHcr1UUQcZV6sfL1VdVewBDgTtF5OJSel3AEkmw9gKB3yxauus8paqHs5smVHURUE1EGpfGa4tINZwP69mq+kEeRTw5ZoXF5eUxc1/zELACiD1lk/94iUhVoD6Q4nVcqpqiqr+7iz7g/FII5yJguIjsBuYCg0TkrVPKeHG8Co3Lo+OFqu51f/4GzAd6nVIkrP+PlkiC8yEw1r3yoQ+QqqpJXgclIs2z24VFpBfO7zPsHz7ua8YB36jq8/kUK/VjFkxcXhwzEWkiIg3c57WAGODbU4p9CIxzn48EPle3l9TLuE5pRx+O0+8UVqr6F1VtqaptcDrSP1fVG08pVurHK5i4vDheIlJbROpmPwcuBU690jOs/4821S4gInNwruZpLCKJwCM4HY+o6v8Bi3CuetgFHAcmlJG4RgK3i0gGcAIYHe5/JtdFwE3A1277OsBDwFkBsXlxzIKJy4tjdgYwU0QicBLXPFX9WEQeA+JV9UOcBPimiOzCucBidJhjCjauySIyHMhw4xpfCnHlqQwcr2Di8uJ4NQPmu9+PqgJvq+oSEbkNSuf/0YZIMcYYExJr2jLGGBMSSyTGGGNCYonEGGNMSCyRGGOMCYklEmOMMSGxRGJMMYlIZsAor1tEZGoJ1t1G8hn12Ziyxu4jMab4TrjDixhTqdkZiTElzJ0b4h/u/BDrRaS9u76NiHzuDui3XETOctc3E5H57kCSCSJyoVtVhIi8Ls5cIUvdu88RkcnizLmyVUTmevQ2jfGzRGJM8dU6pWlrVMC2VFXtBvwLZ8RYcAaJnOkO6DcbeNld/zKwyh1Isgew3V3fAXhVVbsAh4Br3PVTgfPcem4L15szJlh2Z7sxxSQiR1W1Th7rdwODVPVHdxDJX1S1kYjsB85Q1XR3fZKqNhaRZKBlwGB/2cPgL1PVDu7yg0A1VX1CRJYAR3FGLl4QMKeIMZ6wMxJjwkPzeV4Uvwc8z+SPPs3LgVdxzl42uKPfGuMZSyTGhMeogJ9fus/X8sfggjcAa9zny4HbwT/RVP38KhWRKkArVV0BPIgzfHqusyJjSpN9kzGm+GoFjDIMsERVsy8BbigiW3HOKsa46yYBM0TkfiCZP0ZgvRt4TURuxjnzuB3Ib4jvCOAtN9kI8LI7l4gxnrE+EmNKmNtHEq2q+72OxZjSYE1bxhhjQmJnJMYYY0JiZyTGGGNCYonEGGNMSCyRGGOMCYklEmOMMSGxRGKMMSYk/x+qvYk5NFpS8gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f270d4147f0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_history(m1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "def plot_history(history):\n",
    "    loss_list = [s for s in history.history.keys() if 'loss' in s and 'val' not in s]\n",
    "    val_loss_list = [s for s in history.history.keys() if 'loss' in s and 'val' in s]\n",
    "    acc_list = [s for s in history.history.keys() if 'acc' in s and 'val' not in s]\n",
    "    val_acc_list = [s for s in history.history.keys() if 'acc' in s and 'val' in s]\n",
    "    \n",
    "    if len(loss_list) == 0:\n",
    "        print('Loss is missing in history')\n",
    "        return \n",
    "    \n",
    "    ## As loss always exists\n",
    "    epochs = range(1,len(history.history[loss_list[0]]) + 1)\n",
    "    \n",
    "    ## Loss\n",
    "    plt.figure(1)\n",
    "    for l in loss_list:\n",
    "        plt.plot(epochs, history.history[l], 'b', label='Training loss (' + str(str(format(history.history[l][-1],'.5f'))+')'))\n",
    "    for l in val_loss_list:\n",
    "        plt.plot(epochs, history.history[l], 'g', label='Validation loss (' + str(str(format(history.history[l][-1],'.5f'))+')'))\n",
    "    \n",
    "    #plt.xlim(xmin=0)\n",
    "    plt.ylim(ymin=0)\n",
    "    plt.title('Loss')\n",
    "    plt.xlabel('Epochs')\n",
    "    plt.ylabel('Loss')\n",
    "    plt.legend()\n",
    "    \n",
    "    ## Accuracy\n",
    "    plt.figure(2)\n",
    "    for l in acc_list:\n",
    "        plt.plot(epochs, history.history[l], 'b', label='Training accuracy (' + str(format(history.history[l][-1],'.5f'))+')')\n",
    "    for l in val_acc_list:    \n",
    "        plt.plot(epochs, history.history[l], 'g', label='Validation accuracy (' + str(format(history.history[l][-1],'.5f'))+')')\n",
    "\n",
    "    #plt.ylim(0, 1)    \n",
    "    plt.title('Accuracy')\n",
    "    plt.xlabel('Epochs')\n",
    "    plt.ylabel('Accuracy')\n",
    "    plt.legend()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "if tf.test.gpu_device_name():\n",
    "     print(\"Default GPU Device: {}\".format(tf.test.gpu_device_name()))\n",
    "else:\n",
    "     print(\"Please install GPU version of TF\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}