3-2. VGG16 Triplet Network (margin 0.3).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.simplefilter('ignore')\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import sys, os, time, gc, random, pickle\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "import requests, shutil\n",
    "import tensorflow as tf\n",
    "import keras\n",
    "\n",
    "from keras import backend as K\n",
    "from keras.preprocessing.image import load_img, img_to_array\n",
    "from keras.models import Sequential, load_model, Model\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten, Input, Lambda, MaxPooling2D\n",
    "from keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau\n",
    "from keras.applications.vgg16 import VGG16\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train:\t\t (113783, 4)\n",
      "Validation:\t (22255, 4)\n",
      "Test:\t\t (22391, 4)\n",
      "\n",
      "Train Landmarks:\t 14943\n",
      "Validation Landmarks:\t 7674\n",
      "Test Landmarks:\t\t 14436\n"
     ]
    }
   ],
   "source": [
    "train_df = pd.read_csv('./data/triplet/train.csv')\n",
    "val_df = pd.read_csv('./data/triplet/validation.csv')\n",
    "test_df = pd.read_csv('./data/triplet/test.csv')\n",
    "\n",
    "print('Train:\\t\\t', train_df.shape)\n",
    "print('Validation:\\t', val_df.shape)\n",
    "print('Test:\\t\\t', test_df.shape)\n",
    "\n",
    "print('\\nTrain Landmarks:\\t', len(train_df['landmark_id'].unique()))\n",
    "print('Validation Landmarks:\\t', len(val_df['landmark_id'].unique()))\n",
    "print('Test Landmarks:\\t\\t', len(test_df['landmark_id'].unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id</th>\n",
       "      <th>id</th>\n",
       "      <th>url</th>\n",
       "      <th>landmark_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>465272</td>\n",
       "      <td>a2ccf8ed2e969f6a</td>\n",
       "      <td>https://lh4.googleusercontent.com/-TPHkS5gzvm4...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>64516</td>\n",
       "      <td>e205ca7c8dd7c027</td>\n",
       "      <td>https://lh3.googleusercontent.com/-V3RjsZtGpxE...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>928409</td>\n",
       "      <td>4e8ab93c1620e8a3</td>\n",
       "      <td>http://mw2.google.com/mw-panoramio/photos/medi...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>88809</td>\n",
       "      <td>896bf928214d1ca4</td>\n",
       "      <td>http://lh5.ggpht.com/-Cy0l41uUaGA/R--yB8vy41I/...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1001133</td>\n",
       "      <td>375d2a153bdca926</td>\n",
       "      <td>http://lh6.ggpht.com/-UqzFpnqE9bU/S_0u1RovfdI/...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   image_id                id  \\\n",
       "0    465272  a2ccf8ed2e969f6a   \n",
       "1     64516  e205ca7c8dd7c027   \n",
       "2    928409  4e8ab93c1620e8a3   \n",
       "3     88809  896bf928214d1ca4   \n",
       "4   1001133  375d2a153bdca926   \n",
       "\n",
       "                                                 url  landmark_id  \n",
       "0  https://lh4.googleusercontent.com/-TPHkS5gzvm4...            0  \n",
       "1  https://lh3.googleusercontent.com/-V3RjsZtGpxE...            0  \n",
       "2  http://mw2.google.com/mw-panoramio/photos/medi...            0  \n",
       "3  http://lh5.ggpht.com/-Cy0l41uUaGA/R--yB8vy41I/...            0  \n",
       "4  http://lh6.ggpht.com/-UqzFpnqE9bU/S_0u1RovfdI/...            0  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Helper Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# training set triplet generator\n",
    "def train_triplet_generator(df, batch_size=100, img_size=(224, 224), seed=42, \n",
    "                            prefix='./data/triplet/train/'):\n",
    "    \"\"\" training set triplet generator\n",
    "        it will generate 7400 triplet images in total\n",
    "    \"\"\"\n",
    "    # get images with only one training image landmark id and the rest landmark ids\n",
    "    np.random.seed(seed)\n",
    "    grouped = df[['landmark_id', 'image_id']].groupby('landmark_id').count().reset_index()\n",
    "    unique_neg_ids = list(grouped[grouped['image_id'] == 1]['landmark_id'].values)\n",
    "    rest_ids = list(grouped[grouped['image_id'] > 1]['landmark_id'].values)\n",
    "    size = 7400 * 2 - len(unique_neg_ids) \n",
    "    zeros = np.zeros((batch_size, 3, 1), dtype=K.floatx())\n",
    "    \n",
    "    while True:\n",
    "        # get positive and negative image landmark ids\n",
    "        np.random.shuffle(rest_ids)\n",
    "        candidate_ids = list(np.random.choice(rest_ids, size=size, replace=False))\n",
    "        pos_landmark_ids = candidate_ids[:7400]\n",
    "        neg_landmark_ids = candidate_ids[7400:] + unique_neg_ids\n",
    "        np.random.shuffle(neg_landmark_ids)\n",
    "        \n",
    "        # transform landmark id into image id\n",
    "        anc_img_ids = []\n",
    "        pos_img_ids = []\n",
    "        neg_img_ids = []\n",
    "        \n",
    "        for i in range(len(pos_landmark_ids)):\n",
    "            tmp_pos_ids = df[df['landmark_id'] == pos_landmark_ids[i]]['image_id'].values\n",
    "            anc_img_ids.append(tmp_pos_ids[0])\n",
    "            pos_img_ids.append(tmp_pos_ids[1])\n",
    "            \n",
    "            tmp_neg_ids = df[df['landmark_id'] == neg_landmark_ids[i]]['image_id'].values\n",
    "            neg_img_ids.append(tmp_neg_ids[0])\n",
    "        \n",
    "        # iterator to read batch images\n",
    "        for j in range(len(pos_img_ids) // batch_size):\n",
    "            batch_anc_img_ids = anc_img_ids[j * batch_size: (j + 1) * batch_size]\n",
    "            batch_pos_img_ids = pos_img_ids[j * batch_size: (j + 1) * batch_size]\n",
    "            batch_neg_img_ids = neg_img_ids[j * batch_size: (j + 1) * batch_size]\n",
    "            \n",
    "            # get images\n",
    "            anc_imgs = []\n",
    "            pos_imgs = []\n",
    "            neg_imgs = []\n",
    "            \n",
    "            # iteratively read images\n",
    "            for k in range(batch_size):\n",
    "                anc_path = prefix + str(batch_anc_img_ids[k]) + '.jpg'\n",
    "                pos_path = prefix + str(batch_pos_img_ids[k]) + '.jpg'\n",
    "                neg_path = prefix + str(batch_neg_img_ids[k]) + '.jpg'\n",
    "                \n",
    "                tmp_anc_img = load_img(anc_path, target_size=img_size)\n",
    "                tmp_anc_img = img_to_array(tmp_anc_img)\n",
    "                anc_imgs.append(tmp_anc_img)\n",
    "                \n",
    "                tmp_pos_img = load_img(pos_path, target_size=img_size)\n",
    "                tmp_pos_img = img_to_array(tmp_pos_img)\n",
    "                pos_imgs.append(tmp_pos_img)\n",
    "                \n",
    "                tmp_neg_img = load_img(neg_path, target_size=img_size)\n",
    "                tmp_neg_img = img_to_array(tmp_neg_img)\n",
    "                neg_imgs.append(tmp_neg_img)\n",
    "        \n",
    "            # transform list to array\n",
    "            anc_imgs = np.array(anc_imgs, dtype=K.floatx()) / 255.0\n",
    "            pos_imgs = np.array(pos_imgs, dtype=K.floatx()) / 255.0\n",
    "            neg_imgs = np.array(neg_imgs, dtype=K.floatx()) / 255.0\n",
    "\n",
    "            yield [anc_imgs, pos_imgs, neg_imgs], zeros"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# validation set triplet generator\n",
    "def val_triplet_generator(df, batch_size=128, img_size=(224, 224), \n",
    "                          seed=42, prefix='./data/triplet/validation'):\n",
    "    \"\"\" validation set triplet collector \"\"\"\n",
    "    \n",
    "     # get images with only one image landmark id and the rest landmark ids\n",
    "    grouped = df[['landmark_id', 'image_id']].groupby('landmark_id').count().reset_index()\n",
    "    unique_neg_ids = list(grouped[grouped['image_id'] == 1]['landmark_id'].values)\n",
    "    rest_ids = list(grouped[grouped['image_id'] > 1]['landmark_id'].values)\n",
    "    size = 3072 * 2 - len(unique_neg_ids) \n",
    "    zeros = np.zeros((batch_size, 3, 1), dtype=K.floatx())\n",
    "    \n",
    "    while True:\n",
    "        # get positive and negative image landmark ids\n",
    "        np.random.seed(seed)\n",
    "        candidate_ids = list(np.random.choice(rest_ids, size=size, replace=False))\n",
    "        pos_landmark_ids = candidate_ids[:3072]\n",
    "        neg_landmark_ids = candidate_ids[3072:] + unique_neg_ids\n",
    "        np.random.shuffle(neg_landmark_ids)\n",
    "        \n",
    "        # transform landmark id into image id\n",
    "        anc_img_ids = []\n",
    "        pos_img_ids = []\n",
    "        neg_img_ids = []\n",
    "        \n",
    "        for i in range(len(pos_landmark_ids)):\n",
    "            tmp_pos_ids = df[df['landmark_id'] == pos_landmark_ids[i]]['image_id'].values\n",
    "            anc_img_ids.append(tmp_pos_ids[0])\n",
    "            pos_img_ids.append(tmp_pos_ids[1])\n",
    "            \n",
    "            tmp_neg_ids = df[df['landmark_id'] == neg_landmark_ids[i]]['image_id'].values\n",
    "            neg_img_ids.append(tmp_neg_ids[0])\n",
    "        \n",
    "        # iterator to read batch images\n",
    "        for j in range(len(pos_img_ids) // batch_size):\n",
    "            batch_anc_img_ids = anc_img_ids[j * batch_size: (j + 1) * batch_size]\n",
    "            batch_pos_img_ids = pos_img_ids[j * batch_size: (j + 1) * batch_size]\n",
    "            batch_neg_img_ids = neg_img_ids[j * batch_size: (j + 1) * batch_size]\n",
    "            \n",
    "            # get images\n",
    "            anc_imgs = []\n",
    "            pos_imgs = []\n",
    "            neg_imgs = []\n",
    "            \n",
    "            # iteratively read images\n",
    "            for k in range(batch_size):\n",
    "                anc_path = prefix + str(batch_anc_img_ids[k]) + '.jpg'\n",
    "                pos_path = prefix + str(batch_pos_img_ids[k]) + '.jpg'\n",
    "                neg_path = prefix + str(batch_neg_img_ids[k]) + '.jpg'\n",
    "                \n",
    "                tmp_anc_img = load_img(anc_path, target_size=img_size)\n",
    "                tmp_anc_img = img_to_array(tmp_anc_img)\n",
    "                anc_imgs.append(tmp_anc_img)\n",
    "                \n",
    "                tmp_pos_img = load_img(pos_path, target_size=img_size)\n",
    "                tmp_pos_img = img_to_array(tmp_pos_img)\n",
    "                pos_imgs.append(tmp_pos_img)\n",
    "                \n",
    "                tmp_neg_img = load_img(neg_path, target_size=img_size)\n",
    "                tmp_neg_img = img_to_array(tmp_neg_img)\n",
    "                neg_imgs.append(tmp_neg_img)\n",
    "        \n",
    "            # transform list to array\n",
    "            anc_imgs = np.array(anc_imgs, dtype=K.floatx()) / 255.0\n",
    "            pos_imgs = np.array(pos_imgs, dtype=K.floatx()) / 255.0\n",
    "            neg_imgs = np.array(neg_imgs, dtype=K.floatx()) / 255.0\n",
    "            \n",
    "            yield [anc_imgs, pos_imgs, neg_imgs], zeros"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define Triplet Loss Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define base network for triplet network\n",
    "def base_net(input_shape=(224, 224, 3)):\n",
    "    \"\"\" define triplet network \"\"\"\n",
    "    # load pre-trained VGG16 model\n",
    "    vgg16 = VGG16(include_top=False, weights='imagenet', input_shape=input_shape, pooling='avg')\n",
    "    \n",
    "    # frozen shallow layers\n",
    "    vgg16.trainable = True\n",
    "    \n",
    "    set_trainable = False\n",
    "    for layer in vgg16.layers:\n",
    "        if layer.name == 'block5_conv1':\n",
    "            set_trainable = True\n",
    "        if set_trainable:\n",
    "            layer.trainable = True\n",
    "        else:\n",
    "            layer.trainable = False\n",
    "    \n",
    "    # define sequential model\n",
    "    model = Sequential(name='base_net')\n",
    "    model.add(vgg16)\n",
    "    model.add(Lambda(lambda x: K.l2_normalize(x, axis=1), name='l2_norm'))\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define triplet network\n",
    "def triplet_net(base_model, input_shape=(224, 224, 3)):\n",
    "    \"\"\" function to define triplet networks \"\"\"\n",
    "    # define input: anchor, positive, negative\n",
    "    anchor = Input(shape=input_shape, name='anchor_input')\n",
    "    positive = Input(shape=input_shape, name='positive_input')\n",
    "    negative = Input(shape=input_shape, name='negative_input')\n",
    "    \n",
    "    # extract vector represent using CNN based model\n",
    "    anc_vec = base_model(anchor)\n",
    "    pos_vec = base_model(positive)\n",
    "    neg_vec = base_model(negative)\n",
    "    \n",
    "    # stack outputs\n",
    "    stacks = Lambda(lambda x: K.stack(x, axis=1), name='output')([anc_vec, pos_vec, neg_vec])\n",
    "\n",
    "    # define inputs and outputs\n",
    "    inputs=[anchor, positive, negative]\n",
    "    outputs = stacks\n",
    "    \n",
    "    # define the triplet model\n",
    "    model = Model(inputs=inputs, outputs=outputs, name='triplet_net')\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define triplet loss\n",
    "def triplet_loss(y_true, y_pred):\n",
    "    \"\"\" function to compute triplet loss\n",
    "        margin is predefined coded, manually change if needed\n",
    "    \"\"\"\n",
    "    # define triplet margin\n",
    "    margin = K.constant(0.3)\n",
    "    zero = K.constant(0.0)\n",
    "    \n",
    "    # get the prediction vector\n",
    "    anchor, positive, negative = y_pred[:, 0], y_pred[:, 1], y_pred[:, 2]\n",
    "    \n",
    "    # compute distance\n",
    "    pos_distance = K.sum(K.square(anchor - positive), axis=1)\n",
    "    neg_distance = K.sum(K.square(anchor - negative), axis=1)\n",
    "    \n",
    "    # compute loss\n",
    "    partial_loss = pos_distance - neg_distance + margin\n",
    "    full_loss = K.sum(K.maximum(partial_loss, zero), axis=0)\n",
    "    \n",
    "    return full_loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build Triplet Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For reproduciable purpose\n",
    "seed = 42\n",
    "K.clear_session()\n",
    "os.environ['PYTHONHASHSEED'] = '0'\n",
    "np.random.seed(seed)\n",
    "random.seed(seed)\n",
    "session_conf = tf.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)\n",
    "tf.set_random_seed(seed)\n",
    "sess = tf.Session(graph=tf.get_default_graph(), config=session_conf)\n",
    "K.set_session(sess)\n",
    "\n",
    "# Define Parameters\n",
    "img_size = (224, 224, 3)  # target image size\n",
    "\n",
    "# triplet image generator\n",
    "train_generator = train_triplet_generator(train_df, batch_size=100, img_size=img_size[:2], \n",
    "                                          seed=42, prefix='./data/triplet/train/')\n",
    "\n",
    "val_generator = val_triplet_generator(val_df, batch_size=64, img_size=img_size[:2], \n",
    "                                      seed=42, prefix='./data/triplet/validation/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:3148: calling l2_normalize (from tensorflow.python.ops.nn_impl) with dim is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "dim is deprecated, use axis instead\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "vgg16 (Model)                (None, 512)               14714688  \n",
      "_________________________________________________________________\n",
      "l2_norm (Lambda)             (None, 512)               0         \n",
      "=================================================================\n",
      "Total params: 14,714,688\n",
      "Trainable params: 7,079,424\n",
      "Non-trainable params: 7,635,264\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Define triplet network model\n",
    "base_model = base_net(input_shape=img_size)\n",
    "base_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "anchor_input (InputLayer)       (None, 224, 224, 3)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "positive_input (InputLayer)     (None, 224, 224, 3)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "negative_input (InputLayer)     (None, 224, 224, 3)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "base_net (Sequential)           (None, 512)          14714688    anchor_input[0][0]               \n",
      "                                                                 positive_input[0][0]             \n",
      "                                                                 negative_input[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "output (Lambda)                 (None, 3, 512)       0           base_net[1][0]                   \n",
      "                                                                 base_net[2][0]                   \n",
      "                                                                 base_net[3][0]                   \n",
      "==================================================================================================\n",
      "Total params: 14,714,688\n",
      "Trainable params: 7,079,424\n",
      "Non-trainable params: 7,635,264\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "triplet_model = triplet_net(base_model=base_model, input_shape=img_size)\n",
    "triplet_model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fit Triplet Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate:  0.001\n"
     ]
    }
   ],
   "source": [
    "# define learning scheduler\n",
    "def lr_schedule(epoch):\n",
    "    \"\"\" Learning rate schedule \"\"\"\n",
    "    lr = 1e-3\n",
    "    if epoch > 80:\n",
    "        lr *= 2e-1\n",
    "    elif epoch > 60:\n",
    "        lr *= 4e-1\n",
    "    elif epoch > 40:\n",
    "        lr *= 6e-1\n",
    "    elif epoch > 20:\n",
    "        lr *= 8e-1\n",
    "    print('Learning rate: ', lr)\n",
    "    return lr\n",
    "\n",
    "# define optimizer\n",
    "opt = keras.optimizers.Adam(lr=lr_schedule(0))\n",
    "\n",
    "# Create call backs\n",
    "lr_scheduler = LearningRateScheduler(lr_schedule)\n",
    "lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1), cooldown=0, patience=5, min_lr=0.5e-6)\n",
    "callbacks = [lr_reducer, lr_scheduler]\n",
    "\n",
    "# compile the model\n",
    "triplet_model.compile(optimizer=opt, loss=triplet_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "Learning rate:  0.001\n",
      " - 101s - loss: 12.6170 - val_loss: 7.6998\n",
      "Epoch 2/100\n",
      "Learning rate:  0.001\n",
      " - 58s - loss: 8.3585 - val_loss: 5.7255\n",
      "Epoch 3/100\n",
      "Learning rate:  0.001\n",
      " - 58s - loss: 7.3782 - val_loss: 5.5149\n",
      "Epoch 4/100\n",
      "Learning rate:  0.001\n",
      " - 58s - loss: 6.7736 - val_loss: 4.9811\n",
      "Epoch 5/100\n",
      "Learning rate:  0.001\n",
      " - 58s - loss: 6.1316 - val_loss: 4.5373\n",
      "Epoch 6/100\n",
      "Learning rate:  0.001\n",
      " - 60s - loss: 5.5860 - val_loss: 4.2665\n",
      "Epoch 7/100\n",
      "Learning rate:  0.001\n",
      " - 58s - loss: 5.1839 - val_loss: 4.2447\n",
      "Epoch 8/100\n",
      "Learning rate:  0.001\n",
      " - 58s - loss: 4.7850 - val_loss: 4.0661\n",
      "Epoch 9/100\n",
      "Learning rate:  0.001\n",
      " - 60s - loss: 4.6486 - val_loss: 3.7730\n",
      "Epoch 10/100\n",
      "Learning rate:  0.001\n",
      " - 58s - loss: 4.1271 - val_loss: 3.5472\n",
      "Epoch 11/100\n",
      "Learning rate:  0.001\n",
      " - 58s - loss: 4.3459 - val_loss: 3.6962\n",
      "Epoch 12/100\n",
      "Learning rate:  0.001\n",
      " - 58s - loss: 3.9233 - val_loss: 3.5678\n",
      "Epoch 13/100\n",
      "Learning rate:  0.001\n",
      " - 58s - loss: 3.7055 - val_loss: 3.5780\n",
      "Epoch 14/100\n",
      "Learning rate:  0.001\n",
      " - 58s - loss: 3.4200 - val_loss: 3.2311\n",
      "Epoch 15/100\n",
      "Learning rate:  0.001\n",
      " - 58s - loss: 3.3262 - val_loss: 3.5489\n",
      "Epoch 16/100\n",
      "Learning rate:  0.001\n",
      " - 58s - loss: 3.2968 - val_loss: 3.6618\n",
      "Epoch 17/100\n",
      "Learning rate:  0.001\n",
      " - 58s - loss: 3.1597 - val_loss: 3.0896\n",
      "Epoch 18/100\n",
      "Learning rate:  0.001\n",
      " - 58s - loss: 3.1288 - val_loss: 3.0085\n",
      "Epoch 19/100\n",
      "Learning rate:  0.001\n",
      " - 58s - loss: 2.8703 - val_loss: 3.2187\n",
      "Epoch 20/100\n",
      "Learning rate:  0.001\n",
      " - 58s - loss: 2.9179 - val_loss: 2.9102\n",
      "Epoch 21/100\n",
      "Learning rate:  0.001\n",
      " - 58s - loss: 2.8205 - val_loss: 2.9969\n",
      "Epoch 22/100\n",
      "Learning rate:  0.0008\n",
      " - 58s - loss: 2.9275 - val_loss: 3.0330\n",
      "Epoch 23/100\n",
      "Learning rate:  0.0008\n",
      " - 58s - loss: 2.5392 - val_loss: 2.9050\n",
      "Epoch 24/100\n",
      "Learning rate:  0.0008\n",
      " - 58s - loss: 2.4316 - val_loss: 2.9263\n",
      "Epoch 25/100\n",
      "Learning rate:  0.0008\n",
      " - 58s - loss: 2.4688 - val_loss: 2.8683\n",
      "Epoch 26/100\n",
      "Learning rate:  0.0008\n",
      " - 58s - loss: 2.2184 - val_loss: 2.9457\n",
      "Epoch 27/100\n",
      "Learning rate:  0.0008\n",
      " - 58s - loss: 2.3396 - val_loss: 2.6555\n",
      "Epoch 28/100\n",
      "Learning rate:  0.0008\n",
      " - 58s - loss: 2.3799 - val_loss: 2.6770\n",
      "Epoch 29/100\n",
      "Learning rate:  0.0008\n",
      " - 58s - loss: 2.1797 - val_loss: 2.6164\n",
      "Epoch 30/100\n",
      "Learning rate:  0.0008\n",
      " - 58s - loss: 1.9851 - val_loss: 2.6244\n",
      "Epoch 31/100\n",
      "Learning rate:  0.0008\n",
      " - 58s - loss: 1.9228 - val_loss: 2.8834\n",
      "Epoch 32/100\n",
      "Learning rate:  0.0008\n",
      " - 58s - loss: 2.1814 - val_loss: 2.7121\n",
      "Epoch 33/100\n",
      "Learning rate:  0.0008\n",
      " - 58s - loss: 2.1281 - val_loss: 2.6564\n",
      "Epoch 34/100\n",
      "Learning rate:  0.0008\n",
      " - 58s - loss: 1.9888 - val_loss: 2.5838\n",
      "Epoch 35/100\n",
      "Learning rate:  0.0008\n",
      " - 58s - loss: 1.8674 - val_loss: 2.6023\n",
      "Epoch 36/100\n",
      "Learning rate:  0.0008\n",
      " - 58s - loss: 1.9987 - val_loss: 2.3842\n",
      "Epoch 37/100\n",
      "Learning rate:  0.0008\n",
      " - 58s - loss: 1.7135 - val_loss: 2.5272\n",
      "Epoch 38/100\n",
      "Learning rate:  0.0008\n",
      " - 58s - loss: 1.9291 - val_loss: 2.5716\n",
      "Epoch 39/100\n",
      "Learning rate:  0.0008\n",
      " - 58s - loss: 2.0137 - val_loss: 2.6168\n",
      "Epoch 40/100\n",
      "Learning rate:  0.0008\n",
      " - 58s - loss: 1.7273 - val_loss: 2.5266\n",
      "Epoch 41/100\n",
      "Learning rate:  0.0008\n",
      " - 58s - loss: 1.7792 - val_loss: 2.5311\n",
      "Epoch 42/100\n",
      "Learning rate:  0.0006\n",
      " - 58s - loss: 1.5554 - val_loss: 2.4100\n",
      "Epoch 43/100\n",
      "Learning rate:  0.0006\n",
      " - 58s - loss: 1.4686 - val_loss: 2.4219\n",
      "Epoch 44/100\n",
      "Learning rate:  0.0006\n",
      " - 58s - loss: 1.5513 - val_loss: 2.5348\n",
      "Epoch 45/100\n",
      "Learning rate:  0.0006\n",
      " - 58s - loss: 1.5453 - val_loss: 2.4024\n",
      "Epoch 46/100\n",
      "Learning rate:  0.0006\n",
      " - 58s - loss: 1.2737 - val_loss: 2.4201\n",
      "Epoch 47/100\n",
      "Learning rate:  0.0006\n",
      " - 58s - loss: 1.2458 - val_loss: 2.4587\n",
      "Epoch 48/100\n",
      "Learning rate:  0.0006\n",
      " - 58s - loss: 1.3353 - val_loss: 2.4725\n",
      "Epoch 49/100\n",
      "Learning rate:  0.0006\n",
      " - 58s - loss: 1.4931 - val_loss: 2.4302\n",
      "Epoch 50/100\n",
      "Learning rate:  0.0006\n",
      " - 58s - loss: 1.2262 - val_loss: 2.3703\n",
      "Epoch 51/100\n",
      "Learning rate:  0.0006\n",
      " - 58s - loss: 1.2321 - val_loss: 2.2510\n",
      "Epoch 52/100\n",
      "Learning rate:  0.0006\n",
      " - 58s - loss: 1.3065 - val_loss: 2.2557\n",
      "Epoch 53/100\n",
      "Learning rate:  0.0006\n",
      " - 58s - loss: 1.2359 - val_loss: 2.3223\n",
      "Epoch 54/100\n",
      "Learning rate:  0.0006\n",
      " - 58s - loss: 1.1413 - val_loss: 2.3403\n",
      "Epoch 55/100\n",
      "Learning rate:  0.0006\n",
      " - 58s - loss: 1.1534 - val_loss: 2.3193\n",
      "Epoch 56/100\n",
      "Learning rate:  0.0006\n",
      " - 58s - loss: 1.1354 - val_loss: 2.2962\n",
      "Epoch 57/100\n",
      "Learning rate:  0.0006\n",
      " - 58s - loss: 1.2104 - val_loss: 2.3453\n",
      "Epoch 58/100\n",
      "Learning rate:  0.0006\n",
      " - 58s - loss: 1.0815 - val_loss: 2.2705\n",
      "Epoch 59/100\n",
      "Learning rate:  0.0006\n",
      " - 58s - loss: 1.0179 - val_loss: 2.3757\n",
      "Epoch 60/100\n",
      "Learning rate:  0.0006\n",
      " - 58s - loss: 0.9468 - val_loss: 2.5024\n",
      "Epoch 61/100\n",
      "Learning rate:  0.0006\n",
      " - 58s - loss: 0.9275 - val_loss: 2.2978\n",
      "Epoch 62/100\n",
      "Learning rate:  0.0004\n",
      " - 58s - loss: 0.9979 - val_loss: 2.2577\n",
      "Epoch 63/100\n",
      "Learning rate:  0.0004\n",
      " - 58s - loss: 0.9966 - val_loss: 2.1167\n",
      "Epoch 64/100\n",
      "Learning rate:  0.0004\n",
      " - 58s - loss: 0.9118 - val_loss: 2.0843\n",
      "Epoch 65/100\n",
      "Learning rate:  0.0004\n",
      " - 58s - loss: 0.9771 - val_loss: 2.2328\n",
      "Epoch 66/100\n",
      "Learning rate:  0.0004\n",
      " - 58s - loss: 0.8180 - val_loss: 2.3260\n",
      "Epoch 67/100\n",
      "Learning rate:  0.0004\n",
      " - 58s - loss: 0.8473 - val_loss: 2.1389\n",
      "Epoch 68/100\n",
      "Learning rate:  0.0004\n",
      " - 58s - loss: 0.8468 - val_loss: 2.1558\n",
      "Epoch 69/100\n",
      "Learning rate:  0.0004\n",
      " - 58s - loss: 0.8717 - val_loss: 2.2317\n",
      "Epoch 70/100\n",
      "Learning rate:  0.0004\n",
      " - 58s - loss: 0.7426 - val_loss: 2.1625\n",
      "Epoch 71/100\n",
      "Learning rate:  0.0004\n",
      " - 58s - loss: 0.7564 - val_loss: 2.2599\n",
      "Epoch 72/100\n",
      "Learning rate:  0.0004\n",
      " - 58s - loss: 0.7206 - val_loss: 2.1291\n",
      "Epoch 73/100\n",
      "Learning rate:  0.0004\n",
      " - 58s - loss: 0.7395 - val_loss: 2.0567\n",
      "Epoch 74/100\n",
      "Learning rate:  0.0004\n",
      " - 58s - loss: 0.7904 - val_loss: 2.1460\n",
      "Epoch 75/100\n",
      "Learning rate:  0.0004\n",
      " - 58s - loss: 0.7203 - val_loss: 2.1467\n",
      "Epoch 76/100\n",
      "Learning rate:  0.0004\n",
      " - 58s - loss: 0.7153 - val_loss: 2.1630\n",
      "Epoch 77/100\n",
      "Learning rate:  0.0004\n",
      " - 58s - loss: 0.7511 - val_loss: 2.1480\n",
      "Epoch 78/100\n",
      "Learning rate:  0.0004\n",
      " - 58s - loss: 0.7631 - val_loss: 2.0163\n",
      "Epoch 79/100\n",
      "Learning rate:  0.0004\n",
      " - 58s - loss: 0.8451 - val_loss: 2.0885\n",
      "Epoch 80/100\n",
      "Learning rate:  0.0004\n",
      " - 58s - loss: 0.6424 - val_loss: 2.1345\n",
      "Epoch 81/100\n",
      "Learning rate:  0.0004\n",
      " - 58s - loss: 0.6823 - val_loss: 2.1997\n",
      "Epoch 82/100\n",
      "Learning rate:  0.0002\n",
      " - 58s - loss: 0.6041 - val_loss: 2.1020\n",
      "Epoch 83/100\n",
      "Learning rate:  0.0002\n",
      " - 58s - loss: 0.5776 - val_loss: 2.1129\n",
      "Epoch 84/100\n",
      "Learning rate:  0.0002\n",
      " - 58s - loss: 0.6242 - val_loss: 2.0428\n",
      "Epoch 85/100\n",
      "Learning rate:  0.0002\n",
      " - 58s - loss: 0.6042 - val_loss: 2.0145\n",
      "Epoch 86/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.5093 - val_loss: 2.0493\n",
      "Epoch 87/100\n",
      "Learning rate:  0.0002\n",
      " - 58s - loss: 0.5645 - val_loss: 2.0258\n",
      "Epoch 88/100\n",
      "Learning rate:  0.0002\n",
      " - 58s - loss: 0.4806 - val_loss: 2.0670\n",
      "Epoch 89/100\n",
      "Learning rate:  0.0002\n",
      " - 58s - loss: 0.5217 - val_loss: 1.9524\n",
      "Epoch 90/100\n",
      "Learning rate:  0.0002\n",
      " - 58s - loss: 0.4934 - val_loss: 1.9492\n",
      "Epoch 91/100\n",
      "Learning rate:  0.0002\n",
      " - 58s - loss: 0.5282 - val_loss: 2.0001\n",
      "Epoch 92/100\n",
      "Learning rate:  0.0002\n",
      " - 58s - loss: 0.5640 - val_loss: 1.9840\n",
      "Epoch 93/100\n",
      "Learning rate:  0.0002\n",
      " - 58s - loss: 0.4108 - val_loss: 2.0161\n",
      "Epoch 94/100\n",
      "Learning rate:  0.0002\n",
      " - 58s - loss: 0.5150 - val_loss: 1.9795\n",
      "Epoch 95/100\n",
      "Learning rate:  0.0002\n",
      " - 58s - loss: 0.4828 - val_loss: 1.9805\n",
      "Epoch 96/100\n",
      "Learning rate:  0.0002\n",
      " - 58s - loss: 0.5190 - val_loss: 2.1040\n",
      "Epoch 97/100\n",
      "Learning rate:  0.0002\n",
      " - 58s - loss: 0.5145 - val_loss: 2.0430\n",
      "Epoch 98/100\n",
      "Learning rate:  0.0002\n",
      " - 58s - loss: 0.4933 - val_loss: 2.0586\n",
      "Epoch 99/100\n",
      "Learning rate:  0.0002\n",
      " - 58s - loss: 0.4563 - val_loss: 1.9829\n",
      "Epoch 100/100\n",
      "Learning rate:  0.0002\n",
      " - 58s - loss: 0.5363 - val_loss: 2.0298\n"
     ]
    }
   ],
   "source": [
    "# fit the mode\n",
    "history = triplet_model.fit_generator(train_generator, steps_per_epoch=74, epochs=100, \n",
    "                                      validation_data=val_generator, validation_steps=48, \n",
    "                                      verbose=2, callbacks=callbacks)\n",
    "\n",
    "base_model.save('./models/vgg16-base-0.3-model.h5')\n",
    "pickle.dump(history.history, open('./models/vgg16-triplet-0.3-history.p', 'wb'))\n",
    "_ = gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmUAAAG/CAYAAAAZwqDzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvFvnyVgAAIABJREFUeJzs3Xd4VFX+x/H3yaRXAiSB0EIn9BIUkCodK4oNVFBsu7q6uroLrmthdZXf2hbLrqiIgppd7BQLgqigVAULAQQxSO8lkJB2fn/cSaSkTJKZZBI+r+eZJ5k7d879zlyUD+ece66x1iIiIiIiVSugqgsQEREREYUyEREREb+gUCYiIiLiBxTKRERERPyAQpmIiIiIH1AoExEREfEDCmUichpjzDhjjDXGtKjqWvyBMSbJ/X0U9+hchbXpXInUEIFVXYCISDXyKPBBEds3VHYhIlLzKJSJiHjuZ2vt0qouQkRqJg1fiki5GWOuNsasMcZkGWP2GmNmGGPqn7LPaGPMt8aYDGPMYWPM98aYm094vbsxZr4xZp8xJtMY87Mx5vkSjlnPGJNrjLm9iNf+bIzJMcbEuZ8PNcZ8ZYw55D7+emPM/d78Dk45fsEw5++NMU8aY3YbY44ZY+YYY5JO2TfIGPOwMeYXY0y2++fDxpigU/aLMMY8ZozZZIw5bozZaYx52xiTcMrh6xpjXnd/x9uNMVOMMaEntBNojPm7u52C87XYGNPbV9+HiJSNespEpFyMMTcBLwD/BSYCicA/gLONMV2ttRnuv/BnAlOAe3D+IdgGqOVuIxL4GFgOjAOOAElAr+KOa63daYz5FLja3e6JrgE+stbuMcY0wxlqfAuYBGQDLYFmFfjYAcaYU/+/aa21eadsmwisBq4D4nG+l0+MMe2stTnufV4FLne/thjnM//VXd9oAGNMMDAf6AQ8BiwFYoChQCyw64RjzgDeBC4BegIPAgeAB9yv/wW4032M1UA0kALULvvXICI+Ya3VQw899DjpgROQLNCimNddOIHgs1O293a/73b387uB/SUcJ8W9f8cy1jfG/b7WJ2zr7N52ufv5KPfzaC98H0nutop6ZBSx31og4ITt57i3j3c/b+9+/uApx7nvxO8DuN79/EIPztVDp2yfA2w45fk7Vf1nSw899Cj+oeFLESmP1jg9QK+fuNFauxhIB/q5N60AYo0xM40x5xtjap3Szk/AQeAF91BoIw+P/y6QgdMzVuAa4BC/TcRfDeQAqcaYUcaYeA/bLsnDQPdTHn2K2O8ta21+wRNr7RJgK04PFkBf98+Zp7yv4HnB9zcE2GmtLeriglPNPeX590DjE56vAEYYYx4xxvR298KJiB9RKBOR8igY8tpRxGs7C1631n4OXAY0wglSe4wxnxpjOrpfPwQMALYDzwNbjDE/GGMuLeng1tpjwNvAGONwAVcBs6y1We59NuIM8wXgDO3tNMYsNcb0K65dD6Rba1ee8vi2iP12FbOtgfv34r6/nae8XgfY5mFt+095fhwIOeH5P3CGMi8EvgT2GWNeMcbU9bB9EfExhTIRKY+CAFCviNfqnfA61tq3rLX9cOZAjQTqAx8ZYwLcr6+21l6KE0R6ApuA/xlj2pdSwwyc4cLewEB3uzNO3MFa+5m1dhjOHLZBQC4wtxKCyKmT8Au2FQSs4r6/eqe8vpffglyFWGtzrLWTrbUdcL6rO4FLgee80b6IVJxCmYiUx3qcnp8rT9xojOkFNAEWnfoGa22GtXYOzsUB9XF6gU58Pdc6y038Def/Tcml1PAZzpDgNe7HLzg9QKex1h631i4E/g+IAJqW0nZFjSoInQDGmHOAhsDX7k1fuH9eecr7xrh/LnL//ASoZ4y5wJvFWWt3WmtfAj7Fmd8mIn5AV1+KSEmGGWN2nrLtkLV2vntpiReMMTNx5kI1AB7BmSc2DcAYMwmnh+gznCHKhsDtwGrrXCF5PnAT8B6wGScw3Y5zFebXlMBam2+MeR24GQgCnrLW2oLXjTG34Mzdmgf8CtTFuSpyO/CDe59+wALgemvtax58H82MMT2K2L7BWnvi8GEU8J4x5gUgDmfR2Z+A19y1/2CMeRN40H0151c4vYR/A9601n7vbmcmcCPwpjHmUWCZu+2hwNPW2nUe1Iz7s74PrAG+wbkqswswDCcki4gfUCgTkZI8U8S2H4H21tqpxphjOEtdvI8z8X4e8Gdr7VH3vstwQtZTOMOTu3F6f/7mfv0nINP9vD5OGFsBDLbWbvWgvhk4Sz0U/H6iNcBwnEAUjzMkuBgYY63NdO9jcK4k9XTUYKL7carLcJbeKPAo0AKYjhM0PwNus78thwHOVZM/41xheR9OWJwMPFSwg7U2xxgzBGcu2E3un/uAJZw+h6w0X7jrvBUIB7bg9Bw+UsZ2RMRHzAn/sBQRkQpwLxC7GbjRPTwoIuIxzSkTERER8QMKZSIiIiJ+QMOXIiIiIn5APWUiIiIifqBaXn1Zt25dm5SU5NNjHD16lIiICJ8eQ8pH58Y/6bz4L50b/6Tz4r+8fW5WrVq111obV9p+1TKUJSUlsXLlSp8eY9GiRfTv39+nx5Dy0bnxTzov/kvnxj/pvPgvb58bY0y6J/tp+FJERETEDyiUiYiIiPgBhTIRERERP6BQJiIiIuIHFMpERERE/EC1vPpSRESkNIcPH2b37t3k5OSUvnMViImJIS0trarLkCKU5dwEBQURHx9PdHR0hY+rUCYiIjXO4cOH2bVrFw0aNCAsLAxjTFWXdJojR44QFRVV1WVIETw9N9ZaMjMz2bZtG0CFg5mGL0VEpMbZvXs3DRo0IDw83C8DmdQMxhjCw8Np0KABu3fvrnB7CmUiIlLj5OTkEBYWVtVlyBkiLCzMK8PkCmUiIlIjqYdMKou3/qwplImIiIj4AYUyERERET+gUCYiInIGuPLKKxk1alSZ3tOjRw/uvvtuH1Ukp9KSGCIiIn6gtHlJY8eOZfr06eVu/4UXXsBaW6b3zJs3j6CgoHIf01MTJkzg008/ZeXKlT4/lj9TKCvCwWPZbDmcR36+JSBAE0VFRMT3duzYUfj7nDlzuPHGG0/aVtzVpDk5OR4Fp5iYmDLXVLt27TK/R8pPw5dFeGvVVu7/Kosjx3OruhQRETlD1KtXr/BRq1at07bFxMSwbt06jDHMmjWLfv36ERoayquvvsquXbu44oorCtdma9++Pa+//vpJ7Z86fNmjRw/uvPNO7rnnHmrXrk29evWYOHHiSb1ppw5f1qtXj8mTJ3P99dcTFRVFo0aNmDJlyknHWbt2Leeccw6hoaG0bduW+fPnExgYSGpqarm/m7179zJmzBhiY2MJDw9n6NChrF+/vvD1ffv2MXr0aOLi4ggNDaVFixb8+9//Lnz9mWeeoUWLFoSEhBAXF8fw4cPLXYsvqaesCGHBLgCycvKICfN9t62IiPjeQ7N/ZO32w5V6zLaJ0TxwQTuvtzthwgSeeOIJOnXqREhICJmZmfTo0YOJEycSHR3Nhx9+yNixY2nSpAm9e/cutp1p06Zxzz33sGzZMpYvX861117LWWedxciRI4t9z+OPP87f//537r33Xt577z3uuOMOevfuTdeuXcnNzeWiiy6iefPmLF++nMOHD3PnnXeSn59foc87ZswYtm3bxuzZs4mKiuIvf/kLw4cPJy0tjZCQECZMmMBPP/3Ehx9+SN26dfn55585ePAgAEuWLOFPf/oTM2bMoGfPnuzfv58FCxZUqB5fUSgrQrg7lB3LzqviSkRERE531113cfHFF5+07c477yz8/dZbb2X+/PmkpqaWGMq6du3KfffdB0DLli35z3/+w4IFC0oMZeeffz633HILAHfffTf/+te/WLhwIV27dmXu3Lmkp6ezZMkS4uPjAZg8eTIDBw4s92f9/vvv+eSTT1i2bBlnnXUWAK+//jqNGzdm1qxZXH311aSnp5OSkkJKSgoASUlJhe9PT08nOjqaCy64gPDwcBo3bkznzp3LXY8vKZQVISzI+VqOZWv4UkSkpvBFj1VVKQgfBXJzc3nkkUd466232LZtG9nZ2Rw/frzUYbqOHTue9DwxMbHU2wWV9J5169aRlJRUGMgAzj777FI/T0nS0tIIDg6me/fuhdvq1KlDcnIya9euBZwQeuWVV7J06VIGDx7MhRdeWBhGR4wYwcMPP0xSUhJDhw5l6NChjBw5koiIiArV5QuaU1aEgp6yTPWUiYiIHzo1UDzyyCM899xzTJw4kc8++4zVq1czYsQIsrOzS2zn1AsEjDGlDjWW5z2+UnDF6kUXXUR6ejp33HEHO3bsYOjQofzud78DoFatWqxZs4bXX3+dxMREJk2aRNu2bb1yr0pvUygrQpiGL0VEpBpZvHgxI0eOZPTo0XTq1IlmzZqxYcOGSq+jTZs2pKens2fPnsJty5cvr1CbycnJZGdns2LFisJt+/btIy0tjbZt2xZui4+PZ9y4ccyYMYPnn3+el156qTAsBgUFMXjwYCZPnsyaNWvYs2cPH330UYXq8gUNXxYhLEihTEREqo9WrVoxd+5cvv76a2rVqsWTTz7J9u3badKkSaXWcd5559G4cWPGjh3LY489xpEjR5gwYQLGmFLXYcvMzGT16tUnbYuMjKRDhw4MHTqU8ePH85///IfIyEgmTJhAQkICl112GQD33nsvPXr0oG3bthw/fpz33nuP1q1bExAQwDvvvMP27dvp3bs3sbGxfPLJJ2RlZZGcnOyz76G81FNWhMLhyxzNKRMREf/30EMP0bFjRwYPHkz//v2Jj48v8+r93hAYGMj777/PwYMH6d69OzfccAP3338/AKGhoSW+d+3atXTp0uWkx7hx4wCYOXMmHTt25LzzzqNnz57k5+fz4YcfEhwcDDg9YX/5y1/o2LEjffv2JS8vj3feeQeA2NhYZs2axbnnnktycjLPPPMMr7322klz1PyFKevqvv4gJSXF+nLV352Hsujx6AIeGdmeMWdX7r8ypHSLFi2if//+VV2GnELnxX+diecmLS3NL3tCTnTkyBGioqKqugyfW7ZsGT169OCHH36gXbvqcbFFec5NSX/mjDGrrLUpRb54Ag1fFiFME/1FRETKZdasWcTGxtKiRQs2bdrEH//4R84666xqE8iqkkJZEXT1pYiISPkcOnSIiRMnsnXrVurUqcPAgQN58sknq7qsakGhrAhBrgBcBo7lKJSJiIiUxQ033MANN9xQ1WVUS5roX4xgl3rKREREpPIolBUjxGW0or+IiIhUGoWyYoS4tE6ZiIiIVJ5KC2XGmGnGmN3GmB9O2PZPY8w6Y8x3xph3jTG1Kque0oS4jIYvRUREpNJUZk/ZdGDYKdvmA+2ttR2BDcDESqynRCEuyNREfxEREakklRbKrLVfAPtP2faJtbZg4tZSoGFl1VMaZ06ZQpmIiIhUDn9aEuN64L/FvWiMuQm4CSAhIYFFixb5tBgXuew9cMjnx5Gyy8jI0HnxQzov/utMPDcxMTEcOXKkqssoUV5ens9qfPXVV7nvvvv49ddfi3xelCeffJJXX32VNWvWePXY1VF5zk1WVlaF/zvzi1BmjPkrkAu8Xtw+1tqpwFRwbrPk61uGvPDdR+y3wWfcrUmqgzPxljHVgc6L/zoTz01aWprf38Lo1Fv5XHjhhRw9epQFCxactm9aWhpt27bl448/ZsiQIaW2XXCfyYL2x48fz+WXX17idxISEkJAQIDH31tubi5BQUG8++67XHzxxYXbPTmWN9x3333MmTPntJuYe0N5brMUGhpKly5dKnTcKr/60hgzDjgfGGP96EacIQGa6C8iIpVn/PjxfPbZZ/zyyy+nvfbyyy/TpEkTBg0aVK62w8LCiI+Pr2CF/nesmqZKQ5kxZhjwZ+BCa+2xqqzlVFoSQ0REKtN5551HQkICr7zyyknbc3JymDFjBtdffz0BAc5f23fffTetWrUiLCyMpk2bMmHCBI4fP15s2y+99BK1ap28wMGjjz5KQkICUVFRjBs3jmPHTv5reNmyZQwePJi6desSHR1Nnz59WL58eeHrSUlJAIwcORJjDC1atCj2WM8//zzNmzcnODiYli1bMm3atMLXcnNzMcbw0ksvcemllxIREUHz5s158803PfzmirZ//36uueYaYmNjCQ8PZ8iQIaSlpRW+fuDAAcaMGUNcXByhoaE0b96cZ5999qSaW7ZsSUhICHFxcQwbNoz8/PwK1VSaShu+NMa8CfQH6hpjtgIP4FxtGQLMN8YALLXW3lJZNZUkONCQmZOLtRZ3bSIiUp19OAF2fl+5x6zXAYY/5tGugYGBjB07lunTp/PAAw8UBrDZs2ezd+9errvuusJ9o6OjmT59OomJifz444/cfPPNhIWF8cADD3h0rDfeeIMHH3yQZ599ln79+pGamsrjjz9+Ug/XkSNHGDt2LFOmTAHgmWeeYfjw4WzcuJHY2FhWrFhBYmIir7zyCsOGDSMwsOhIMWvWLP74xz/y9NNPM2jQIObNm8dNN91E/fr1GT58eOF+Dz30EJMnT2by5Mm88MILjBs3jj59+tCwYfmuAbzmmmvYvHkzH3zwATExMUycOJFhw4axfv16QkNDuffee1m3bh3z5s0jPj6en3/+mX379gGwYsUK7rjjDl577TV69erFgQMHWLhwYbnqKItKC2XW2quK2PxyZR2/rEJcYC1k5eQT5r5BuYiIiC+NHz+eyZMn8+mnnxbOHXv55ZcZMmQIjRo1Ktzv/vvvL/w9KSmJCRMm8Oyzz3ocyp5++mmuv/56brzxxsL2Fi5cyNatWwv3OXWo9LnnnmPWrFl8/PHHXHnllcTFxQFQq1Yt6tWrV+yxHn/8ccaNG8fvf/97AFq1asXKlSuZPHnySaFs3LhxjB49GoBHHnmEKVOmsHjxYq688kqPPtOJ0tLSmDdvHkuWLKFXr14AzJw5k8aNG5Oamsq4ceNIT0+na9eudO/eHYAmTZoUvv/XX38lKiqKCy64gMjISJo0aULnzp3LXEdZ+cVEf38U4nJ6xzJz8hTKRERqAg97rKpSy5Yt6devH9OmTWPIkCFs376djz/+mNTU1JP2++9//8uUKVPYtGkTGRkZ5ObmFvaseSItLY3bbrvtpG09e/Zk1qxZhc937drF3/72NxYtWsSuXbvIy8vj2LFjbNmypUyfKS0trTCQFejdu/dJwRKgY8eOhb8HBwdTt25ddu/eXaZjnXjMwMBAzj777MJtsbGxtGvXjrVr1wLw+9//nssvv5wVK1YwePBgLrjgAvr27Qs4gTQxMZGmTZsydOhQhgwZwiWXXEJkZGS56vFUlU/091ch7hym+1+KiEhlGj9+PO+99x779+9n+vTp1K5dm4suuqjw9cWLFzNmzBhGjBjB7Nmz+fbbb5k0aRLZ2dlerePqq6/m22+/5emnn+arr75i9erVJCYmeu04p04NCgoKOu11X8zhKjju+eefT3p6OnfddRe7du1i+PDhhT2H0dHRrF69mtTUVBo2bMgjjzxCcnIyO3fu9Ho9J1IoK0ZhT5km+4uISCUaNWoUoaGhzJw5k2nTpnHttdeeFFiWLFlCkyZN+Otf/0r37t1p2bJlkVdsliQ5OZmlS5eetO3U54sXL+b2229nxIgRtGvXjoiIiJNCicvlwuVykZdX8t+TycnJLFmy5LS227ZtW6aayyI5OZnc3FyWLVtWuO3gwYP8+OOPJx03Li6Oa6+9ltdee42pU6cybdo0cnJyAGeO38CBA3nsscdYs2YNBw8eZN68eT6rGTR8WazfesoUykREpPKEhYUxevRoHnzwQQ4cOMD48eNPer1Vq1Zs2bKFN998k7POOosPP/yQ//3vf2U6xh133MH48ePp1q0bffr04X//+x+rVq06aaJ/q1atmDFjBikpKRw5coR77rmHkJCQwteNMTRu3JgFCxZwzjnnEBISQmxs7GnHuueeexg9ejRdunRh0KBBzJ07l9TUVGbPnl3Gb+Z0mZmZp61TFhERQXJyMueddx433ngjL7zwAtHR0UycOJHatWtzxRVXAM46Z927d6dt27bk5OTw7rvv0rJlS4KCgnj77bfZvXs3ffv2JTY2lgULFnDs2DGSk5MrXHNJ1FNWjGB3T5lCmYiIVLYbbriBAwcO0KtXr9OCwMiRI7nzzju5/fbb6dy5M4sWLeKhhx4qU/tjxozhvvvuY+LEiXTt2pX169dzxx13nLTP9OnTOXjwIF26dGH06NHcfPPNJ11sAM5dAObPn0+jRo0KJ8yfatSoUTz11FM8/vjjtGvXjueee44XXnjhpEn+5bVhwwa6dOly0uOaa64B4LXXXqNr166cf/759OjRg+zsbD766KPChXWDg4OZOHEinTp1onfv3mRlZfH+++8DzsUL77zzDgMHDqRNmzY89dRTvPLKK/Ts2bPCNZfE+NF6rR5LSUmxK1eu9OkxXn5vAX9fmsW0cSmc2ybBp8eSsjkTVyevDnRe/NeZeG7S0tJ83qtRUeVZNV4qR3nOTUl/5owxq6y1KaW1oZ6yYoSop0xEREQqkUJZMQrmlGmiv4iIiFQGhbJinLhOmYiIiIivKZQVQ1dfioiISGVSKCtGkEKZiEi15uubR4sU8NafNYWyYgQYQ2hQAJla0V9EpNqJiIhg27ZtZGdnUx1XGZDqwVpLdnY227ZtIyIiosLtafHYEoQHB6qnTESkGmrYsCF79+4lPT2d3Fz//Md1VlZW4ZpZ4l/Kcm4CAwOJiYmhbt26FT6uQlkJwoJcuvpSRKQaCggIID4+/qQV6v3NokWL6NKlS1WXIUWoqnOj4csShAe7dPWliIiIVAqFshKEB7s0fCkiIiKVQqGsBGHBGr4UERGRyqFQVoLw4ECO5fjnBFERERGpWRTKShCm4UsRERGpJAplJdDVlyIiIlJZFMpKoIn+IiIiUlkUykqgif4iIiJSWRTKShAeFEh2Xj65ebp/moiIiPiWQlkJwoOdu5JrAVkRERHxNYWyEoQVhDINYYqIiIiPKZSVoKCnTJP9RURExNcUykqgUCYiIiKVRaGsBKFBBXPKtKq/iIiI+JZCWQnCgwMB9ZSJiIiI7ymUlSBcE/1FRESkkiiUlSBMS2KIiIhIJVEoK4Em+ouIiEhlUSgrQXiQ5pSJiIhI5VAoK8Fvi8fq6ksRERHxLYWyEgS5DK4Ao54yERER8TmFshIYYwgPcimUiYiIiM8plJUiLNilJTFERETE5xTKShEe7NKSGCIiIuJzCmWlCAsO1PCliIiI+JxCWSmcnjJdfSkiIiK+pVBWivBgTfQXERER31MoK0VYkCb6i4iIiO8plJUiTD1lIiIiUgkUykqh4UsRERGpDAplpQgLCiRLS2KIiIiIjymUlcLpKcvFWlvVpYiIiEgNplBWirBgF/kWjufmV3UpIiIiUoMplJUiPNgFoCswRURExKcqLZQZY6YZY3YbY344YVttY8x8Y8xP7p+xlVWPpwpC2THNKxMREREfqsyesunAsFO2TQAWWGtbAgvcz/1KWHAgAJnZWtVfREREfKfSQpm19gtg/ymbLwJedf/+KnBxZdXjqbAgd0+Zhi9FRETEh0xlXlVojEkC5lhr27ufH7TW1nL/boADBc+LeO9NwE0ACQkJ3VJTU31aa0ZGBpGRkazdl8f/rchiwlmhtKnt8ukxxTMF50b8i86L/9K58U86L/7L2+dmwIABq6y1KaXtF+i1I1aQtdYaY4pNiNbaqcBUgJSUFNu/f3+f1rNo0SL69+9P9JYDsOIrWrfrQP/W8T49pnim4NyIf9F58V86N/5J58V/VdW5qeqrL3cZY+oDuH/uruJ6TqOrL0VERKQyVHUo+wAY6/59LPB+FdZSpPAgpzNRc8pERETElypzSYw3ga+B1saYrcaY8cBjwGBjzE/AIPdzvxJW2FOmqy9FRETEdyptTpm19qpiXhpYWTWUR+E6ZeopExERER+q6uFLvxeqJTFERESkEiiUlcIVYAgJDCBTK/qLiIiIDymUeSA82KWrL0VERMSnFMo8EB4cqOFLERER8SmFMg+EBbvIzNHVlyIiIuI7CmUeCA92qadMREREfEqhzANhQQplIiIi4lsKZR7QRH8RERHxNYUyD4QFuzimFf1FRETEhxTKPBAWFKieMhEREfEphTIPhAe7tHisiIiI+JRCmQd09aWIiIj4mkKZB8KCXRzPzScv31Z1KSIiIlJDKZR5IDzYuSm5hjBFRETEVxTKPBAWHAigKzBFRETEZxTKPBAe5O4p07wyERER8RGFMg+EuYcvNdlfREREfEWhzANhmlMmIiIiPqZQ5gENX4qIiIivKZR5ILxwor9CmYiIiPiGQpkHfptTpqsvRURExDcUyjxQuE6ZespERETERxTKPBCuqy9FRETExxTKPBAapKsvRURExLcUyjwQEhhAgNGcMhEREfEdhTIPGGMIDw4kMzu/qksRERGRGkqhzENhwS4yc9RTJiIiIr6hUOah8GCXJvqLiIiIzyiUeSgsSKFMREREfEehzEPhwS6tUyYiIiI+o1DmofDgQF19KSIiIj6jUOahUA1fioiIiA8plHkoPNhFlhaPFRERER9RKPOQrr4UERERX1Io81CYJvqLiIiIDymUeSg82MWxnDystVVdioiIiNRACmUeCg8OJC/fkp2nWy2JiIiI9ymUeSgsyAWgIUwRERHxCYUyD4UHO6FMk/1FRETEFxTKPBSmUCYiIiI+pFDmoVrhwQDszThexZWIiIhITaRQ5qE29aIAWLfjcBVXIiIiIjWRQpmH4qNCqB0RTNqOI1VdioiIiNRACmUeMsbQtn40a9VTJiIiIj6gUFYGyfWjWL/rCLlaq0xERES8TKGsDJLrR5Odm8/mvUeruhQRERGpYRTKyiC5fjSAhjBFRETE6/wilBlj7jTG/GiM+cEY86YxJrSqaypK87hIglxGoUxERES8rspDmTGmAXA7kGKtbQ+4gCurtqqiBQcG0DI+SldgioiIiNdVeShzCwTCjDGBQDiwvYrrKVZy/WjS1FMmIiIiXmastVVdA8aYO4BHgEzgE2vtmCL2uQm4CSAhIaFbamqqT2vKyMggMjIML3ggAAAgAElEQVTytO0f/5LDm+uy+deAcGJCjE9rkKIVd26kaum8+C+dG/+k8+K/vH1uBgwYsMpam1LafoFeO2I5GWNigYuApsBBYJYx5mpr7cwT97PWTgWmAqSkpNj+/fv7tK5FixZR1DGCN+7lzXXLiG3anr6t4nxagxStuHMjVUvnxX/p3PgnnRf/VVXnxh+GLwcBm621e6y1OcA7QK8qrqlYBVdgaghTREREvMkfQtkWoIcxJtwYY4CBQFoV11Ss2Ihg6seEKpSJiIiIV1V5KLPWLgPeAr4BvsepaWqVFlUKZ7K/rsAUERER76nyUAZgrX3AWtvGWtveWnuNtfZ4VddUkuT6UWzck0FWTl5VlyIiIiI1hF+EsuomuX40efmWjbszqroUERERqSEUysqhrW63JCIiIl6mUFaUNal0X34r5GQV+XKTOhGEBbk02V9ERES8RqGsKLnHiTi2FY7uKfJlV4Chdb0ohTIRERHxGoWyokQmOD8zdhe7S3L9aNZuP4w/3BFBREREqj+FsqJEulfqP1p8KGubGM3hrFy2Hyp6iFNERESkLBTKilLYU7ar2F3a1o8CIG27hjBFRESk4hTKihLh7inLKHpOGUDrerrdkoiIiHiPQllRAkPICYwscfgyMiSQJnXCtSyGiIiIeIVCWTGyg2uVOHwJkFwvWj1lIiIi4hUKZcVwQlnxw5fgTPZP33+Mo8dzK6kqERERqakUyoqRE+RBT1n9aKyFdTvVWyYiIiIVo1BWjOzgWsUuHlugc6NaAHy9aV9llCQiIiI1mEJZMbKDY+H4YcjJLHafuKgQOjWMYcG64i8IEBEREfGEQlkxsoOdXrCSVvUHGJicwOpfD7I343glVCUiIiI1lUJZMTwNZee2icdaWKjeMhEREakAhbJiFIayEtYqA2iXGE296FAWpimUiYiISPkplBUjOzjW+aWUnjJjDOcmx/PlT3s4nptXCZWJiIhITaRQVoycoBjnl1JCGcCg5HiOZuex7Of9Pq5KREREaiqFsmLYgEAIiy11+BKgV/O6hAYFsCCt5HXNRERERIqjUFaSyIRSF5AFCA1y0btFXRas2421thIKExERkZpGoawkEXGl3mqpwLltEth6IJMNuzJ8XJSIiIjURAplJfGwpwycpTEAFqzTEKaIiIiUnUJZSSLjS73VUoF6MaG0bxDNAi2NISIiIuWgUFaSyHjIzoDsox7tPrBNAt9sOcD+o9k+LkxERERqmgqFMmNMmDFmkDGmibcK8isRzpCkJ8tiAAxMdlb3/0yr+4uIiEgZlSmUGWOmG2N+7/49GFgOfAKsN8YM90F9VSsywfnpYShrnxhDfFSI5pWJiIhImZW1p2wosNT9+4VAFFAPeND9qFki45yfHqxVBhAQYBiYHM8XG/aSnZvvw8JERESkpilrKIsFChLKMOBta+1uIBVo683C/EIZhy/BWRoj43guyzdrdX8RERHxXFlD2U6gvTHGhdNr9ql7eySQ483C/EJEXcCUKZT1blGX4MAAPluveWUiIiLiubKGsmnAf4EfgDxggXv72cA6L9blH1xBEF7b4+FLgLBgF2c3rc3nGzxbSkNEREQEyhjKrLWTgOuBqUBva23B2g+5wGQv1+YfIhPK1FMG0K9VHBt3Z7D1wDEfFSUiIiI1TZmXxLDWvm2tfcpau/WEba9aa9/3bml+IiKuzKGsf2vnAoEvNuz1RUUiIiJSA5V1SYzLjTFDTnh+vzFmqzHmY2NMfe+X5wfKcKulAs3jImlQK4zPN2hemYiIiHimrD1lDxb8YozpCtwLTAGCgCe8V5YfKbjVkrUev8UYQ99WcSzZuI+cPC2NISIiIqUrayhrAqx3/z4SeM9a+3/AXcBAbxbmNyLiIOeYc7ulMujXKo6M47l8k37AR4WJiIhITVLWUJaFs2AsOCGsYEmMQydsr1nKuKp/gV4t6hAYYHQVpoiIiHikrKHsS+AJY8zfgBRgnnt7K+BXbxbmNwpX9S9buIoODaJrk1iFMhEREfFIWUPZbUA2MAq4xVq73b19OPCxNwvzG4U9ZWW/n2W/VnH8uP0wu49kebkoERERqWnKuk7ZVmvtBdbaTtbaaSds/6O19nbvl+cHynGrpQL9WmlpDBEREfFMmdcpAzDGnGuMuc0Yc6sxZoC3i/IrEXXBBJQrlLVLjCYuKkRDmCIiIlKqwLLsbIxpALwLdAMKhi4TjTErgZEnDGfWHAEuCK9TplstFTDG0LdlHAvW7SIv3+IKMD4oUERERGqCsvaUTcG552ULa20ja20joKV72xRvF+c3ynGrpQL9Wsdx8FgO32096OWiREREpCYpaygbDNxqrd1csMFa+zNwu/u1mqkct1oq0KdFXYxBQ5giIiJSovLMKStqaXvPl7uvjiLjyx3KYiOC6dSwlkKZiIiIlKisoWwB8IwxplHBBmNMY+BpYKE3C/MrkfHOnLIy3GrpRP1axbHm14McOJrt5cJERESkpihrKLsdiAB+NsakG2PSgU1AOPAHbxfnNyLiITcLjh8u19v7tY4j38LijVoaQ0RERIpWpqsvrbW/um9EPgho496cBmwEngQu9255fqJwAdk9EBpT5rd3aliL2PAgPk3bxQWdEr1cnIiIiNQEZQplANZaC8x3PwAwxnQCLi1vEcaYWsBLQHuc+WnXW2u/Lm97Xld4q6XdULdFmd/uCjAMaVuPud/vICsnj9Agl5cLFBERkequXIvH+sC/gI+stW2ATji9b/6jArdaKjCiY30yjufy5U8awhQREZHTVXkoM8bEAH2BlwGstdnWWv9a1KvwVkvlv4KyV/M6xIQFMe/7HV4qSkRERGoSY8t5ReFJjTjDl99Ya8s8LmeM6QxMBdbi9JKtAu6w1h49Zb+bgJsAEhISuqWmpla47pJkZGQQGRnpPLF59Pt8FFsaX8rmZleXu82Xvz/Oyl25TDk3nCCt7l9uJ50b8Rs6L/5L58Y/6bz4L2+fmwEDBqyy1qaUtp9Hc8qMMR+Usku0R1UVX0NX4A/W2mXGmH8BE4C/nbiTtXYqTngjJSXF9u/fvwKHLN2iRYs46Rir4mhSN4wmFTiurb+bL19ZQUD9tvRPTqhwjWeq086N+AWdF/+lc+OfdF78V1WdG08n+u/z4PXNpexTnK3AVmvtMvfzt3BCmX+JKP8CsgXOaV6X6NBA5n6/g4EKZSIiInICj0KZtfY6XxVgrd1pjPnVGNPaWrseGIgzlOlfKrCqf4HgwACGtKvHxz/u5HhuHiGBugpTREREHFU+0d/tD8DrxpjvgM7AP6q4ntN5IZQBnNehPkeyclmihWRFRETkBGVep8wXrLWrgVInwFWpE2+1ZMo/Sf+cFnWJCg1k7nc7ObeNhjBFRETE4S89Zf4vIh7ysiHrUIWaCQ4MYHDbBOav3Ul2br6XihMREZHqTqHMUwULyO5ZX+GmzutQn8NZuSzZpCFMERERcSiUeapZP6e37J0bKjy3rHfLukSFBDLvOy0kKyIiIg6FMk9FxsNVqc6q/m9eBTmZ5W4qJNDF4LYJfLJ2Fzl5GsIUERERhbKyadgNLn0Rtq2Cd2+G/PIHqhEd6nMoM0dXYYqIiAigUFZ2yRfAkL/D2vdh4aRyN9OnlTOEOXuNhjBFREREoax8et4GKdfD4qdg1avlaiIk0MUFnROZ/d129hw57uUCRUREpLpRKCsPY2D4P6HFIJhzJ+xYU65mxvduSnZuPjOWpnu5QBEREaluFMrKyxUIl74MriD4dma5mmgeF8mg5HhmLk0nKyfPywWKiIhIdaJQVhFhtaDlYGd+WX75QtUNfZqx/2g2b3+z1cvFiYiISHWiUFZR7S6BjF2Q/lW53n5209p0aBDDy19uJj/ferk4ERERqS4Uyiqq1VAICocf3ynX240x3NCnKT/vPcrCdRW/4bmIiIhUTwplFRUc4QSztR9AXm65mhjRoT4NaoUx9cufvVyciIiIVBcKZd7Q7hI4thd++bJcbw9yBXDdOUks37yf77Ye9HJxIiIiUh0olHlDy8EQHFnuIUyAK7o3IiokkBe/3OzFwkRERKS6UCjzhqAwaD0C0mZDXk65mogKDeLKsxox7/sdbDtY/vtqioiISPWkUOYt7UZC5gH4+fNyNzHunKYAvLJYvWUiIiJnGoUyb2kxEEJiKjSE2aBWGOd3rE/qil85nFW+HjcRERGpnhTKvCUwBNqcB2lzIDe73M3c2KcZGcdzSV2+xYvFiYiIiL9TKPOm9pfA8UOwaWH5m2gQQ6/mdZi2+Beyc/O9WJyIiIj4M4Uyb2raD0JrVWgIE+DGvs3YeTiLud9v91JhIiIi4u8UyrwpMBiSL4B18yAnq9zN9G8VR8v4SKZ+sRlrdeslERGRM4FCmbe1vwSyj8DG+eVuwhjDjX2bkbbjMEs27vNicSIiIuKvFMq8LakvRNaDb2ZUqJmLOicSFxWiWy+JiIicIRTKvM0VCF2vhZ8+gQPp5W4mJNDFuF5JfLFhD2k7DnuxQBEREfFHCmW+0G0sGAPfvFqhZsac3ZjwYBcv6dZLIiIiNZ5CmS/ENIRWw5whzAqsWVYrPJjLUxrxwZpt7DxU/gsHRERExP8plPlKyvVwdDesm1OhZsb3bkpevmX6V794py4RERHxSwplvtL8XKjVGFZOq1AzjWqHM6JDfaYt3szMpelaIkNERKSGUijzlQAXdBsHv3wJezZUqKm/X9Sens3rcN97P3B76moyjud6p0YRERHxGwplvtTlGggIglWvVKiZ2IhgXhnXnXuGtmbud9u58JnFuiJTRESkhlEo86XIeGeF/9WvQ05mhZoKCDDcOqAFb97Yg4zjuVz83BL+t/JXLxUqIiIiVU2hzNdSroesQ/Dju15p7uxmdZh3Rx9SkmL581vfkb7vqFfaFRERkaqlUOZrSb2hTktY8bLXmqwbGcL/jeoEwOw1umm5iIhITaBQ5mvGOL1l21bCjjVea7ZBrTBSmsTygUKZiIhIjaBQVhk6XwWBYfD5/4EXl7S4sHMiG3ZlsH7nEa+1KSIiIlVDoawyhMVC/784C8n++I7Xmh3RoT6uAMMHa7Z5rU0RERGpGgpllaXnH6BBN5h7N2Ts9kqTdSND6NW8DrPX7NCisiIiItWcQlllcQXCRc9DdgbMvctrw5gXdEpky/5jrNl6yCvtiYiISNVQKKtM8W1gwL2QNht+eNsrTQ5tV49gVwAfrNaEfxERkepMoayyFQxjzrvHK8OYMWFB9G8dx5zvtpOXf3rv27aDmdz539VsPXCswscSERER31Eoq2yuQLj435B9FObc6ZVhzAs7J7L7yHGWbd530vZj2bnc8OpK3v12G/9etKnCxxERERHfUSirCnGtnWHMdXPgk/vgh3dg60o4shPy88vc3MA2CYQHu5i9Zkfhtvx8y5/+t4b1Ow/TsWEM73yzjUOZOd78FCIiIuJFCmVVpdcfoPlA+PpZeOs6eGkgPNEaHqkHqWMg2/PhxrBgF4PbJvDhDzvIznVC3dMLfuLDH3Zy74hkHr2kA5k5eczSvTJFRET8VmBVF3DGCnDBNe9A1mE49Csc/NX5uWc9rHgJ3rgcRv8XgiM8au7CTom8v3o7izfu4Vh2HlMW/MRl3RoyvndTjDGclVSb175O57pzmuIKMD7+cCIiIlJWCmVVLTQaQttBQrvftjU6C969GV53B7OQyFKb6dMyjpiwIJ5duJG1Ow7TrUksD49sjzFOABvbK4lb3/iGz9btZlDbBF99GhERESknDV/6o46XwyUvwpav4PXL4Hjpt1EKDgxgRId6fLPlIHUiQvjP1d0ICXQVvj6kXQL1Y0J59etffFe3iIiIlJtCmb/qMAoufRl+XQYzRznDnKW4ukcTOjaMYeq13YiLCjnptSBXAFf3aMKXP+1l427dK1NERMTf+E0oM8a4jDHfGmPmVHUtfqP9JTBqGmxbCW9cAfl5Je7eLjGGD27rTbvEmCJfv7J7I4IDA3j1q3RfVCsiIiIV4DehDLgDSKvqIvxOu4vhwmecocwVL1WoqTqRIVzYKZG3v9nK4SwtjyEiIuJP/CKUGWMaAucBFUsdNVWnq6D5ubDg73B4R+n7l2BcrySOZecxa+VWLxUnIiIi3mCsl26MXaEijHkLeBSIAu621p5fxD43ATcBJCQkdEtNTfVpTRkZGURGln7VY2UJO7aD7iv+wN66Z7G23Z8r1NYjSzM5lG15rE8YAab6LY/hb+dGHDov/kvnxj/pvPgvb5+bAQMGrLLWppS2X5UviWGMOR/Yba1dZYzpX9x+1tqpwFSAlJQU279/sbt6xaJFi/D1Mcos4lfiP3uE+Ia50GJQuZvJqL2d2974lmN1WnN+x0QvFlg5/PLciM6LH9O58U86L/6rqs6NPwxfngNcaIz5BUgFzjXGzKzakvzUOXdAnRYw927IySx3M0Pb1aNdYjQT3/5eV2KKiIj4iSoPZdbaidbahtbaJOBKYKG19uoqLss/BYbAeU/Cgc3w5ZPlbibIFcCL16YQEuRi/KsrOXA024tFioiISHlUeSiTMmrWDzpeAYufgj0byt1MYq0wpl7bjR2Hsvjd66vIySv7jdBFRETEe/wqlFlrFxU1yV9OMeRhCA6HuXdBfvnDVNfGsUy+tANLf97PAx/8iD9c9CEiInKm8qtQJh6KjIfBk+CXL+GLf1aoqZFdGvK7/s15Y9kWXvtai8qKiIhUFYWy6qrrWGf9skX/gPUfFr9ffh5sWgg5WcXucs+Q1gxKjmfSnLUs2bjXB8WKiIhIaRTKqitj4PynoH5neOcm2PvT6ftkHYY3r4IZI+HFAbDrxyKbCggwPH1lF5LqhPPnt77j6PHcstdzcAvsWFP294mIiAigUFa9BYXBFTPBFQypo0++afm+TfDSINi0AHr9AY7uhakDYOm/i5yHFhkSyP+N6si2g5k8Ob8MFxDkZsOXT8Cz3WHaMMg+5oUPJiIicuZRKKvuajWCy6Y7Iezdm53A9fPn8OK5cHQ3XPOuc2HA776C5gPgownw+ig4svO0pro1juXqHo15Zclmvtt6sPRjb1kKL/SFBZOgXgfIOeaEQBERESkzhbKaoGkfGPoPWD8P3rjcGa6Mqgc3LoSmfZ19IuPgqlQ47wlI/wqe7wkvD3V+PtUeHm0Mk+pwX/SH1I0MYcLb35Nb3DIZmQdg9h0wbShkZ8BV/4XrPoKwWEibU3mfW0REpAZRKKspzr7Zmfi/cT60HALj50PtZifvYwx0vwFu/hwane0sRlu7GST1gc5XQYNuhH79NI8NrcfaHYeZtmTz6cexFt64Er6Z4QyL3roMWg8DVyC0Gg4bPoK8nMr5zCIiIjVIld/7UrzEGLhgirOwbNO+EOAqft+41jC6iBu6790Iz53FgD0zGdx2FE/O38Dw9vVpVDv8t31+eBt+XQoXPgNdrz35/W3OgzVvQPoSaNbfG59KRETkjKGespokMNiZN1ZSICtJ3RbQeTRm5cs8fG4tXMbw1/d++G1R2ZxMmP8A1OsInYu4E1bzcyEwTEOYIiIi5aBQJifr9xcAEr6Zwp+HteGLDXt4Zckv5Odb+OpZOLwVhj0KAUX80QkOhxYDYd3cCt1pQERE5EykUCYnq9UIUq6Hb2dydctcejarw6Q5axk75X1yv3wSm3wBJPUu/v3JF8CR7bD928qrWUREpAZQKJPT9fkTBIbg+uIxZt5wNk9c1omrjrxKXk42t+6+mK837Sv+va2GQkAgrJtdefWKiIjUAJroL6eLjHeu5lz8NK7ed3Jp/WzIW8jaZuP4Zlss815cSuuEKNrUj6JFXCQtEyJpER9JkzoRBIXFOj1paXNg0INV/UlERESqDYUyKVqv22HFNFj4CGTuh/C6tL1iEotckbyxbAtf/LSHlb8c4P3V2wvfUi86lClXdeGsNufDvLthz3rnSk8REREplUKZFC28trMO2WcPO8/PfwpCYwgFru/dlOt7NwXg6PFcNu3JYMOuDJ77bCNXvbiUB/q151qAtNkKZSIiIh7SnDIpXo9bICIO4ttBl2uL3CUiJJCODWsxqltDPrjtHIa1q8f9n+3n5+A25K71cF7Z3p/gjSucnjUREZEzlEKZFC8kCm78DMbNcVbsL0VUaBDPju7CpIva8XZmZwJ3rmb1Dz84y2kU5+he516cGz6COXc5dwwQERE5A2n4UkpWq1GZdjfGcG3PJNZF3gRvp/LOm1MZ7RpBy/hIWiVE0bpeFF0a16Jbk9qQkwVvXuXcHD1lPKx8GX58F9pf4qMPIyIi4r8UysQn2nToRt6iVtzKOlxJN7Nh1xE+W7+HWau2AvD05R25eNP9sHU5XPaqs77Z1uXwyd+cZTWCI6r4E4iIiFQuDV+Kz7jaXUzCvmU8cGQSrw/MYeVfB/LN3wbTrUksuz/4G/z4DgyeBO0udm4NNfyfzh0DFj9d1aWLiIhUOvWUie/0ucsJW8tfhFfPh/qdqN3zNqa2PUCdXe/yafhwzu3xh9/+ZdCkJ3S4DJb8C7qMgdikkts/sgt2rIad30G9TtBqiI8/kIiIiO8olInvBIVB/wlwzh3w3X/h6+fgnRupA+ys24tbtl7Fn5ds5qa+zX97z+BJsG4efPxXuPL1k9vLPAjfzqD99+/DqlvgyI6TX+9yDQx7DEIiff7RREREvE2hTHwvKAy6jXOW1dg4HzZ/QUK/P3Pu/37i8Y830KdlHMn1o519oxOh759gwSTYtBCan+tcofn1c7DiJTh+mLDwhtCiL9TvDIldIL4NLJkCi5+C9CVwyUvQsFvFas7NhsDgCn90ERERTymUSeUJCHAm8bcaigEevaQDQ5/+kj+mrub9284hNMjl7NfjVvJXzeD4B/cQ2HIgQatfg9wsaHsR9PkTK9bvp3///gDk5OXz1qqttGjxB7q3GATv3gwvD4b+E50euqO74dBW9+NX576crYZD3Ran15eX4yx4u3yqc0P16+ZBgwqGOxEREQ8plEmVqRMZwj9HdeS66SuY/NE6zutQnyUb97Fk415i917KC4GPk7tyMxsTz6PJRfcRlOC+O8D6RQAs/mkvD87+kY27M2hQK4xF9/Qn6JbFzi2ePnv4t7sRnOqT+yC+rXPFZ/IFEFkPvpnu3FbqyHZnLltIFMz+o7NOmwdrtJXL4R2QnwO1GvumfRERqVYUyqRKDWgTz9U9GvPKkl94ZckvGAPtE2Po0utSlgY0Y8amcOb+HEzSa9u5e2gUI9rXZ29mPr+buYoPf9hJkzrh3NKvOf/5fBNzvtvOyC4N4dKXIPlC2LEGYhpCTCP3z4aQddC5WXrabPjin/D5ZMAAFpoNgPOfhJZDIO0DmDUOlr8APW/1/gfPOuT06GUegNH/dW7iLiIiZzSFMqlyfx3RlvoxYTSPi6BHszrUCi+Yy9WWs61l1Po9PPbhOm5741taJ2xk855MAlzHuXtIK27o04xgVwAL0nbxwuc/c3HnBhhjoO2FzuNUIZHO7aN63AIZe2D9XDi4BTpecfJ9Otte7ISzhY84w6YxDb37oef9GQ5vd3rJZl4Kl8/Q1aMiImc4rVMmVS4s2MWtA1owrH39EwKZwxjDgDbxzLujD09c1gljoHO8iwV/6s9t57YkNMhFQIDhln7NWbfzCIvW7/H8wJFxzgUIA+8//cbpxsCIx8HmOwHKm358F75Lhb73wA0LnGOnjna2i4jIGUuhTKoFV4Dh0m4N+eiPffl951Aa1Ao76fULOyeSGBPKvxdt8t5BY5s4S3qsn+sMeXrD4e3OXLXErtD3boioA2NnOxcUvHU9fDvTO8cREZFqR6FMaoQgVwDj+zRj+S/7WZV+wHsN97wV4tvBh3+G40cq1lZ+Prz3e8jLhkteBFeQsz00Bq55B5r1h/dvhUWTnYVxRUTkjKJQJjXGld0bERMWxH8+92JvmSsILngaDm+Dzx51gtXh7fDrcvjhbfj6ediy1NlemhUvws+fwZCHT1+SIzgCrkp1LlBY9A94ohW8OBC+fAJ2rwNrvfeZRETEL2miv9QYESGBjO3ZhCkLN7Jx9xFaxEd5p+FGZ0G362Dp884aZvk5p+8Tlejcw7PtxdCwu7Mm24l2r4P59zsXD6RcX/RxAkPg8tdg91rnrgbr5zqL6C6YBHVaQtdroPMYiKjrnc8lIiJ+RaFMapSxvZKY+uXPvPD5z/zzsk7ea3jQg87CsyGR7uU1Gjs/w+vAL186k/RXvOwEt+gGEJkAOccg+6jzOH7YGaa88FnnIoLiGAMJ7ZxHv3ucXrn1H8L3bzmhbuHDztpq3cZBUh9n/+MZzi2njuxwhkab9q/Y2mrWwt6fnM+VvgRCouGc26F2s/K3KSIipVIokxqlTmQIl6c04s3lW7hrSCvqx4SV/iZPhNWC8x4v+rUOo5xH1mHY8JGzBlrOMQhqCMGREBwOQeHQ/lKISijbcaMToft457F7HayaDmvecIZOI+IhJxOyT5nrljLeWW/NU9bCnvWQvhh+WQK/LHbuhABOD2DmfvjmNeh0JfT5E9RpXnJ7IiJSLgplUuPc2KcZry/bwtPzf+Ifl3TAFVBCz5Q3hUZDx8udhy/Et4Hhj8GgB+DH95z5aWG1IaqeE96i6jm9akufh/hkOOvG4tvatwl+mu8EsfSv4Ng+Z3tUIjQf4Cxmm9QbYptCxi7n3qIrX4Y1qc6abp1HO710ednOfULzsok5+CvYfiX3BBYl9zis/cC50CEyrrzfjohItadQJjVOo9rhjD6rMTOWpvP9tkNMuqgdKUm1q7os7wkKg85XOY9TNTkH9m+GD/8CdVo4AetE1jrz4j6+F/JznWHYlkMh6RznvbFJp4eqqHow7B//3959h0dVpQ8c/56ZSTLphfRGQguEFEooUhQQVIq6AiK2xYLYVte17LqudddddV111Z9rb4gKFkRUrCiK0nvvPQlJIKTXyfWz8NQAACAASURBVJzfH2foCSSQMAO8n+e5TzJ37tw5M9cbX84573vMWqK/Pe8Kzj446q27AlTPg+HPNj642vYbfHkX7Nlgev5GvmoWoRdNV10Gy9438xqb2iMrhPAIEpSJM9LfL+1M7zatePyrNYx+ZS4ju8Vx/9CORAbaAaiqrWN1TgkrdxVRU+dkfL82WE5Vj1pLslhh1Ovw5gXw8Tizduf+4caaCvjyT6ZwbcowuOhJU4utsQKjTHDW7y7YvQKs3q7NC6w+bPn2FdpsmGLmoQ1/1iQ+NKSiEL5/yNRlC0mES16EuS/BeyPN+Qf+7WDJkGOpqzWJEfu2md6+ikKzVRaa1yf0gsRzzHy4pvbgnU52LoTPJkDhFjPUfP0MM4dRCHFakaBMnJGUUgzPiGFASgQv/bSJ12dv4bvVeQzuFMmGvDI25JXicB4sM2FRivH9z5CJ7D6BcOWH8NpA+OAKGP+DWfNzyjWwexUMfNDMDTsyQ7SxAiKh3eCjdu9oPZo2Q2+Hz24xAeGakWZVBL8wk+xQVQSVRZCzBH541Kz/2fcuOO8vZt5d2mj45n749Tkzr23UG6bnDkwPX1UxlO6G/NWwazFkLzbrmzoqD2+Il79JwKguMQEKgH8EJPaG1v2g04jmXzbLXepqzRquv/zHJJgM+QfMfAymXAtXfwI27+OfQwjhMSQoE2c0fx8bf76oI6O7x/P4V2uZvXEPqbFB3NyxDRnxIWTEB/PI56t56pt19G7TirS4pvUuaK3NWpueJjQJrpgEEy81a2vu3QRouOqjll1jM7KTWTrqt//CrCdh3Veg68xQ6aHie5r6b1GdD+7z9oNLXjBzy774I7xyrjlf2W5TTPfQ4Mtmh5hMU14krptZqsqvlZlj52V6Q3E6zbDoznmmltyOuSYJ45u/mLIlqb8z65qGJLTc99GS9myCqTeZIDdjLAz7t+kd8w+HabfC9D/AZa+e2T2EQpxhJCgTZ4U2EQG8dV2Pep97alQGQ5+fzZ0fLuWLO/rh71P/baG1ZtveCpZs38eSHftYsqOIrXvKuGNQe24f2K7e17hVUl+ThTn9DrMqwdhJp6ashdVmlpBKGWqGJ212k71qDwZ7iOlpS+jdcE9d2kgTaH3zgOntiu9hSowERkNANIS3N8Hc8YY3LRaTHBHZ0ZQQAZPgsGaaSZT47m9mS+hthmXjujfr13BCtAZn3bFLmtRWwbyXTO+Y1RtGv22+s/26XAXF2fDT46ZH8PyHW77dQohmIUGZOOuF+nvz7BWZXP3GfB77YjX/Hn14fTOtNZ8uyeapb9ZRUFoNQKCPjS6JIbTyD+Ppb9cT6ufNVb0S3dH8Y+v2exPARHQyPVGnUlRnuOiJE3ttaBJceXQywUlr1dYM3fa/xxWgfQ4LXoc3BkOfO2DAAwd72o6nuhTWf2N6Ict2m6HV0t0mW9XqDcEJrpp2+7cEM4cvOOHw9ygrgC2zYPOPJqO2cp8Zyu1xowlO99Ma1s8wSRr7tkHKcFOmJSj26Ladey+U7DIrQgTFAS1cxiRvtRlyzrqhcXMBhRD1kqBMCKBP23BuG9CWl37azLkdIhiRYf5Hl1dSxQNTVzJzXT5ZrUO5e0gHuiWG0i4yAKtFUVvnZMLERTw4bSWtAry5sHO0mz9JPTyhB8gTtWoL/e82wc93D5rM0nUz4NKXILFX/a9x1sHWn01pkP316MDMWQuINr150WmmzEfxLpP0UJJjhnAPFRBtArSaCshbafb5hkLyeWZO4KqpsGwSxHaFHuMhKs3Mw9vyE4SnwLWfHTtLVSkY9owJEmfcS4foIRCaBwk9TJmT5hrS3LMRZj1h2os25VVGvXlyxYuFOIvJnSOEy12DO/Dbpr38depKuiSEsHBbIY9OX0O1o46HRqRyfZ+kozI0vawWXrq6G1e9Pp87P1zKpPG96HEmld84G9iDTfZn58tg+p3w1oUmEApvb4KrOlOHjcp9sPZLKM0xr8m4whTUje127An1zjqz2kLRTijaAUXbYd9289PPBwY9ZAKsmEyTPQtw4T9hxUew8A2zSD2AT7DJmO0xvnG9UVYbjH4Lpt1G5Ppv4LNvzX6/cDMknHU9dLjwxL6zwq3w879NJq/N1/Q8evnCj/8wy4X97uWDn6W5VJeZci7e/tDjphNPVBHCg0lQJoSLl9XCC2O7MuyF2Qx7fjYlVQ66tw7l6dEZtIkIaPB1ft423rquB6NfmcON7yzk41v6kBLdTOtuilOn7SC4bS58/4hZPP5INrtJQrjoCehwUeOHOS3Wg0OYrc9p3Gvswab4b4/xprctZylkXtn0dU+9/WHMu/z600wGdIqCXQtM+Yxtv8IHY6Df3TDowcYHUFqbuWw/P2mWHet9G/T70+Ht2h+YjXj++IFTTTms/NgEoKHJ0PUakyV7aE9eXa1ZyeLnp6C8wOxb/zVc9orpmRTiDCJBmRCHSGzlxxMj03lw2ir+NqwTN/RLbtSKAGH+3ky8oScj/zeHcW8t4NPb+hAX0vQlnnYWVrB8VxF92oYT5i/lDE45n0CTHHH+QyYAcdVgw+rlnixGpQ6urnBS57GaYdXoNDPvq7bKZKH++qzJ3hz15vEDvppymHabSZRIGw0X/OPo+Wzn3guOKlOmw2aHof+u/3vbu9n0Ai59H6qLIbyDKW+ybBKEtTXBWeZYkzH74+Om/lrrvjD2A1OX7uv74eU+pkfuRHv73KW61Ax9L59MVv42CH0A0i9v/p5FcVqSoEyII1ycGcuIjJgml7qID/Xj3Rt6MubVuVz75nw+vvkcWgX4NPr1lTV1jHt7AVsKyrEo6JoYyqCOkQzqGEnH6EDPLL1xpvINdXcLWpaXHS5+HuKy4Kt74NXzYMxEiG9g/mHRTph8pZnQf8HjcM4fGg5SB/7NBGZzXgRcQWVZHpTlm597N5vlvSw2U5Kkx02md6y2wiReLJ1kaq3NfMycLzIVrvoY2g8x75nQ0xQE/uRG09vX6xYY/Njxey6Ld5m1aescgDZBN9qUaynfY3rhyvJM4kVloVm31j/clFrxjzBZw+mXn9jar3W1sPknWDHFlIlxVLpq8Fngs5vNdzX4MWh3vpQwOctJUCZEPU40AOoUE8Rb1/Xg2jfnM+7tBXx4U28C7Y3LRvvXjLVsKSjnyZHp5BZX8eO6fJ7+dj1Pf7ueNuH+PHdFFzITQk6oXULUq9u1EJ0OH11r5tL1utkkhkSlmfIpVhtsn2sKD9fVmDp37Ycc+5xKmSK2jmpY8KrZzBMmuAmMMlmu3ccdPvzo7W/KeXS5ygRuq6aa1R7SRx/dixSRYooi//AIzH8FNnxrEjYyrwL/VocfW7QDZj9rgj1nbf1ttvqYoMs/AoLjICbD9GhV7IWCdWa4t7LQrAE77OmDa782xFkHuctg62zYNtvUyaspM8F+l6vMfMSEniya9RMDwgvNkO/7oyCpvxkOtnqbcjBVxVBVYtrdYSiEe2DpnVOputR8J2dK8ed6uD0oU0olABOBKEADr2mtn3dvq4Q4cT2Swnj56u7cNHERN767iIk39MTudeyhiZ/W5/PevO2M75fM2J6mtMafhnQgv8QEZy/+uInLX5nLwxencnWvROk1E80ntgtM+NnUs5v38sFMUZvdDCvmrzXB0ZWTIaJD486plBm6zBhrhn4DokyPU2OzMlu1hfPuO/YxXnYY+hS0GwK//Ntk0M78O3S6xNSlC443w7PLPgCUCUB7325WmNjfRpQJ+LwDjt9DVZwNUyfA57eZLNjhz4I96ODzTids+wUWvwubfjBBFZhs2cyx0PZ8sxLGoUkhymKCzk6XwOK3zby5SSOp13cPQmIf8zlSLzVBbHOqKDTt8fXQf/it+8oUlS4vMEkxnUeapdz2r/pxhnB7UAY4gHu01kuUUoHAYqXU91rrNe5umBAnamDHSJ4Zk8ldU5Zx+/tLeOXa7nhZ65/0XFhew58/WUHH6EDuvTDlsOcig+yM7ZnIhZ2juWvKMh6ctorF2/fxz8vS8PP2hNtXnBH8wmDs+6Z3q2C9mbeVt8osy5U20gQ/TR3SVarh4dDm1H6w2fLWmISA5ZNh1SfmOas3dL/erKd6sr0rwXEwbrrpdZv1BOxaCKPeMgHrskkmGNu31XxPnS+D5HNNz1djFoe3eZteyswrTVkRL19XseUgU3C5thJWfgRL3jOrNcz4s7kuieeYwDm83eFrnVYVw+6VkLvC/LRYTfASmgQhrU05lsois9rFzvmwc4FZ/cLL38yn7DnBc+a4VRWbOYTLPzC9ur1vg3Vfml7SHx4xPbuJ55jPU17g2vaAdkLGGDOHsqFVO7Q2Nf/8I8Cn4WSuU8ntf9W11rlAruv3UqXUWiAOkKBMnNYu7RJHSZWDh6at4r6Pl/PsmC5HldTQWvPXqSsorqg9Zo9aqL83b1/Xgxd/3MR/Z25gTU4JL1/T7ZhZoftt3VNO6zC/Ji+4vresmknzdhAV5HOg906c4Ww+ZuguJsPdLWm6qFSz1NTgR00yQtFO06tUX3HdE2Wxmh685P7w6Xh46wJAmeHF1n1h4AOm16uxmblHsgdBykX1P9fvT2at2O1zYOl7JmN1ybsHnw+INkPOpTkm0DiwP8oEKPszV4/kGwoJvUxv3va5Zv3ZFR+ZOYeN/e+gssgkavgEHhwGtrnm0zqqTcCXt9psezaYNsV2NVtkasMlZbbMgmm3m5Iy594H5/7ZHNv/blNWZs00M8y94HUz/88/3Lx3RIopYfPbf83WYSj0HA/JA8y8wW2zYcvPsPUXKN4Bl79ret08gNJaH/+oU0QplQT8AqRprUuOeG4CMAEgKiqq++TJk1u0LWVlZQQEeEbkLA53ul2bLzbX8OnGWjqFWegf70XXSCu+NhMgzd5Vy5urargixZuhyY2be7aywMGrK6pxOOHKjt6cG2+rdzizuk7z4boaZu10MCzZizEpjcvmLKhw8s22WmbvclDjBAU8co6dpOBj/8v5dLsuZxO5Ns3PVltG0rYP0cpKbswFVPg3vSfuZK6LcjqwV+XhV7ELv4ps/Cp24VuZS413CGUBbQ5sNT6mh9NSV4W9Kh/fyjzsVXnUWe0UB3ek0jfu4NCt1kTmz6bdpjfwqi1lZ8Lv2JY0Fqf1iIQl7SSwdDNhhUsJ3beU4OJ1KJyHHVJr88dh88enei8W15C4U9mo9I3Fu2YvXo7yA/vKApKp9QoCNEqb81ictYQUr6bCN461ne6iNKiRQ+eH8KkqIDbnG2Jyv8O7toRaWyBejlJX+wIoCklnX2gGe1v1pNp+ePZxc98zAwcOXKy1zjrecR4TlCmlAoCfgX9qrace69isrCy9aNGiFm3PrFmzGDBgQIu+hzgxp9u10Vrz5q9befu3bWQXVeJjs3B+p0gGdIjksS9Wkx4fzAfjezepJyu7qJJ7PlrGvC2FnNchgidHpRMTfLAEx/rdpdzx4RI25JXRISqAzQXlfH5732MuuL4pv4yXftrE9OU5WBT8rkscV/ZKZMLExcQE2/nstj7YGhiChdPvupxN5Np4Jo+9LhWF8P3DpkfO5muWaLN4mYxZq80kH1QWmmNju5r5cq37mKzbsvyDw4gVhWaoNDLVJI+0amvmGO4fNsxZarbcZWYSP8rMa9s/36/1OTDgr2Y492TUVpletU0zzRBo8rkQnXHMOnrNfW2UUo0Kytw+fAmglPICPgXeP15AJsTpRinF+P5tuKFvMkt37mP6shy+WpnLjJW7CbTbeKaeYc3jiQvx5YPxvXlv3nae/HodFzz3C49c3JlR3eKYNG87j3+1lkC7FxNv6ElmfAjnP/szf526kmm396237trGvFJG/m8OdVpzfZ8kbuyffCDIe/SSVP7wwVLembON8f1PwYLmQgj38guDS//PzHFb+4XJvHU6Dm5WbxPYtBkIARFNP79SEJZstrQGEhuak5fdDM9mjm359zpJbg/KlBl3eRNYq7V+1t3tEaKlWCyK7q3D6N46jIdGpLJgayHBfl4nVGR2//nG9UnivA4R3PfJcu79eDnPz9zAzsJKzusQwTNjMgl31Ul75OJU7vjQBFY39ks+7Dx7y6q54d2F+HhZmXZ7H+JDD1+4fHh6DFM7ZvPs9xu4KC36qOeFEGeopL5mE6eMJywe1he4FhiklFrm2oa5u1FCtCSb1UKfduF0jm14OLGxksL9mTLhHB4akUqNw8mDwzvx9nU9DgRkACMyYhjUMZJnvlvPrn0VB/ZXO+q4ZdJi8kqqef333esNuJRS/P3SzmgND3++Gk+Z8iCEEGcatwdlWutftdZKa52hte7i2ma4u11CnE4sFsWN/ZKZ/8Bgxvdvc9Rw6P7ACuDBaavQWqO15m+frWLhtn08PTqDrokNlzyID/Xjngs68OO6fGas3N2in0UIIc5Wbh++FEKcGiawSuEfX67hyxW5ZBdV8sniXdx5fnsu7RJ33Ndf1yeJz5fl8OgXq+nXPhy7l4WFW/cxa30+szYUUFBcwRfpFSS2kuFNIYQ4ERKUCXEWMYFVNg98tpKyagfDM2K46/z2jXqtzWrhiZHpXPrSb1z2v9/YXVxFRU0d3lYLvdqEkVNYxoT3FvHprX3w95E/LUII0VRuH74UQpw6VoviyZEZVNTUkREXzDOXZzYp8zMtLpg7B7VHaxjZLY43x2Wx7JEhvHdjL27r4sOGvFLu+2S5zDsTQogTIP+cFeIskxobxIw7+xMbYj/umpz1+ePg9vxx8NG9a2nhNu4fmsy/ZqzjpZ828YdBjeuBE0IIYUhQJsRZKCU6sEXOe1P/NqzOKeGZ7zfQKSaI8zs1Yt0/IYQQgAxfCiGakVJmeDQ1Joi7Ji9jU36Zu5skhBCnDekpE0I0K19vK6/9PotLXvyVmyYu4qlRGfRMDmvw+O17y/l08S6qHE7sNgs+XlZ8bBb8vG0MS48mxK9xa3YKIcTpToIyIUSziwvx5ZVru3PLe4sZ8+pceiaFcfugdpzbPvzA4unLdxbx2i9b+HpVLkopvKyKaoeTQ3MEpi3NZvKEpq0LKoQQpysJyoQQLaJHUhi//mUQkxfu4LVftjDurQWkxwUzuns8M1bmMn9rIYF2Gzef15br+yQRGWRHa01NnZNqh5Mvl+fywGcreXfuNq7vm3zc9zsWrTUv/riJ2RsLeGNcD4J9vZrnQwohRDOSoEwI0WJ8va1c3zeZq3u1ZuqSXbz882Yemb6a2GA7Dw7vxNieiQQcUtNMKYWPzYqPzcqVPRP4YW0eT32zjoEpkSSF+59QG2ocTu6fuoKpS7IB+PsXa3hmTGazfD4hhGhOEpQJIVqct83C2J6JjO4ez7rdpaREB+JlPXaekVKKf12WzpDnfubPn65g8k1NH8YsqarllvcWM2fzXu4e0oEah5P/+2kTF6VFMyRVMkOFEJ5Fsi+FEKeMzWohLS74uAHZftHBdh65uDMLthYyce62Jr1XTlEll788lwVbC3nm8kzuPL89d57fno7Rgfx16kr2ldc0/QMIIUQLkqBMCOHRRnWLY2BKBE99s57te8sb9ZqF2wq57H+/kVNUybs39GRU93jA9Ng9O6YLxZU1PPT5qpZsthBCNJkEZUIIj6aU4omRGdisivs+WYHT2fASTou2FXLtm/O5/JW52CwWPrm1D33bhR92TGpsEHcOas+XK3L5akXucd9fa83i7YX848s1fL4sm6raupP+TEIIUR+ZUyaE8HjRwXYeHpHKfZ+s4JZJi+maGErbCH/aRATQupUfS7bv4/mZG5mzeS+t/L3569COXNO7dYMLo986oC0/rM3jwWkr6ZkcRkSgz1HHlFc7mLYsm/fmbmfd7lIsCpwagn29uKxrHFf0SKBTTFBLf3QhxFlEgjIhxGlhdPd4VueUMGNlLt+tyTuw32pR1Dk14QE+PDi8E1f1SsTP+9h/2mxWC8+MyWTYC79y90fLGJYeQ3VtHdUOJ1W1TnKLK/lyRS5l1Q5SY4J4YmQ6F2fGsmJnEZMX7uSD+Tt4Z842MuODGd+/DcPTY6SWmhDipElQJoQ4LSilePSSzjx6SWdKqmrZUlDOloIyNheUERVk5/LuCfh6N36B9XaRgdx/UUf+/uUaZm/cc9hzvl5WhqZFc805remaEHKg4G2fduH0aRfOvvIaPluazfvzt3PHh0v536zN3HdhBwamRB44tim01izYWsh787aTV1LF34an0iUhpMnnEUKc3iQoE0KcdoLsXnRJCDnpwOWGfskMz4jBqTV2mxUfLws+NivW4/R6hfp7c0O/ZMb1SeKL5Tk898MGbnhnEd1bh3LfhSn0TAqjtNpBSWUtxa7NohSRQT5EBdkP1GYrrapl2tJs3pu3nQ15ZQTZbfh6Wxn18hz+NLg9tw5od9y2CCHOHBKUCSHOalFB9hN+rdWi+F3XOIZnxPDxol28MHMjY1+bd2D+WUP8va1EBtnJL6mivKaO9Lhg/j06g4szYqmpc/LgtFX857sN/LJhD8+N7UJciO8Jt1EIcfqQoEwIIU6Sl9XCVb0SGdktjo8X7SS/tJpgXy+CfL0Idm11Tk1+aRV5JdXkl1STV1pFr+QwxvZMPKzHzxcrL4ztwoAOETz8+Sou+u8v/OPSNC7OjJVeMyHOcBKUCSFEM7F7Wbn2nKSTPo9SilHd48lKCuWuKcu4a8oy/jVjLRdnxnJpl1jS44JPaO6aEMKzSVAmhBAeqnUrfz6++Ry+XZ3H567yHG/+upXkcH8uyYxldPd4EsL83N1MIUQzkaBMCCE8mM1qYXhGDMMzYiiuqOWb1bl8viyHF37cyAs/bqRfu3DG9khkSGoU3rZj1wMvrqxl+vIcPl28i/ySKpRSWCxgUQoFpEQHMuHctnRvHdrgOQpKqymvdpzwAvFCiIZJUCaEEKeJYD8vruiRyBU9EskpquSjRTv5aOFObv9gCWH+3ozsGkd6fDBRQXZigu1EBdnxsVlYX1jH9CnLmLEql6paJx2jA+nTLhytTTkOp9Y4nJpfN+3h29V59EwO49YBbRnQIQKlFBU1Dr5fk8fUJdn8umkPTq3527BOjO/fxt1fiRBnFAnKhBDiNBQb4stdgztwx6D2zN5YwOQFO3lnzjYcR6R9+nlbqaipI8Anj5Hd4rkiK4GM+PrnpJVXO5iycCevz97C9W8vpFNMEClRAXy/Jo/ymjriQny55bw2bCko5/Gv1pJdVMmDw1MlAUGIZiJBmRBCnMasFsWAlEgGpERSXu0gt7iS3cXV5BZXkldSRUFpNbbSXO4ZM/C4Kx34+9i4oV8y1/RuzfTlObzy82Zmrsvn4sxYLusaR4+kMCwWhdOp+eeMtbz561Zyiip5fmxX7F6NL9y7X43Dya59FewuriI9PphAu9eJfg1CnBEkKBNCiDOEv4+NdpGBtIsMPGz/rFl7jhuQHcrbZmF093hGd49Ha31Ur5rFonhoRCqxIb48/tUarnp9Hm+M60GQ3cbOfZVszCtlU0EZOwsrcNSZnjulQKGo05qcokq2760gt7jyQD23uBBfnruiCz2Tw07uS3Cjypo6Pl+WTVSwnYEpke5ujjgNSVAmhBCiQccqvXFjv2Rigu3cNWUZA57+iapaJzV1zgPPhwd44221oAGt958PIoPsZCWF0josjsRW/gT4WHni63WMfW0utw9sx53nt8fLeuykBU+yp6yaiXO3897cbeyrqMVmUbwxLosBEpiJJpKgTAghxAkblh5DZKAPE+duJybYTrvIANpFBtA2MoCgJgxH9msfwaPTV/Pij5uYvXEPz4/tQutWh2d4OuqcWJRq9OLvjjony3YW8fOGAn7eUMC2PeVc0DmasT0S6N469IRqvTnqnJRUOSiurGVvWTWfLsnm0yW7qHE4GdwpimvPac2TX6/jtveXMGXCOaTHBzf5PcTZS4IyIYQQJyUrKYyspJMbdgzwsfGfyzMZkBLBA1NXMuz52fRq04p9FTXsK6+hsLyGkioH/t5WUmODSIsLJj0umLS4YMIDfMgrqWJ3SRX5JVXsLq5mfV4JszfuobTKgUVBt8RQBnWM5OuVuXyyeBdtI/wZk5XAZd3iiAjwOSpAK6qoYVV2CSuzi1mVXcya3BIKSqspq3Ycdpy3zcKobvGM759M24gAADpGBzLyf3O4/p0FfHprn6OCSyEaIkGZEEIIjzEiI5auiaE88vlqcosrCfP3JiHUj1A/L0L8vCmqqGFldjEfLtjB27XOBs8TG2xnWFoM56VE0LdtOMF+pteuvNrBVytzmbJwJ098vY4nvl6HUuBtteBtMwvSK2Xqse2XEOZL55hgBqREHFg2a/+WmRBCeIDPYe8dFWTn3Rt6MvqVOYx7ywRmrY44Roj6SFAmhBDCo8SF+PLGuKxjHuOoc7JlTzkrdxVTXFlLVJCd6GAfooLsRAT64GOrPxvU38fGmKwExmQlsCm/lB/W5lNR7aDa4aTaYebE1dVpkiP8SYsNJi0uiBA/7yZ/hnaRAbw5LourXp/PDe8s5IOberOnrJolO/axdEcRS3bsY09RBdepzYztkXBC7yHOPBKUCSGEOO3YrBY6RAXSISrw+Ac3oL5M1ebUvXUYL17ZlVsmLabrP76nxmF69vy9rWQmhFDrrXjy63X894cNXNY1nuv7JjXq85RW1bIyu5gAHxsxwb608vdu9Dw74dkkKBNCCCFayAWdo/nf1d2Ytb6A9PhguiWG0iEqEKtFMWvWLKJSuvHOb9uYumQXHy7YQY+kUDrHBpMc7n9gC/P3ZumOIuZs3sOczXtZmV1M3SFFgr2siqggO7EhvozIiGFsj8TjLrklPJMEZUIIIUQLuigthovSYup9rlNMEE+NzuAvQzvy4YIdzFiZy0eLdlJRU3fUsVaLIjM+mFvPa0tWUig1Die5xVXkFlexu7iSjfllPPz5al77ZQt3De7AZV3jDlttQWvN6pwSflqXT5CvF6O7x+Pv07xhehxDNQAADGhJREFUQI3DyYa8UlZlF7NzXwVXZCWS2MqvWd+jOdVXh8+dJCgTQggh3CzM35vbB7bj9oHt0FpTUFrNlj3lbNtTTn5pNenxwfRICiPgGEGU1prZG/fw9Lfruffj5bzy82buHtKBAB8b36/JY+baPHKKqw4c/+z3Gxh3TmvG9Uk6KhGhsqaO5buKyC+tJrmVP20i/I8K4Kpq61i3u5TVOcWsyi5hdU4x63JLD6tV98H8Hbz2+yx6HCM7d09ZNU6nJiLw6CzYI9+roLQaL6s6kJjhZbXg72MjNsTepALJVbV1TF2SzVu/beWZyzPJTAhp9GtbkgRlQgghhAdRShEZZCcyyE7vNq2a9LpzO0TQv30436zazX++W89t7y8BwO5loX/7CO4a0oFBHSPZUVjBK7M288KPm3j1ly2MyUogKyn0QBLCmpySo9ZRjQm20zYigFB/bzbsNqs27B9GDfb1Ii0uiOv7JZEWa8qVOLVm/LuLuPr1+Tw5Kp2R3eIPO19JVS0vztzIO3O2UVunCfP3JiUqkI4xgXSMDqTG4WRldjErs0vYmFd6VHuOFOrnRWyIL7EhviSE+pEWF0RGfDDJ4QEHegzzS6p4b952Js3bzr6KWtLigqh2NJzFe6pJUCaEEEKcQZRSDE2PYUhqFD+szcPLaqFvu/DD1icND/Dhtd9nsSm/jNd+2czkhTt4b9527F4WMuNDuPm8NnRvHUpsiC/b9pSzuaCczfllbC4oY9vecjpEBXJh5yhSY4PpHBtEfKhvvb1cU2/rw62TlnD3R8vZUlDO3UM6oIGPF+3k6W/XU1hRw+Xd40mNCWLd7lLW7i5l8oKdVNaa4dtQPy/S40MY1DGC9Lhg4kL8qHU6qXE4qa0zW0mlg5ziSnKKKskpqmJnYQW/btxz4Bz+3lbS4oIJ8/fmh7V5OJyawZ2iuLFfMr2Sw2T4UgghhBAty2a1NDiXbb92kQH8e3Qm916QQn5pNSnRgUctcdUxOuiE2xDi583EG3vy0LRV/N9Pm1i3u5ScokrW5JbQIymUd0b0PGrVA6dTs6OwAptVERdSf7B3PHVOzeaCMlbsKmbFriJW7Cpm4bZ9XN2rNdf1SSIp3DML+kpQJoQQQpzl9g+XtgQvq4UnRqbTJsKfJ75eR0yQnRev7MqIjJh6Ay6LRZ100GS1qAMlU0Z3jz/+CzyEBGVCCCGEaFFKKSac25YhqdHEBNsPG0oVB0lQJoQQQohTItlDhw09hVSXE0IIIYTwABKUCSGEEEJ4AAnKhBBCCCE8gARlQgghhBAeQIIyIYQQQggPIEGZEEIIIYQH8IigTCl1kVJqvVJqk1Lqfne3RwghhBDiVHN7UKaUsgIvAUOBVOBKpVSqe1slhBBCCHFquT0oA3oCm7TWW7TWNcBk4FI3t0kIIYQQ4pRSWmv3NkCp0cBFWuvxrsfXAr201n844rgJwASAqKio7pMnT27RdpWVlREQENCi7yFOjFwbzyTXxXPJtfFMcl08V3Nfm4EDBy7WWmcd77jTZpklrfVrwGsAWVlZesCAAS36frNmzaKl30OcGLk2nkmui+eSa+OZ5Lp4LnddG08YvswGEg55HO/aJ4QQQghx1vCEoGwh0F4playU8gbGAtPd3CYhhBBCiFPK7cOXWmuHUuoPwLeAFXhLa73azc0SQgghhDil3B6UAWitZwAz3N0OIYQQQgh3cXv25YlQShUA21v4bcKBPS38HuLEyLXxTHJdPJdcG88k18VzNfe1aa21jjjeQadlUHYqKKUWNSZ9VZx6cm08k1wXzyXXxjPJdfFc7ro2njDRXwghhBDirCdBmRBCCCGEB5CgrGGvubsBokFybTyTXBfPJdfGM8l18VxuuTYyp0wIIYQQwgNIT5kQQgghhAeQoEwIIYQQwgNIUFYPpdRFSqn1SqlNSqn73d2es5VSKkEp9ZNSao1SarVS6o+u/WFKqe+VUhtdP0Pd3dazkVLKqpRaqpT60vU4WSk133XfTHEtmyZOMaVUiFLqE6XUOqXUWqXUOXLPeAal1J9cf8tWKaU+VErZ5b459ZRSbyml8pVSqw7ZV+89oowXXNdnhVKqW0u2TYKyIyilrMBLwFAgFbhSKZXq3ladtRzAPVrrVKA3cLvrWtwPzNRatwdmuh6LU++PwNpDHj8FPKe1bgfsA250S6vE88A3WuuOQCbmGsk942ZKqTjgTiBLa52GWVZwLHLfuMM7wEVH7GvoHhkKtHdtE4CXW7JhEpQdrSewSWu9RWtdA0wGLnVzm85KWutcrfUS1++lmP+5xGGux7uuw94FfueeFp69lFLxwHDgDddjBQwCPnEdItfFDZRSwcC5wJsAWusarXURcs94Chvgq5SyAX5ALnLfnHJa61+AwiN2N3SPXApM1MY8IEQpFdNSbZOg7GhxwM5DHu9y7RNupJRKAroC84EorXWu66ndQJSbmnU2+y/wZ8DpetwKKNJaO1yP5b5xj2SgAHjbNbT8hlLKH7ln3E5rnQ38B9iBCcaKgcXIfeMpGrpHTmlMIEGZ8HhKqQDgU+AurXXJoc9pU9NF6rqcQkqpEUC+1nqxu9sijmIDugEva627AuUcMVQp94x7uOYoXYoJnGMBf44eQhMewJ33iARlR8sGEg55HO/aJ9xAKeWFCcje11pPde3O29997PqZ7672naX6ApcopbZhhvcHYeYxhbiGZUDuG3fZBezSWs93Pf4EE6TJPeN+g4GtWusCrXUtMBVzL8l94xkaukdOaUwgQdnRFgLtXRkx3piJmNPd3Kazkmue0pvAWq31s4c8NR0Y5/p9HPD5qW7b2Uxr/VetdbzWOglzf/yotb4a+AkY7TpMrosbaK13AzuVUimuXecDa5B7xhPsAHorpfxcf9v2Xxu5bzxDQ/fIdOD3rizM3kDxIcOczU4q+tdDKTUMM2fGCryltf6nm5t0VlJK9QNmAys5OHfpAcy8so+ARGA7MEZrfeSkTXEKKKUGAPdqrUcopdpges7CgKXANVrrane272yklOqCScDwBrYA12P+AS73jJsppR4DrsBkli8FxmPmJ8l9cwoppT4EBgDhQB7wCDCNeu4RVwD9f5ih5grgeq31ohZrmwRlQgghhBDuJ8OXQgghhBAeQIIyIYQQQggPIEGZEEIIIYQHkKBMCCGEEMIDSFAmhBBCCOEBJCgTQogmUEpppdTo4x8phBBNI0GZEOK0oZR6xxUUHbnNc3fbhBDiZNmOf4gQQniUH4Brj9hX446GCCFEc5KeMiHE6aZaa737iK0QDgwt/kEp9ZVSqkIptV0pdc2hL1ZKpSulflBKVSqlCl29b8FHHDNOKbVSKVWtlMpTSr17RBvClFIfK6XKlVJb6nmPh13vXa2U2q2Umtgi34QQ4owiQZkQ4kzzGGa9ui7Aa8BEpVQWgFLKH/gWKAN6ApcBfYC39r9YKXUz8CrwNpABDANWHfEeD2PWxssEpgBvKaUSXa8fBdwL3Aa0B0YAC1rgcwohzjCyzJIQ4rShlHoHuAaoOuKpl7TWf1FKaeANrfVNh7zmB2C31voapdRNwH+AeK11qev5AZhFodtrrTcppXYBk7TW9zfQBg08qbX+q+uxDSgBJmitJyml7gZuBtK01rXN9uGFEGc8mVMmhDjd/AJMOGJf0SG/zz3iubnAcNfvnYAV+wMylzmYBe9TlVIlmAWiZx6nDSv2/6K1diilCoBI166PgT8CW5VS3wLfANNlkWkhxPHI8KUQ4nRTobXedMS2pxnO25RhgyN7wDSuv6da651ACqa3rAR4BljsGjoVQogGSVAmhDjT9K7n8VrX72uBdKVU4CHP98H8LVyrtc4HsoHzT6YBWusqrfVXWus/AT2AzkDfkzmnEOLMJ8OXQojTjY9SKvqIfXVa6wLX7yOVUguBWcBoTIDVy/Xc+5hEgIlKqYeBUMyk/qla602uY/4JPKeUygO+AvyA87XWzzSmcUqp6zB/W+djEgquwPSsbWzi5xRCnGUkKBNCnG4GA7lH7MsG4l2/PwqMAl4ACoDrtdYLAbTWFUqpC4H/YjIiqzBZlH/cfyKt9ctKqRrgHuApoBCY0YT2FQF/wSQUeAFrgJFa661NOIcQ4iwk2ZdCiDOGKzPycq31J+5uixBCNJXMKRNCCCGE8AASlAkhhBBCeAAZvhRCCCGE8ADSUyaEEEII4QEkKBNCCCGE8AASlAkhhBBCeAAJyoQQQgghPIAEZUIIIYQQHuD/AaXTMbem0kK8AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x504 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Visualize the training process\n",
    "train_loss = history.history['loss']\n",
    "val_loss = history.history['val_loss']\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(10, 7))\n",
    "ax.plot(train_loss, label='Training Loss')\n",
    "ax.plot(val_loss, label='Validation Loss')\n",
    "ax.set_title('Loss vs. Epochs', fontsize=16)\n",
    "ax.set_xlabel('Epochs', fontsize=14)\n",
    "ax.set_ylabel('Loss', fontsize=14)\n",
    "ax.legend(fontsize=14)\n",
    "ax.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract Features using Triplet Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train:\t\t (113783, 4)\n",
      "Validation:\t (22255, 4)\n",
      "Test:\t\t (22391, 4)\n",
      "\n",
      "Train Landmarks:\t 14943\n",
      "Validation Landmarks:\t 7674\n",
      "Test Landmarks:\t\t 14436\n"
     ]
    }
   ],
   "source": [
    "train_df = pd.read_csv('./data/triplet/train.csv')\n",
    "val_df = pd.read_csv('./data/triplet/validation.csv')\n",
    "test_df = pd.read_csv('./data/triplet/test.csv')\n",
    "\n",
    "print('Train:\\t\\t', train_df.shape)\n",
    "print('Validation:\\t', val_df.shape)\n",
    "print('Test:\\t\\t', test_df.shape)\n",
    "\n",
    "print('\\nTrain Landmarks:\\t', len(train_df['landmark_id'].unique()))\n",
    "print('Validation Landmarks:\\t', len(val_df['landmark_id'].unique()))\n",
    "print('Test Landmarks:\\t\\t', len(test_df['landmark_id'].unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "vgg16 (Model)                (None, 512)               14714688  \n",
      "_________________________________________________________________\n",
      "l2_norm (Lambda)             (None, 512)               0         \n",
      "=================================================================\n",
      "Total params: 14,714,688\n",
      "Trainable params: 7,079,424\n",
      "Non-trainable params: 7,635,264\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Load trained model\n",
    "base_model = load_model('./models/vgg16-base-0.3-model.h5')\n",
    "base_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define train_imgs and test_imgs\n",
    "train_imgs = np.zeros(shape=(len(train_df), 512), dtype=np.float32)\n",
    "val_imgs = np.zeros(shape=(len(val_df), 512), dtype=np.float32)\n",
    "test_imgs = np.zeros(shape=(len(test_df), 512), dtype=np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Process:          0\n",
      "================================================================================\n",
      "Process:      20000\n",
      "================================================================================\n",
      "Process:      40000\n",
      "================================================================================\n",
      "Process:      60000\n",
      "================================================================================\n",
      "Process:      80000\n",
      "================================================================================\n",
      "Process:     100000\n",
      "========================================================"
     ]
    }
   ],
   "source": [
    "# Process training images\n",
    "img_ids = train_df['image_id'].values\n",
    "steps = 20000\n",
    "for i in range(0, len(train_df), steps):\n",
    "    tmp_imgs = []\n",
    "    print('\\nProcess: {:10d}'.format(i))\n",
    "    \n",
    "    start = i\n",
    "    end = min(len(train_df), i + steps)\n",
    "    for idx in range(start, end):\n",
    "        if idx % 250 == 0:\n",
    "            print('=', end='')\n",
    "            \n",
    "        img_id = img_ids[idx]\n",
    "        path = './data/triplet/train/' + str(img_id) + '.jpg'\n",
    "        img = load_img(path, target_size=img_size[:2])\n",
    "        img = img_to_array(img)\n",
    "        tmp_imgs.append(img)\n",
    "        \n",
    "    tmp_imgs = np.array(tmp_imgs, dtype=np.float32) / 255.0\n",
    "    tmp_prediction = base_model.predict(tmp_imgs)\n",
    "    train_imgs[start: end, ] = tmp_prediction\n",
    "    _ = gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Process:          0\n",
      "================================================================================\n",
      "Process:       4000\n",
      "================================================================================\n",
      "Process:       8000\n",
      "================================================================================\n",
      "Process:      12000\n",
      "================================================================================\n",
      "Process:      16000\n",
      "================================================================================\n",
      "Process:      20000\n",
      "=============================================="
     ]
    }
   ],
   "source": [
    "# Process validation images\n",
    "img_ids = val_df['image_id'].values\n",
    "steps = 4000\n",
    "for i in range(0, len(val_df), steps):\n",
    "    tmp_imgs = []\n",
    "    print('\\nProcess: {:10d}'.format(i))\n",
    "    \n",
    "    start = i\n",
    "    end = min(len(val_df), i + steps)\n",
    "    for idx in range(start, end):\n",
    "        if idx % 50 == 0:\n",
    "            print('=', end='')\n",
    "            \n",
    "        img_id = img_ids[idx]\n",
    "        path = './data/triplet/validation/' + str(img_id) + '.jpg'\n",
    "        img = load_img(path, target_size=img_size[:2])\n",
    "        img = img_to_array(img)\n",
    "        tmp_imgs.append(img)\n",
    "        \n",
    "    tmp_imgs = np.array(tmp_imgs, dtype=np.float32) / 255.0\n",
    "    tmp_prediction = base_model.predict(tmp_imgs)\n",
    "    val_imgs[start: end, ] = tmp_prediction\n",
    "    _ = gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Process:          0\n",
      "================================================================================\n",
      "Process:       4000\n",
      "================================================================================\n",
      "Process:       8000\n",
      "================================================================================\n",
      "Process:      12000\n",
      "================================================================================\n",
      "Process:      16000\n",
      "================================================================================\n",
      "Process:      20000\n",
      "================================================"
     ]
    }
   ],
   "source": [
    "# Process test images\n",
    "img_ids = test_df['image_id'].values\n",
    "steps = 4000\n",
    "for i in range(0, len(test_df), steps):\n",
    "    tmp_imgs = []\n",
    "    print('\\nProcess: {:10d}'.format(i))\n",
    "    \n",
    "    start = i\n",
    "    end = min(len(test_df), i + steps)\n",
    "    for idx in range(start, end):\n",
    "        if idx % 50 == 0:\n",
    "            print('=', end='')\n",
    "            \n",
    "        img_id = img_ids[idx]\n",
    "        path = './data/triplet/test/' + str(img_id) + '.jpg'\n",
    "        img = load_img(path, target_size=img_size[:2])\n",
    "        img = img_to_array(img)\n",
    "        tmp_imgs.append(img)\n",
    "        \n",
    "    tmp_imgs = np.array(tmp_imgs, dtype=np.float32) / 255.0\n",
    "    tmp_prediction = base_model.predict(tmp_imgs)\n",
    "    test_imgs[start: end, ] = tmp_prediction\n",
    "    _ = gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train:\t\t (113783, 512)\n",
      "Validation:\t (22255, 512)\n",
      "Test:\t\t (22391, 512)\n"
     ]
    }
   ],
   "source": [
    "print('Train:\\t\\t', train_imgs.shape)\n",
    "print('Validation:\\t', val_imgs.shape)\n",
    "print('Test:\\t\\t', test_imgs.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save to disk\n",
    "np.save('./data/triplet/train-triplet-vgg16-0.3-features.npy', train_imgs)\n",
    "np.save('./data/triplet/validation-triplet-vgg16-0.3-features.npy', val_imgs)\n",
    "np.save('./data/triplet/test-triplet-vgg16-0.3-features.npy', test_imgs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Features and Labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train:\t\t (113783, 512) (113783, 4)\n",
      "Validation:\t (22255, 512) (22255, 4)\n",
      "Test:\t\t (22391, 512) (22391, 4)\n"
     ]
    }
   ],
   "source": [
    "# Already normalized\n",
    "train_feature = np.load('./data/triplet/train-triplet-vgg16-0.3-features.npy')\n",
    "val_feature = np.load('./data/triplet/validation-triplet-vgg16-0.3-features.npy')\n",
    "test_feature = np.load('./data/triplet/test-triplet-vgg16-0.3-features.npy')\n",
    "\n",
    "train_df = pd.read_csv('./data/triplet/train.csv')\n",
    "val_df = pd.read_csv('./data/triplet/validation.csv')\n",
    "test_df = pd.read_csv('./data/triplet/test.csv')\n",
    "\n",
    "print('Train:\\t\\t', train_feature.shape, train_df.shape)\n",
    "print('Validation:\\t', val_feature.shape, val_df.shape)\n",
    "print('Test:\\t\\t', test_feature.shape, test_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Helper function\n",
    "def accuracy(true_label, prediction, top=1):\n",
    "    \"\"\" function to calculate the prediction accuracy \"\"\"\n",
    "    prediction = prediction[:, :top]\n",
    "    count = 0\n",
    "    for i in range(len(true_label)):\n",
    "        if true_label[i] in prediction[i]:\n",
    "            count += 1\n",
    "            \n",
    "    return count / len(true_label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Implement KNN Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Merge train and validation features\n",
    "train_val_feature = np.concatenate((train_feature, val_feature), axis=0)\n",
    "train_val_df = pd.concat((train_df, val_df), axis=0)\n",
    "train_val_df = train_val_df.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "NearestNeighbors(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "         metric_params=None, n_jobs=-1, n_neighbors=50, p=2, radius=1.0)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Implement KNN model\n",
    "knn = NearestNeighbors(n_neighbors=50, algorithm='auto', leaf_size=30, \n",
    "                       metric='minkowski', p=2, n_jobs=-1)\n",
    "knn.fit(train_val_feature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Search the first 50 neighbors\n",
    "distance, neighbor_index = knn.kneighbors(test_feature, return_distance=True)\n",
    "\n",
    "# Save the results\n",
    "np.save('./result/knn-triplet-vgg16-0.3-distance.npy', distance)\n",
    "np.save('./result/knn-triplet-vgg16-0.3-neighbor.npy', neighbor_index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Search Neighbors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn_distance = np.load('./result/knn-triplet-vgg16-0.3-distance.npy')\n",
    "knn_neighbor = np.load('./result/knn-triplet-vgg16-0.3-neighbor.npy')\n",
    "\n",
    "# Get the first 50 neighbors\n",
    "predictions = []\n",
    "for neighbors in knn_neighbor:\n",
    "    predictions.append(train_val_df.loc[neighbors]['landmark_id'].values)\n",
    "\n",
    "predictions = np.array(predictions)\n",
    "np.save('./result/knn-triplet-vgg16-0.3-test-prediction.npy', predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top  1 accuracy:\t 0.35978741458621766\n",
      "Top  5 accuracy:\t 0.5340538609262651\n",
      "Top 10 accuracy:\t 0.6056897860747622\n",
      "Top 20 accuracy:\t 0.6743334375418695\n"
     ]
    }
   ],
   "source": [
    "print('Top  1 accuracy:\\t', accuracy(test_df['landmark_id'].values, predictions, top=1))\n",
    "print('Top  5 accuracy:\\t', accuracy(test_df['landmark_id'].values, predictions, top=5))\n",
    "print('Top 10 accuracy:\\t', accuracy(test_df['landmark_id'].values, predictions, top=10))\n",
    "print('Top 20 accuracy:\\t', accuracy(test_df['landmark_id'].values, predictions, top=20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn_acc = []\n",
    "for i in range(1, 51):\n",
    "    tmp_acc = accuracy(test_df['landmark_id'].values, predictions, top=i)\n",
    "    knn_acc.append(tmp_acc)\n",
    "\n",
    "np.save('./result/knn-triplet-vgg16-0.3-accuracy.npy', knn_acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Environment (conda_tensorflow_p36)",
   "language": "python",
   "name": "conda_tensorflow_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}