tune_fashion_network.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tuning your Tensorflow model using Automatic Model Tuning\n",
    "In this notebook, we will tune our Tensorflow model through Sagemaker. The model is written in an [*entry_point* file](https://docs.aws.amazon.com/sagemaker/latest/dg/tf-training-inference-code-template.html) and consists of two important definitions: \n",
    "1. Model definition using the **model_fn**\n",
    "2. Data feeding definition using the **train_input_fn** and **eval_input_fn**.\n",
    "\n",
    "To optimise our training time we use Sagemaker's [Pipe input mode](https://aws.amazon.com/blogs/machine-learning/using-pipe-input-mode-for-amazon-sagemaker-algorithms/). *Pipe input mode* streams datasets directly to the training instances instead of being downloaded first. This paradigm improves performance on multiple facets: \n",
    "1. Training jobs can start sooner since they don't need to wait for the full dataset\n",
    "2. Training instances require less storage space\n",
    "3. Streaming data from S3 is faster than streaming from a local file since s3 filehandles are highly optimised and multi-threaded."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sagemaker\n",
    "\n",
    "bucket = sagemaker.Session().default_bucket() \n",
    "prefix = 'radix/mnist_fashion_tutorial' \n",
    "\n",
    "role = sagemaker.get_execution_role() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import boto3\n",
    "from time import gmtime, strftime\n",
    "from sagemaker.tensorflow import TensorFlow\n",
    "from sagemaker.tuner import IntegerParameter, CategoricalParameter, ContinuousParameter, HyperparameterTuner"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model definition in an entrypoint file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "import os\r\n",
      "import tensorflow as tf\r\n",
      "from tensorflow.python.estimator.model_fn import ModeKeys as Modes\r\n",
      "from sagemaker_tensorflow import PipeModeDataset\r\n",
      "from tensorflow.contrib.data import map_and_batch\r\n",
      "\r\n",
      "INPUT_TENSOR_NAME = 'inputs'\r\n",
      "SIGNATURE_NAME = 'predictions'\r\n",
      "PREFETCH_SIZE = 10\r\n",
      "BATCH_SIZE = 256\r\n",
      "NUM_PARALLEL_BATCHES = 10\r\n",
      "MAX_EPOCHS = 20\r\n",
      "\r\n",
      "\r\n",
      "def _conv_pool(inputs, kernel_shape, kernel_count, padding_type):\r\n",
      "    # Convolutional Layer \r\n",
      "    conv = tf.layers.conv2d(\r\n",
      "      inputs=inputs,\r\n",
      "      filters=kernel_count,\r\n",
      "      kernel_size=kernel_shape,\r\n",
      "      padding=padding_type,\r\n",
      "      activation=tf.nn.relu)\r\n",
      "\r\n",
      "    # Pooling Layer \r\n",
      "    pool = tf.layers.max_pooling2d(inputs=conv, pool_size=[2, 2], strides=2)\r\n",
      "    return pool\r\n",
      "\r\n",
      "    \r\n",
      "\r\n",
      "def model_fn(features, labels, mode, params):\r\n",
      "    learning_rate = params.get(\"learning_rate\", 0.0001)\r\n",
      "    dropout_rate = params.get(\"dropout_rate\", 0.8)\r\n",
      "    nw_depth = params.get(\"nw_depth\", 2)\r\n",
      "    optimizer_type = params.get(\"optimizer_type\", 'adam')\r\n",
      "\r\n",
      "    # Input Layer\r\n",
      "    X = tf.reshape(features[INPUT_TENSOR_NAME], [-1, 28, 28, 1])\r\n",
      "    \r\n",
      "    # Series of convolutional layers\r\n",
      "    for i in range(nw_depth):\r\n",
      "        X = _conv_pool(X, [5,5], 2^(5+i), 'same')\r\n",
      "    \r\n",
      "    # Dense Layer\r\n",
      "    X_flat = tf.layers.flatten(X)\r\n",
      "    dense = tf.layers.dense(inputs=X_flat, units=1024, activation=tf.nn.relu)\r\n",
      "    dropout = tf.layers.dropout(\r\n",
      "      inputs=dense, rate=dropout_rate, training=mode == tf.estimator.ModeKeys.TRAIN)\r\n",
      "\r\n",
      "    # Logits Layer\r\n",
      "    logits = tf.layers.dense(inputs=dropout, units=10) # default activation is linear combination\r\n",
      "\r\n",
      "    predictions = {\r\n",
      "      \"classes\": tf.argmax(input=logits, axis=1),\r\n",
      "      \"probabilities\": tf.nn.softmax(logits)\r\n",
      "    }\r\n",
      "    if mode == tf.estimator.ModeKeys.PREDICT:\r\n",
      "        export_outputs = {\r\n",
      "            SIGNATURE_NAME: tf.estimator.export.PredictOutput(predictions)\r\n",
      "        }\r\n",
      "        return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions, export_outputs=export_outputs)\r\n",
      "\r\n",
      "    loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)\r\n",
      "    tf.summary.scalar('loss', loss)\r\n",
      "\r\n",
      "\r\n",
      "    if mode == tf.estimator.ModeKeys.TRAIN:\r\n",
      "        if optimizer_type == 'adam':\r\n",
      "            optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\r\n",
      "        elif optimizer_type == 'sgd':\r\n",
      "            optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001)\r\n",
      "            \r\n",
      "        train_op = optimizer.minimize(\r\n",
      "            loss=loss,\r\n",
      "            global_step=tf.train.get_global_step())\r\n",
      "        return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)\r\n",
      "\r\n",
      "    if mode == tf.estimator.ModeKeys.EVAL:\r\n",
      "        eval_metric_ops = {\r\n",
      "          \"accuracy\": tf.metrics.accuracy(\r\n",
      "              labels=labels, predictions=predictions[\"classes\"])}\r\n",
      "        return tf.estimator.EstimatorSpec(\r\n",
      "          mode=mode, loss=loss, eval_metric_ops=eval_metric_ops)\r\n",
      "\r\n",
      "\r\n",
      "def _input_fn(channel):\r\n",
      "    \"\"\"Returns a Dataset which reads from a SageMaker PipeMode channel.\"\"\"\r\n",
      "    features={\r\n",
      "            'image_raw': tf.FixedLenFeature([], tf.string),\r\n",
      "            'label': tf.FixedLenFeature([], tf.int64),\r\n",
      "            'height': tf.FixedLenFeature([], tf.int64),\r\n",
      "            'width': tf.FixedLenFeature([], tf.int64),\r\n",
      "            'channels': tf.FixedLenFeature([], tf.int64) \r\n",
      "        }\r\n",
      "\r\n",
      "    def parse(record):\r\n",
      "        parsed = tf.parse_single_example(record, features)\r\n",
      "        \r\n",
      "        image = tf.decode_raw(parsed['image_raw'], tf.uint8)\r\n",
      "        image.set_shape([784])\r\n",
      "        image = tf.cast(image, tf.float32) * (1. / 255)\r\n",
      "        label = tf.cast(parsed['label'], tf.int32)\r\n",
      "        return ({INPUT_TENSOR_NAME: image}, label)\r\n",
      "\r\n",
      "    ds = PipeModeDataset(channel=channel, record_format='TFRecord')\r\n",
      "\r\n",
      "    ds = ds.repeat(MAX_EPOCHS)\r\n",
      "    ds = ds.prefetch(PREFETCH_SIZE)\r\n",
      "    ds = ds.map(parse, num_parallel_calls=NUM_PARALLEL_BATCHES)\r\n",
      "    ds = ds.batch(BATCH_SIZE)\r\n",
      "    \r\n",
      "    return ds\r\n",
      "\r\n",
      "def train_input_fn(training_dir, params):\r\n",
      "    \"\"\"Returns input function that feeds the model during training\"\"\"\r\n",
      "    return _input_fn('train')\r\n",
      "\r\n",
      "def eval_input_fn(training_dir, params):\r\n",
      "    \"\"\"Returns input function that feeds the model during evaluation\"\"\"\r\n",
      "    return _input_fn('eval')\r\n",
      "\r\n",
      "\r\n"
     ]
    }
   ],
   "source": [
    "!cat 'cnn_fashion_mnist.py'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use Tensorflow model through Sagemaker's Tensorflow estimator\n",
    "We use Sagemaker's Estimator wrapper called Tensorflow to make our own tensorflow model compatible with Sagemaker's services (more information can be found [here](https://sagemaker.readthedocs.io/en/latest/sagemaker.tensorflow.html)). Note that we explicitly set the *input_mode* to 'Pipe' to force the usage of Pipe file mode. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "estimator = TensorFlow(entry_point='cnn_fashion_mnist.py',\n",
    "                       role=role,\n",
    "                       input_mode='Pipe',\n",
    "                       training_steps=20_000, \n",
    "                       evaluation_steps=100,\n",
    "                       train_instance_count=1,\n",
    "                       train_instance_type='ml.c5.2xlarge',\n",
    "                       base_job_name='radix_mnist_fashion')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Automatic Model Tuning\n",
    "Automatic Model Tuning is a service that automatically optimises the provided model through its hyperparameters. The service finds the best hyperparameter configuration through Bayesian Optimisation. The tuning process parallelises its computations across multiple instances which improve performance significantly. \n",
    "\n",
    "Using automatic model tuning involves three steps.\n",
    "1. Define which objective that has to be optimised. If you want to use an alternative objective such as accuracy you need to log this metric during training since the objective is fetched from the logs using a regular expression.\n",
    "2. Define hyperparameter ranges. Avoid using IntegerParameter for continuous variables since this will limit hyperparameter exploration. \n",
    "3. Define a HyperparameterTuner instance which defines the number of (parallel) jobs that will be used during the optimisation process.\n",
    "\n",
    "More information on how to use automatic model tuning can be found using [this link](https://docs.aws.amazon.com/sagemaker/latest/dg/automatic-model-tuning.html).\n",
    "\n",
    "The results of the hyperparameter search can be viewed in the Amazon SageMaker console. AWS also provides a Jupiter notebook to analyse the results of the hyperparameter search which can be found [here](https://github.com/awslabs/amazon-sagemaker-examples/blob/master/hyperparameter_tuning/analyze_results/HPO_Analyze_TuningJob_Results.ipynb)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. Define which objective has to be optimised\n",
    "objective_metric_name = 'loss'\n",
    "objective_type = 'Minimize'\n",
    "metric_definitions = [{'Name': 'loss',\n",
    "                       'Regex': 'loss = ([0-9\\\\.]+)'}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2. Define hyperparameter ranges\n",
    "hyperparameter_ranges = {\n",
    "                            'learning_rate': ContinuousParameter(0.0001, 0.001), \n",
    "                            'dropout_rate': ContinuousParameter(0.3, 1.0),\n",
    "                            'nw_depth': IntegerParameter(1, 4),\n",
    "                            'optimizer_type': CategoricalParameter(['sgd', 'adam']),\n",
    "                        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3. Instantiate a HyperparameterTuner instance\n",
    "tuner = HyperparameterTuner(estimator,\n",
    "                            objective_metric_name,\n",
    "                            hyperparameter_ranges,\n",
    "                            metric_definitions,\n",
    "                            max_jobs=16,\n",
    "                            max_parallel_jobs=4,\n",
    "                            objective_type=objective_type)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:sagemaker:Creating hyperparameter tuning job with name: sagemaker-tensorflow-180824-0803\n"
     ]
    }
   ],
   "source": [
    "# Fit the HyperparameterTuner to start the hyperparameter optimisation process\n",
    "train_data = 's3://sagemaker-eu-central-1-959924085179/radix/mnist_fashion_tutorial/data/mnist/train.tfrecords'\n",
    "eval_data = 's3://sagemaker-eu-central-1-959924085179/radix/mnist_fashion_tutorial/data/mnist/validation.tfrecords'\n",
    "\n",
    "tuner.fit({'train':train_data, 'eval':eval_data}, logs=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'InProgress'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Sanity check if the optimisation process has started \n",
    "boto3.client('sagemaker').describe_hyper_parameter_tuning_job(\n",
    "    HyperParameterTuningJobName=tuner.latest_tuning_job.job_name)['HyperParameterTuningJobStatus']"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_tensorflow_p36",
   "language": "python",
   "name": "conda_tensorflow_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  },
  "notice": "Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.  Licensed under the Apache License, Version 2.0 (the \"License\"). You may not use this file except in compliance with the License. A copy of the License is located at http://aws.amazon.com/apache2.0/ or in the \"license\" file accompanying this file. This file is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License."
 },
 "nbformat": 4,
 "nbformat_minor": 2
}