project_part2_start.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "project_part2_start.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "l6pwbk2cSp7m"
      },
      "outputs": [],
      "source": [
        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
        "# import libraries\n",
        "%matplotlib inline\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# TensorFlow and tf.keras\n",
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "\n",
        "from tensorflow.keras.datasets import cifar10\n",
        "from tensorflow.keras.models import Sequential \n",
        "from tensorflow.keras import models\n",
        "from tensorflow.keras import layers\n",
        "from tensorflow.keras import optimizers\n",
        "# Helper libraries\n",
        "from sklearn.model_selection import train_test_split\n",
        "from tensorflow.keras.utils import to_categorical\n",
        "from google.colab import files\n",
        "import io"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# load mostly preprocessed data (preprocessing accomplished before 3/25 proposal)\n",
        "\n",
        "# would use regular numpy syntax to load locally:\n",
        "#X = np.load('<path>/X_cleaned_attempt_new.npy',allow_pickle=True)\n",
        "#Y = np.load('<path>/Y_cleaned_attempt_new.npy',allow_pickle=True)\n",
        "#X = np.asarray(X).astype('float32')\n",
        "#Y = np.asarray(Y).astype('float32')\n",
        "\n",
        "# i'm using colab, so it's slightly more complicated\n",
        "ytrainval_byteseq = files.upload() # choose 'Y_cleaned_attempt_new.npy' from local system\n",
        "ytrainval_filelike = io.BytesIO(ytrainval_byteseq['Y_cleaned_attempt_new.npy']) # create file-like object\n",
        "ytrainval = np.load(ytrainval_filelike,allow_pickle=True) # create regular numpy array\n",
        "xtrainval_byteseq = files.upload() # choose 'X_cleaned_attempt_new.npy' from local system\n",
        "xtrainval_filelike = io.BytesIO(xtrainval_byteseq['X_cleaned_attempt_new.npy']) # create file-like object\n",
        "xtrainval = np.load(xtrainval_filelike,allow_pickle=True) # create regular numpy array\n",
        "ytest_byteseq = files.upload() # choose 'Y_test.npy' from local system\n",
        "ytest_filelike = io.BytesIO(ytest_byteseq['Y_test.npy']) # create file-like object\n",
        "ytest = np.load(ytest_filelike,allow_pickle=True) # create regular numpy array\n",
        "xtest_byteseq = files.upload() # choose 'X_test.npy' from local system\n",
        "xtest_filelike = io.BytesIO(xtest_byteseq['X_test.npy']) # create file-like object\n",
        "xtest = np.load(xtest_filelike,allow_pickle=True) # create regular numpy array"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 240,
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7CgpmdW5jdGlvbiBfdXBsb2FkRmlsZXMoaW5wdXRJZCwgb3V0cHV0SWQpIHsKICBjb25zdCBzdGVwcyA9IHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCk7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICAvLyBDYWNoZSBzdGVwcyBvbiB0aGUgb3V0cHV0RWxlbWVudCB0byBtYWtlIGl0IGF2YWlsYWJsZSBmb3IgdGhlIG5leHQgY2FsbAogIC8vIHRvIHVwbG9hZEZpbGVzQ29udGludWUgZnJvbSBQeXRob24uCiAgb3V0cHV0RWxlbWVudC5zdGVwcyA9IHN0ZXBzOwoKICByZXR1cm4gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpOwp9CgovLyBUaGlzIGlzIHJvdWdobHkgYW4gYXN5bmMgZ2VuZXJhdG9yIChub3Qgc3VwcG9ydGVkIGluIHRoZSBicm93c2VyIHlldCksCi8vIHdoZXJlIHRoZXJlIGFyZSBtdWx0aXBsZSBhc3luY2hyb25vdXMgc3RlcHMgYW5kIHRoZSBQeXRob24gc2lkZSBpcyBnb2luZwovLyB0byBwb2xsIGZvciBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcC4KLy8gVGhpcyB1c2VzIGEgUHJvbWlzZSB0byBibG9jayB0aGUgcHl0aG9uIHNpZGUgb24gY29tcGxldGlvbiBvZiBlYWNoIHN0ZXAsCi8vIHRoZW4gcGFzc2VzIHRoZSByZXN1bHQgb2YgdGhlIHByZXZpb3VzIHN0ZXAgYXMgdGhlIGlucHV0IHRvIHRoZSBuZXh0IHN0ZXAuCmZ1bmN0aW9uIF91cGxvYWRGaWxlc0NvbnRpbnVlKG91dHB1dElkKSB7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICBjb25zdCBzdGVwcyA9IG91dHB1dEVsZW1lbnQuc3RlcHM7CgogIGNvbnN0IG5leHQgPSBzdGVwcy5uZXh0KG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSk7CiAgcmV0dXJuIFByb21pc2UucmVzb2x2ZShuZXh0LnZhbHVlLnByb21pc2UpLnRoZW4oKHZhbHVlKSA9PiB7CiAgICAvLyBDYWNoZSB0aGUgbGFzdCBwcm9taXNlIHZhbHVlIHRvIG1ha2UgaXQgYXZhaWxhYmxlIHRvIHRoZSBuZXh0CiAgICAvLyBzdGVwIG9mIHRoZSBnZW5lcmF0b3IuCiAgICBvdXRwdXRFbGVtZW50Lmxhc3RQcm9taXNlVmFsdWUgPSB2YWx1ZTsKICAgIHJldHVybiBuZXh0LnZhbHVlLnJlc3BvbnNlOwogIH0pOwp9CgovKioKICogR2VuZXJhdG9yIGZ1bmN0aW9uIHdoaWNoIGlzIGNhbGxlZCBiZXR3ZWVuIGVhY2ggYXN5bmMgc3RlcCBvZiB0aGUgdXBsb2FkCiAqIHByb2Nlc3MuCiAqIEBwYXJhbSB7c3RyaW5nfSBpbnB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIGlucHV0IGZpbGUgcGlja2VyIGVsZW1lbnQuCiAqIEBwYXJhbSB7c3RyaW5nfSBvdXRwdXRJZCBFbGVtZW50IElEIG9mIHRoZSBvdXRwdXQgZGlzcGxheS4KICogQHJldHVybiB7IUl0ZXJhYmxlPCFPYmplY3Q+fSBJdGVyYWJsZSBvZiBuZXh0IHN0ZXBzLgogKi8KZnVuY3Rpb24qIHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IGlucHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKGlucHV0SWQpOwogIGlucHV0RWxlbWVudC5kaXNhYmxlZCA9IGZhbHNlOwoKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIG91dHB1dEVsZW1lbnQuaW5uZXJIVE1MID0gJyc7CgogIGNvbnN0IHBpY2tlZFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgaW5wdXRFbGVtZW50LmFkZEV2ZW50TGlzdGVuZXIoJ2NoYW5nZScsIChlKSA9PiB7CiAgICAgIHJlc29sdmUoZS50YXJnZXQuZmlsZXMpOwogICAgfSk7CiAgfSk7CgogIGNvbnN0IGNhbmNlbCA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2J1dHRvbicpOwogIGlucHV0RWxlbWVudC5wYXJlbnRFbGVtZW50LmFwcGVuZENoaWxkKGNhbmNlbCk7CiAgY2FuY2VsLnRleHRDb250ZW50ID0gJ0NhbmNlbCB1cGxvYWQnOwogIGNvbnN0IGNhbmNlbFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgY2FuY2VsLm9uY2xpY2sgPSAoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9OwogIH0pOwoKICAvLyBXYWl0IGZvciB0aGUgdXNlciB0byBwaWNrIHRoZSBmaWxlcy4KICBjb25zdCBmaWxlcyA9IHlpZWxkIHsKICAgIHByb21pc2U6IFByb21pc2UucmFjZShbcGlja2VkUHJvbWlzZSwgY2FuY2VsUHJvbWlzZV0pLAogICAgcmVzcG9uc2U6IHsKICAgICAgYWN0aW9uOiAnc3RhcnRpbmcnLAogICAgfQogIH07CgogIGNhbmNlbC5yZW1vdmUoKTsKCiAgLy8gRGlzYWJsZSB0aGUgaW5wdXQgZWxlbWVudCBzaW5jZSBmdXJ0aGVyIHBpY2tzIGFyZSBub3QgYWxsb3dlZC4KICBpbnB1dEVsZW1lbnQuZGlzYWJsZWQgPSB0cnVlOwoKICBpZiAoIWZpbGVzKSB7CiAgICByZXR1cm4gewogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgICAgfQogICAgfTsKICB9CgogIGZvciAoY29uc3QgZmlsZSBvZiBmaWxlcykgewogICAgY29uc3QgbGkgPSBkb2N1bWVudC5jcmVhdGVFbGVtZW50KCdsaScpOwogICAgbGkuYXBwZW5kKHNwYW4oZmlsZS5uYW1lLCB7Zm9udFdlaWdodDogJ2JvbGQnfSkpOwogICAgbGkuYXBwZW5kKHNwYW4oCiAgICAgICAgYCgke2ZpbGUudHlwZSB8fCAnbi9hJ30pIC0gJHtmaWxlLnNpemV9IGJ5dGVzLCBgICsKICAgICAgICBgbGFzdCBtb2RpZmllZDogJHsKICAgICAgICAgICAgZmlsZS5sYXN0TW9kaWZpZWREYXRlID8gZmlsZS5sYXN0TW9kaWZpZWREYXRlLnRvTG9jYWxlRGF0ZVN0cmluZygpIDoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ24vYSd9IC0gYCkpOwogICAgY29uc3QgcGVyY2VudCA9IHNwYW4oJzAlIGRvbmUnKTsKICAgIGxpLmFwcGVuZENoaWxkKHBlcmNlbnQpOwoKICAgIG91dHB1dEVsZW1lbnQuYXBwZW5kQ2hpbGQobGkpOwoKICAgIGNvbnN0IGZpbGVEYXRhUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICAgIGNvbnN0IHJlYWRlciA9IG5ldyBGaWxlUmVhZGVyKCk7CiAgICAgIHJlYWRlci5vbmxvYWQgPSAoZSkgPT4gewogICAgICAgIHJlc29sdmUoZS50YXJnZXQucmVzdWx0KTsKICAgICAgfTsKICAgICAgcmVhZGVyLnJlYWRBc0FycmF5QnVmZmVyKGZpbGUpOwogICAgfSk7CiAgICAvLyBXYWl0IGZvciB0aGUgZGF0YSB0byBiZSByZWFkeS4KICAgIGxldCBmaWxlRGF0YSA9IHlpZWxkIHsKICAgICAgcHJvbWlzZTogZmlsZURhdGFQcm9taXNlLAogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbnRpbnVlJywKICAgICAgfQogICAgfTsKCiAgICAvLyBVc2UgYSBjaHVua2VkIHNlbmRpbmcgdG8gYXZvaWQgbWVzc2FnZSBzaXplIGxpbWl0cy4gU2VlIGIvNjIxMTU2NjAuCiAgICBsZXQgcG9zaXRpb24gPSAwOwogICAgZG8gewogICAgICBjb25zdCBsZW5ndGggPSBNYXRoLm1pbihmaWxlRGF0YS5ieXRlTGVuZ3RoIC0gcG9zaXRpb24sIE1BWF9QQVlMT0FEX1NJWkUpOwogICAgICBjb25zdCBjaHVuayA9IG5ldyBVaW50OEFycmF5KGZpbGVEYXRhLCBwb3NpdGlvbiwgbGVuZ3RoKTsKICAgICAgcG9zaXRpb24gKz0gbGVuZ3RoOwoKICAgICAgY29uc3QgYmFzZTY0ID0gYnRvYShTdHJpbmcuZnJvbUNoYXJDb2RlLmFwcGx5KG51bGwsIGNodW5rKSk7CiAgICAgIHlpZWxkIHsKICAgICAgICByZXNwb25zZTogewogICAgICAgICAgYWN0aW9uOiAnYXBwZW5kJywKICAgICAgICAgIGZpbGU6IGZpbGUubmFtZSwKICAgICAgICAgIGRhdGE6IGJhc2U2NCwKICAgICAgICB9LAogICAgICB9OwoKICAgICAgbGV0IHBlcmNlbnREb25lID0gZmlsZURhdGEuYnl0ZUxlbmd0aCA9PT0gMCA/CiAgICAgICAgICAxMDAgOgogICAgICAgICAgTWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCk7CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPSBgJHtwZXJjZW50RG9uZX0lIGRvbmVgOwoKICAgIH0gd2hpbGUgKHBvc2l0aW9uIDwgZmlsZURhdGEuYnl0ZUxlbmd0aCk7CiAgfQoKICAvLyBBbGwgZG9uZS4KICB5aWVsZCB7CiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICB9CiAgfTsKfQoKc2NvcGUuZ29vZ2xlID0gc2NvcGUuZ29vZ2xlIHx8IHt9OwpzY29wZS5nb29nbGUuY29sYWIgPSBzY29wZS5nb29nbGUuY29sYWIgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYi5fZmlsZXMgPSB7CiAgX3VwbG9hZEZpbGVzLAogIF91cGxvYWRGaWxlc0NvbnRpbnVlLAp9Owp9KShzZWxmKTsK",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          }
        },
        "id": "2h7ozrbgSrlm",
        "outputId": "72927b66-a37f-49d0-f229-273adf173316"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-36c342aa-6cd3-4a5e-8948-04f94e738745\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-36c342aa-6cd3-4a5e-8948-04f94e738745\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving Y_cleaned_attempt_new.npy to Y_cleaned_attempt_new.npy\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-3cbf8fb8-24ff-4aed-b12f-47421969fc1c\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-3cbf8fb8-24ff-4aed-b12f-47421969fc1c\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving X_cleaned_attempt_new.npy to X_cleaned_attempt_new.npy\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-34589f7f-87ec-4ce3-a2e9-03610d67037d\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-34589f7f-87ec-4ce3-a2e9-03610d67037d\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving Y_test.npy to Y_test.npy\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-b89e7b3b-4d64-4ea2-aa44-2ecc05ef5317\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-b89e7b3b-4d64-4ea2-aa44-2ecc05ef5317\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving X_test.npy to X_test.npy\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#check shapes\n",
        "print(xtrainval.shape)\n",
        "print(ytrainval.shape)\n",
        "print(xtest.shape)\n",
        "print(ytest.shape)\n",
        "\n",
        "#ytrainval needs to be converted to number representation of labels\n",
        "feature_map = {'Rock':0,'Electronic':1,'Experimental':2,'Hip-Hop':3,'Folk':4,'Instrumental':5,'Pop':6,\n",
        "              'International':7,'Classical':8,'Old-Time / Historic':9, 'Jazz':10,'Country':11,'Soul-RnB':12,\n",
        "              'Spoken':13,'Blues':14,'Easy Listening':15}\n",
        "for i in range(ytrainval.shape[0]):\n",
        "    ytrainval[i] = float(feature_map[ytrainval[i]])\n",
        "ytrainval[i] = np.asarray(ytrainval[i]).astype('float32')\n",
        "\n",
        "print(ytrainval)\n",
        "print(ytest)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AQFpXvEFvhiL",
        "outputId": "dbe0c998-18ad-493b-b231-d426797cfade"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(9349, 520)\n",
            "(9349,)\n",
            "(40249, 520)\n",
            "(40249,)\n",
            "[3.0 3.0 3.0 ... 4.0 0.0 array(0., dtype=float32)]\n",
            "[3.0 6.0 0.0 ... 0.0 0.0 array(0., dtype=float32)]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# for the preliminary report, i did a roughly 80-20 split, but used the 20% as the training set to improve speed\n",
        "# so will recombine and do a 90-10 split, using the 90% as training/validation\n",
        "xfull = np.concatenate((xtrainval,xtest),axis=0)\n",
        "yfull = np.concatenate((ytrainval,ytest),axis=0)\n",
        "print(xfull.shape)\n",
        "print(yfull.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vGIYeIxp7TnH",
        "outputId": "efbb3744-ad61-408d-ac04-1c996513e0d6"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(49598, 520)\n",
            "(49598,)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# finish preprocessing\n",
        "'''\n",
        "Look out for improbable event that not all classes included in validation set\n",
        "resulting in incompatible shape (15 categories for one hot)\n",
        "just run again if this happens\n",
        "'''\n",
        "\n",
        "# ~50000 instances in full set, so could get away with 90-10 split probably\n",
        "xtrainval, xtest, ytrainval, ytest = train_test_split(xfull,yfull,train_size=int(0.9*(xtest.shape[0]+xtrainval.shape[0])))\n",
        "\n",
        "# split into training and validation\n",
        "xtrain, xval, ytrain, yval = train_test_split(xtrainval,ytrainval,train_size=int(0.9*xtrainval.shape[0]))\n",
        "\n",
        "# one-hot encoding\n",
        "ytrain_1hot = to_categorical(ytrain)\n",
        "yval_1hot = to_categorical(yval)\n",
        "ytest_1hot = to_categorical(ytest)\n",
        "\n",
        "print(ytrain_1hot.shape)\n",
        "print(yval_1hot.shape)\n",
        "print(ytest_1hot.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3cCnat6IWIOm",
        "outputId": "95dc2ce0-2395-470f-cf00-3dae5d33e56f"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(40174, 16)\n",
            "(4464, 16)\n",
            "(4960, 16)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(xtrain.shape)\n",
        "print(xtrain[0,:])\n",
        "print(xval[0,:])\n",
        "print(xtest[0,:])\n",
        "# starting with MLP; will need to un-flatten features into time-series signal for cnn implementation\n",
        "# looks like data isn't normalized"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9j4VwR0AM9EV",
        "outputId": "b694f7bb-f491-48a0-f076-27e08685ee8f"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(40174, 520)\n",
            "[-3.09899390e-01  2.00813651e-01 -4.19419289e-01 -4.83177871e-01\n",
            " -4.88094240e-01 -5.28498113e-01  9.30651426e-01  3.37793541e+00\n",
            "  2.23753214e+00  6.26284003e-01 -5.40368676e-01  1.61023021e-01\n",
            "  6.80902779e-01  6.91087365e-01  6.85647011e-01  6.81586206e-01\n",
            "  6.35318160e-01  6.81717515e-01  6.43619061e-01  6.64180756e-01\n",
            "  6.64180756e-01  6.68109953e-01  6.91748261e-01  6.64101720e-01\n",
            "  2.69591242e-01  2.71138757e-01  2.70915002e-01  2.66046971e-01\n",
            "  2.59812683e-01  2.91525364e-01  2.03960612e-01  1.26284078e-01\n",
            "  1.68915257e-01  2.64811546e-01  2.90303648e-01  2.76634455e-01\n",
            "  2.68004894e-01  2.64780700e-01  2.53414512e-01  2.63145864e-01\n",
            "  2.72416413e-01  3.08607847e-01  2.02602342e-01  8.82767066e-02\n",
            "  1.51804999e-01  2.63520867e-01  2.75879860e-01  2.82813013e-01\n",
            "  0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00\n",
            "  0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00\n",
            "  0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00\n",
            "  2.96847284e-01  2.79067338e-01  4.60668743e-01  1.00259654e-01\n",
            " -1.97370023e-01 -1.42215073e-01  6.74390733e-01  1.66514504e+00\n",
            "  1.17637134e+00  1.94595978e-01  2.27466419e-01 -4.00884040e-02\n",
            "  1.54895678e-01  1.38172165e-01  1.66122481e-01  1.47472531e-01\n",
            "  1.34695947e-01  1.54558808e-01  1.27135172e-01  1.27071798e-01\n",
            "  1.19817950e-01  1.15912773e-01  1.65873930e-01  1.34363145e-01\n",
            " -1.18156826e+00 -6.38713241e-01 -1.19966483e+00 -9.53003585e-01\n",
            " -9.86691773e-01 -1.35495424e+00 -4.44563210e-01  1.59987092e+00\n",
            "  5.39223671e-01 -5.67450345e-01 -1.26355839e+00 -8.35334659e-01\n",
            "  1.00000000e+00  1.00000000e+00  1.00000000e+00  1.00000000e+00\n",
            "  1.00000000e+00  1.00000000e+00  1.00000000e+00  1.00000000e+00\n",
            "  1.00000000e+00  1.00000000e+00  1.00000000e+00  1.00000000e+00\n",
            "  4.91124123e-01  4.67332900e-01  4.98721540e-01  4.77258265e-01\n",
            "  4.70998645e-01  5.28694034e-01  3.92283171e-01  3.10558766e-01\n",
            "  3.51706266e-01  4.63764429e-01  5.20808399e-01  4.80431229e-01\n",
            "  4.32764530e-01  4.55019981e-01  4.33092475e-01  4.56308305e-01\n",
            "  4.41819876e-01  4.90223885e-01  3.46768051e-01  2.35959649e-01\n",
            "  2.89625615e-01  4.21119511e-01  4.62099224e-01  4.76087332e-01\n",
            "  1.69214737e-02  2.45338511e-02  2.24947557e-02  2.32264902e-02\n",
            "  2.09428500e-02  1.71029735e-02  2.23407932e-02  1.90205667e-02\n",
            "  2.28331536e-02  2.43923906e-02  2.40570828e-02  3.14960331e-02\n",
            "  3.68294299e-01  3.25176269e-01  3.92126948e-01  3.23850006e-01\n",
            "  3.15499216e-01  1.64661571e-01  6.28732920e-01  1.50451279e+00\n",
            "  1.09333885e+00  5.47047675e-01  2.93984979e-01  1.90428659e-01\n",
            "  3.10902208e-01  2.44538277e-01  3.13695192e-01  2.85767108e-01\n",
            "  2.84984142e-01  3.24650705e-01  2.41926193e-01  2.39956126e-01\n",
            "  2.29084954e-01  2.55381435e-01  3.12422276e-01  2.54355639e-01\n",
            " -1.17459893e+00  5.89234173e-01 -1.01814592e+00  1.49133742e-01\n",
            " -1.16065353e-01 -1.32436490e+00  2.29448795e+00  2.84257913e+00\n",
            "  2.72561765e+00  2.58608729e-01 -7.49654651e-01  3.42550844e-01\n",
            "  1.00000000e+00  1.00000000e+00  1.00000000e+00  1.00000000e+00\n",
            "  1.00000000e+00  1.00000000e+00  1.00000000e+00  1.00000000e+00\n",
            "  1.00000000e+00  1.00000000e+00  1.00000000e+00  1.00000000e+00\n",
            "  3.69195729e-01  2.77542382e-01  3.52335423e-01  2.85299301e-01\n",
            "  2.96325892e-01  3.99709135e-01  1.98421597e-01  1.76432222e-01\n",
            "  1.89247951e-01  2.95416534e-01  3.35716665e-01  2.77804017e-01\n",
            "  1.84176520e-01  1.93961009e-01  1.63063541e-01  1.71318024e-01\n",
            "  1.81603700e-01  2.80907214e-01  1.07776731e-01  5.46213798e-02\n",
            "  8.83853286e-02  1.85423404e-01  1.60826325e-01  1.92623138e-01\n",
            "  5.89929754e-04  4.43175872e-04  2.97052116e-04  3.28319409e-04\n",
            "  3.31855263e-04  6.57056691e-04  5.80539461e-04  2.97073566e-04\n",
            "  3.96768533e-04  9.03376902e-04  4.62453114e-04  7.55454414e-04\n",
            "  6.86532199e-01  1.17814958e+00  7.87350595e-01  1.14015210e+00\n",
            "  1.01870644e+00  4.71389502e-01  1.66871703e+00  1.95337403e+00\n",
            "  1.83408916e+00  1.17852533e+00  9.08756495e-01  1.11453712e+00\n",
            "  3.81814986e-01  2.68672377e-01  3.78657669e-01  3.06761235e-01\n",
            "  3.05493712e-01  3.71297389e-01  2.30972648e-01  2.61911452e-01\n",
            "  2.37347394e-01  3.00127983e-01  3.59700620e-01  2.73684412e-01\n",
            "  1.95300305e+00  3.36432368e-01 -7.25562930e-01 -7.29909182e-01\n",
            "  1.67202926e+00  6.93952963e-02 -6.46448255e-01 -4.98573363e-01\n",
            " -8.84125158e-02  1.77676928e+00  5.50517797e-01  3.89835119e+00\n",
            "  1.17844009e+00  8.36427629e-01  1.00583589e+00  7.04783440e-01\n",
            "  1.42794394e+00  4.71820474e-01  4.83779222e-01  5.06919622e-01\n",
            " -7.48972397e+01  1.97006454e+02  2.40409966e+01  4.15602417e+01\n",
            "  3.09339504e+01  3.13264046e+01  1.78848190e+01  2.66010933e+01\n",
            "  1.43846569e+01  3.31093941e+01  2.39871101e+01  5.04230537e+01\n",
            "  3.48760147e+01  4.27968674e+01  3.74712982e+01  4.78578339e+01\n",
            "  4.24141731e+01  3.94012222e+01  4.12943802e+01  3.47536545e+01\n",
            " -2.01088242e+02  1.29162476e+02 -3.54655952e+01 -1.42157659e-01\n",
            " -2.58816833e+01  2.60501838e+00 -1.93008575e+01 -1.48839605e+00\n",
            " -2.14993362e+01 -4.17309856e+00 -1.23256750e+01  2.85992718e+00\n",
            " -6.53082991e+00  4.76850897e-01 -1.41814399e+00  5.21571636e+00\n",
            "  7.82890737e-01  7.34485760e-02 -2.39737749e+00 -4.19670963e+00\n",
            " -1.96186478e+02  1.31454819e+02 -3.36637421e+01 -1.45840156e+00\n",
            " -2.73660603e+01  2.07529569e+00 -2.01020603e+01 -1.54953909e+00\n",
            " -2.23523121e+01 -4.89578009e+00 -1.27090559e+01  2.07605100e+00\n",
            " -6.79155397e+00  1.18804857e-01 -2.01029921e+00  4.08246708e+00\n",
            "  2.46954545e-01 -4.34296191e-01 -2.86743402e+00 -4.82864141e+00\n",
            " -5.46076233e+02 -5.86197757e-14 -1.00460106e+02 -3.72389641e+01\n",
            " -5.62108383e+01 -2.46673355e+01 -5.07050247e+01 -2.78418674e+01\n",
            " -4.57658386e+01 -2.66697674e+01 -3.37171669e+01 -1.93370171e+01\n",
            " -3.65289116e+01 -3.15064278e+01 -3.78710098e+01 -3.14066925e+01\n",
            " -3.96601562e+01 -3.52305565e+01 -3.55373955e+01 -3.72020531e+01\n",
            " -8.19379866e-01 -5.48186660e-01 -1.92975193e-01  2.27254450e-01\n",
            "  1.03851509e+00  2.61595875e-01  1.96710095e-01  3.66199315e-02\n",
            "  3.83488476e-01  8.20894301e-01  4.49299037e-01  1.31516540e+00\n",
            "  3.97566885e-01  2.97335029e-01  3.17257673e-01  4.90294725e-01\n",
            "  1.73190445e-01  2.11982429e-01  2.47590378e-01  4.13381666e-01\n",
            "  6.10304451e+01  2.70201225e+01  2.21856213e+01  1.36896858e+01\n",
            "  1.17434759e+01  7.94296980e+00  1.19082289e+01  8.04881477e+00\n",
            "  8.26932716e+00  6.91602325e+00  7.51237392e+00  7.73270512e+00\n",
            "  7.94118214e+00  8.73795223e+00  9.60913563e+00  1.10670204e+01\n",
            "  1.05812950e+01  9.90828609e+00  1.08759842e+01  1.02093077e+01\n",
            " -2.98032999e-01  1.05747309e+01  3.18734527e+00  2.79133558e+00\n",
            "  1.66451300e-05  6.66976154e-01  2.18886113e+00  2.96836585e-01\n",
            "  3.33679126e+03  1.71249597e+03  1.64144495e+03  9.53576477e+02\n",
            "  8.23147595e-01  3.09919403e+02  6.51266193e+00  5.52176660e+03\n",
            "  1.39764673e+03  1.31337561e+03  7.06570374e+02  2.15131211e+00\n",
            "  4.05797424e+02  1.30660045e+00  1.55838490e-01 -1.82549879e-01\n",
            " -3.74024868e-01 -1.42792061e-01  9.22929585e-01  7.50937271e+00\n",
            "  6.79102173e+01  4.27004433e+01  4.30382347e+01  3.72873993e+01\n",
            "  2.90228901e+01  2.20277004e+01  4.36816940e+01  2.39625435e+01\n",
            "  1.86711311e+01  2.08778534e+01  2.00752029e+01  1.65275249e+01\n",
            "  1.46342106e+01  3.58265953e+01  2.33616676e+01  1.83830395e+01\n",
            "  2.07691021e+01  2.01806583e+01  1.58640842e+01  1.44874029e+01\n",
            "  3.61248932e+01  5.48233080e+00  4.02121210e+00  6.16992569e+00\n",
            "  6.50073195e+00  8.71711922e+00  1.02727985e+01  1.17660675e+01\n",
            "  7.41629064e-01  3.10808241e-01  1.33273497e-01 -4.08731960e-02\n",
            "  6.65250242e-01  6.69783771e-01 -1.40328383e+00  6.09583998e+00\n",
            "  4.95932293e+00  5.27586699e+00  4.38387728e+00  3.39253020e+00\n",
            "  1.44949007e+00  2.42206812e+00  1.63475156e+00  9.32387695e+03\n",
            "  2.68097070e+03  2.41171875e+03  8.72094727e+02  1.34409285e+00\n",
            "  1.12120874e+03  8.81916881e-02  1.13751388e+00  7.45263338e-01\n",
            " -3.26131225e-01  8.74445438e-01  1.02998607e-01  8.84033442e-02\n",
            "  1.12335481e-01  3.18714410e-01  3.87963921e-01  7.74243847e-02\n",
            "  9.56339315e-02 -6.60661049e-03  5.94905159e-03 -5.40604815e-02\n",
            " -2.60572471e-02 -8.76145810e-03  3.97249917e-03 -7.37102283e-03\n",
            "  5.02047688e-03 -6.07315861e-02 -1.02274464e-02 -9.98247601e-03\n",
            "  5.58374496e-03 -8.85050818e-02 -1.18884355e-01 -3.67414683e-01\n",
            " -4.05416816e-01 -9.15385634e-02 -7.92327002e-02  9.26852748e-02\n",
            "  6.85808528e-03  3.79680127e-01 -2.81394005e-01  4.26423728e-01\n",
            " -2.91847587e-01  2.56450884e-02  2.59852689e-02  1.07796311e-01\n",
            "  1.49982944e-01  2.34234668e-02  2.48171315e-02  1.21653767e+01\n",
            "  2.50976562e-01  5.43694235e-02  4.78515625e-02  5.85937500e-03\n",
            "  3.15650773e+00  2.70169545e-02  2.01100000e+03  2.13000000e+02]\n",
            "[-8.67672265e-01 -9.49622214e-01 -7.69648373e-01 -8.33724082e-01\n",
            " -7.88737595e-01 -7.87861884e-01 -4.52275813e-01 -7.43407488e-01\n",
            " -1.01973104e+00 -9.79906559e-01 -8.30152571e-01 -4.70538884e-01\n",
            "  6.49601758e-01  7.07447469e-01  6.77751303e-01  6.84412181e-01\n",
            "  7.19166577e-01  6.75768971e-01  6.95088327e-01  6.94760323e-01\n",
            "  7.01998174e-01  7.10103750e-01  6.64917648e-01  6.98755980e-01\n",
            "  2.15263337e-01  2.41333425e-01  2.30966717e-01  2.58496314e-01\n",
            "  2.68417507e-01  2.14166179e-01  1.94529668e-01  2.06147715e-01\n",
            "  2.72277147e-01  2.66560316e-01  2.30883181e-01  2.07017869e-01\n",
            "  2.14249790e-01  2.08307952e-01  2.19706818e-01  2.52074122e-01\n",
            "  2.45400280e-01  2.12715372e-01  1.45909339e-01  2.00296611e-01\n",
            "  2.60237873e-01  2.49731079e-01  2.27786496e-01  1.78240821e-01\n",
            "  0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00\n",
            "  0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00\n",
            "  0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00\n",
            "  2.42790878e-01  5.00705183e-01  3.71848851e-01  1.60561845e-01\n",
            "  3.73085409e-01  2.73895055e-01  7.81557560e-01  3.23838592e-01\n",
            "  2.01379254e-01  3.04839402e-01  1.30607679e-01  6.57298863e-01\n",
            "  1.46728545e-01  1.99122608e-01  1.62672788e-01  1.56106547e-01\n",
            "  1.79889038e-01  1.45622000e-01  1.77221090e-01  1.42340422e-01\n",
            "  1.78499550e-01  1.89876989e-01  1.39963582e-01  1.72171474e-01\n",
            " -2.28750721e-01 -1.00517082e+00 -5.05675554e-01 -7.66006470e-01\n",
            " -1.12897480e+00 -3.67363602e-01 -4.71930265e-01 -3.55160356e-01\n",
            " -1.03679109e+00 -1.06455004e+00 -3.61174196e-01 -5.17936885e-01\n",
            "  1.00000000e+00  1.00000000e+00  1.00000000e+00  1.00000000e+00\n",
            "  1.00000000e+00  1.00000000e+00  1.00000000e+00  1.00000000e+00\n",
            "  1.00000000e+00  1.00000000e+00  1.00000000e+00  1.00000000e+00\n",
            "  3.64951640e-01  4.27241892e-01  3.89532924e-01  4.32910800e-01\n",
            "  4.59770322e-01  3.66054833e-01  3.75101715e-01  3.57141614e-01\n",
            "  4.59139824e-01  4.54489499e-01  3.85517836e-01  3.87969524e-01\n",
            "  3.35551083e-01  3.10353935e-01  3.38788420e-01  3.76820028e-01\n",
            "  3.59926045e-01  3.32348526e-01  2.47548252e-01  3.12941462e-01\n",
            "  3.79720151e-01  3.75655562e-01  3.49468619e-01  2.75587350e-01\n",
            "  0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00\n",
            "  0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00\n",
            "  0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00\n",
            "  6.03099287e-01  6.55966580e-01  6.17954433e-01  5.77898085e-01\n",
            "  5.35532653e-01  5.67038774e-01  9.58294213e-01  6.38231695e-01\n",
            "  5.40315568e-01  5.09177983e-01  5.84727466e-01  8.90757203e-01\n",
            "  2.26498738e-01  3.30278933e-01  2.52821743e-01  2.83318490e-01\n",
            "  3.22192431e-01  2.27890655e-01  3.16590160e-01  2.28247747e-01\n",
            "  3.10151786e-01  3.10919374e-01  2.35491157e-01  3.08785439e-01\n",
            "  6.02363539e+00 -8.31665456e-01  3.20420647e+00  3.47724557e-02\n",
            " -6.09724700e-01  4.73014688e+00  6.22708678e-01  4.66093493e+00\n",
            "  2.58720249e-01 -1.81353241e-01  3.39114451e+00  1.78954348e-01\n",
            "  1.00000000e+00  1.00000000e+00  1.00000000e+00  1.00000000e+00\n",
            "  1.00000000e+00  1.00000000e+00  1.00000000e+00  1.00000000e+00\n",
            "  1.00000000e+00  1.00000000e+00  1.00000000e+00  1.00000000e+00\n",
            "  1.60522819e-01  3.21274072e-01  1.90113455e-01  2.90635407e-01\n",
            "  3.25239927e-01  1.69571906e-01  2.33958393e-01  1.54009581e-01\n",
            "  2.72051573e-01  2.85967857e-01  1.95617080e-01  2.61257559e-01\n",
            "  9.12613496e-02  1.09340645e-01  9.51722339e-02  1.55835167e-01\n",
            "  1.53644338e-01  9.01438892e-02  5.76027185e-02  6.83151186e-02\n",
            "  1.15157761e-01  1.01178490e-01  1.18148588e-01  8.69009271e-02\n",
            "  0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00\n",
            "  0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00\n",
            "  0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00\n",
            "  2.24413967e+00  9.18937087e-01  1.83815455e+00  1.17365360e+00\n",
            "  9.59702730e-01  2.09255385e+00  1.45232439e+00  2.07699323e+00\n",
            "  1.29740822e+00  1.14027584e+00  1.85775507e+00  1.28989053e+00\n",
            "  1.90938115e-01  3.83812547e-01  2.30819553e-01  3.27335328e-01\n",
            "  3.57651889e-01  2.10088447e-01  3.31195533e-01  1.99828476e-01\n",
            "  3.31115335e-01  3.45801800e-01  2.24825561e-01  3.38142514e-01\n",
            "  3.31961513e-01  2.76931107e-01 -1.45086080e-01  3.59049588e-02\n",
            "  4.36890602e-01 -7.17589855e-02  4.42177564e-01  2.93051273e-01\n",
            "  7.33787715e-01  2.00204432e-01 -6.69935122e-02 -3.13698232e-01\n",
            " -1.17818102e-01 -1.72078580e-01 -1.41197950e-01  1.08131215e-01\n",
            " -3.11219156e-01 -2.81980425e-01  6.67220056e-02  6.14561081e-01\n",
            " -1.12657188e+02  1.95304733e+02  6.72105865e+01  5.45935554e+01\n",
            "  5.46895065e+01  4.06597557e+01  2.84615326e+01  4.44289284e+01\n",
            "  6.21553993e+01  5.97497902e+01  6.32485886e+01  6.45277023e+01\n",
            "  5.61633568e+01  6.57388840e+01  6.05496788e+01  6.23597298e+01\n",
            "  5.81020355e+01  6.24018860e+01  6.27149239e+01  5.71533928e+01\n",
            " -3.77884277e+02  1.04477867e+02 -3.83178177e+01  6.39249802e-01\n",
            "  9.49509525e+00 -4.95470762e+00 -1.68611832e+01 -1.17378750e+01\n",
            " -4.62021923e+00 -2.14341354e+00 -8.52583796e-02  6.00400877e+00\n",
            "  3.02547693e+00  2.61812878e+00  3.02486324e+00  2.54969883e+00\n",
            "  4.94319010e+00  9.23010635e+00  6.43861103e+00  1.88082591e-01\n",
            " -3.74414429e+02  1.04661598e+02 -4.28219910e+01  4.46471334e-01\n",
            "  1.11685410e+01 -5.43762255e+00 -1.73236046e+01 -1.23216181e+01\n",
            " -5.96586227e+00 -4.54318953e+00 -2.76000261e+00  2.95761132e+00\n",
            "  1.66629636e+00  1.27077782e+00  2.29047465e+00  1.41854358e+00\n",
            "  3.99973750e+00  8.85649872e+00  5.21080542e+00 -1.30646408e+00\n",
            " -5.76016968e+02 -5.86197757e-14 -1.07499832e+02 -5.42078857e+01\n",
            " -6.32180099e+01 -4.48454971e+01 -6.11206627e+01 -4.80428696e+01\n",
            " -4.37296486e+01 -4.61161537e+01 -4.76036301e+01 -3.80820770e+01\n",
            " -5.31657104e+01 -5.17453995e+01 -5.04234924e+01 -5.09404678e+01\n",
            " -5.06225243e+01 -4.27594604e+01 -3.74843292e+01 -4.36902733e+01\n",
            " -1.29960775e-01 -2.87510723e-01  6.27486348e-01  7.29836971e-02\n",
            " -5.21813691e-01  2.15498775e-01  3.08940798e-01  3.89911830e-01\n",
            "  6.49432003e-01  6.45607710e-01  5.57397127e-01  5.51524460e-01\n",
            "  2.24452212e-01  1.65207848e-01  4.93681207e-02  2.84900427e-01\n",
            "  7.94819072e-02  7.39819407e-02  3.86464745e-01  6.37438536e-01\n",
            "  6.82409134e+01  2.79654961e+01  2.76281109e+01  1.19190540e+01\n",
            "  1.45102396e+01  1.02342920e+01  1.13981533e+01  1.04720640e+01\n",
            "  1.17626324e+01  1.46514502e+01  1.59114056e+01  1.60116196e+01\n",
            "  1.48706379e+01  1.64335136e+01  1.54910898e+01  1.50167227e+01\n",
            "  1.55272694e+01  1.48756981e+01  1.27913418e+01  1.28481636e+01\n",
            "  3.66578674e+00  6.86149454e+00  8.15187275e-01  6.43361390e-01\n",
            "  0.00000000e+00  1.56673837e+00  6.75086498e-01  5.08105612e+00\n",
            "  3.33249854e+03  1.09452319e+03  1.04165637e+03  0.00000000e+00\n",
            "  1.43651676e+00  2.79138794e+02  8.87287235e+00  5.65521826e+03\n",
            "  1.08900439e+03  1.06290930e+03  0.00000000e+00  1.00416934e+00\n",
            "  3.01255127e+02  1.47080886e+00 -8.73377100e-02  5.10765612e-01\n",
            "  1.65576920e-01  4.79865640e-01 -2.00007841e-01 -8.71770740e-01\n",
            "  7.68898010e+01  5.00305290e+01  5.52534065e+01  4.78896904e+01\n",
            "  5.09735527e+01  4.89590187e+01  4.68035889e+01  1.56992254e+01\n",
            "  2.22213383e+01  2.54263821e+01  2.61578999e+01  2.99393673e+01\n",
            "  2.93977928e+01  2.74751358e+01  1.45532742e+01  2.25037403e+01\n",
            "  2.54765701e+01  2.63566475e+01  3.04489594e+01  2.99517269e+01\n",
            "  2.77519588e+01  3.46727180e+00  3.41332316e+00  3.67132688e+00\n",
            "  6.07383728e+00  8.20200920e+00  7.92412758e+00  8.31162739e+00\n",
            "  1.06214035e+00 -1.22106627e-01 -9.60429907e-02 -2.15893060e-01\n",
            " -5.50680816e-01 -4.07033592e-01 -1.68331727e-01  5.36849642e+00\n",
            "  5.98716640e+00  5.54485750e+00  5.15868664e+00  5.83053446e+00\n",
            "  5.93657303e+00  6.75804567e+00  1.46295929e+01  9.52844238e+03\n",
            "  1.80886157e+03  1.72265625e+03  0.00000000e+00  2.06261373e+00\n",
            "  5.29313110e+02  9.35408592e-01  9.40726817e-01 -2.58945942e-01\n",
            " -1.15919277e-01  3.24383006e-02  3.12135220e-02  1.58893138e-01\n",
            "  1.79180607e-01  4.58972007e-01  4.67624515e-01  1.18695721e-01\n",
            "  9.26522017e-02 -5.88576589e-03 -1.44788446e-02 -2.05901414e-02\n",
            "  3.76697034e-02  3.41582904e-03 -3.40221799e-04 -5.39906695e-03\n",
            " -1.53441476e-02 -1.00875879e-02  3.90136503e-02  3.73313017e-03\n",
            "  1.67297781e-03 -2.30769232e-01 -1.59527987e-01 -4.41862732e-01\n",
            " -4.35406089e-01 -9.53546092e-02 -1.04980007e-01 -1.30591020e-01\n",
            "  3.29293817e-01 -7.35113770e-02 -2.44704355e-02 -4.10193205e-03\n",
            " -2.92129576e-01  3.08545008e-02  3.30569968e-02  1.51631877e-01\n",
            "  1.42211497e-01  2.90707853e-02  2.78936792e-02  1.35065103e+00\n",
            "  4.14062500e-01  6.86819106e-02  6.83593750e-02  0.00000000e+00\n",
            "  3.62475246e-01  2.88142804e-02  2.01000000e+03  1.16500000e+03]\n",
            "[ 2.18883917e-01  3.60998437e-02  9.06098270e+00  6.14961433e+00\n",
            "  1.81050479e+00 -7.54021406e-02  2.35621616e-01 -6.05575740e-01\n",
            " -3.58201861e-02  2.03211260e+00  1.28433275e+00  6.83052492e+00\n",
            "  5.02033412e-01  3.83664250e-01  4.47346747e-01  4.04287219e-01\n",
            "  3.66575837e-01  5.47252536e-01  5.43965459e-01  6.75495863e-01\n",
            "  5.61587334e-01  5.25193214e-01  3.80959004e-01  4.01950121e-01\n",
            "  2.63327718e-01  1.83507949e-01  5.47561310e-02  6.59858212e-02\n",
            "  1.49544016e-01  2.86482334e-01  4.30023611e-01  5.43624222e-01\n",
            "  4.17608976e-01  2.42493272e-01  1.06424458e-01  1.07228860e-01\n",
            "  2.60729432e-01  1.78828523e-01  4.36112732e-02  6.19659387e-02\n",
            "  1.40976772e-01  2.55986601e-01  4.34292883e-01  5.48392177e-01\n",
            "  4.18723911e-01  2.18550146e-01  8.58756974e-02  9.81811658e-02\n",
            "  7.70358145e-02  3.05905417e-02  0.00000000e+00  0.00000000e+00\n",
            "  2.51951776e-02  1.38126582e-01  3.08580279e-01  3.34358692e-01\n",
            "  2.80884564e-01  1.09372839e-01  0.00000000e+00  2.90092407e-03\n",
            "  4.13862437e-01  5.20904124e-01  2.46197319e+00  1.69043303e+00\n",
            "  1.01200759e+00  1.00462651e+00 -4.00419056e-01  6.40357062e-02\n",
            "  7.25660697e-02  1.70466113e+00  1.35208476e+00  2.12548470e+00\n",
            "  7.84646794e-02  6.49859980e-02  5.21081127e-02  4.29400653e-02\n",
            "  5.11988997e-02  8.31723288e-02  3.23435366e-02  5.65618128e-02\n",
            "  3.95424664e-02  8.65872949e-02  8.86861235e-02  5.54430000e-02\n",
            " -8.40510011e-01 -3.40168536e-01  5.66344357e+00  3.48849392e+00\n",
            "  8.25997055e-01 -1.91854104e-01  6.67619050e-01  2.76575470e+00\n",
            "  1.55478239e+00  1.65805316e+00  1.37544441e+00  1.66418290e+00\n",
            "  1.00000000e+00  1.00000000e+00  1.00000000e+00  1.00000000e+00\n",
            "  1.00000000e+00  1.00000000e+00  1.00000000e+00  1.00000000e+00\n",
            "  1.00000000e+00  1.00000000e+00  1.00000000e+00  1.00000000e+00\n",
            "  4.79954481e-01  3.73629332e-01  2.28388324e-01  2.43532196e-01\n",
            "  3.34353507e-01  4.92469996e-01  7.12112904e-01  9.08171952e-01\n",
            "  6.90751433e-01  4.25389856e-01  2.84429610e-01  2.87053913e-01\n",
            "  4.01453257e-01  3.14275861e-01  1.91245303e-01  2.05524459e-01\n",
            "  2.91291386e-01  4.39251482e-01  7.24180341e-01  1.00000000e+00\n",
            "  7.29219556e-01  3.80755812e-01  2.38933235e-01  2.43366614e-01\n",
            "  4.70591113e-02  3.93938012e-02  4.10871953e-02  3.97870056e-02\n",
            "  2.26560626e-02  1.03229620e-01  1.20888330e-01  8.98901448e-02\n",
            "  7.45362863e-02  7.59145338e-03  2.79704276e-02  3.66513096e-02\n",
            "  6.42023206e-01  7.77695596e-01  1.93643093e+00  1.61568296e+00\n",
            "  1.02445471e+00  8.18472624e-01 -6.24353528e-01 -1.97693455e+00\n",
            " -1.28893459e+00  1.37147975e+00  1.27536321e+00  1.21726346e+00\n",
            "  2.79289901e-01  2.13298902e-01  1.35524705e-01  1.39716908e-01\n",
            "  1.90608412e-01  2.25394025e-01  1.61106154e-01  1.82380617e-01\n",
            "  1.50691360e-01  1.99110910e-01  1.79283917e-01  1.66944236e-01\n",
            " -8.41650665e-01  1.07900143e+00  2.45859413e+01  1.77815914e+01\n",
            "  4.68905306e+00  4.16495323e+00  3.74186516e-01  3.46805954e+00\n",
            "  1.70080036e-01  5.12141085e+00  7.94035482e+00  1.10138502e+01\n",
            "  1.00000000e+00  1.00000000e+00  1.00000000e+00  1.00000000e+00\n",
            "  1.00000000e+00  1.00000000e+00  1.00000000e+00  1.00000000e+00\n",
            "  1.00000000e+00  1.00000000e+00  1.00000000e+00  1.00000000e+00\n",
            "  3.70284230e-01  2.14449286e-01  7.00046271e-02  8.05880055e-02\n",
            "  1.86552048e-01  2.46245369e-01  5.03143370e-01  8.99765015e-01\n",
            "  5.69791377e-01  2.43542373e-01  1.36138067e-01  1.34913281e-01\n",
            "  2.21017182e-01  1.21604390e-01  3.55875790e-02  4.47007753e-02\n",
            "  1.09669618e-01  1.80124298e-01  5.19864559e-01  1.00000000e+00\n",
            "  6.15910649e-01  2.00690135e-01  8.21575448e-02  8.14082474e-02\n",
            "  0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00\n",
            "  0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00\n",
            "  0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00\n",
            "  8.10183942e-01  1.33379793e+00  4.19113016e+00  3.45811296e+00\n",
            "  2.13285685e+00  2.03832912e+00  8.27189349e-03 -2.16072202e+00\n",
            " -7.29556978e-01  1.97060299e+00  2.50693321e+00  2.81783223e+00\n",
            "  3.44744653e-01  2.19020218e-01  1.03341579e-01  1.03084058e-01\n",
            "  2.08431974e-01  2.02324376e-01  1.89272746e-01  2.18684375e-01\n",
            "  1.91942930e-01  1.65636212e-01  1.55514807e-01  1.54623330e-01\n",
            "  3.55762863e+00  9.02033138e+00  5.98521650e-01  2.24112913e-01\n",
            " -1.81521177e-01  4.83339280e-03  1.59442052e-01  1.43013388e-01\n",
            " -4.35385555e-02  1.28826290e-01 -6.14611916e-02  8.96800496e-03\n",
            " -4.69844267e-02  6.28945306e-02  2.04664707e-01  7.47500807e-02\n",
            " -3.99683928e-03  1.37764469e-01 -7.05285892e-02 -1.37655521e-02\n",
            " -2.58923798e+02  2.39124420e+02  5.31975632e+01  4.87631645e+01\n",
            "  2.25584660e+01  3.01599636e+01  3.46962662e+01  2.69405823e+01\n",
            "  2.29591999e+01  2.26527386e+01  1.83402328e+01  2.29432220e+01\n",
            "  2.35699272e+01  1.61963291e+01  1.83412971e+01  2.21148968e+01\n",
            "  3.49999084e+01  2.09652271e+01  3.24868546e+01  2.81743202e+01\n",
            " -4.23281677e+02  1.82147141e+02 -1.34613285e+01 -1.31797111e+00\n",
            " -2.29876480e+01 -1.35007677e+01 -5.84128499e-01 -1.03480330e+01\n",
            " -1.86938305e+01 -9.11856365e+00 -1.88890953e+01 -5.68067694e+00\n",
            " -1.14619560e+01 -8.92402363e+00 -9.16255760e+00 -3.54160309e+00\n",
            "  6.12229729e+00 -2.19139695e+00  8.14439869e+00  5.34944248e+00\n",
            " -3.95333893e+02  1.90727600e+02 -1.46159439e+01 -1.11920047e+00\n",
            " -2.25787411e+01 -1.36351357e+01 -5.97960234e-01 -1.00656080e+01\n",
            " -1.90818310e+01 -9.09732151e+00 -1.93528500e+01 -5.72141218e+00\n",
            " -1.15897732e+01 -8.94652176e+00 -9.16722107e+00 -3.54172087e+00\n",
            "  5.89918137e+00 -2.20995140e+00  8.16561508e+00  5.31893444e+00\n",
            " -7.35012695e+02  5.32907052e-15 -1.00115959e+02 -7.91775513e+01\n",
            " -7.87278976e+01 -5.21588745e+01 -3.66949005e+01 -4.79039116e+01\n",
            " -5.31993370e+01 -4.19203758e+01 -5.42237625e+01 -3.27859306e+01\n",
            " -4.28774681e+01 -3.88359451e+01 -3.47003899e+01 -3.17165241e+01\n",
            " -1.91155014e+01 -2.37000141e+01 -1.58410473e+01 -1.74465370e+01\n",
            " -2.00196147e+00 -2.79340434e+00  2.82334298e-01 -2.39598721e-01\n",
            " -1.93943188e-01  8.93952698e-02 -4.76563200e-02 -8.90471786e-02\n",
            "  2.12577567e-01 -6.93664933e-03  1.81886911e-01  7.23828971e-02\n",
            "  8.46568644e-02 -1.08972460e-03 -2.60889228e-03  7.49136135e-02\n",
            "  2.11793527e-01  4.36982028e-02  7.34013831e-03  3.17748748e-02\n",
            "  8.42395630e+01  3.68941689e+01  2.02670326e+01  1.57125731e+01\n",
            "  1.39765453e+01  1.05486069e+01  8.53332043e+00  9.24761486e+00\n",
            "  9.49777985e+00  7.81260967e+00  9.59437466e+00  7.31982231e+00\n",
            "  7.99957895e+00  6.95864630e+00  6.74513197e+00  6.73418522e+00\n",
            "  6.47452784e+00  5.88302279e+00  6.36087608e+00  6.01982975e+00\n",
            " -1.06804824e+00  1.53522074e+00  6.26301289e-01  5.78410625e-01\n",
            "  0.00000000e+00  1.38256148e-01  3.80479366e-01  2.08883171e+01\n",
            "  3.69395117e+03  1.16544495e+03  1.08635413e+03  0.00000000e+00\n",
            "  4.20140123e+00  3.71365540e+02  6.15239563e+01  5.81215039e+03\n",
            "  7.64710510e+02  6.82978210e+02  0.00000000e+00  7.29784107e+00\n",
            "  4.56491333e+02  1.17128289e+00  2.62961537e-01  8.81003499e-01\n",
            "  4.54978317e-01  3.91229659e-01  3.79989535e-01  7.15886593e-01\n",
            "  4.70658150e+01  4.15370140e+01  3.96423302e+01  3.22124557e+01\n",
            "  3.32008018e+01  4.25601501e+01  3.86212578e+01  2.13108025e+01\n",
            "  1.75672398e+01  1.69777088e+01  1.89282379e+01  1.80361767e+01\n",
            "  2.33864841e+01  2.93953228e+01  2.11137638e+01  1.75462551e+01\n",
            "  1.67345638e+01  1.88174191e+01  1.78266659e+01  2.31683292e+01\n",
            "  3.09676685e+01  3.27611208e-01  3.22572261e-01  3.27053010e-01\n",
            "  3.24083090e-01  3.14040989e-01  2.84598649e-01  1.82780147e-01\n",
            "  4.46031004e-01  2.77027041e-01  5.34262300e-01  1.90290585e-01\n",
            "  4.31249112e-01  1.81274444e-01 -1.11674571e+00  3.84491706e+00\n",
            "  4.12118244e+00  3.45950580e+00  2.77398157e+00  2.80902910e+00\n",
            "  4.21983910e+00  5.62136412e+00  3.59728241e+01  9.62534180e+03\n",
            "  1.20714124e+03  9.36694336e+02  0.00000000e+00  5.63848352e+00\n",
            "  1.04698816e+03  5.62010780e-02 -4.38362472e-02 -7.10641444e-01\n",
            " -1.22958191e-01 -4.74605232e-01  3.96383740e-02  6.51463643e-02\n",
            "  9.61986259e-02  1.70017540e-01  2.10948735e-01  7.15912059e-02\n",
            "  5.29164039e-02 -3.52215371e-03  3.44068408e-02  1.87656563e-03\n",
            "  8.40745419e-02  1.75376367e-02  1.33683467e-02 -3.32618109e-03\n",
            "  3.45672183e-02  1.13798464e-02  8.60436112e-02  1.75990798e-02\n",
            "  1.28176343e-02 -5.85600808e-02 -5.45452498e-02 -1.36418432e-01\n",
            " -6.79284409e-02 -3.74471061e-02 -2.08298117e-02  7.52280802e-02\n",
            " -1.29969686e-01 -1.32729225e-02 -2.15406910e-01  4.90090437e-02\n",
            "  2.22718880e-01  1.79673638e-02  1.99803319e-02  6.47000670e-02\n",
            "  4.66539636e-02  1.83730163e-02  1.13377096e-02  1.05211182e+02\n",
            "  3.53515625e-01  3.49576473e-02  3.22265625e-02  0.00000000e+00\n",
            "  9.14428902e+00  1.82740819e-02  2.01200000e+03  2.30000000e+02]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# this is a modification of what was previously here, which wasn't very successful\n",
        "# further modifications are down at the bottom\n",
        "layer_list = [ #520 parameters in input\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Dropout(0.3),\n",
        "          layers.Dense(100,kernel_initializer='lecun_normal'), #let's try dimensionality reduction\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(256,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(128,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(64,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(32,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(16,activation='softmax',kernel_initializer='lecun_normal'),\n",
        "          ]\n",
        "model = Sequential(layer_list)\n",
        "model.compile(optimizer=keras.optimizers.Adam(\n",
        "    learning_rate=keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.001,decay_steps=2000,decay_rate=0.99)),\n",
        "    loss=\"categorical_crossentropy\", metrics=['accuracy'],)\n",
        "history = model.fit(xtrain, ytrain_1hot, batch_size=32, epochs=150, validation_data=(xval, yval_1hot))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wPp3_3ogNqQa",
        "outputId": "4f3d22d6-b4d7-4ca7-ec46-ddc84e23288c"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/150\n",
            "1256/1256 [==============================] - 48s 5ms/step - loss: 1.4474 - accuracy: 0.5604 - val_loss: 1.1439 - val_accuracy: 0.6371\n",
            "Epoch 2/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 1.2389 - accuracy: 0.5994 - val_loss: 1.1164 - val_accuracy: 0.6470\n",
            "Epoch 3/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 1.2062 - accuracy: 0.6074 - val_loss: 1.0895 - val_accuracy: 0.6541\n",
            "Epoch 4/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 1.1839 - accuracy: 0.6149 - val_loss: 1.0678 - val_accuracy: 0.6599\n",
            "Epoch 5/150\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 1.1572 - accuracy: 0.6228 - val_loss: 1.0494 - val_accuracy: 0.6606\n",
            "Epoch 6/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 1.1445 - accuracy: 0.6276 - val_loss: 1.0456 - val_accuracy: 0.6653\n",
            "Epoch 7/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 1.1317 - accuracy: 0.6312 - val_loss: 1.0386 - val_accuracy: 0.6651\n",
            "Epoch 8/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 1.1177 - accuracy: 0.6346 - val_loss: 1.0354 - val_accuracy: 0.6707\n",
            "Epoch 9/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 1.1058 - accuracy: 0.6384 - val_loss: 1.0237 - val_accuracy: 0.6680\n",
            "Epoch 10/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 1.0884 - accuracy: 0.6455 - val_loss: 1.0132 - val_accuracy: 0.6812\n",
            "Epoch 11/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 1.0777 - accuracy: 0.6446 - val_loss: 1.0052 - val_accuracy: 0.6750\n",
            "Epoch 12/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 1.0704 - accuracy: 0.6497 - val_loss: 1.0044 - val_accuracy: 0.6767\n",
            "Epoch 13/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 1.0589 - accuracy: 0.6524 - val_loss: 0.9996 - val_accuracy: 0.6781\n",
            "Epoch 14/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 1.0469 - accuracy: 0.6554 - val_loss: 0.9976 - val_accuracy: 0.6747\n",
            "Epoch 15/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 1.0396 - accuracy: 0.6582 - val_loss: 0.9832 - val_accuracy: 0.6855\n",
            "Epoch 16/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 1.0291 - accuracy: 0.6622 - val_loss: 0.9819 - val_accuracy: 0.6785\n",
            "Epoch 17/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 1.0218 - accuracy: 0.6630 - val_loss: 0.9762 - val_accuracy: 0.6855\n",
            "Epoch 18/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 1.0102 - accuracy: 0.6671 - val_loss: 0.9915 - val_accuracy: 0.6877\n",
            "Epoch 19/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 1.0044 - accuracy: 0.6681 - val_loss: 0.9848 - val_accuracy: 0.6837\n",
            "Epoch 20/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.9972 - accuracy: 0.6733 - val_loss: 0.9772 - val_accuracy: 0.6873\n",
            "Epoch 21/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.9848 - accuracy: 0.6773 - val_loss: 0.9609 - val_accuracy: 0.6893\n",
            "Epoch 22/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.9769 - accuracy: 0.6759 - val_loss: 0.9667 - val_accuracy: 0.6879\n",
            "Epoch 23/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.9739 - accuracy: 0.6778 - val_loss: 0.9567 - val_accuracy: 0.6891\n",
            "Epoch 24/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.9653 - accuracy: 0.6812 - val_loss: 0.9645 - val_accuracy: 0.6927\n",
            "Epoch 25/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.9635 - accuracy: 0.6818 - val_loss: 0.9428 - val_accuracy: 0.6942\n",
            "Epoch 26/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.9527 - accuracy: 0.6848 - val_loss: 0.9565 - val_accuracy: 0.6953\n",
            "Epoch 27/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.9486 - accuracy: 0.6890 - val_loss: 0.9537 - val_accuracy: 0.6960\n",
            "Epoch 28/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.9369 - accuracy: 0.6896 - val_loss: 0.9544 - val_accuracy: 0.6924\n",
            "Epoch 29/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.9323 - accuracy: 0.6916 - val_loss: 0.9696 - val_accuracy: 0.6888\n",
            "Epoch 30/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.9252 - accuracy: 0.6946 - val_loss: 0.9581 - val_accuracy: 0.6949\n",
            "Epoch 31/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.9192 - accuracy: 0.6931 - val_loss: 0.9451 - val_accuracy: 0.6902\n",
            "Epoch 32/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.9141 - accuracy: 0.6946 - val_loss: 0.9442 - val_accuracy: 0.6962\n",
            "Epoch 33/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.9014 - accuracy: 0.6994 - val_loss: 0.9665 - val_accuracy: 0.6967\n",
            "Epoch 34/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.8979 - accuracy: 0.7004 - val_loss: 0.9815 - val_accuracy: 0.6938\n",
            "Epoch 35/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.8938 - accuracy: 0.7004 - val_loss: 0.9459 - val_accuracy: 0.6956\n",
            "Epoch 36/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.8850 - accuracy: 0.7039 - val_loss: 0.9562 - val_accuracy: 0.6915\n",
            "Epoch 37/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.8884 - accuracy: 0.7000 - val_loss: 0.9477 - val_accuracy: 0.7014\n",
            "Epoch 38/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.8776 - accuracy: 0.7079 - val_loss: 0.9443 - val_accuracy: 0.6983\n",
            "Epoch 39/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.8789 - accuracy: 0.7064 - val_loss: 0.9505 - val_accuracy: 0.6893\n",
            "Epoch 40/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.8725 - accuracy: 0.7074 - val_loss: 0.9455 - val_accuracy: 0.6938\n",
            "Epoch 41/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.8652 - accuracy: 0.7110 - val_loss: 0.9457 - val_accuracy: 0.6971\n",
            "Epoch 42/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.8588 - accuracy: 0.7097 - val_loss: 0.9496 - val_accuracy: 0.6978\n",
            "Epoch 43/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.8567 - accuracy: 0.7130 - val_loss: 0.9374 - val_accuracy: 0.6971\n",
            "Epoch 44/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.8563 - accuracy: 0.7131 - val_loss: 0.9385 - val_accuracy: 0.7003\n",
            "Epoch 45/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.8502 - accuracy: 0.7153 - val_loss: 0.9280 - val_accuracy: 0.7005\n",
            "Epoch 46/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.8436 - accuracy: 0.7147 - val_loss: 0.9421 - val_accuracy: 0.7005\n",
            "Epoch 47/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.8341 - accuracy: 0.7214 - val_loss: 0.9430 - val_accuracy: 0.7061\n",
            "Epoch 48/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.8382 - accuracy: 0.7177 - val_loss: 0.9579 - val_accuracy: 0.6931\n",
            "Epoch 49/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.8370 - accuracy: 0.7157 - val_loss: 0.9451 - val_accuracy: 0.7018\n",
            "Epoch 50/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.8278 - accuracy: 0.7227 - val_loss: 0.9546 - val_accuracy: 0.7005\n",
            "Epoch 51/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.8265 - accuracy: 0.7203 - val_loss: 0.9506 - val_accuracy: 0.6958\n",
            "Epoch 52/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.8193 - accuracy: 0.7222 - val_loss: 0.9538 - val_accuracy: 0.6918\n",
            "Epoch 53/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.8211 - accuracy: 0.7229 - val_loss: 0.9407 - val_accuracy: 0.7003\n",
            "Epoch 54/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.8163 - accuracy: 0.7232 - val_loss: 0.9467 - val_accuracy: 0.7041\n",
            "Epoch 55/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.8119 - accuracy: 0.7293 - val_loss: 0.9433 - val_accuracy: 0.6962\n",
            "Epoch 56/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.8052 - accuracy: 0.7277 - val_loss: 0.9419 - val_accuracy: 0.7050\n",
            "Epoch 57/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.7982 - accuracy: 0.7302 - val_loss: 0.9355 - val_accuracy: 0.7043\n",
            "Epoch 58/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.7882 - accuracy: 0.7347 - val_loss: 0.9409 - val_accuracy: 0.7018\n",
            "Epoch 59/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.8017 - accuracy: 0.7257 - val_loss: 0.9395 - val_accuracy: 0.7061\n",
            "Epoch 60/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.7897 - accuracy: 0.7312 - val_loss: 0.9505 - val_accuracy: 0.6971\n",
            "Epoch 61/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.7895 - accuracy: 0.7310 - val_loss: 0.9478 - val_accuracy: 0.7018\n",
            "Epoch 62/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.7863 - accuracy: 0.7353 - val_loss: 0.9511 - val_accuracy: 0.7059\n",
            "Epoch 63/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.7855 - accuracy: 0.7331 - val_loss: 0.9501 - val_accuracy: 0.6974\n",
            "Epoch 64/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.7824 - accuracy: 0.7354 - val_loss: 0.9676 - val_accuracy: 0.6960\n",
            "Epoch 65/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.7768 - accuracy: 0.7380 - val_loss: 0.9423 - val_accuracy: 0.7047\n",
            "Epoch 66/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.7771 - accuracy: 0.7372 - val_loss: 0.9521 - val_accuracy: 0.7047\n",
            "Epoch 67/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.7711 - accuracy: 0.7382 - val_loss: 0.9404 - val_accuracy: 0.7065\n",
            "Epoch 68/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.7679 - accuracy: 0.7393 - val_loss: 0.9536 - val_accuracy: 0.7009\n",
            "Epoch 69/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.7648 - accuracy: 0.7403 - val_loss: 0.9471 - val_accuracy: 0.7063\n",
            "Epoch 70/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.7667 - accuracy: 0.7397 - val_loss: 0.9401 - val_accuracy: 0.7072\n",
            "Epoch 71/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.7569 - accuracy: 0.7429 - val_loss: 0.9526 - val_accuracy: 0.6974\n",
            "Epoch 72/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.7586 - accuracy: 0.7399 - val_loss: 0.9582 - val_accuracy: 0.7000\n",
            "Epoch 73/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.7609 - accuracy: 0.7404 - val_loss: 0.9422 - val_accuracy: 0.7052\n",
            "Epoch 74/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.7526 - accuracy: 0.7433 - val_loss: 0.9524 - val_accuracy: 0.6971\n",
            "Epoch 75/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.7459 - accuracy: 0.7450 - val_loss: 0.9527 - val_accuracy: 0.7027\n",
            "Epoch 76/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.7449 - accuracy: 0.7451 - val_loss: 0.9609 - val_accuracy: 0.7014\n",
            "Epoch 77/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.7404 - accuracy: 0.7480 - val_loss: 0.9455 - val_accuracy: 0.7023\n",
            "Epoch 78/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.7373 - accuracy: 0.7490 - val_loss: 0.9509 - val_accuracy: 0.7050\n",
            "Epoch 79/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.7435 - accuracy: 0.7445 - val_loss: 0.9575 - val_accuracy: 0.7005\n",
            "Epoch 80/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.7328 - accuracy: 0.7493 - val_loss: 0.9715 - val_accuracy: 0.7005\n",
            "Epoch 81/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.7407 - accuracy: 0.7457 - val_loss: 0.9555 - val_accuracy: 0.7025\n",
            "Epoch 82/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.7405 - accuracy: 0.7480 - val_loss: 0.9541 - val_accuracy: 0.7021\n",
            "Epoch 83/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.7320 - accuracy: 0.7505 - val_loss: 0.9756 - val_accuracy: 0.6944\n",
            "Epoch 84/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.7261 - accuracy: 0.7510 - val_loss: 0.9654 - val_accuracy: 0.6969\n",
            "Epoch 85/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.7283 - accuracy: 0.7504 - val_loss: 0.9591 - val_accuracy: 0.7000\n",
            "Epoch 86/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.7231 - accuracy: 0.7523 - val_loss: 0.9661 - val_accuracy: 0.6980\n",
            "Epoch 87/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.7179 - accuracy: 0.7538 - val_loss: 0.9642 - val_accuracy: 0.6953\n",
            "Epoch 88/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.7184 - accuracy: 0.7540 - val_loss: 0.9550 - val_accuracy: 0.6991\n",
            "Epoch 89/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.7162 - accuracy: 0.7540 - val_loss: 0.9618 - val_accuracy: 0.6927\n",
            "Epoch 90/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.7147 - accuracy: 0.7550 - val_loss: 0.9565 - val_accuracy: 0.7014\n",
            "Epoch 91/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.7115 - accuracy: 0.7557 - val_loss: 0.9650 - val_accuracy: 0.7061\n",
            "Epoch 92/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.7076 - accuracy: 0.7564 - val_loss: 0.9661 - val_accuracy: 0.7047\n",
            "Epoch 93/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.7142 - accuracy: 0.7537 - val_loss: 0.9651 - val_accuracy: 0.6991\n",
            "Epoch 94/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.7082 - accuracy: 0.7601 - val_loss: 0.9596 - val_accuracy: 0.7063\n",
            "Epoch 95/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.7094 - accuracy: 0.7548 - val_loss: 0.9734 - val_accuracy: 0.7016\n",
            "Epoch 96/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.7007 - accuracy: 0.7610 - val_loss: 0.9722 - val_accuracy: 0.6980\n",
            "Epoch 97/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.7038 - accuracy: 0.7595 - val_loss: 0.9649 - val_accuracy: 0.7000\n",
            "Epoch 98/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.7029 - accuracy: 0.7588 - val_loss: 0.9795 - val_accuracy: 0.6965\n",
            "Epoch 99/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.6917 - accuracy: 0.7628 - val_loss: 0.9628 - val_accuracy: 0.7034\n",
            "Epoch 100/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.6943 - accuracy: 0.7590 - val_loss: 0.9661 - val_accuracy: 0.7014\n",
            "Epoch 101/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.7030 - accuracy: 0.7582 - val_loss: 0.9588 - val_accuracy: 0.7009\n",
            "Epoch 102/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.6940 - accuracy: 0.7596 - val_loss: 0.9778 - val_accuracy: 0.6983\n",
            "Epoch 103/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.6904 - accuracy: 0.7618 - val_loss: 0.9756 - val_accuracy: 0.7021\n",
            "Epoch 104/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.6849 - accuracy: 0.7638 - val_loss: 0.9802 - val_accuracy: 0.7009\n",
            "Epoch 105/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.6867 - accuracy: 0.7625 - val_loss: 0.9722 - val_accuracy: 0.6956\n",
            "Epoch 106/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.6848 - accuracy: 0.7624 - val_loss: 0.9772 - val_accuracy: 0.6985\n",
            "Epoch 107/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.6853 - accuracy: 0.7653 - val_loss: 0.9647 - val_accuracy: 0.7059\n",
            "Epoch 108/150\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6838 - accuracy: 0.7652 - val_loss: 0.9819 - val_accuracy: 0.7032\n",
            "Epoch 109/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.6767 - accuracy: 0.7658 - val_loss: 0.9796 - val_accuracy: 0.7000\n",
            "Epoch 110/150\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.6734 - accuracy: 0.7678 - val_loss: 0.9718 - val_accuracy: 0.7030\n",
            "Epoch 111/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.6783 - accuracy: 0.7645 - val_loss: 0.9701 - val_accuracy: 0.7003\n",
            "Epoch 112/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.6729 - accuracy: 0.7660 - val_loss: 0.9891 - val_accuracy: 0.6991\n",
            "Epoch 113/150\n",
            "1256/1256 [==============================] - 8s 6ms/step - loss: 0.6750 - accuracy: 0.7665 - val_loss: 0.9870 - val_accuracy: 0.6951\n",
            "Epoch 114/150\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.6671 - accuracy: 0.7699 - val_loss: 0.9963 - val_accuracy: 0.7000\n",
            "Epoch 115/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.6692 - accuracy: 0.7696 - val_loss: 0.9804 - val_accuracy: 0.6987\n",
            "Epoch 116/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.6677 - accuracy: 0.7693 - val_loss: 0.9888 - val_accuracy: 0.6942\n",
            "Epoch 117/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.6620 - accuracy: 0.7725 - val_loss: 0.9837 - val_accuracy: 0.6969\n",
            "Epoch 118/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.6633 - accuracy: 0.7682 - val_loss: 0.9854 - val_accuracy: 0.7059\n",
            "Epoch 119/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.6638 - accuracy: 0.7705 - val_loss: 0.9805 - val_accuracy: 0.7000\n",
            "Epoch 120/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.6623 - accuracy: 0.7714 - val_loss: 0.9815 - val_accuracy: 0.7086\n",
            "Epoch 121/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.6650 - accuracy: 0.7700 - val_loss: 0.9800 - val_accuracy: 0.7016\n",
            "Epoch 122/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.6590 - accuracy: 0.7700 - val_loss: 0.9809 - val_accuracy: 0.6971\n",
            "Epoch 123/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.6602 - accuracy: 0.7713 - val_loss: 0.9799 - val_accuracy: 0.7074\n",
            "Epoch 124/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.6548 - accuracy: 0.7743 - val_loss: 0.9992 - val_accuracy: 0.6967\n",
            "Epoch 125/150\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.6480 - accuracy: 0.7749 - val_loss: 0.9923 - val_accuracy: 0.6960\n",
            "Epoch 126/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.6555 - accuracy: 0.7738 - val_loss: 0.9982 - val_accuracy: 0.6987\n",
            "Epoch 127/150\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.6511 - accuracy: 0.7714 - val_loss: 0.9908 - val_accuracy: 0.7032\n",
            "Epoch 128/150\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.6449 - accuracy: 0.7761 - val_loss: 0.9862 - val_accuracy: 0.7000\n",
            "Epoch 129/150\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.6537 - accuracy: 0.7732 - val_loss: 0.9948 - val_accuracy: 0.7005\n",
            "Epoch 130/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.6485 - accuracy: 0.7757 - val_loss: 0.9905 - val_accuracy: 0.6991\n",
            "Epoch 131/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.6459 - accuracy: 0.7760 - val_loss: 0.9919 - val_accuracy: 0.6983\n",
            "Epoch 132/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.6475 - accuracy: 0.7760 - val_loss: 0.9979 - val_accuracy: 0.6978\n",
            "Epoch 133/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.6401 - accuracy: 0.7799 - val_loss: 0.9863 - val_accuracy: 0.7041\n",
            "Epoch 134/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.6466 - accuracy: 0.7748 - val_loss: 0.9840 - val_accuracy: 0.6989\n",
            "Epoch 135/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.6403 - accuracy: 0.7757 - val_loss: 0.9908 - val_accuracy: 0.6991\n",
            "Epoch 136/150\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.6428 - accuracy: 0.7744 - val_loss: 0.9907 - val_accuracy: 0.6980\n",
            "Epoch 137/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.6362 - accuracy: 0.7777 - val_loss: 1.0044 - val_accuracy: 0.6969\n",
            "Epoch 138/150\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.6319 - accuracy: 0.7786 - val_loss: 0.9893 - val_accuracy: 0.7021\n",
            "Epoch 139/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.6320 - accuracy: 0.7802 - val_loss: 0.9899 - val_accuracy: 0.6953\n",
            "Epoch 140/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.6333 - accuracy: 0.7779 - val_loss: 0.9982 - val_accuracy: 0.7036\n",
            "Epoch 141/150\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.6369 - accuracy: 0.7794 - val_loss: 0.9902 - val_accuracy: 0.7018\n",
            "Epoch 142/150\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.6363 - accuracy: 0.7789 - val_loss: 0.9908 - val_accuracy: 0.7027\n",
            "Epoch 143/150\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.6328 - accuracy: 0.7794 - val_loss: 0.9950 - val_accuracy: 0.6983\n",
            "Epoch 144/150\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6322 - accuracy: 0.7787 - val_loss: 0.9946 - val_accuracy: 0.6996\n",
            "Epoch 145/150\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6315 - accuracy: 0.7802 - val_loss: 0.9850 - val_accuracy: 0.7061\n",
            "Epoch 146/150\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.6306 - accuracy: 0.7823 - val_loss: 0.9880 - val_accuracy: 0.6989\n",
            "Epoch 147/150\n",
            "1256/1256 [==============================] - 8s 6ms/step - loss: 0.6333 - accuracy: 0.7789 - val_loss: 0.9907 - val_accuracy: 0.7023\n",
            "Epoch 148/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.6262 - accuracy: 0.7823 - val_loss: 0.9924 - val_accuracy: 0.7005\n",
            "Epoch 149/150\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.6205 - accuracy: 0.7846 - val_loss: 1.0136 - val_accuracy: 0.6974\n",
            "Epoch 150/150\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.6205 - accuracy: 0.7833 - val_loss: 0.9949 - val_accuracy: 0.7032\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# >98% accuracy on the training set is pretty good, suggesting sufficient model complexity\n",
        "# but validation accuracy is stuck around 68%\n",
        "# hopefully, adding regularization will improve this\n",
        "layer_list = [ #520 parameters in input\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Dense(512,kernel_initializer='lecun_normal',kernel_regularizer='l2',bias_regularizer='l2',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(256,kernel_initializer='lecun_normal',kernel_regularizer='l2',bias_regularizer='l2',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(128,kernel_initializer='lecun_normal',kernel_regularizer='l2',bias_regularizer='l2',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(64,kernel_initializer='lecun_normal',kernel_regularizer='l2',bias_regularizer='l2',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(32,kernel_initializer='lecun_normal',kernel_regularizer='l2',bias_regularizer='l2',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(16,activation='softmax',kernel_initializer='lecun_normal',kernel_regularizer='l2',bias_regularizer='l2',),\n",
        "          ]\n",
        "model = Sequential(layer_list)\n",
        "model.compile(optimizer=keras.optimizers.Adam(\n",
        "    learning_rate=keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.001,decay_steps=2000,decay_rate=0.99)),\n",
        "    loss=\"categorical_crossentropy\", metrics=['accuracy'],)\n",
        "history = model.fit(xtrain, ytrain_1hot, batch_size=32, epochs=150, validation_data=(xval, yval_1hot))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4kMlBworhA_K",
        "outputId": "a1fd7476-5d88-40d9-c589-c749f5e10e44"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/150\n",
            "1256/1256 [==============================] - 13s 8ms/step - loss: 2.8038 - accuracy: 0.5412 - val_loss: 1.7316 - val_accuracy: 0.5421\n",
            "Epoch 2/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.6599 - accuracy: 0.5577 - val_loss: 1.5745 - val_accuracy: 0.5878\n",
            "Epoch 3/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5971 - accuracy: 0.5654 - val_loss: 1.5197 - val_accuracy: 0.5912\n",
            "Epoch 4/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5391 - accuracy: 0.5757 - val_loss: 1.5722 - val_accuracy: 0.5591\n",
            "Epoch 5/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.4767 - accuracy: 0.5882 - val_loss: 1.4549 - val_accuracy: 0.6013\n",
            "Epoch 6/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.4359 - accuracy: 0.5985 - val_loss: 1.4082 - val_accuracy: 0.5981\n",
            "Epoch 7/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.4080 - accuracy: 0.6004 - val_loss: 1.3716 - val_accuracy: 0.6158\n",
            "Epoch 8/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.3873 - accuracy: 0.6078 - val_loss: 1.4137 - val_accuracy: 0.5892\n",
            "Epoch 9/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.3691 - accuracy: 0.6091 - val_loss: 1.3161 - val_accuracy: 0.6266\n",
            "Epoch 10/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.3528 - accuracy: 0.6136 - val_loss: 1.3052 - val_accuracy: 0.6373\n",
            "Epoch 11/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.3364 - accuracy: 0.6174 - val_loss: 1.2935 - val_accuracy: 0.6340\n",
            "Epoch 12/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.3281 - accuracy: 0.6201 - val_loss: 1.3031 - val_accuracy: 0.6286\n",
            "Epoch 13/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.3196 - accuracy: 0.6218 - val_loss: 1.3336 - val_accuracy: 0.6259\n",
            "Epoch 14/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.3076 - accuracy: 0.6267 - val_loss: 1.2878 - val_accuracy: 0.6254\n",
            "Epoch 15/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.3049 - accuracy: 0.6215 - val_loss: 1.2777 - val_accuracy: 0.6387\n",
            "Epoch 16/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.2998 - accuracy: 0.6235 - val_loss: 1.2871 - val_accuracy: 0.6237\n",
            "Epoch 17/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.2922 - accuracy: 0.6247 - val_loss: 1.2889 - val_accuracy: 0.6250\n",
            "Epoch 18/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.2838 - accuracy: 0.6276 - val_loss: 1.2500 - val_accuracy: 0.6422\n",
            "Epoch 19/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.2817 - accuracy: 0.6281 - val_loss: 1.3048 - val_accuracy: 0.6246\n",
            "Epoch 20/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.2746 - accuracy: 0.6313 - val_loss: 1.2447 - val_accuracy: 0.6436\n",
            "Epoch 21/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.2749 - accuracy: 0.6283 - val_loss: 1.2438 - val_accuracy: 0.6458\n",
            "Epoch 22/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.2711 - accuracy: 0.6312 - val_loss: 1.2859 - val_accuracy: 0.6212\n",
            "Epoch 23/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.2648 - accuracy: 0.6310 - val_loss: 1.2476 - val_accuracy: 0.6393\n",
            "Epoch 24/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.2561 - accuracy: 0.6354 - val_loss: 1.2790 - val_accuracy: 0.6295\n",
            "Epoch 25/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.2549 - accuracy: 0.6367 - val_loss: 1.2328 - val_accuracy: 0.6431\n",
            "Epoch 26/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.2446 - accuracy: 0.6372 - val_loss: 1.2518 - val_accuracy: 0.6335\n",
            "Epoch 27/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.2502 - accuracy: 0.6375 - val_loss: 1.2557 - val_accuracy: 0.6364\n",
            "Epoch 28/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.2436 - accuracy: 0.6383 - val_loss: 1.2693 - val_accuracy: 0.6351\n",
            "Epoch 29/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.2418 - accuracy: 0.6379 - val_loss: 1.2184 - val_accuracy: 0.6461\n",
            "Epoch 30/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.2343 - accuracy: 0.6414 - val_loss: 1.2393 - val_accuracy: 0.6378\n",
            "Epoch 31/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.2335 - accuracy: 0.6393 - val_loss: 1.2091 - val_accuracy: 0.6467\n",
            "Epoch 32/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.2331 - accuracy: 0.6396 - val_loss: 1.2272 - val_accuracy: 0.6429\n",
            "Epoch 33/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.2282 - accuracy: 0.6403 - val_loss: 1.2294 - val_accuracy: 0.6438\n",
            "Epoch 34/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.2273 - accuracy: 0.6427 - val_loss: 1.2349 - val_accuracy: 0.6360\n",
            "Epoch 35/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.2300 - accuracy: 0.6380 - val_loss: 1.2397 - val_accuracy: 0.6259\n",
            "Epoch 36/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.2214 - accuracy: 0.6426 - val_loss: 1.2448 - val_accuracy: 0.6239\n",
            "Epoch 37/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.2247 - accuracy: 0.6422 - val_loss: 1.2340 - val_accuracy: 0.6402\n",
            "Epoch 38/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.2162 - accuracy: 0.6429 - val_loss: 1.2072 - val_accuracy: 0.6472\n",
            "Epoch 39/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.2157 - accuracy: 0.6438 - val_loss: 1.2214 - val_accuracy: 0.6452\n",
            "Epoch 40/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.2151 - accuracy: 0.6445 - val_loss: 1.2026 - val_accuracy: 0.6503\n",
            "Epoch 41/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.2149 - accuracy: 0.6446 - val_loss: 1.2322 - val_accuracy: 0.6485\n",
            "Epoch 42/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.2148 - accuracy: 0.6427 - val_loss: 1.2321 - val_accuracy: 0.6364\n",
            "Epoch 43/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.2122 - accuracy: 0.6458 - val_loss: 1.2136 - val_accuracy: 0.6402\n",
            "Epoch 44/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.2115 - accuracy: 0.6455 - val_loss: 1.2276 - val_accuracy: 0.6407\n",
            "Epoch 45/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.2038 - accuracy: 0.6463 - val_loss: 1.2109 - val_accuracy: 0.6449\n",
            "Epoch 46/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.2068 - accuracy: 0.6459 - val_loss: 1.1992 - val_accuracy: 0.6478\n",
            "Epoch 47/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.2032 - accuracy: 0.6458 - val_loss: 1.2325 - val_accuracy: 0.6369\n",
            "Epoch 48/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.2005 - accuracy: 0.6455 - val_loss: 1.2151 - val_accuracy: 0.6445\n",
            "Epoch 49/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.1994 - accuracy: 0.6467 - val_loss: 1.2004 - val_accuracy: 0.6436\n",
            "Epoch 50/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.2018 - accuracy: 0.6476 - val_loss: 1.2430 - val_accuracy: 0.6355\n",
            "Epoch 51/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.2025 - accuracy: 0.6463 - val_loss: 1.1961 - val_accuracy: 0.6517\n",
            "Epoch 52/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.2003 - accuracy: 0.6504 - val_loss: 1.1944 - val_accuracy: 0.6458\n",
            "Epoch 53/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1918 - accuracy: 0.6512 - val_loss: 1.2099 - val_accuracy: 0.6487\n",
            "Epoch 54/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.1885 - accuracy: 0.6505 - val_loss: 1.1936 - val_accuracy: 0.6487\n",
            "Epoch 55/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1960 - accuracy: 0.6480 - val_loss: 1.1925 - val_accuracy: 0.6537\n",
            "Epoch 56/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1903 - accuracy: 0.6509 - val_loss: 1.2053 - val_accuracy: 0.6425\n",
            "Epoch 57/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.1857 - accuracy: 0.6518 - val_loss: 1.2010 - val_accuracy: 0.6496\n",
            "Epoch 58/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1920 - accuracy: 0.6487 - val_loss: 1.1891 - val_accuracy: 0.6517\n",
            "Epoch 59/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.1875 - accuracy: 0.6499 - val_loss: 1.1904 - val_accuracy: 0.6431\n",
            "Epoch 60/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1874 - accuracy: 0.6509 - val_loss: 1.1904 - val_accuracy: 0.6526\n",
            "Epoch 61/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1851 - accuracy: 0.6498 - val_loss: 1.2040 - val_accuracy: 0.6485\n",
            "Epoch 62/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.1816 - accuracy: 0.6514 - val_loss: 1.2073 - val_accuracy: 0.6425\n",
            "Epoch 63/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.1889 - accuracy: 0.6501 - val_loss: 1.2167 - val_accuracy: 0.6443\n",
            "Epoch 64/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1834 - accuracy: 0.6514 - val_loss: 1.2164 - val_accuracy: 0.6449\n",
            "Epoch 65/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.1829 - accuracy: 0.6508 - val_loss: 1.1984 - val_accuracy: 0.6465\n",
            "Epoch 66/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1807 - accuracy: 0.6528 - val_loss: 1.1724 - val_accuracy: 0.6512\n",
            "Epoch 67/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1806 - accuracy: 0.6517 - val_loss: 1.2017 - val_accuracy: 0.6445\n",
            "Epoch 68/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1782 - accuracy: 0.6527 - val_loss: 1.1877 - val_accuracy: 0.6476\n",
            "Epoch 69/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1790 - accuracy: 0.6519 - val_loss: 1.1844 - val_accuracy: 0.6550\n",
            "Epoch 70/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1740 - accuracy: 0.6547 - val_loss: 1.2017 - val_accuracy: 0.6510\n",
            "Epoch 71/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.1729 - accuracy: 0.6532 - val_loss: 1.1900 - val_accuracy: 0.6568\n",
            "Epoch 72/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1714 - accuracy: 0.6543 - val_loss: 1.2249 - val_accuracy: 0.6389\n",
            "Epoch 73/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.1723 - accuracy: 0.6517 - val_loss: 1.1861 - val_accuracy: 0.6474\n",
            "Epoch 74/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1724 - accuracy: 0.6521 - val_loss: 1.2055 - val_accuracy: 0.6492\n",
            "Epoch 75/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1715 - accuracy: 0.6530 - val_loss: 1.2008 - val_accuracy: 0.6512\n",
            "Epoch 76/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.1663 - accuracy: 0.6555 - val_loss: 1.1882 - val_accuracy: 0.6499\n",
            "Epoch 77/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1680 - accuracy: 0.6566 - val_loss: 1.1644 - val_accuracy: 0.6582\n",
            "Epoch 78/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1669 - accuracy: 0.6553 - val_loss: 1.1865 - val_accuracy: 0.6508\n",
            "Epoch 79/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.1701 - accuracy: 0.6538 - val_loss: 1.1915 - val_accuracy: 0.6483\n",
            "Epoch 80/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1640 - accuracy: 0.6551 - val_loss: 1.1634 - val_accuracy: 0.6543\n",
            "Epoch 81/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1617 - accuracy: 0.6577 - val_loss: 1.1903 - val_accuracy: 0.6510\n",
            "Epoch 82/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.1630 - accuracy: 0.6552 - val_loss: 1.1803 - val_accuracy: 0.6478\n",
            "Epoch 83/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1630 - accuracy: 0.6556 - val_loss: 1.1824 - val_accuracy: 0.6526\n",
            "Epoch 84/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1624 - accuracy: 0.6553 - val_loss: 1.1816 - val_accuracy: 0.6555\n",
            "Epoch 85/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1608 - accuracy: 0.6546 - val_loss: 1.1651 - val_accuracy: 0.6579\n",
            "Epoch 86/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1545 - accuracy: 0.6573 - val_loss: 1.1636 - val_accuracy: 0.6579\n",
            "Epoch 87/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1597 - accuracy: 0.6554 - val_loss: 1.1666 - val_accuracy: 0.6655\n",
            "Epoch 88/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.1542 - accuracy: 0.6578 - val_loss: 1.1699 - val_accuracy: 0.6566\n",
            "Epoch 89/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1557 - accuracy: 0.6573 - val_loss: 1.1746 - val_accuracy: 0.6478\n",
            "Epoch 90/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1521 - accuracy: 0.6563 - val_loss: 1.1734 - val_accuracy: 0.6528\n",
            "Epoch 91/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1555 - accuracy: 0.6560 - val_loss: 1.1663 - val_accuracy: 0.6521\n",
            "Epoch 92/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.1539 - accuracy: 0.6571 - val_loss: 1.1766 - val_accuracy: 0.6503\n",
            "Epoch 93/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.1542 - accuracy: 0.6568 - val_loss: 1.1733 - val_accuracy: 0.6537\n",
            "Epoch 94/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.1475 - accuracy: 0.6586 - val_loss: 1.2139 - val_accuracy: 0.6398\n",
            "Epoch 95/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1546 - accuracy: 0.6580 - val_loss: 1.1885 - val_accuracy: 0.6526\n",
            "Epoch 96/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1465 - accuracy: 0.6591 - val_loss: 1.1599 - val_accuracy: 0.6519\n",
            "Epoch 97/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.1435 - accuracy: 0.6600 - val_loss: 1.1840 - val_accuracy: 0.6452\n",
            "Epoch 98/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.1442 - accuracy: 0.6612 - val_loss: 1.1721 - val_accuracy: 0.6534\n",
            "Epoch 99/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1503 - accuracy: 0.6584 - val_loss: 1.1812 - val_accuracy: 0.6456\n",
            "Epoch 100/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1446 - accuracy: 0.6587 - val_loss: 1.1768 - val_accuracy: 0.6517\n",
            "Epoch 101/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.1448 - accuracy: 0.6594 - val_loss: 1.1674 - val_accuracy: 0.6550\n",
            "Epoch 102/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1418 - accuracy: 0.6616 - val_loss: 1.1636 - val_accuracy: 0.6579\n",
            "Epoch 103/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1442 - accuracy: 0.6602 - val_loss: 1.1497 - val_accuracy: 0.6626\n",
            "Epoch 104/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.1438 - accuracy: 0.6622 - val_loss: 1.1702 - val_accuracy: 0.6461\n",
            "Epoch 105/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.1441 - accuracy: 0.6600 - val_loss: 1.1698 - val_accuracy: 0.6570\n",
            "Epoch 106/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.1413 - accuracy: 0.6609 - val_loss: 1.1748 - val_accuracy: 0.6512\n",
            "Epoch 107/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.1392 - accuracy: 0.6612 - val_loss: 1.1683 - val_accuracy: 0.6530\n",
            "Epoch 108/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.1343 - accuracy: 0.6633 - val_loss: 1.1623 - val_accuracy: 0.6606\n",
            "Epoch 109/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1384 - accuracy: 0.6604 - val_loss: 1.1525 - val_accuracy: 0.6599\n",
            "Epoch 110/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.1352 - accuracy: 0.6614 - val_loss: 1.1558 - val_accuracy: 0.6584\n",
            "Epoch 111/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1331 - accuracy: 0.6661 - val_loss: 1.1707 - val_accuracy: 0.6465\n",
            "Epoch 112/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1345 - accuracy: 0.6636 - val_loss: 1.1611 - val_accuracy: 0.6519\n",
            "Epoch 113/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.1297 - accuracy: 0.6633 - val_loss: 1.1572 - val_accuracy: 0.6532\n",
            "Epoch 114/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1314 - accuracy: 0.6638 - val_loss: 1.1702 - val_accuracy: 0.6550\n",
            "Epoch 115/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1347 - accuracy: 0.6606 - val_loss: 1.1560 - val_accuracy: 0.6532\n",
            "Epoch 116/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1261 - accuracy: 0.6657 - val_loss: 1.1514 - val_accuracy: 0.6595\n",
            "Epoch 117/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1327 - accuracy: 0.6625 - val_loss: 1.1597 - val_accuracy: 0.6564\n",
            "Epoch 118/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1303 - accuracy: 0.6629 - val_loss: 1.1824 - val_accuracy: 0.6476\n",
            "Epoch 119/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1303 - accuracy: 0.6590 - val_loss: 1.1474 - val_accuracy: 0.6644\n",
            "Epoch 120/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1300 - accuracy: 0.6627 - val_loss: 1.1647 - val_accuracy: 0.6505\n",
            "Epoch 121/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1258 - accuracy: 0.6618 - val_loss: 1.1575 - val_accuracy: 0.6532\n",
            "Epoch 122/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1274 - accuracy: 0.6616 - val_loss: 1.1655 - val_accuracy: 0.6523\n",
            "Epoch 123/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1260 - accuracy: 0.6633 - val_loss: 1.1514 - val_accuracy: 0.6588\n",
            "Epoch 124/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1235 - accuracy: 0.6636 - val_loss: 1.1599 - val_accuracy: 0.6534\n",
            "Epoch 125/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1238 - accuracy: 0.6647 - val_loss: 1.1405 - val_accuracy: 0.6602\n",
            "Epoch 126/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1203 - accuracy: 0.6671 - val_loss: 1.1426 - val_accuracy: 0.6644\n",
            "Epoch 127/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.1220 - accuracy: 0.6638 - val_loss: 1.1550 - val_accuracy: 0.6548\n",
            "Epoch 128/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1209 - accuracy: 0.6641 - val_loss: 1.1425 - val_accuracy: 0.6629\n",
            "Epoch 129/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1195 - accuracy: 0.6644 - val_loss: 1.1732 - val_accuracy: 0.6568\n",
            "Epoch 130/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1175 - accuracy: 0.6652 - val_loss: 1.1529 - val_accuracy: 0.6615\n",
            "Epoch 131/150\n",
            "1256/1256 [==============================] - 12s 9ms/step - loss: 1.1215 - accuracy: 0.6641 - val_loss: 1.1363 - val_accuracy: 0.6575\n",
            "Epoch 132/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1166 - accuracy: 0.6649 - val_loss: 1.1525 - val_accuracy: 0.6539\n",
            "Epoch 133/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1142 - accuracy: 0.6675 - val_loss: 1.1686 - val_accuracy: 0.6405\n",
            "Epoch 134/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1153 - accuracy: 0.6674 - val_loss: 1.1471 - val_accuracy: 0.6586\n",
            "Epoch 135/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1132 - accuracy: 0.6657 - val_loss: 1.1404 - val_accuracy: 0.6584\n",
            "Epoch 136/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1162 - accuracy: 0.6637 - val_loss: 1.1635 - val_accuracy: 0.6541\n",
            "Epoch 137/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1141 - accuracy: 0.6654 - val_loss: 1.1656 - val_accuracy: 0.6537\n",
            "Epoch 138/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.1120 - accuracy: 0.6673 - val_loss: 1.1540 - val_accuracy: 0.6566\n",
            "Epoch 139/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.1105 - accuracy: 0.6690 - val_loss: 1.1528 - val_accuracy: 0.6526\n",
            "Epoch 140/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1065 - accuracy: 0.6697 - val_loss: 1.1573 - val_accuracy: 0.6519\n",
            "Epoch 141/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.1150 - accuracy: 0.6650 - val_loss: 1.1394 - val_accuracy: 0.6606\n",
            "Epoch 142/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1108 - accuracy: 0.6672 - val_loss: 1.1357 - val_accuracy: 0.6651\n",
            "Epoch 143/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1098 - accuracy: 0.6685 - val_loss: 1.1426 - val_accuracy: 0.6550\n",
            "Epoch 144/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.1074 - accuracy: 0.6673 - val_loss: 1.1524 - val_accuracy: 0.6503\n",
            "Epoch 145/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1043 - accuracy: 0.6686 - val_loss: 1.1471 - val_accuracy: 0.6564\n",
            "Epoch 146/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1119 - accuracy: 0.6655 - val_loss: 1.1390 - val_accuracy: 0.6599\n",
            "Epoch 147/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1059 - accuracy: 0.6677 - val_loss: 1.1451 - val_accuracy: 0.6595\n",
            "Epoch 148/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1048 - accuracy: 0.6679 - val_loss: 1.1496 - val_accuracy: 0.6532\n",
            "Epoch 149/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1065 - accuracy: 0.6679 - val_loss: 1.1361 - val_accuracy: 0.6608\n",
            "Epoch 150/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.1066 - accuracy: 0.6680 - val_loss: 1.1474 - val_accuracy: 0.6559\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# that didn't work, will try dropout regularization\n",
        "layer_list = [ #520 parameters in input\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Dropout(rate=0.2),\n",
        "          layers.Dense(512,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dropout(rate=0.2),\n",
        "          layers.Dense(256,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(128,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(64,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(32,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(16,activation='softmax',kernel_initializer='lecun_normal'),\n",
        "          ]\n",
        "model = Sequential(layer_list)\n",
        "model.compile(optimizer=keras.optimizers.Adam(\n",
        "    learning_rate=keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.001,decay_steps=2000,decay_rate=0.99)),\n",
        "    loss=\"categorical_crossentropy\", metrics=['accuracy'],)\n",
        "history = model.fit(xtrain, ytrain_1hot, batch_size=32, epochs=150, validation_data=(xval, yval_1hot))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z7wFfIxxwLIs",
        "outputId": "ac7ae69a-f52e-4ac7-9905-282f69086fd3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/150\n",
            "1256/1256 [==============================] - 12s 8ms/step - loss: 1.4129 - accuracy: 0.5750 - val_loss: 1.1817 - val_accuracy: 0.6228\n",
            "Epoch 2/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.2216 - accuracy: 0.6089 - val_loss: 1.1611 - val_accuracy: 0.6279\n",
            "Epoch 3/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1878 - accuracy: 0.6157 - val_loss: 1.1161 - val_accuracy: 0.6440\n",
            "Epoch 4/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1619 - accuracy: 0.6219 - val_loss: 1.1200 - val_accuracy: 0.6387\n",
            "Epoch 5/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1445 - accuracy: 0.6300 - val_loss: 1.0963 - val_accuracy: 0.6463\n",
            "Epoch 6/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1321 - accuracy: 0.6318 - val_loss: 1.0800 - val_accuracy: 0.6503\n",
            "Epoch 7/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1154 - accuracy: 0.6393 - val_loss: 1.1011 - val_accuracy: 0.6485\n",
            "Epoch 8/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1024 - accuracy: 0.6406 - val_loss: 1.0708 - val_accuracy: 0.6546\n",
            "Epoch 9/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0843 - accuracy: 0.6464 - val_loss: 1.0666 - val_accuracy: 0.6537\n",
            "Epoch 10/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0707 - accuracy: 0.6508 - val_loss: 1.0496 - val_accuracy: 0.6577\n",
            "Epoch 11/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0610 - accuracy: 0.6522 - val_loss: 1.0439 - val_accuracy: 0.6582\n",
            "Epoch 12/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0521 - accuracy: 0.6574 - val_loss: 1.0284 - val_accuracy: 0.6664\n",
            "Epoch 13/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0390 - accuracy: 0.6583 - val_loss: 1.0345 - val_accuracy: 0.6608\n",
            "Epoch 14/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0252 - accuracy: 0.6626 - val_loss: 1.0224 - val_accuracy: 0.6640\n",
            "Epoch 15/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0156 - accuracy: 0.6674 - val_loss: 1.0405 - val_accuracy: 0.6685\n",
            "Epoch 16/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0026 - accuracy: 0.6718 - val_loss: 1.0063 - val_accuracy: 0.6797\n",
            "Epoch 17/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9925 - accuracy: 0.6730 - val_loss: 1.0081 - val_accuracy: 0.6767\n",
            "Epoch 18/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9816 - accuracy: 0.6762 - val_loss: 0.9925 - val_accuracy: 0.6808\n",
            "Epoch 19/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9736 - accuracy: 0.6797 - val_loss: 1.0025 - val_accuracy: 0.6776\n",
            "Epoch 20/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9617 - accuracy: 0.6824 - val_loss: 0.9975 - val_accuracy: 0.6763\n",
            "Epoch 21/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9524 - accuracy: 0.6877 - val_loss: 0.9923 - val_accuracy: 0.6826\n",
            "Epoch 22/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9425 - accuracy: 0.6896 - val_loss: 0.9878 - val_accuracy: 0.6743\n",
            "Epoch 23/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9315 - accuracy: 0.6902 - val_loss: 0.9767 - val_accuracy: 0.6783\n",
            "Epoch 24/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9176 - accuracy: 0.6969 - val_loss: 0.9967 - val_accuracy: 0.6759\n",
            "Epoch 25/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9075 - accuracy: 0.6993 - val_loss: 0.9731 - val_accuracy: 0.6810\n",
            "Epoch 26/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.8996 - accuracy: 0.7026 - val_loss: 0.9598 - val_accuracy: 0.6857\n",
            "Epoch 27/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.8953 - accuracy: 0.7020 - val_loss: 0.9706 - val_accuracy: 0.6846\n",
            "Epoch 28/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.8767 - accuracy: 0.7087 - val_loss: 0.9734 - val_accuracy: 0.6866\n",
            "Epoch 29/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.8726 - accuracy: 0.7104 - val_loss: 0.9739 - val_accuracy: 0.6904\n",
            "Epoch 30/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.8665 - accuracy: 0.7114 - val_loss: 0.9810 - val_accuracy: 0.6868\n",
            "Epoch 31/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.8548 - accuracy: 0.7158 - val_loss: 0.9656 - val_accuracy: 0.6922\n",
            "Epoch 32/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.8499 - accuracy: 0.7170 - val_loss: 0.9703 - val_accuracy: 0.6837\n",
            "Epoch 33/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.8406 - accuracy: 0.7209 - val_loss: 0.9654 - val_accuracy: 0.6931\n",
            "Epoch 34/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.8306 - accuracy: 0.7234 - val_loss: 0.9612 - val_accuracy: 0.6920\n",
            "Epoch 35/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.8242 - accuracy: 0.7237 - val_loss: 0.9660 - val_accuracy: 0.6893\n",
            "Epoch 36/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.8149 - accuracy: 0.7274 - val_loss: 0.9691 - val_accuracy: 0.6911\n",
            "Epoch 37/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.8059 - accuracy: 0.7308 - val_loss: 0.9618 - val_accuracy: 0.6978\n",
            "Epoch 38/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.7981 - accuracy: 0.7322 - val_loss: 0.9722 - val_accuracy: 0.6911\n",
            "Epoch 39/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.7978 - accuracy: 0.7307 - val_loss: 0.9832 - val_accuracy: 0.6866\n",
            "Epoch 40/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.7852 - accuracy: 0.7359 - val_loss: 0.9744 - val_accuracy: 0.6953\n",
            "Epoch 41/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.7835 - accuracy: 0.7352 - val_loss: 0.9705 - val_accuracy: 0.6967\n",
            "Epoch 42/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.7749 - accuracy: 0.7376 - val_loss: 0.9645 - val_accuracy: 0.6987\n",
            "Epoch 43/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.7631 - accuracy: 0.7410 - val_loss: 0.9868 - val_accuracy: 0.6940\n",
            "Epoch 44/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.7532 - accuracy: 0.7450 - val_loss: 0.9658 - val_accuracy: 0.6949\n",
            "Epoch 45/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.7484 - accuracy: 0.7450 - val_loss: 0.9689 - val_accuracy: 0.6895\n",
            "Epoch 46/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.7407 - accuracy: 0.7495 - val_loss: 0.9695 - val_accuracy: 0.6994\n",
            "Epoch 47/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.7327 - accuracy: 0.7507 - val_loss: 0.9757 - val_accuracy: 0.6931\n",
            "Epoch 48/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.7269 - accuracy: 0.7530 - val_loss: 0.9691 - val_accuracy: 0.6985\n",
            "Epoch 49/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.7140 - accuracy: 0.7569 - val_loss: 0.9732 - val_accuracy: 0.7005\n",
            "Epoch 50/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.7143 - accuracy: 0.7564 - val_loss: 0.9852 - val_accuracy: 0.6940\n",
            "Epoch 51/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.7155 - accuracy: 0.7559 - val_loss: 0.9866 - val_accuracy: 0.6960\n",
            "Epoch 52/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.7045 - accuracy: 0.7609 - val_loss: 0.9745 - val_accuracy: 0.6942\n",
            "Epoch 53/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.7003 - accuracy: 0.7602 - val_loss: 0.9743 - val_accuracy: 0.6931\n",
            "Epoch 54/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.6872 - accuracy: 0.7654 - val_loss: 0.9911 - val_accuracy: 0.6951\n",
            "Epoch 55/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.6886 - accuracy: 0.7623 - val_loss: 0.9851 - val_accuracy: 0.6971\n",
            "Epoch 56/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.6787 - accuracy: 0.7691 - val_loss: 0.9801 - val_accuracy: 0.6989\n",
            "Epoch 57/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.6703 - accuracy: 0.7710 - val_loss: 0.9957 - val_accuracy: 0.6960\n",
            "Epoch 58/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.6695 - accuracy: 0.7699 - val_loss: 0.9828 - val_accuracy: 0.6989\n",
            "Epoch 59/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.6650 - accuracy: 0.7720 - val_loss: 0.9910 - val_accuracy: 0.6942\n",
            "Epoch 60/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.6629 - accuracy: 0.7711 - val_loss: 1.0089 - val_accuracy: 0.6866\n",
            "Epoch 61/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.6472 - accuracy: 0.7761 - val_loss: 0.9914 - val_accuracy: 0.6949\n",
            "Epoch 62/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.6463 - accuracy: 0.7786 - val_loss: 0.9958 - val_accuracy: 0.7000\n",
            "Epoch 63/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.6392 - accuracy: 0.7813 - val_loss: 0.9896 - val_accuracy: 0.7007\n",
            "Epoch 64/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.6338 - accuracy: 0.7816 - val_loss: 0.9976 - val_accuracy: 0.6958\n",
            "Epoch 65/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.6306 - accuracy: 0.7828 - val_loss: 1.0107 - val_accuracy: 0.6987\n",
            "Epoch 66/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.6288 - accuracy: 0.7835 - val_loss: 1.0105 - val_accuracy: 0.6909\n",
            "Epoch 67/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.6241 - accuracy: 0.7842 - val_loss: 1.0194 - val_accuracy: 0.6976\n",
            "Epoch 68/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.6178 - accuracy: 0.7892 - val_loss: 1.0018 - val_accuracy: 0.6996\n",
            "Epoch 69/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.6075 - accuracy: 0.7893 - val_loss: 1.0241 - val_accuracy: 0.6967\n",
            "Epoch 70/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.6054 - accuracy: 0.7926 - val_loss: 1.0341 - val_accuracy: 0.6911\n",
            "Epoch 71/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.6004 - accuracy: 0.7917 - val_loss: 1.0133 - val_accuracy: 0.7005\n",
            "Epoch 72/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.5991 - accuracy: 0.7906 - val_loss: 1.0069 - val_accuracy: 0.6931\n",
            "Epoch 73/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.5945 - accuracy: 0.7952 - val_loss: 1.0247 - val_accuracy: 0.6976\n",
            "Epoch 74/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.5978 - accuracy: 0.7921 - val_loss: 1.0140 - val_accuracy: 0.6944\n",
            "Epoch 75/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.5872 - accuracy: 0.7985 - val_loss: 1.0213 - val_accuracy: 0.6994\n",
            "Epoch 76/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.5793 - accuracy: 0.7991 - val_loss: 1.0305 - val_accuracy: 0.6924\n",
            "Epoch 77/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.5797 - accuracy: 0.8002 - val_loss: 1.0239 - val_accuracy: 0.6947\n",
            "Epoch 78/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.5690 - accuracy: 0.8020 - val_loss: 1.0280 - val_accuracy: 0.6974\n",
            "Epoch 79/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.5734 - accuracy: 0.8006 - val_loss: 1.0384 - val_accuracy: 0.6978\n",
            "Epoch 80/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.5616 - accuracy: 0.8051 - val_loss: 1.0336 - val_accuracy: 0.6991\n",
            "Epoch 81/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.5608 - accuracy: 0.8046 - val_loss: 1.0472 - val_accuracy: 0.6978\n",
            "Epoch 82/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.5691 - accuracy: 0.8010 - val_loss: 1.0383 - val_accuracy: 0.6989\n",
            "Epoch 83/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.5535 - accuracy: 0.8071 - val_loss: 1.0351 - val_accuracy: 0.6991\n",
            "Epoch 84/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.5526 - accuracy: 0.8074 - val_loss: 1.0333 - val_accuracy: 0.6942\n",
            "Epoch 85/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.5459 - accuracy: 0.8087 - val_loss: 1.0322 - val_accuracy: 0.6938\n",
            "Epoch 86/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.5469 - accuracy: 0.8095 - val_loss: 1.0287 - val_accuracy: 0.7014\n",
            "Epoch 87/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.5447 - accuracy: 0.8104 - val_loss: 1.0344 - val_accuracy: 0.6927\n",
            "Epoch 88/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.5354 - accuracy: 0.8105 - val_loss: 1.0288 - val_accuracy: 0.7000\n",
            "Epoch 89/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.5293 - accuracy: 0.8157 - val_loss: 1.0404 - val_accuracy: 0.6927\n",
            "Epoch 90/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.5340 - accuracy: 0.8125 - val_loss: 1.0407 - val_accuracy: 0.6949\n",
            "Epoch 91/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.5335 - accuracy: 0.8125 - val_loss: 1.0358 - val_accuracy: 0.6953\n",
            "Epoch 92/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.5227 - accuracy: 0.8178 - val_loss: 1.0469 - val_accuracy: 0.6922\n",
            "Epoch 93/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.5234 - accuracy: 0.8168 - val_loss: 1.0460 - val_accuracy: 0.7032\n",
            "Epoch 94/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.5215 - accuracy: 0.8181 - val_loss: 1.0517 - val_accuracy: 0.6989\n",
            "Epoch 95/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.5143 - accuracy: 0.8211 - val_loss: 1.0481 - val_accuracy: 0.7023\n",
            "Epoch 96/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.5167 - accuracy: 0.8184 - val_loss: 1.0508 - val_accuracy: 0.7003\n",
            "Epoch 97/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.5149 - accuracy: 0.8205 - val_loss: 1.0568 - val_accuracy: 0.6987\n",
            "Epoch 98/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.5091 - accuracy: 0.8207 - val_loss: 1.0483 - val_accuracy: 0.6983\n",
            "Epoch 99/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.5082 - accuracy: 0.8223 - val_loss: 1.0671 - val_accuracy: 0.6960\n",
            "Epoch 100/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4954 - accuracy: 0.8246 - val_loss: 1.0396 - val_accuracy: 0.7023\n",
            "Epoch 101/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4972 - accuracy: 0.8253 - val_loss: 1.0609 - val_accuracy: 0.7018\n",
            "Epoch 102/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4957 - accuracy: 0.8250 - val_loss: 1.0645 - val_accuracy: 0.6962\n",
            "Epoch 103/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.5021 - accuracy: 0.8238 - val_loss: 1.0552 - val_accuracy: 0.6971\n",
            "Epoch 104/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4912 - accuracy: 0.8276 - val_loss: 1.0702 - val_accuracy: 0.6920\n",
            "Epoch 105/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4846 - accuracy: 0.8309 - val_loss: 1.0740 - val_accuracy: 0.6985\n",
            "Epoch 106/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4871 - accuracy: 0.8297 - val_loss: 1.0737 - val_accuracy: 0.6918\n",
            "Epoch 107/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.4797 - accuracy: 0.8311 - val_loss: 1.0823 - val_accuracy: 0.6895\n",
            "Epoch 108/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.4808 - accuracy: 0.8319 - val_loss: 1.0834 - val_accuracy: 0.7014\n",
            "Epoch 109/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.4816 - accuracy: 0.8301 - val_loss: 1.0598 - val_accuracy: 0.6974\n",
            "Epoch 110/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.4747 - accuracy: 0.8335 - val_loss: 1.0654 - val_accuracy: 0.6933\n",
            "Epoch 111/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.4701 - accuracy: 0.8344 - val_loss: 1.0785 - val_accuracy: 0.6947\n",
            "Epoch 112/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.4676 - accuracy: 0.8360 - val_loss: 1.0938 - val_accuracy: 0.6967\n",
            "Epoch 113/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4701 - accuracy: 0.8375 - val_loss: 1.0826 - val_accuracy: 0.6994\n",
            "Epoch 114/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4654 - accuracy: 0.8376 - val_loss: 1.0758 - val_accuracy: 0.6989\n",
            "Epoch 115/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.4603 - accuracy: 0.8376 - val_loss: 1.0776 - val_accuracy: 0.7030\n",
            "Epoch 116/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.4578 - accuracy: 0.8378 - val_loss: 1.0795 - val_accuracy: 0.7009\n",
            "Epoch 117/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4569 - accuracy: 0.8385 - val_loss: 1.0887 - val_accuracy: 0.7050\n",
            "Epoch 118/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.4644 - accuracy: 0.8364 - val_loss: 1.0855 - val_accuracy: 0.6983\n",
            "Epoch 119/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.4537 - accuracy: 0.8420 - val_loss: 1.1071 - val_accuracy: 0.6980\n",
            "Epoch 120/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.4516 - accuracy: 0.8419 - val_loss: 1.0982 - val_accuracy: 0.7018\n",
            "Epoch 121/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.4558 - accuracy: 0.8391 - val_loss: 1.0960 - val_accuracy: 0.6983\n",
            "Epoch 122/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.4502 - accuracy: 0.8411 - val_loss: 1.0999 - val_accuracy: 0.7052\n",
            "Epoch 123/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.4498 - accuracy: 0.8424 - val_loss: 1.1006 - val_accuracy: 0.7003\n",
            "Epoch 124/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.4461 - accuracy: 0.8447 - val_loss: 1.0878 - val_accuracy: 0.7056\n",
            "Epoch 125/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4460 - accuracy: 0.8442 - val_loss: 1.1050 - val_accuracy: 0.6974\n",
            "Epoch 126/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.4421 - accuracy: 0.8444 - val_loss: 1.1082 - val_accuracy: 0.7016\n",
            "Epoch 127/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4399 - accuracy: 0.8448 - val_loss: 1.1263 - val_accuracy: 0.7047\n",
            "Epoch 128/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.4359 - accuracy: 0.8468 - val_loss: 1.1170 - val_accuracy: 0.7023\n",
            "Epoch 129/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.4406 - accuracy: 0.8449 - val_loss: 1.1060 - val_accuracy: 0.6978\n",
            "Epoch 130/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.4318 - accuracy: 0.8487 - val_loss: 1.1261 - val_accuracy: 0.7012\n",
            "Epoch 131/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4301 - accuracy: 0.8485 - val_loss: 1.1082 - val_accuracy: 0.6967\n",
            "Epoch 132/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4295 - accuracy: 0.8479 - val_loss: 1.1092 - val_accuracy: 0.6998\n",
            "Epoch 133/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.4285 - accuracy: 0.8476 - val_loss: 1.1146 - val_accuracy: 0.7050\n",
            "Epoch 134/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4288 - accuracy: 0.8502 - val_loss: 1.1108 - val_accuracy: 0.6983\n",
            "Epoch 135/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4293 - accuracy: 0.8476 - val_loss: 1.1201 - val_accuracy: 0.6951\n",
            "Epoch 136/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.4266 - accuracy: 0.8491 - val_loss: 1.1051 - val_accuracy: 0.7023\n",
            "Epoch 137/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.4184 - accuracy: 0.8519 - val_loss: 1.1258 - val_accuracy: 0.7041\n",
            "Epoch 138/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.4245 - accuracy: 0.8511 - val_loss: 1.1079 - val_accuracy: 0.6949\n",
            "Epoch 139/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4221 - accuracy: 0.8509 - val_loss: 1.1118 - val_accuracy: 0.7023\n",
            "Epoch 140/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4198 - accuracy: 0.8509 - val_loss: 1.1207 - val_accuracy: 0.7025\n",
            "Epoch 141/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.4184 - accuracy: 0.8525 - val_loss: 1.1011 - val_accuracy: 0.7059\n",
            "Epoch 142/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.4131 - accuracy: 0.8553 - val_loss: 1.1090 - val_accuracy: 0.7016\n",
            "Epoch 143/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.4208 - accuracy: 0.8515 - val_loss: 1.1301 - val_accuracy: 0.7074\n",
            "Epoch 144/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4085 - accuracy: 0.8567 - val_loss: 1.1180 - val_accuracy: 0.7000\n",
            "Epoch 145/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.4047 - accuracy: 0.8566 - val_loss: 1.1149 - val_accuracy: 0.6998\n",
            "Epoch 146/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.4083 - accuracy: 0.8548 - val_loss: 1.1363 - val_accuracy: 0.6976\n",
            "Epoch 147/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.4039 - accuracy: 0.8586 - val_loss: 1.1186 - val_accuracy: 0.6985\n",
            "Epoch 148/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.4040 - accuracy: 0.8573 - val_loss: 1.1243 - val_accuracy: 0.7018\n",
            "Epoch 149/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.4063 - accuracy: 0.8574 - val_loss: 1.1291 - val_accuracy: 0.7041\n",
            "Epoch 150/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.4003 - accuracy: 0.8584 - val_loss: 1.1331 - val_accuracy: 0.7012\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# that didn't work, will try dropout regularization\n",
        "layer_list = [ #520 parameters in input\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Dropout(rate=0.8),\n",
        "          layers.Dense(512,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dropout(rate=0.8),\n",
        "          layers.Dense(256,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dropout(rate=0.8),\n",
        "          layers.Dense(128,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(64,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(32,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(16,activation='softmax',kernel_initializer='lecun_normal'),\n",
        "          ]\n",
        "model = Sequential(layer_list)\n",
        "model.compile(optimizer=keras.optimizers.Adam(\n",
        "    learning_rate=keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.001,decay_steps=2000,decay_rate=0.99)),\n",
        "    loss=\"categorical_crossentropy\", metrics=['accuracy'],)\n",
        "history = model.fit(xtrain, ytrain_1hot, batch_size=32, epochs=150, validation_data=(xval, yval_1hot))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "9ZcVY3SE5qZT",
        "outputId": "34bef29b-ef5c-43e6-9ef5-fc3be0ea3f5d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/150\n",
            "1256/1256 [==============================] - 12s 8ms/step - loss: 1.8519 - accuracy: 0.4137 - val_loss: 1.4552 - val_accuracy: 0.5426\n",
            "Epoch 2/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.6124 - accuracy: 0.4661 - val_loss: 1.4001 - val_accuracy: 0.5562\n",
            "Epoch 3/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5813 - accuracy: 0.4786 - val_loss: 1.3630 - val_accuracy: 0.5699\n",
            "Epoch 4/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5632 - accuracy: 0.4827 - val_loss: 1.3476 - val_accuracy: 0.5726\n",
            "Epoch 5/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5504 - accuracy: 0.4865 - val_loss: 1.3405 - val_accuracy: 0.5744\n",
            "Epoch 6/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5410 - accuracy: 0.4884 - val_loss: 1.3364 - val_accuracy: 0.5744\n",
            "Epoch 7/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.5372 - accuracy: 0.4920 - val_loss: 1.3222 - val_accuracy: 0.5820\n",
            "Epoch 8/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5275 - accuracy: 0.4937 - val_loss: 1.3128 - val_accuracy: 0.5797\n",
            "Epoch 9/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5227 - accuracy: 0.4946 - val_loss: 1.3115 - val_accuracy: 0.5789\n",
            "Epoch 10/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5175 - accuracy: 0.4976 - val_loss: 1.3066 - val_accuracy: 0.5746\n",
            "Epoch 11/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5122 - accuracy: 0.4993 - val_loss: 1.3085 - val_accuracy: 0.5789\n",
            "Epoch 12/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5021 - accuracy: 0.5044 - val_loss: 1.2982 - val_accuracy: 0.5853\n",
            "Epoch 13/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5051 - accuracy: 0.5019 - val_loss: 1.3014 - val_accuracy: 0.5771\n",
            "Epoch 14/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5020 - accuracy: 0.5047 - val_loss: 1.2942 - val_accuracy: 0.5824\n",
            "Epoch 15/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.4965 - accuracy: 0.5067 - val_loss: 1.2876 - val_accuracy: 0.5838\n",
            "Epoch 16/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.4930 - accuracy: 0.5050 - val_loss: 1.2974 - val_accuracy: 0.5784\n",
            "Epoch 17/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.4913 - accuracy: 0.5072 - val_loss: 1.2912 - val_accuracy: 0.5791\n",
            "Epoch 18/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.4917 - accuracy: 0.5088 - val_loss: 1.2841 - val_accuracy: 0.5827\n",
            "Epoch 19/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.4867 - accuracy: 0.5076 - val_loss: 1.2850 - val_accuracy: 0.5840\n",
            "Epoch 20/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.4885 - accuracy: 0.5102 - val_loss: 1.2860 - val_accuracy: 0.5849\n",
            "Epoch 21/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.4864 - accuracy: 0.5097 - val_loss: 1.2802 - val_accuracy: 0.5862\n",
            "Epoch 22/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.4833 - accuracy: 0.5127 - val_loss: 1.2783 - val_accuracy: 0.5883\n",
            "Epoch 23/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.4828 - accuracy: 0.5138 - val_loss: 1.2665 - val_accuracy: 0.5914\n",
            "Epoch 24/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.4766 - accuracy: 0.5122 - val_loss: 1.2817 - val_accuracy: 0.5871\n",
            "Epoch 25/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.4805 - accuracy: 0.5129 - val_loss: 1.2773 - val_accuracy: 0.5912\n",
            "Epoch 26/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.4791 - accuracy: 0.5134 - val_loss: 1.2799 - val_accuracy: 0.5833\n",
            "Epoch 27/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.4771 - accuracy: 0.5126 - val_loss: 1.2751 - val_accuracy: 0.5847\n",
            "Epoch 28/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.4775 - accuracy: 0.5133 - val_loss: 1.2758 - val_accuracy: 0.5874\n",
            "Epoch 29/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.4717 - accuracy: 0.5189 - val_loss: 1.2682 - val_accuracy: 0.5901\n",
            "Epoch 30/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.4689 - accuracy: 0.5152 - val_loss: 1.2682 - val_accuracy: 0.5905\n",
            "Epoch 31/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.4691 - accuracy: 0.5144 - val_loss: 1.2742 - val_accuracy: 0.5840\n",
            "Epoch 32/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.4705 - accuracy: 0.5133 - val_loss: 1.2691 - val_accuracy: 0.5867\n",
            "Epoch 33/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.4741 - accuracy: 0.5139 - val_loss: 1.2728 - val_accuracy: 0.5869\n",
            "Epoch 34/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.4647 - accuracy: 0.5150 - val_loss: 1.2658 - val_accuracy: 0.5914\n",
            "Epoch 35/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.4700 - accuracy: 0.5164 - val_loss: 1.2621 - val_accuracy: 0.5883\n",
            "Epoch 36/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.4685 - accuracy: 0.5173 - val_loss: 1.2648 - val_accuracy: 0.5874\n",
            "Epoch 37/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.4663 - accuracy: 0.5213 - val_loss: 1.2689 - val_accuracy: 0.5894\n",
            "Epoch 38/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.4682 - accuracy: 0.5141 - val_loss: 1.2701 - val_accuracy: 0.5833\n",
            "Epoch 39/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.4642 - accuracy: 0.5193 - val_loss: 1.2631 - val_accuracy: 0.5883\n",
            "Epoch 40/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.4623 - accuracy: 0.5188 - val_loss: 1.2726 - val_accuracy: 0.5856\n",
            "Epoch 41/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.4652 - accuracy: 0.5140 - val_loss: 1.2651 - val_accuracy: 0.5883\n",
            "Epoch 42/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.4655 - accuracy: 0.5171 - val_loss: 1.2647 - val_accuracy: 0.5898\n",
            "Epoch 43/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.4597 - accuracy: 0.5192 - val_loss: 1.2663 - val_accuracy: 0.5885\n",
            "Epoch 44/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.4612 - accuracy: 0.5175 - val_loss: 1.2570 - val_accuracy: 0.5930\n",
            "Epoch 45/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.4625 - accuracy: 0.5183 - val_loss: 1.2594 - val_accuracy: 0.5959\n",
            "Epoch 46/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.4632 - accuracy: 0.5174 - val_loss: 1.2597 - val_accuracy: 0.5945\n",
            "Epoch 47/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.4632 - accuracy: 0.5166 - val_loss: 1.2690 - val_accuracy: 0.5885\n",
            "Epoch 48/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.4635 - accuracy: 0.5178 - val_loss: 1.2634 - val_accuracy: 0.5878\n",
            "Epoch 49/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.4626 - accuracy: 0.5192 - val_loss: 1.2623 - val_accuracy: 0.5925\n",
            "Epoch 50/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.4584 - accuracy: 0.5192 - val_loss: 1.2607 - val_accuracy: 0.5923\n",
            "Epoch 51/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.4574 - accuracy: 0.5201 - val_loss: 1.2564 - val_accuracy: 0.5959\n",
            "Epoch 52/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.4586 - accuracy: 0.5196 - val_loss: 1.2609 - val_accuracy: 0.5939\n",
            "Epoch 53/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.4542 - accuracy: 0.5215 - val_loss: 1.2626 - val_accuracy: 0.5903\n",
            "Epoch 54/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.4570 - accuracy: 0.5190 - val_loss: 1.2563 - val_accuracy: 0.5923\n",
            "Epoch 55/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.4520 - accuracy: 0.5206 - val_loss: 1.2567 - val_accuracy: 0.5930\n",
            "Epoch 56/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.4554 - accuracy: 0.5201 - val_loss: 1.2645 - val_accuracy: 0.5934\n",
            "Epoch 57/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.4532 - accuracy: 0.5232 - val_loss: 1.2541 - val_accuracy: 0.5932\n",
            "Epoch 58/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.4554 - accuracy: 0.5191 - val_loss: 1.2564 - val_accuracy: 0.5950\n",
            "Epoch 59/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.4552 - accuracy: 0.5201 - val_loss: 1.2612 - val_accuracy: 0.5894\n",
            "Epoch 60/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.4543 - accuracy: 0.5208 - val_loss: 1.2486 - val_accuracy: 0.5986\n",
            "Epoch 61/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.4501 - accuracy: 0.5236 - val_loss: 1.2514 - val_accuracy: 0.5903\n",
            "Epoch 62/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.4509 - accuracy: 0.5217 - val_loss: 1.2550 - val_accuracy: 0.5932\n",
            "Epoch 63/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.4478 - accuracy: 0.5228 - val_loss: 1.2521 - val_accuracy: 0.5927\n",
            "Epoch 64/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.4572 - accuracy: 0.5213 - val_loss: 1.2546 - val_accuracy: 0.5927\n",
            "Epoch 65/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.4518 - accuracy: 0.5232 - val_loss: 1.2475 - val_accuracy: 0.5945\n",
            "Epoch 66/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.4483 - accuracy: 0.5217 - val_loss: 1.2527 - val_accuracy: 0.5966\n",
            "Epoch 67/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.4505 - accuracy: 0.5214 - val_loss: 1.2541 - val_accuracy: 0.5936\n",
            "Epoch 68/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.4494 - accuracy: 0.5232 - val_loss: 1.2453 - val_accuracy: 0.5977\n",
            "Epoch 69/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.4506 - accuracy: 0.5237 - val_loss: 1.2530 - val_accuracy: 0.5961\n",
            "Epoch 70/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.4468 - accuracy: 0.5239 - val_loss: 1.2535 - val_accuracy: 0.5957\n",
            "Epoch 71/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.4534 - accuracy: 0.5200 - val_loss: 1.2546 - val_accuracy: 0.5912\n",
            "Epoch 72/150\n",
            "  41/1256 [..............................] - ETA: 9s - loss: 1.4690 - accuracy: 0.5168 "
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-76-50559317e1cd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     26\u001b[0m     learning_rate=keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.001,decay_steps=2000,decay_rate=0.99)),\n\u001b[1;32m     27\u001b[0m     loss=\"categorical_crossentropy\", metrics=['accuracy'],)\n\u001b[0;32m---> 28\u001b[0;31m \u001b[0mhistory\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mytrain_1hot\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m150\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myval_1hot\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/utils/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     65\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint: disable=broad-except\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m       \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1382\u001b[0m                 _r=1):\n\u001b[1;32m   1383\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1384\u001b[0;31m               \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1385\u001b[0m               \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1386\u001b[0m                 \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/util/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    148\u001b[0m     \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    149\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 150\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    151\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    152\u001b[0m       \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    913\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    914\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0mOptionalXlaContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jit_compile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 915\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    916\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    917\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    945\u001b[0m       \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    946\u001b[0m       \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 947\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    948\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    949\u001b[0m       \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   2955\u001b[0m        filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[1;32m   2956\u001b[0m     return graph_function._call_flat(\n\u001b[0;32m-> 2957\u001b[0;31m         filtered_flat_args, captured_inputs=graph_function.captured_inputs)  # pylint: disable=protected-access\n\u001b[0m\u001b[1;32m   2958\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2959\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1852\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1853\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1854\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m   1855\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1856\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    502\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    503\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 504\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    505\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    506\u001b[0m           outputs = execute.execute_with_cancellation(\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     53\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0;32m---> 55\u001b[0;31m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[1;32m     56\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# i suspect some irrelevant parameters, so trying l1 regulation\n",
        "layer_list = [ #520 parameters in input\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Dense(512,kernel_initializer='lecun_normal',kernel_regularizer='l1',bias_regularizer='l1',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(256,kernel_initializer='lecun_normal',kernel_regularizer='l1',bias_regularizer='l1',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(128,kernel_initializer='lecun_normal',kernel_regularizer='l1',bias_regularizer='l1',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(64,kernel_initializer='lecun_normal',kernel_regularizer='l1',bias_regularizer='l1',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(32,kernel_initializer='lecun_normal',kernel_regularizer='l1',bias_regularizer='l1',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(16,activation='softmax',kernel_initializer='lecun_normal',kernel_regularizer='l1',bias_regularizer='l1',),\n",
        "          ]\n",
        "model = Sequential(layer_list)\n",
        "model.compile(optimizer=keras.optimizers.Adam(\n",
        "    learning_rate=keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.001,decay_steps=2000,decay_rate=0.95)),\n",
        "    loss=\"categorical_crossentropy\", metrics=['accuracy'],)\n",
        "history = model.fit(xtrain, ytrain_1hot, batch_size=32, epochs=150, validation_data=(xval, yval_1hot))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "03xLeVJv8qvf",
        "outputId": "236890ff-b7d6-4fc1-8a63-15ef3855a0b2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/150\n",
            "1256/1256 [==============================] - 13s 9ms/step - loss: 10.3641 - accuracy: 0.4803 - val_loss: 3.9701 - val_accuracy: 0.4733\n",
            "Epoch 2/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 3.1762 - accuracy: 0.5197 - val_loss: 2.9228 - val_accuracy: 0.5233\n",
            "Epoch 3/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.8422 - accuracy: 0.5354 - val_loss: 2.3132 - val_accuracy: 0.5441\n",
            "Epoch 4/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.5269 - accuracy: 0.5466 - val_loss: 2.8277 - val_accuracy: 0.5488\n",
            "Epoch 5/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.4113 - accuracy: 0.5474 - val_loss: 2.1340 - val_accuracy: 0.5656\n",
            "Epoch 6/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.2687 - accuracy: 0.5548 - val_loss: 2.2618 - val_accuracy: 0.5677\n",
            "Epoch 7/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.2004 - accuracy: 0.5566 - val_loss: 2.0772 - val_accuracy: 0.5616\n",
            "Epoch 8/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.1236 - accuracy: 0.5606 - val_loss: 2.1252 - val_accuracy: 0.5383\n",
            "Epoch 9/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 2.0646 - accuracy: 0.5596 - val_loss: 2.0378 - val_accuracy: 0.5674\n",
            "Epoch 10/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 2.0364 - accuracy: 0.5616 - val_loss: 2.0314 - val_accuracy: 0.5540\n",
            "Epoch 11/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.9847 - accuracy: 0.5630 - val_loss: 1.9347 - val_accuracy: 0.5712\n",
            "Epoch 12/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.9648 - accuracy: 0.5626 - val_loss: 1.9282 - val_accuracy: 0.5724\n",
            "Epoch 13/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.9400 - accuracy: 0.5638 - val_loss: 1.9527 - val_accuracy: 0.5663\n",
            "Epoch 14/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.9101 - accuracy: 0.5645 - val_loss: 1.9547 - val_accuracy: 0.5506\n",
            "Epoch 15/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.8942 - accuracy: 0.5637 - val_loss: 1.8607 - val_accuracy: 0.5670\n",
            "Epoch 16/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.8682 - accuracy: 0.5639 - val_loss: 1.8364 - val_accuracy: 0.5710\n",
            "Epoch 17/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.8484 - accuracy: 0.5651 - val_loss: 1.8023 - val_accuracy: 0.5802\n",
            "Epoch 18/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.8349 - accuracy: 0.5667 - val_loss: 1.8147 - val_accuracy: 0.5663\n",
            "Epoch 19/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.8238 - accuracy: 0.5664 - val_loss: 1.7823 - val_accuracy: 0.5755\n",
            "Epoch 20/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.8022 - accuracy: 0.5686 - val_loss: 1.7843 - val_accuracy: 0.5717\n",
            "Epoch 21/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.7928 - accuracy: 0.5677 - val_loss: 1.7663 - val_accuracy: 0.5809\n",
            "Epoch 22/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.7785 - accuracy: 0.5704 - val_loss: 1.7455 - val_accuracy: 0.5685\n",
            "Epoch 23/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.7662 - accuracy: 0.5678 - val_loss: 1.7415 - val_accuracy: 0.5791\n",
            "Epoch 24/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.7547 - accuracy: 0.5681 - val_loss: 1.7200 - val_accuracy: 0.5791\n",
            "Epoch 25/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.7455 - accuracy: 0.5717 - val_loss: 1.7125 - val_accuracy: 0.5885\n",
            "Epoch 26/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.7317 - accuracy: 0.5710 - val_loss: 1.7421 - val_accuracy: 0.5748\n",
            "Epoch 27/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.7205 - accuracy: 0.5711 - val_loss: 1.6788 - val_accuracy: 0.5795\n",
            "Epoch 28/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.7107 - accuracy: 0.5715 - val_loss: 1.6918 - val_accuracy: 0.5768\n",
            "Epoch 29/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.7021 - accuracy: 0.5737 - val_loss: 1.7066 - val_accuracy: 0.5634\n",
            "Epoch 30/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6924 - accuracy: 0.5720 - val_loss: 1.6657 - val_accuracy: 0.5719\n",
            "Epoch 31/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.6849 - accuracy: 0.5721 - val_loss: 1.6595 - val_accuracy: 0.5829\n",
            "Epoch 32/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6787 - accuracy: 0.5745 - val_loss: 1.6481 - val_accuracy: 0.5777\n",
            "Epoch 33/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6653 - accuracy: 0.5745 - val_loss: 1.6352 - val_accuracy: 0.5789\n",
            "Epoch 34/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6584 - accuracy: 0.5753 - val_loss: 1.6174 - val_accuracy: 0.5936\n",
            "Epoch 35/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6544 - accuracy: 0.5724 - val_loss: 1.6295 - val_accuracy: 0.5889\n",
            "Epoch 36/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6446 - accuracy: 0.5735 - val_loss: 1.6310 - val_accuracy: 0.5804\n",
            "Epoch 37/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6386 - accuracy: 0.5746 - val_loss: 1.6313 - val_accuracy: 0.5833\n",
            "Epoch 38/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6305 - accuracy: 0.5749 - val_loss: 1.6051 - val_accuracy: 0.5876\n",
            "Epoch 39/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6236 - accuracy: 0.5753 - val_loss: 1.6635 - val_accuracy: 0.5672\n",
            "Epoch 40/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6233 - accuracy: 0.5736 - val_loss: 1.5886 - val_accuracy: 0.5847\n",
            "Epoch 41/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6105 - accuracy: 0.5765 - val_loss: 1.5872 - val_accuracy: 0.5818\n",
            "Epoch 42/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6071 - accuracy: 0.5767 - val_loss: 1.5933 - val_accuracy: 0.5824\n",
            "Epoch 43/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.5975 - accuracy: 0.5763 - val_loss: 1.5643 - val_accuracy: 0.5898\n",
            "Epoch 44/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.5944 - accuracy: 0.5749 - val_loss: 1.5598 - val_accuracy: 0.5818\n",
            "Epoch 45/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.5879 - accuracy: 0.5781 - val_loss: 1.5634 - val_accuracy: 0.5793\n",
            "Epoch 46/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.5805 - accuracy: 0.5782 - val_loss: 1.5418 - val_accuracy: 0.5916\n",
            "Epoch 47/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.5798 - accuracy: 0.5777 - val_loss: 1.5544 - val_accuracy: 0.5818\n",
            "Epoch 48/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.5717 - accuracy: 0.5772 - val_loss: 1.5498 - val_accuracy: 0.5847\n",
            "Epoch 49/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.5638 - accuracy: 0.5788 - val_loss: 1.5386 - val_accuracy: 0.5827\n",
            "Epoch 50/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.5608 - accuracy: 0.5796 - val_loss: 1.5491 - val_accuracy: 0.5874\n",
            "Epoch 51/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.5566 - accuracy: 0.5793 - val_loss: 1.5383 - val_accuracy: 0.5842\n",
            "Epoch 52/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5532 - accuracy: 0.5787 - val_loss: 1.5257 - val_accuracy: 0.5878\n",
            "Epoch 53/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5511 - accuracy: 0.5790 - val_loss: 1.5295 - val_accuracy: 0.5802\n",
            "Epoch 54/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5447 - accuracy: 0.5789 - val_loss: 1.5185 - val_accuracy: 0.5869\n",
            "Epoch 55/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5379 - accuracy: 0.5798 - val_loss: 1.5265 - val_accuracy: 0.5885\n",
            "Epoch 56/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5320 - accuracy: 0.5806 - val_loss: 1.4962 - val_accuracy: 0.5939\n",
            "Epoch 57/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5331 - accuracy: 0.5805 - val_loss: 1.5032 - val_accuracy: 0.5885\n",
            "Epoch 58/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5252 - accuracy: 0.5825 - val_loss: 1.5071 - val_accuracy: 0.5883\n",
            "Epoch 59/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5241 - accuracy: 0.5800 - val_loss: 1.5079 - val_accuracy: 0.5800\n",
            "Epoch 60/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5216 - accuracy: 0.5807 - val_loss: 1.4978 - val_accuracy: 0.5918\n",
            "Epoch 61/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5171 - accuracy: 0.5801 - val_loss: 1.5008 - val_accuracy: 0.5856\n",
            "Epoch 62/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5148 - accuracy: 0.5818 - val_loss: 1.4952 - val_accuracy: 0.5851\n",
            "Epoch 63/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5054 - accuracy: 0.5833 - val_loss: 1.4860 - val_accuracy: 0.5914\n",
            "Epoch 64/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.5088 - accuracy: 0.5835 - val_loss: 1.4739 - val_accuracy: 0.5909\n",
            "Epoch 65/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5016 - accuracy: 0.5830 - val_loss: 1.4801 - val_accuracy: 0.5907\n",
            "Epoch 66/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.4992 - accuracy: 0.5829 - val_loss: 1.4778 - val_accuracy: 0.5905\n",
            "Epoch 67/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.4975 - accuracy: 0.5830 - val_loss: 1.4718 - val_accuracy: 0.5865\n",
            "Epoch 68/150\n",
            "1034/1256 [=======================>......] - ETA: 1s - loss: 1.4908 - accuracy: 0.5834"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-77-3cbe7f171d7b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     23\u001b[0m     learning_rate=keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.001,decay_steps=2000,decay_rate=0.95)),\n\u001b[1;32m     24\u001b[0m     loss=\"categorical_crossentropy\", metrics=['accuracy'],)\n\u001b[0;32m---> 25\u001b[0;31m \u001b[0mhistory\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mytrain_1hot\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m150\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myval_1hot\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/utils/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     65\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint: disable=broad-except\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m       \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1382\u001b[0m                 _r=1):\n\u001b[1;32m   1383\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1384\u001b[0;31m               \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1385\u001b[0m               \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1386\u001b[0m                 \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/util/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    148\u001b[0m     \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    149\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 150\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    151\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    152\u001b[0m       \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    913\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    914\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0mOptionalXlaContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jit_compile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 915\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    916\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    917\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    945\u001b[0m       \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    946\u001b[0m       \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 947\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    948\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    949\u001b[0m       \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   2955\u001b[0m        filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[1;32m   2956\u001b[0m     return graph_function._call_flat(\n\u001b[0;32m-> 2957\u001b[0;31m         filtered_flat_args, captured_inputs=graph_function.captured_inputs)  # pylint: disable=protected-access\n\u001b[0m\u001b[1;32m   2958\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2959\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1852\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1853\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1854\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m   1855\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1856\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    502\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    503\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 504\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    505\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    506\u001b[0m           outputs = execute.execute_with_cancellation(\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     53\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0;32m---> 55\u001b[0;31m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[1;32m     56\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# i suspect some irrelevant parameters, so trying l1 regulation\n",
        "layer_list = [ #520 parameters in input\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Dense(512,kernel_initializer='lecun_normal',kernel_regularizer='l1',bias_regularizer='l1',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(256,kernel_initializer='lecun_normal',kernel_regularizer='l1',bias_regularizer='l1',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(128,kernel_initializer='lecun_normal',kernel_regularizer='l1',bias_regularizer='l1',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(64,kernel_initializer='lecun_normal',kernel_regularizer='l1',bias_regularizer='l1',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(32,kernel_initializer='lecun_normal',kernel_regularizer='l1',bias_regularizer='l1',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(16,activation='softmax',kernel_initializer='lecun_normal',kernel_regularizer='l1',bias_regularizer='l1',),\n",
        "          ]\n",
        "model = Sequential(layer_list)\n",
        "model.compile(optimizer=keras.optimizers.Adam(\n",
        "    learning_rate=keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.001,decay_steps=2000,decay_rate=0.99)),\n",
        "    loss=\"categorical_crossentropy\", metrics=['accuracy'],)\n",
        "history = model.fit(xtrain, ytrain_1hot, batch_size=32, epochs=150, validation_data=(xval, yval_1hot))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "wlJfa9On_1GM",
        "outputId": "422676c1-32b1-4e55-d8b5-1ae203410d3a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/150\n",
            "1256/1256 [==============================] - 24s 8ms/step - loss: 10.4413 - accuracy: 0.4816 - val_loss: 3.5299 - val_accuracy: 0.5336\n",
            "Epoch 2/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 3.2842 - accuracy: 0.5245 - val_loss: 2.5593 - val_accuracy: 0.5372\n",
            "Epoch 3/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 2.9627 - accuracy: 0.5390 - val_loss: 2.8630 - val_accuracy: 0.5394\n",
            "Epoch 4/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 2.6978 - accuracy: 0.5560 - val_loss: 2.5063 - val_accuracy: 0.5753\n",
            "Epoch 5/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 2.5270 - accuracy: 0.5668 - val_loss: 2.4881 - val_accuracy: 0.5784\n",
            "Epoch 6/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 2.3587 - accuracy: 0.5724 - val_loss: 2.4303 - val_accuracy: 0.5737\n",
            "Epoch 7/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 2.2690 - accuracy: 0.5766 - val_loss: 2.2033 - val_accuracy: 0.5780\n",
            "Epoch 8/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 2.2139 - accuracy: 0.5748 - val_loss: 2.1510 - val_accuracy: 0.5952\n",
            "Epoch 9/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 2.1613 - accuracy: 0.5801 - val_loss: 2.1407 - val_accuracy: 0.5934\n",
            "Epoch 10/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 2.1250 - accuracy: 0.5839 - val_loss: 2.0940 - val_accuracy: 0.6026\n",
            "Epoch 11/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 2.0923 - accuracy: 0.5850 - val_loss: 2.0416 - val_accuracy: 0.5972\n",
            "Epoch 12/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 2.0561 - accuracy: 0.5861 - val_loss: 2.0505 - val_accuracy: 0.5892\n",
            "Epoch 13/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 2.0462 - accuracy: 0.5866 - val_loss: 2.0399 - val_accuracy: 0.5979\n",
            "Epoch 14/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 2.0291 - accuracy: 0.5886 - val_loss: 1.9698 - val_accuracy: 0.6013\n",
            "Epoch 15/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.0203 - accuracy: 0.5912 - val_loss: 1.9676 - val_accuracy: 0.6028\n",
            "Epoch 16/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 2.0091 - accuracy: 0.5890 - val_loss: 1.9633 - val_accuracy: 0.5950\n",
            "Epoch 17/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.9983 - accuracy: 0.5875 - val_loss: 1.9726 - val_accuracy: 0.5851\n",
            "Epoch 18/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.9808 - accuracy: 0.5916 - val_loss: 1.9309 - val_accuracy: 0.5961\n",
            "Epoch 19/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.9811 - accuracy: 0.5893 - val_loss: 1.9599 - val_accuracy: 0.5961\n",
            "Epoch 20/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.9681 - accuracy: 0.5901 - val_loss: 1.9883 - val_accuracy: 0.5977\n",
            "Epoch 21/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.9653 - accuracy: 0.5909 - val_loss: 1.9174 - val_accuracy: 0.6066\n",
            "Epoch 22/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.9607 - accuracy: 0.5898 - val_loss: 1.9748 - val_accuracy: 0.5939\n",
            "Epoch 23/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.9521 - accuracy: 0.5934 - val_loss: 1.8992 - val_accuracy: 0.6062\n",
            "Epoch 24/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.9462 - accuracy: 0.5938 - val_loss: 1.8875 - val_accuracy: 0.6129\n",
            "Epoch 25/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.9397 - accuracy: 0.5932 - val_loss: 1.9100 - val_accuracy: 0.5997\n",
            "Epoch 26/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.9357 - accuracy: 0.5942 - val_loss: 1.9016 - val_accuracy: 0.6053\n",
            "Epoch 27/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.9321 - accuracy: 0.5934 - val_loss: 1.9265 - val_accuracy: 0.5959\n",
            "Epoch 28/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.9193 - accuracy: 0.5940 - val_loss: 1.8981 - val_accuracy: 0.5981\n",
            "Epoch 29/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.9246 - accuracy: 0.5913 - val_loss: 1.9037 - val_accuracy: 0.5963\n",
            "Epoch 30/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.9127 - accuracy: 0.5947 - val_loss: 1.9097 - val_accuracy: 0.5905\n",
            "Epoch 31/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.9105 - accuracy: 0.5924 - val_loss: 1.8967 - val_accuracy: 0.6057\n",
            "Epoch 32/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.9072 - accuracy: 0.5926 - val_loss: 1.8676 - val_accuracy: 0.6129\n",
            "Epoch 33/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.9024 - accuracy: 0.5915 - val_loss: 1.8625 - val_accuracy: 0.6022\n",
            "Epoch 34/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.9009 - accuracy: 0.5914 - val_loss: 1.8648 - val_accuracy: 0.6078\n",
            "Epoch 35/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.8946 - accuracy: 0.5956 - val_loss: 1.9121 - val_accuracy: 0.5791\n",
            "Epoch 36/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.8872 - accuracy: 0.5960 - val_loss: 1.8687 - val_accuracy: 0.6042\n",
            "Epoch 37/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.8887 - accuracy: 0.5939 - val_loss: 1.8980 - val_accuracy: 0.5901\n",
            "Epoch 38/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.8782 - accuracy: 0.5923 - val_loss: 1.8818 - val_accuracy: 0.5988\n",
            "Epoch 39/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.8795 - accuracy: 0.5955 - val_loss: 1.8470 - val_accuracy: 0.6062\n",
            "Epoch 40/150\n",
            "1256/1256 [==============================] - 12s 10ms/step - loss: 1.8740 - accuracy: 0.5938 - val_loss: 1.8188 - val_accuracy: 0.6111\n",
            "Epoch 41/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.8710 - accuracy: 0.5951 - val_loss: 1.8446 - val_accuracy: 0.5997\n",
            "Epoch 42/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.8667 - accuracy: 0.5941 - val_loss: 1.8524 - val_accuracy: 0.5925\n",
            "Epoch 43/150\n",
            " 706/1256 [===============>..............] - ETA: 4s - loss: 1.8724 - accuracy: 0.5883"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-78-8f03ef02b04b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     23\u001b[0m     learning_rate=keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.001,decay_steps=2000,decay_rate=0.99)),\n\u001b[1;32m     24\u001b[0m     loss=\"categorical_crossentropy\", metrics=['accuracy'],)\n\u001b[0;32m---> 25\u001b[0;31m \u001b[0mhistory\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mytrain_1hot\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m150\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myval_1hot\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/utils/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     65\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint: disable=broad-except\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m       \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1382\u001b[0m                 _r=1):\n\u001b[1;32m   1383\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1384\u001b[0;31m               \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1385\u001b[0m               \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1386\u001b[0m                 \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/util/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    148\u001b[0m     \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    149\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 150\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    151\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    152\u001b[0m       \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    913\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    914\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0mOptionalXlaContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jit_compile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 915\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    916\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    917\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    945\u001b[0m       \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    946\u001b[0m       \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 947\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    948\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    949\u001b[0m       \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   2955\u001b[0m        filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[1;32m   2956\u001b[0m     return graph_function._call_flat(\n\u001b[0;32m-> 2957\u001b[0;31m         filtered_flat_args, captured_inputs=graph_function.captured_inputs)  # pylint: disable=protected-access\n\u001b[0m\u001b[1;32m   2958\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2959\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1852\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1853\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1854\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m   1855\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1856\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    502\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    503\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 504\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    505\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    506\u001b[0m           outputs = execute.execute_with_cancellation(\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     53\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0;32m---> 55\u001b[0;31m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[1;32m     56\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# i suspect some irrelevant parameters, so trying l1 regulation\n",
        "layer_list = [ #520 parameters in input\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Dense(512,kernel_initializer='lecun_normal',kernel_regularizer='l1',bias_regularizer='l1',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(256,kernel_initializer='lecun_normal',kernel_regularizer='l1',bias_regularizer='l1',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(128,kernel_initializer='lecun_normal',kernel_regularizer='l1',bias_regularizer='l1',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(64,kernel_initializer='lecun_normal',kernel_regularizer='l1',bias_regularizer='l1',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(32,kernel_initializer='lecun_normal',kernel_regularizer='l1',bias_regularizer='l1',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(16,activation='softmax',kernel_initializer='lecun_normal',kernel_regularizer='l1',bias_regularizer='l1',),\n",
        "          ]\n",
        "model = Sequential(layer_list)\n",
        "model.compile(optimizer=keras.optimizers.Adam(\n",
        "    learning_rate=keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.01,decay_steps=2000,decay_rate=0.3,)),\n",
        "    loss=\"categorical_crossentropy\", metrics=['accuracy'],)\n",
        "history = model.fit(xtrain, ytrain_1hot, batch_size=32, epochs=150, validation_data=(xval, yval_1hot))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "0wkubnyLBinW",
        "outputId": "df2df149-4d22-456f-f0f7-d30666f89a0f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/150\n",
            "1256/1256 [==============================] - 12s 8ms/step - loss: 9.2636 - accuracy: 0.4375 - val_loss: 5.0132 - val_accuracy: 0.5067\n",
            "Epoch 2/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 3.7299 - accuracy: 0.5231 - val_loss: 2.8536 - val_accuracy: 0.5314\n",
            "Epoch 3/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 2.4462 - accuracy: 0.5423 - val_loss: 2.0749 - val_accuracy: 0.5551\n",
            "Epoch 4/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.9433 - accuracy: 0.5513 - val_loss: 1.7787 - val_accuracy: 0.5659\n",
            "Epoch 5/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.7155 - accuracy: 0.5599 - val_loss: 1.6260 - val_accuracy: 0.5753\n",
            "Epoch 6/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.6071 - accuracy: 0.5635 - val_loss: 1.5425 - val_accuracy: 0.5836\n",
            "Epoch 7/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5553 - accuracy: 0.5656 - val_loss: 1.5057 - val_accuracy: 0.5800\n",
            "Epoch 8/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.5273 - accuracy: 0.5677 - val_loss: 1.4842 - val_accuracy: 0.5860\n",
            "Epoch 9/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5117 - accuracy: 0.5683 - val_loss: 1.4768 - val_accuracy: 0.5836\n",
            "Epoch 10/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5073 - accuracy: 0.5690 - val_loss: 1.4711 - val_accuracy: 0.5851\n",
            "Epoch 11/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.5082 - accuracy: 0.5667 - val_loss: 1.4726 - val_accuracy: 0.5827\n",
            "Epoch 12/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.5054 - accuracy: 0.5697 - val_loss: 1.4711 - val_accuracy: 0.5865\n",
            "Epoch 13/150\n",
            "1256/1256 [==============================] - 12s 10ms/step - loss: 1.5038 - accuracy: 0.5678 - val_loss: 1.4689 - val_accuracy: 0.5869\n",
            "Epoch 14/150\n",
            "1256/1256 [==============================] - 12s 9ms/step - loss: 1.5042 - accuracy: 0.5691 - val_loss: 1.4684 - val_accuracy: 0.5862\n",
            "Epoch 15/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.5019 - accuracy: 0.5684 - val_loss: 1.4678 - val_accuracy: 0.5865\n",
            "Epoch 16/150\n",
            "1256/1256 [==============================] - 13s 11ms/step - loss: 1.5017 - accuracy: 0.5698 - val_loss: 1.4696 - val_accuracy: 0.5858\n",
            "Epoch 17/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.5009 - accuracy: 0.5681 - val_loss: 1.4684 - val_accuracy: 0.5853\n",
            "Epoch 18/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5005 - accuracy: 0.5682 - val_loss: 1.4696 - val_accuracy: 0.5865\n",
            "Epoch 19/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5000 - accuracy: 0.5702 - val_loss: 1.4673 - val_accuracy: 0.5869\n",
            "Epoch 20/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5024 - accuracy: 0.5680 - val_loss: 1.4693 - val_accuracy: 0.5851\n",
            "Epoch 21/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5002 - accuracy: 0.5694 - val_loss: 1.4679 - val_accuracy: 0.5847\n",
            "Epoch 22/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5010 - accuracy: 0.5682 - val_loss: 1.4686 - val_accuracy: 0.5856\n",
            "Epoch 23/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.5026 - accuracy: 0.5682 - val_loss: 1.4676 - val_accuracy: 0.5856\n",
            "Epoch 24/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.5014 - accuracy: 0.5688 - val_loss: 1.4697 - val_accuracy: 0.5840\n",
            "Epoch 25/150\n",
            "1256/1256 [==============================] - 12s 9ms/step - loss: 1.5000 - accuracy: 0.5704 - val_loss: 1.4675 - val_accuracy: 0.5867\n",
            "Epoch 26/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.5007 - accuracy: 0.5704 - val_loss: 1.4689 - val_accuracy: 0.5847\n",
            "Epoch 27/150\n",
            "1256/1256 [==============================] - 12s 9ms/step - loss: 1.5029 - accuracy: 0.5704 - val_loss: 1.4678 - val_accuracy: 0.5858\n",
            "Epoch 28/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.5022 - accuracy: 0.5707 - val_loss: 1.4689 - val_accuracy: 0.5858\n",
            "Epoch 29/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.5023 - accuracy: 0.5708 - val_loss: 1.4687 - val_accuracy: 0.5856\n",
            "Epoch 30/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.5034 - accuracy: 0.5692 - val_loss: 1.4692 - val_accuracy: 0.5856\n",
            "Epoch 31/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.5012 - accuracy: 0.5695 - val_loss: 1.4684 - val_accuracy: 0.5860\n",
            "Epoch 32/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.5017 - accuracy: 0.5694 - val_loss: 1.4691 - val_accuracy: 0.5858\n",
            "Epoch 33/150\n",
            "1256/1256 [==============================] - 12s 9ms/step - loss: 1.5026 - accuracy: 0.5691 - val_loss: 1.4681 - val_accuracy: 0.5845\n",
            "Epoch 34/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.4998 - accuracy: 0.5685 - val_loss: 1.4687 - val_accuracy: 0.5856\n",
            "Epoch 35/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.5031 - accuracy: 0.5688 - val_loss: 1.4699 - val_accuracy: 0.5836\n",
            "Epoch 36/150\n",
            "1256/1256 [==============================] - 12s 9ms/step - loss: 1.5034 - accuracy: 0.5683 - val_loss: 1.4699 - val_accuracy: 0.5869\n",
            "Epoch 37/150\n",
            "1067/1256 [========================>.....] - ETA: 1s - loss: 1.5036 - accuracy: 0.5667"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-79-38c1335640d2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     23\u001b[0m     learning_rate=keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.01,decay_steps=2000,decay_rate=0.3)),\n\u001b[1;32m     24\u001b[0m     loss=\"categorical_crossentropy\", metrics=['accuracy'],)\n\u001b[0;32m---> 25\u001b[0;31m \u001b[0mhistory\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mytrain_1hot\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m150\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myval_1hot\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/utils/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     65\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint: disable=broad-except\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m       \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1382\u001b[0m                 _r=1):\n\u001b[1;32m   1383\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1384\u001b[0;31m               \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1385\u001b[0m               \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1386\u001b[0m                 \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/util/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    148\u001b[0m     \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    149\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 150\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    151\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    152\u001b[0m       \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    913\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    914\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0mOptionalXlaContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jit_compile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 915\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    916\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    917\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    945\u001b[0m       \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    946\u001b[0m       \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 947\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    948\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    949\u001b[0m       \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   2955\u001b[0m        filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[1;32m   2956\u001b[0m     return graph_function._call_flat(\n\u001b[0;32m-> 2957\u001b[0;31m         filtered_flat_args, captured_inputs=graph_function.captured_inputs)  # pylint: disable=protected-access\n\u001b[0m\u001b[1;32m   2958\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2959\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1852\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1853\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1854\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m   1855\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1856\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    502\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    503\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 504\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    505\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    506\u001b[0m           outputs = execute.execute_with_cancellation(\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     53\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0;32m---> 55\u001b[0;31m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[1;32m     56\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# combining l1 with dropout\n",
        "layer_list = [ #520 parameters in input\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Dropout(rate=0.4),\n",
        "          layers.Dense(512,kernel_initializer='lecun_normal',kernel_regularizer='l1',bias_regularizer='l1',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dropout(rate=0.4),\n",
        "          layers.Dense(256,kernel_initializer='lecun_normal',kernel_regularizer='l1',bias_regularizer='l1',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(128,kernel_initializer='lecun_normal',kernel_regularizer='l1',bias_regularizer='l1',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(64,kernel_initializer='lecun_normal',kernel_regularizer='l1',bias_regularizer='l1',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(32,kernel_initializer='lecun_normal',kernel_regularizer='l1',bias_regularizer='l1',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(16,activation='softmax',kernel_initializer='lecun_normal',kernel_regularizer='l1',bias_regularizer='l1',),\n",
        "          ]\n",
        "model = Sequential(layer_list)\n",
        "model.compile(optimizer=keras.optimizers.Adam(\n",
        "    learning_rate=keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.01,decay_steps=2000,decay_rate=0.3,)),\n",
        "    loss=\"categorical_crossentropy\", metrics=['accuracy'],)\n",
        "history = model.fit(xtrain, ytrain_1hot, batch_size=32, epochs=150, validation_data=(xval, yval_1hot))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "MwZJXYOxCOSz",
        "outputId": "46ea2b13-c9d6-4244-cf78-7246797a00ab"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/150\n",
            "1256/1256 [==============================] - 13s 9ms/step - loss: 11.7728 - accuracy: 0.4032 - val_loss: 7.3489 - val_accuracy: 0.4364\n",
            "Epoch 2/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 5.4752 - accuracy: 0.4459 - val_loss: 4.5335 - val_accuracy: 0.4895\n",
            "Epoch 3/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 3.7206 - accuracy: 0.4703 - val_loss: 3.0647 - val_accuracy: 0.5213\n",
            "Epoch 4/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 2.7128 - accuracy: 0.4852 - val_loss: 2.2973 - val_accuracy: 0.5231\n",
            "Epoch 5/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.2062 - accuracy: 0.4950 - val_loss: 1.9500 - val_accuracy: 0.5417\n",
            "Epoch 6/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.9372 - accuracy: 0.5014 - val_loss: 1.7380 - val_accuracy: 0.5565\n",
            "Epoch 7/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.7935 - accuracy: 0.5115 - val_loss: 1.6381 - val_accuracy: 0.5643\n",
            "Epoch 8/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.7080 - accuracy: 0.5175 - val_loss: 1.5801 - val_accuracy: 0.5694\n",
            "Epoch 9/150\n",
            "1256/1256 [==============================] - 12s 9ms/step - loss: 1.6737 - accuracy: 0.5193 - val_loss: 1.5480 - val_accuracy: 0.5672\n",
            "Epoch 10/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.6482 - accuracy: 0.5187 - val_loss: 1.5321 - val_accuracy: 0.5710\n",
            "Epoch 11/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6344 - accuracy: 0.5215 - val_loss: 1.5229 - val_accuracy: 0.5730\n",
            "Epoch 12/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6334 - accuracy: 0.5198 - val_loss: 1.5189 - val_accuracy: 0.5735\n",
            "Epoch 13/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6276 - accuracy: 0.5236 - val_loss: 1.5173 - val_accuracy: 0.5724\n",
            "Epoch 14/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6215 - accuracy: 0.5253 - val_loss: 1.5177 - val_accuracy: 0.5735\n",
            "Epoch 15/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6241 - accuracy: 0.5218 - val_loss: 1.5160 - val_accuracy: 0.5741\n",
            "Epoch 16/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6227 - accuracy: 0.5224 - val_loss: 1.5163 - val_accuracy: 0.5719\n",
            "Epoch 17/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6241 - accuracy: 0.5210 - val_loss: 1.5157 - val_accuracy: 0.5739\n",
            "Epoch 18/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6230 - accuracy: 0.5237 - val_loss: 1.5151 - val_accuracy: 0.5733\n",
            "Epoch 19/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6208 - accuracy: 0.5247 - val_loss: 1.5154 - val_accuracy: 0.5730\n",
            "Epoch 20/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6210 - accuracy: 0.5233 - val_loss: 1.5151 - val_accuracy: 0.5748\n",
            "Epoch 21/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6187 - accuracy: 0.5238 - val_loss: 1.5158 - val_accuracy: 0.5726\n",
            "Epoch 22/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6262 - accuracy: 0.5239 - val_loss: 1.5144 - val_accuracy: 0.5730\n",
            "Epoch 23/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6251 - accuracy: 0.5209 - val_loss: 1.5158 - val_accuracy: 0.5735\n",
            "Epoch 24/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6245 - accuracy: 0.5241 - val_loss: 1.5164 - val_accuracy: 0.5737\n",
            "Epoch 25/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6238 - accuracy: 0.5254 - val_loss: 1.5165 - val_accuracy: 0.5724\n",
            "Epoch 26/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6198 - accuracy: 0.5223 - val_loss: 1.5156 - val_accuracy: 0.5730\n",
            "Epoch 27/150\n",
            "1256/1256 [==============================] - 12s 9ms/step - loss: 1.6234 - accuracy: 0.5234 - val_loss: 1.5153 - val_accuracy: 0.5746\n",
            "Epoch 28/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 1.6248 - accuracy: 0.5199 - val_loss: 1.5155 - val_accuracy: 0.5750\n",
            "Epoch 29/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6231 - accuracy: 0.5229 - val_loss: 1.5163 - val_accuracy: 0.5739\n",
            "Epoch 30/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6271 - accuracy: 0.5204 - val_loss: 1.5165 - val_accuracy: 0.5717\n",
            "Epoch 31/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6222 - accuracy: 0.5241 - val_loss: 1.5153 - val_accuracy: 0.5739\n",
            "Epoch 32/150\n",
            "1256/1256 [==============================] - 13s 11ms/step - loss: 1.6196 - accuracy: 0.5238 - val_loss: 1.5157 - val_accuracy: 0.5733\n",
            "Epoch 33/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6241 - accuracy: 0.5237 - val_loss: 1.5159 - val_accuracy: 0.5748\n",
            "Epoch 34/150\n",
            "1256/1256 [==============================] - 14s 11ms/step - loss: 1.6249 - accuracy: 0.5201 - val_loss: 1.5147 - val_accuracy: 0.5741\n",
            "Epoch 35/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6226 - accuracy: 0.5228 - val_loss: 1.5158 - val_accuracy: 0.5724\n",
            "Epoch 36/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6230 - accuracy: 0.5242 - val_loss: 1.5158 - val_accuracy: 0.5726\n",
            "Epoch 37/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6188 - accuracy: 0.5241 - val_loss: 1.5149 - val_accuracy: 0.5739\n",
            "Epoch 38/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6262 - accuracy: 0.5228 - val_loss: 1.5158 - val_accuracy: 0.5744\n",
            "Epoch 39/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6215 - accuracy: 0.5242 - val_loss: 1.5154 - val_accuracy: 0.5737\n",
            "Epoch 40/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6246 - accuracy: 0.5214 - val_loss: 1.5167 - val_accuracy: 0.5726\n",
            "Epoch 41/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.6242 - accuracy: 0.5231 - val_loss: 1.5147 - val_accuracy: 0.5739\n",
            "Epoch 42/150\n",
            "1250/1256 [============================>.] - ETA: 0s - loss: 1.6191 - accuracy: 0.5229"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-80-41f5f13cc4d6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     25\u001b[0m     learning_rate=keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.01,decay_steps=2000,decay_rate=0.3,)),\n\u001b[1;32m     26\u001b[0m     loss=\"categorical_crossentropy\", metrics=['accuracy'],)\n\u001b[0;32m---> 27\u001b[0;31m \u001b[0mhistory\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mytrain_1hot\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m150\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myval_1hot\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/utils/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     65\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint: disable=broad-except\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m       \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1382\u001b[0m                 _r=1):\n\u001b[1;32m   1383\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1384\u001b[0;31m               \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1385\u001b[0m               \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1386\u001b[0m                 \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/util/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    148\u001b[0m     \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    149\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 150\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    151\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    152\u001b[0m       \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    913\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    914\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0mOptionalXlaContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jit_compile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 915\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    916\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    917\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    945\u001b[0m       \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    946\u001b[0m       \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 947\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    948\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    949\u001b[0m       \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   2955\u001b[0m        filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[1;32m   2956\u001b[0m     return graph_function._call_flat(\n\u001b[0;32m-> 2957\u001b[0;31m         filtered_flat_args, captured_inputs=graph_function.captured_inputs)  # pylint: disable=protected-access\n\u001b[0m\u001b[1;32m   2958\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2959\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1852\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1853\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1854\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m   1855\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1856\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    502\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    503\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 504\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    505\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    506\u001b[0m           outputs = execute.execute_with_cancellation(\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     53\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0;32m---> 55\u001b[0;31m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[1;32m     56\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# combining l1 with dropout, adjusting learning rates to what appears to be optimal, removing bias regularization\n",
        "layer_list = [ #520 parameters in input\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Dropout(rate=0.4),\n",
        "          layers.Dense(512,kernel_initializer='lecun_normal',kernel_regularizer='l1',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dropout(rate=0.4),\n",
        "          layers.Dense(256,kernel_initializer='lecun_normal',kernel_regularizer='l1',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(128,kernel_initializer='lecun_normal',kernel_regularizer='l1',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(64,kernel_initializer='lecun_normal',kernel_regularizer='l1',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(32,kernel_initializer='lecun_normal',kernel_regularizer='l1',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(16,activation='softmax',kernel_initializer='lecun_normal',kernel_regularizer='l1',),\n",
        "          ]\n",
        "model = Sequential(layer_list)\n",
        "model.compile(optimizer=keras.optimizers.Adam(\n",
        "    learning_rate=keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.001,decay_steps=2000,decay_rate=0.99,)),\n",
        "    loss=\"categorical_crossentropy\", metrics=['accuracy'],)\n",
        "history = model.fit(xtrain, ytrain_1hot, batch_size=32, epochs=150, validation_data=(xval, yval_1hot))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "4sLYRqZxGzQq",
        "outputId": "6786e676-7440-41da-a7cf-e7e2a1c41fd6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/150\n",
            "1256/1256 [==============================] - 13s 9ms/step - loss: 12.0282 - accuracy: 0.4343 - val_loss: 4.8906 - val_accuracy: 0.4467\n",
            "Epoch 2/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 4.5223 - accuracy: 0.4605 - val_loss: 4.3225 - val_accuracy: 0.5081\n",
            "Epoch 3/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 4.1898 - accuracy: 0.4710 - val_loss: 3.8579 - val_accuracy: 0.5177\n",
            "Epoch 4/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 3.9195 - accuracy: 0.4764 - val_loss: 3.6148 - val_accuracy: 0.4857\n",
            "Epoch 5/150\n",
            "1256/1256 [==============================] - 12s 9ms/step - loss: 3.5627 - accuracy: 0.4817 - val_loss: 3.3943 - val_accuracy: 0.5181\n",
            "Epoch 6/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 3.3063 - accuracy: 0.4790 - val_loss: 3.0029 - val_accuracy: 0.5202\n",
            "Epoch 7/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 3.1808 - accuracy: 0.4878 - val_loss: 3.0086 - val_accuracy: 0.5004\n",
            "Epoch 8/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 3.0826 - accuracy: 0.4879 - val_loss: 2.9667 - val_accuracy: 0.4969\n",
            "Epoch 9/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.9764 - accuracy: 0.4883 - val_loss: 2.8132 - val_accuracy: 0.5170\n",
            "Epoch 10/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.9214 - accuracy: 0.4913 - val_loss: 2.7191 - val_accuracy: 0.5314\n",
            "Epoch 11/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.8316 - accuracy: 0.4912 - val_loss: 2.7994 - val_accuracy: 0.5276\n",
            "Epoch 12/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 2.7952 - accuracy: 0.4922 - val_loss: 2.6802 - val_accuracy: 0.5484\n",
            "Epoch 13/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.7346 - accuracy: 0.4918 - val_loss: 2.6659 - val_accuracy: 0.5439\n",
            "Epoch 14/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.6987 - accuracy: 0.4939 - val_loss: 2.6930 - val_accuracy: 0.5181\n",
            "Epoch 15/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.6776 - accuracy: 0.4973 - val_loss: 2.5540 - val_accuracy: 0.5383\n",
            "Epoch 16/150\n",
            "1256/1256 [==============================] - 14s 11ms/step - loss: 2.6467 - accuracy: 0.4933 - val_loss: 2.5515 - val_accuracy: 0.5287\n",
            "Epoch 17/150\n",
            "1256/1256 [==============================] - 13s 11ms/step - loss: 2.6194 - accuracy: 0.4939 - val_loss: 2.5152 - val_accuracy: 0.5228\n",
            "Epoch 18/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.5841 - accuracy: 0.4953 - val_loss: 2.4618 - val_accuracy: 0.5332\n",
            "Epoch 19/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.5518 - accuracy: 0.4966 - val_loss: 2.5338 - val_accuracy: 0.5354\n",
            "Epoch 20/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.5460 - accuracy: 0.4947 - val_loss: 2.4091 - val_accuracy: 0.5403\n",
            "Epoch 21/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.5310 - accuracy: 0.4964 - val_loss: 2.3993 - val_accuracy: 0.5426\n",
            "Epoch 22/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.5173 - accuracy: 0.4964 - val_loss: 2.4380 - val_accuracy: 0.5392\n",
            "Epoch 23/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.4932 - accuracy: 0.4947 - val_loss: 2.4368 - val_accuracy: 0.5343\n",
            "Epoch 24/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.4975 - accuracy: 0.4945 - val_loss: 2.3664 - val_accuracy: 0.5533\n",
            "Epoch 25/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.4682 - accuracy: 0.4966 - val_loss: 2.4130 - val_accuracy: 0.5491\n",
            "Epoch 26/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.4673 - accuracy: 0.4978 - val_loss: 2.3354 - val_accuracy: 0.5450\n",
            "Epoch 27/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.4657 - accuracy: 0.4988 - val_loss: 2.3599 - val_accuracy: 0.5367\n",
            "Epoch 28/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.4444 - accuracy: 0.4984 - val_loss: 2.3493 - val_accuracy: 0.5493\n",
            "Epoch 29/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.4324 - accuracy: 0.4988 - val_loss: 2.4073 - val_accuracy: 0.5392\n",
            "Epoch 30/150\n",
            " 874/1256 [===================>..........] - ETA: 3s - loss: 2.4472 - accuracy: 0.4986"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-81-d56a41c5699d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     25\u001b[0m     learning_rate=keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.001,decay_steps=2000,decay_rate=0.99,)),\n\u001b[1;32m     26\u001b[0m     loss=\"categorical_crossentropy\", metrics=['accuracy'],)\n\u001b[0;32m---> 27\u001b[0;31m \u001b[0mhistory\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mytrain_1hot\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m150\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myval_1hot\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/utils/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     65\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint: disable=broad-except\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m       \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1382\u001b[0m                 _r=1):\n\u001b[1;32m   1383\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1384\u001b[0;31m               \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1385\u001b[0m               \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1386\u001b[0m                 \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/util/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    148\u001b[0m     \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    149\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 150\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    151\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    152\u001b[0m       \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    913\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    914\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0mOptionalXlaContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jit_compile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 915\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    916\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    917\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    945\u001b[0m       \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    946\u001b[0m       \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 947\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    948\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    949\u001b[0m       \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   2955\u001b[0m        filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[1;32m   2956\u001b[0m     return graph_function._call_flat(\n\u001b[0;32m-> 2957\u001b[0;31m         filtered_flat_args, captured_inputs=graph_function.captured_inputs)  # pylint: disable=protected-access\n\u001b[0m\u001b[1;32m   2958\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2959\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1852\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1853\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1854\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m   1855\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1856\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    502\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    503\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 504\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    505\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    506\u001b[0m           outputs = execute.execute_with_cancellation(\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     53\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0;32m---> 55\u001b[0;31m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[1;32m     56\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# combining l1 with dropout, adjusting learning rates to what appears to be optimal, removing bias regularization\n",
        "# also shouldn't regularize final layer\n",
        "layer_list = [ #520 parameters in input\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Dropout(rate=0.4),\n",
        "          layers.Dense(512,kernel_initializer='lecun_normal',kernel_regularizer='l1',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dropout(rate=0.4),\n",
        "          layers.Dense(256,kernel_initializer='lecun_normal',kernel_regularizer='l1',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(128,kernel_initializer='lecun_normal',kernel_regularizer='l1',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(64,kernel_initializer='lecun_normal',kernel_regularizer='l1',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(32,kernel_initializer='lecun_normal',kernel_regularizer='l1',),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(16,activation='softmax',kernel_initializer='lecun_normal',),\n",
        "          ]\n",
        "model = Sequential(layer_list)\n",
        "model.compile(optimizer=keras.optimizers.Adam(\n",
        "    learning_rate=keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.001,decay_steps=2000,decay_rate=0.99,)),\n",
        "    loss=\"categorical_crossentropy\", metrics=['accuracy'],)\n",
        "history = model.fit(xtrain, ytrain_1hot, batch_size=32, epochs=150, validation_data=(xval, yval_1hot))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TmwIRQ6dIMEH",
        "outputId": "b49d9a57-fd91-41ad-945a-28935d79b257"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/150\n",
            "1256/1256 [==============================] - 40s 9ms/step - loss: 11.9001 - accuracy: 0.4372 - val_loss: 5.1039 - val_accuracy: 0.4778\n",
            "Epoch 2/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 4.8369 - accuracy: 0.4595 - val_loss: 4.6961 - val_accuracy: 0.4854\n",
            "Epoch 3/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 4.3689 - accuracy: 0.4750 - val_loss: 4.1814 - val_accuracy: 0.5284\n",
            "Epoch 4/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 4.0096 - accuracy: 0.4899 - val_loss: 3.7745 - val_accuracy: 0.5352\n",
            "Epoch 5/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 3.6160 - accuracy: 0.5002 - val_loss: 3.5151 - val_accuracy: 0.5421\n",
            "Epoch 6/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 3.3717 - accuracy: 0.4998 - val_loss: 3.1828 - val_accuracy: 0.5087\n",
            "Epoch 7/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 3.2052 - accuracy: 0.5077 - val_loss: 3.0242 - val_accuracy: 0.5448\n",
            "Epoch 8/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 3.0766 - accuracy: 0.5080 - val_loss: 2.9586 - val_accuracy: 0.5437\n",
            "Epoch 9/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 3.0069 - accuracy: 0.5103 - val_loss: 2.8556 - val_accuracy: 0.5513\n",
            "Epoch 10/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 2.8992 - accuracy: 0.5128 - val_loss: 2.7019 - val_accuracy: 0.5681\n",
            "Epoch 11/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 2.8370 - accuracy: 0.5162 - val_loss: 2.7569 - val_accuracy: 0.5468\n",
            "Epoch 12/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 2.7997 - accuracy: 0.5136 - val_loss: 2.6708 - val_accuracy: 0.5603\n",
            "Epoch 13/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 2.7083 - accuracy: 0.5153 - val_loss: 2.5026 - val_accuracy: 0.5750\n",
            "Epoch 14/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 2.6553 - accuracy: 0.5160 - val_loss: 2.5482 - val_accuracy: 0.5612\n",
            "Epoch 15/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.5968 - accuracy: 0.5151 - val_loss: 2.4488 - val_accuracy: 0.5560\n",
            "Epoch 16/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.5826 - accuracy: 0.5184 - val_loss: 2.4914 - val_accuracy: 0.5712\n",
            "Epoch 17/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 2.5607 - accuracy: 0.5162 - val_loss: 2.4376 - val_accuracy: 0.5721\n",
            "Epoch 18/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 2.5330 - accuracy: 0.5202 - val_loss: 2.3579 - val_accuracy: 0.5522\n",
            "Epoch 19/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 2.5022 - accuracy: 0.5208 - val_loss: 2.4498 - val_accuracy: 0.5661\n",
            "Epoch 20/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 2.4838 - accuracy: 0.5202 - val_loss: 2.3676 - val_accuracy: 0.5504\n",
            "Epoch 21/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.4780 - accuracy: 0.5203 - val_loss: 2.3359 - val_accuracy: 0.5627\n",
            "Epoch 22/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 2.4585 - accuracy: 0.5192 - val_loss: 2.3650 - val_accuracy: 0.5464\n",
            "Epoch 23/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 2.4486 - accuracy: 0.5208 - val_loss: 2.3101 - val_accuracy: 0.5645\n",
            "Epoch 24/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.4573 - accuracy: 0.5208 - val_loss: 2.3023 - val_accuracy: 0.5715\n",
            "Epoch 25/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.4418 - accuracy: 0.5222 - val_loss: 2.3074 - val_accuracy: 0.5553\n",
            "Epoch 26/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 2.4264 - accuracy: 0.5199 - val_loss: 2.3609 - val_accuracy: 0.5643\n",
            "Epoch 27/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.4143 - accuracy: 0.5231 - val_loss: 2.2636 - val_accuracy: 0.5730\n",
            "Epoch 28/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 2.4136 - accuracy: 0.5169 - val_loss: 2.2868 - val_accuracy: 0.5470\n",
            "Epoch 29/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 2.4108 - accuracy: 0.5206 - val_loss: 2.2468 - val_accuracy: 0.5706\n",
            "Epoch 30/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.4017 - accuracy: 0.5198 - val_loss: 2.2750 - val_accuracy: 0.5556\n",
            "Epoch 31/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.3942 - accuracy: 0.5187 - val_loss: 2.3415 - val_accuracy: 0.5598\n",
            "Epoch 32/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.3825 - accuracy: 0.5217 - val_loss: 2.2860 - val_accuracy: 0.5690\n",
            "Epoch 33/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.3827 - accuracy: 0.5205 - val_loss: 2.2928 - val_accuracy: 0.5708\n",
            "Epoch 34/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 2.3832 - accuracy: 0.5197 - val_loss: 2.2618 - val_accuracy: 0.5632\n",
            "Epoch 35/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 2.3727 - accuracy: 0.5204 - val_loss: 2.2933 - val_accuracy: 0.5721\n",
            "Epoch 36/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 2.3736 - accuracy: 0.5217 - val_loss: 2.1989 - val_accuracy: 0.5694\n",
            "Epoch 37/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 2.3540 - accuracy: 0.5233 - val_loss: 2.2405 - val_accuracy: 0.5795\n",
            "Epoch 38/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.3559 - accuracy: 0.5183 - val_loss: 2.2841 - val_accuracy: 0.5650\n",
            "Epoch 39/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 2.3417 - accuracy: 0.5203 - val_loss: 2.2279 - val_accuracy: 0.5712\n",
            "Epoch 40/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 2.3462 - accuracy: 0.5213 - val_loss: 2.2645 - val_accuracy: 0.5600\n",
            "Epoch 41/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 2.3478 - accuracy: 0.5256 - val_loss: 2.2449 - val_accuracy: 0.5643\n",
            "Epoch 42/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.3492 - accuracy: 0.5190 - val_loss: 2.2450 - val_accuracy: 0.5645\n",
            "Epoch 43/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 2.3376 - accuracy: 0.5240 - val_loss: 2.1756 - val_accuracy: 0.5766\n",
            "Epoch 44/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 2.3300 - accuracy: 0.5228 - val_loss: 2.2657 - val_accuracy: 0.5654\n",
            "Epoch 45/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 2.3270 - accuracy: 0.5236 - val_loss: 2.1915 - val_accuracy: 0.5724\n",
            "Epoch 46/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 2.3081 - accuracy: 0.5228 - val_loss: 2.2057 - val_accuracy: 0.5733\n",
            "Epoch 47/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.3174 - accuracy: 0.5251 - val_loss: 2.1603 - val_accuracy: 0.5699\n",
            "Epoch 48/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.3040 - accuracy: 0.5247 - val_loss: 2.1742 - val_accuracy: 0.5690\n",
            "Epoch 49/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.3093 - accuracy: 0.5226 - val_loss: 2.1522 - val_accuracy: 0.5800\n",
            "Epoch 50/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.3087 - accuracy: 0.5231 - val_loss: 2.1779 - val_accuracy: 0.5724\n",
            "Epoch 51/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.2976 - accuracy: 0.5234 - val_loss: 2.1446 - val_accuracy: 0.5739\n",
            "Epoch 52/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.2874 - accuracy: 0.5221 - val_loss: 2.1447 - val_accuracy: 0.5833\n",
            "Epoch 53/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.2825 - accuracy: 0.5259 - val_loss: 2.1721 - val_accuracy: 0.5690\n",
            "Epoch 54/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.2706 - accuracy: 0.5243 - val_loss: 2.1384 - val_accuracy: 0.5694\n",
            "Epoch 55/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.2689 - accuracy: 0.5255 - val_loss: 2.1250 - val_accuracy: 0.5654\n",
            "Epoch 56/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.2705 - accuracy: 0.5215 - val_loss: 2.1955 - val_accuracy: 0.5654\n",
            "Epoch 57/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 2.2660 - accuracy: 0.5230 - val_loss: 2.1422 - val_accuracy: 0.5755\n",
            "Epoch 58/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.2659 - accuracy: 0.5242 - val_loss: 2.1352 - val_accuracy: 0.5607\n",
            "Epoch 59/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.2565 - accuracy: 0.5230 - val_loss: 2.1337 - val_accuracy: 0.5607\n",
            "Epoch 60/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.2426 - accuracy: 0.5263 - val_loss: 2.1599 - val_accuracy: 0.5663\n",
            "Epoch 61/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.2610 - accuracy: 0.5220 - val_loss: 2.1660 - val_accuracy: 0.5721\n",
            "Epoch 62/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.2609 - accuracy: 0.5278 - val_loss: 2.1506 - val_accuracy: 0.5766\n",
            "Epoch 63/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.2350 - accuracy: 0.5271 - val_loss: 2.1124 - val_accuracy: 0.5737\n",
            "Epoch 64/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.2436 - accuracy: 0.5235 - val_loss: 2.1251 - val_accuracy: 0.5688\n",
            "Epoch 65/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.2271 - accuracy: 0.5237 - val_loss: 2.1031 - val_accuracy: 0.5677\n",
            "Epoch 66/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.2223 - accuracy: 0.5271 - val_loss: 2.1029 - val_accuracy: 0.5636\n",
            "Epoch 67/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.2264 - accuracy: 0.5246 - val_loss: 2.0900 - val_accuracy: 0.5768\n",
            "Epoch 68/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.2112 - accuracy: 0.5253 - val_loss: 2.0764 - val_accuracy: 0.5822\n",
            "Epoch 69/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.2051 - accuracy: 0.5297 - val_loss: 2.0935 - val_accuracy: 0.5853\n",
            "Epoch 70/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.2066 - accuracy: 0.5275 - val_loss: 2.0640 - val_accuracy: 0.5762\n",
            "Epoch 71/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 2.2157 - accuracy: 0.5274 - val_loss: 2.1273 - val_accuracy: 0.5656\n",
            "Epoch 72/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 2.1953 - accuracy: 0.5271 - val_loss: 2.0316 - val_accuracy: 0.5685\n",
            "Epoch 73/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 2.1914 - accuracy: 0.5246 - val_loss: 2.0828 - val_accuracy: 0.5627\n",
            "Epoch 74/150\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 2.1962 - accuracy: 0.5273 - val_loss: 2.0584 - val_accuracy: 0.5708\n",
            "Epoch 75/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.1896 - accuracy: 0.5283 - val_loss: 2.0842 - val_accuracy: 0.5605\n",
            "Epoch 76/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 2.1752 - accuracy: 0.5278 - val_loss: 2.1007 - val_accuracy: 0.5453\n",
            "Epoch 77/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.1823 - accuracy: 0.5296 - val_loss: 2.0238 - val_accuracy: 0.5656\n",
            "Epoch 78/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.1809 - accuracy: 0.5273 - val_loss: 2.0589 - val_accuracy: 0.5697\n",
            "Epoch 79/150\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 2.1744 - accuracy: 0.5293 - val_loss: 2.0344 - val_accuracy: 0.5791\n",
            "Epoch 80/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.1630 - accuracy: 0.5240 - val_loss: 2.0441 - val_accuracy: 0.5654\n",
            "Epoch 81/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.1550 - accuracy: 0.5279 - val_loss: 2.0062 - val_accuracy: 0.5733\n",
            "Epoch 82/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.1588 - accuracy: 0.5267 - val_loss: 2.0356 - val_accuracy: 0.5786\n",
            "Epoch 83/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.1445 - accuracy: 0.5282 - val_loss: 2.0412 - val_accuracy: 0.5804\n",
            "Epoch 84/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.1591 - accuracy: 0.5293 - val_loss: 2.0386 - val_accuracy: 0.5836\n",
            "Epoch 85/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.1463 - accuracy: 0.5295 - val_loss: 2.0319 - val_accuracy: 0.5728\n",
            "Epoch 86/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.1514 - accuracy: 0.5317 - val_loss: 2.0440 - val_accuracy: 0.5701\n",
            "Epoch 87/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.1533 - accuracy: 0.5277 - val_loss: 2.0152 - val_accuracy: 0.5748\n",
            "Epoch 88/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.1491 - accuracy: 0.5307 - val_loss: 2.0015 - val_accuracy: 0.5842\n",
            "Epoch 89/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.1393 - accuracy: 0.5282 - val_loss: 2.0425 - val_accuracy: 0.5701\n",
            "Epoch 90/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.1292 - accuracy: 0.5310 - val_loss: 2.0813 - val_accuracy: 0.5661\n",
            "Epoch 91/150\n",
            "1256/1256 [==============================] - 12s 9ms/step - loss: 2.1310 - accuracy: 0.5264 - val_loss: 2.0041 - val_accuracy: 0.5724\n",
            "Epoch 92/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.1286 - accuracy: 0.5279 - val_loss: 2.0282 - val_accuracy: 0.5797\n",
            "Epoch 93/150\n",
            "1256/1256 [==============================] - 12s 9ms/step - loss: 2.1207 - accuracy: 0.5291 - val_loss: 2.0004 - val_accuracy: 0.5629\n",
            "Epoch 94/150\n",
            "1256/1256 [==============================] - 12s 9ms/step - loss: 2.1195 - accuracy: 0.5307 - val_loss: 1.9786 - val_accuracy: 0.5806\n",
            "Epoch 95/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.1122 - accuracy: 0.5286 - val_loss: 2.0004 - val_accuracy: 0.5701\n",
            "Epoch 96/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.1134 - accuracy: 0.5272 - val_loss: 1.9987 - val_accuracy: 0.5764\n",
            "Epoch 97/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.1123 - accuracy: 0.5292 - val_loss: 2.0041 - val_accuracy: 0.5753\n",
            "Epoch 98/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.1017 - accuracy: 0.5263 - val_loss: 1.9469 - val_accuracy: 0.5822\n",
            "Epoch 99/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.1035 - accuracy: 0.5306 - val_loss: 1.9616 - val_accuracy: 0.5771\n",
            "Epoch 100/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.0966 - accuracy: 0.5297 - val_loss: 1.9719 - val_accuracy: 0.5764\n",
            "Epoch 101/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.1008 - accuracy: 0.5302 - val_loss: 1.9532 - val_accuracy: 0.5762\n",
            "Epoch 102/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.0947 - accuracy: 0.5356 - val_loss: 2.0259 - val_accuracy: 0.5710\n",
            "Epoch 103/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.0929 - accuracy: 0.5296 - val_loss: 1.9451 - val_accuracy: 0.5741\n",
            "Epoch 104/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.0894 - accuracy: 0.5303 - val_loss: 1.9565 - val_accuracy: 0.5690\n",
            "Epoch 105/150\n",
            "1256/1256 [==============================] - 12s 9ms/step - loss: 2.0743 - accuracy: 0.5291 - val_loss: 1.9630 - val_accuracy: 0.5764\n",
            "Epoch 106/150\n",
            "1256/1256 [==============================] - 12s 9ms/step - loss: 2.0905 - accuracy: 0.5296 - val_loss: 1.9865 - val_accuracy: 0.5710\n",
            "Epoch 107/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.0885 - accuracy: 0.5310 - val_loss: 1.9491 - val_accuracy: 0.5797\n",
            "Epoch 108/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.0813 - accuracy: 0.5285 - val_loss: 1.9396 - val_accuracy: 0.5800\n",
            "Epoch 109/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.0741 - accuracy: 0.5308 - val_loss: 1.9453 - val_accuracy: 0.5741\n",
            "Epoch 110/150\n",
            "1256/1256 [==============================] - 12s 9ms/step - loss: 2.0896 - accuracy: 0.5255 - val_loss: 1.9252 - val_accuracy: 0.5853\n",
            "Epoch 111/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.0743 - accuracy: 0.5297 - val_loss: 1.9334 - val_accuracy: 0.5797\n",
            "Epoch 112/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.0680 - accuracy: 0.5315 - val_loss: 1.9535 - val_accuracy: 0.5665\n",
            "Epoch 113/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.0676 - accuracy: 0.5319 - val_loss: 1.9475 - val_accuracy: 0.5813\n",
            "Epoch 114/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.0603 - accuracy: 0.5303 - val_loss: 1.9472 - val_accuracy: 0.5748\n",
            "Epoch 115/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.0601 - accuracy: 0.5316 - val_loss: 1.9443 - val_accuracy: 0.5746\n",
            "Epoch 116/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.0520 - accuracy: 0.5297 - val_loss: 1.9216 - val_accuracy: 0.5771\n",
            "Epoch 117/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.0397 - accuracy: 0.5315 - val_loss: 1.9308 - val_accuracy: 0.5755\n",
            "Epoch 118/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.0504 - accuracy: 0.5306 - val_loss: 1.9037 - val_accuracy: 0.5840\n",
            "Epoch 119/150\n",
            "1256/1256 [==============================] - 12s 9ms/step - loss: 2.0380 - accuracy: 0.5313 - val_loss: 1.9084 - val_accuracy: 0.5804\n",
            "Epoch 120/150\n",
            "1256/1256 [==============================] - 12s 9ms/step - loss: 2.0498 - accuracy: 0.5288 - val_loss: 1.9163 - val_accuracy: 0.5896\n",
            "Epoch 121/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.0410 - accuracy: 0.5300 - val_loss: 1.9130 - val_accuracy: 0.5730\n",
            "Epoch 122/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.0442 - accuracy: 0.5303 - val_loss: 1.9204 - val_accuracy: 0.5907\n",
            "Epoch 123/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.0365 - accuracy: 0.5319 - val_loss: 1.9044 - val_accuracy: 0.5800\n",
            "Epoch 124/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.0278 - accuracy: 0.5361 - val_loss: 1.8857 - val_accuracy: 0.5755\n",
            "Epoch 125/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.0304 - accuracy: 0.5330 - val_loss: 1.8969 - val_accuracy: 0.5795\n",
            "Epoch 126/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.0295 - accuracy: 0.5282 - val_loss: 1.8795 - val_accuracy: 0.5784\n",
            "Epoch 127/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.0282 - accuracy: 0.5340 - val_loss: 1.9254 - val_accuracy: 0.5775\n",
            "Epoch 128/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.0249 - accuracy: 0.5332 - val_loss: 1.8943 - val_accuracy: 0.5806\n",
            "Epoch 129/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.0192 - accuracy: 0.5320 - val_loss: 1.9227 - val_accuracy: 0.5668\n",
            "Epoch 130/150\n",
            "1256/1256 [==============================] - 12s 9ms/step - loss: 2.0123 - accuracy: 0.5322 - val_loss: 1.8968 - val_accuracy: 0.5829\n",
            "Epoch 131/150\n",
            "1256/1256 [==============================] - 12s 9ms/step - loss: 2.0054 - accuracy: 0.5335 - val_loss: 1.8813 - val_accuracy: 0.5815\n",
            "Epoch 132/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.0203 - accuracy: 0.5358 - val_loss: 1.8947 - val_accuracy: 0.5780\n",
            "Epoch 133/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.0006 - accuracy: 0.5335 - val_loss: 1.8791 - val_accuracy: 0.5889\n",
            "Epoch 134/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.9990 - accuracy: 0.5313 - val_loss: 1.8751 - val_accuracy: 0.5811\n",
            "Epoch 135/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.9976 - accuracy: 0.5351 - val_loss: 1.8595 - val_accuracy: 0.5755\n",
            "Epoch 136/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 2.0003 - accuracy: 0.5346 - val_loss: 1.8917 - val_accuracy: 0.5697\n",
            "Epoch 137/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.9907 - accuracy: 0.5338 - val_loss: 1.8573 - val_accuracy: 0.5771\n",
            "Epoch 138/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.9995 - accuracy: 0.5329 - val_loss: 1.8689 - val_accuracy: 0.5836\n",
            "Epoch 139/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.9893 - accuracy: 0.5339 - val_loss: 1.8772 - val_accuracy: 0.5782\n",
            "Epoch 140/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.9824 - accuracy: 0.5343 - val_loss: 1.9032 - val_accuracy: 0.5802\n",
            "Epoch 141/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.9906 - accuracy: 0.5298 - val_loss: 1.9072 - val_accuracy: 0.5560\n",
            "Epoch 142/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.9796 - accuracy: 0.5320 - val_loss: 1.8277 - val_accuracy: 0.5889\n",
            "Epoch 143/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.9817 - accuracy: 0.5324 - val_loss: 1.8416 - val_accuracy: 0.5818\n",
            "Epoch 144/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.9830 - accuracy: 0.5345 - val_loss: 1.8511 - val_accuracy: 0.5889\n",
            "Epoch 145/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.9735 - accuracy: 0.5336 - val_loss: 1.8633 - val_accuracy: 0.5842\n",
            "Epoch 146/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.9741 - accuracy: 0.5313 - val_loss: 1.8869 - val_accuracy: 0.5659\n",
            "Epoch 147/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.9681 - accuracy: 0.5343 - val_loss: 1.8297 - val_accuracy: 0.5898\n",
            "Epoch 148/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.9614 - accuracy: 0.5363 - val_loss: 1.8168 - val_accuracy: 0.5880\n",
            "Epoch 149/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.9645 - accuracy: 0.5330 - val_loss: 1.8373 - val_accuracy: 0.5771\n",
            "Epoch 150/150\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.9638 - accuracy: 0.5364 - val_loss: 1.8351 - val_accuracy: 0.5728\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# let's see how these compare to random guessing accuracy\n",
        "yval_sum = np.sum(yval_1hot, axis=0)\n",
        "print(yval_sum)\n",
        "print(yval_1hot.shape)\n",
        "print(yval_sum.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p-h5-odBeRk-",
        "outputId": "3d50cef4-b8b7-4022-e570-1809a4b910b4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1282.  841.  985.  332.  250.  154.  215.  126.  115.   37.   38.   15.\n",
            "   15.   40.   17.    2.]\n",
            "(4464, 16)\n",
            "(16,)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "layer_list = [ #520 parameters in input\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Dropout(0.3),\n",
        "          layers.Dense(100,kernel_initializer='lecun_normal'), #let's try dimensionality reduction\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(256,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          #layers.Dense(128,kernel_initializer='lecun_normal'),\n",
        "          #layers.BatchNormalization(),\n",
        "          #layers.Activation('selu'),\n",
        "          layers.Dense(64,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(32,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(16,activation='softmax',kernel_initializer='lecun_normal'),\n",
        "          ]\n",
        "model = Sequential(layer_list)\n",
        "model.compile(optimizer=keras.optimizers.Adam(\n",
        "    learning_rate=keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.001,decay_steps=2000,decay_rate=0.99)),\n",
        "    loss=\"categorical_crossentropy\", metrics=['accuracy'],)\n",
        "history = model.fit(xtrain, ytrain_1hot, batch_size=32, epochs=50, validation_data=(xval, yval_1hot))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j0e45l3PPj0d",
        "outputId": "79337100-470e-47a7-a5e1-69ea90858aae"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/50\n",
            "1256/1256 [==============================] - 8s 5ms/step - loss: 1.4333 - accuracy: 0.5647 - val_loss: 1.1566 - val_accuracy: 0.6290\n",
            "Epoch 2/50\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 1.2349 - accuracy: 0.6016 - val_loss: 1.1059 - val_accuracy: 0.6465\n",
            "Epoch 3/50\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 1.2012 - accuracy: 0.6104 - val_loss: 1.0938 - val_accuracy: 0.6456\n",
            "Epoch 4/50\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 1.1739 - accuracy: 0.6188 - val_loss: 1.0896 - val_accuracy: 0.6514\n",
            "Epoch 5/50\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 1.1577 - accuracy: 0.6235 - val_loss: 1.0543 - val_accuracy: 0.6647\n",
            "Epoch 6/50\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 1.1408 - accuracy: 0.6283 - val_loss: 1.0261 - val_accuracy: 0.6647\n",
            "Epoch 7/50\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 1.1259 - accuracy: 0.6313 - val_loss: 1.0258 - val_accuracy: 0.6676\n",
            "Epoch 8/50\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 1.1114 - accuracy: 0.6361 - val_loss: 1.0366 - val_accuracy: 0.6629\n",
            "Epoch 9/50\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 1.0961 - accuracy: 0.6414 - val_loss: 1.0154 - val_accuracy: 0.6700\n",
            "Epoch 10/50\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 1.0833 - accuracy: 0.6437 - val_loss: 1.0036 - val_accuracy: 0.6682\n",
            "Epoch 11/50\n",
            "1256/1256 [==============================] - 6s 4ms/step - loss: 1.0755 - accuracy: 0.6481 - val_loss: 1.0099 - val_accuracy: 0.6709\n",
            "Epoch 12/50\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 1.0617 - accuracy: 0.6511 - val_loss: 1.0019 - val_accuracy: 0.6676\n",
            "Epoch 13/50\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 1.0542 - accuracy: 0.6521 - val_loss: 0.9831 - val_accuracy: 0.6752\n",
            "Epoch 14/50\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 1.0439 - accuracy: 0.6567 - val_loss: 0.9840 - val_accuracy: 0.6754\n",
            "Epoch 15/50\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 1.0359 - accuracy: 0.6586 - val_loss: 0.9847 - val_accuracy: 0.6835\n",
            "Epoch 16/50\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 1.0221 - accuracy: 0.6609 - val_loss: 0.9796 - val_accuracy: 0.6747\n",
            "Epoch 17/50\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 1.0117 - accuracy: 0.6626 - val_loss: 0.9690 - val_accuracy: 0.6808\n",
            "Epoch 18/50\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 1.0028 - accuracy: 0.6702 - val_loss: 0.9683 - val_accuracy: 0.6763\n",
            "Epoch 19/50\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.9934 - accuracy: 0.6707 - val_loss: 0.9664 - val_accuracy: 0.6830\n",
            "Epoch 20/50\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.9899 - accuracy: 0.6719 - val_loss: 0.9578 - val_accuracy: 0.6857\n",
            "Epoch 21/50\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.9821 - accuracy: 0.6747 - val_loss: 0.9664 - val_accuracy: 0.6815\n",
            "Epoch 22/50\n",
            "1256/1256 [==============================] - 6s 4ms/step - loss: 0.9718 - accuracy: 0.6764 - val_loss: 0.9738 - val_accuracy: 0.6846\n",
            "Epoch 23/50\n",
            "1256/1256 [==============================] - 6s 4ms/step - loss: 0.9654 - accuracy: 0.6778 - val_loss: 0.9625 - val_accuracy: 0.6835\n",
            "Epoch 24/50\n",
            "1256/1256 [==============================] - 6s 4ms/step - loss: 0.9589 - accuracy: 0.6799 - val_loss: 0.9610 - val_accuracy: 0.6879\n",
            "Epoch 25/50\n",
            "1256/1256 [==============================] - 6s 4ms/step - loss: 0.9473 - accuracy: 0.6839 - val_loss: 0.9562 - val_accuracy: 0.6868\n",
            "Epoch 26/50\n",
            "1256/1256 [==============================] - 6s 4ms/step - loss: 0.9464 - accuracy: 0.6833 - val_loss: 0.9511 - val_accuracy: 0.6859\n",
            "Epoch 27/50\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.9411 - accuracy: 0.6854 - val_loss: 0.9617 - val_accuracy: 0.6828\n",
            "Epoch 28/50\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.9335 - accuracy: 0.6893 - val_loss: 0.9489 - val_accuracy: 0.6859\n",
            "Epoch 29/50\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.9246 - accuracy: 0.6888 - val_loss: 0.9520 - val_accuracy: 0.6879\n",
            "Epoch 30/50\n",
            "1256/1256 [==============================] - 6s 4ms/step - loss: 0.9199 - accuracy: 0.6965 - val_loss: 0.9600 - val_accuracy: 0.6799\n",
            "Epoch 31/50\n",
            "1256/1256 [==============================] - 6s 4ms/step - loss: 0.9152 - accuracy: 0.6934 - val_loss: 0.9438 - val_accuracy: 0.6871\n",
            "Epoch 32/50\n",
            "1256/1256 [==============================] - 6s 4ms/step - loss: 0.9051 - accuracy: 0.6989 - val_loss: 0.9488 - val_accuracy: 0.6877\n",
            "Epoch 33/50\n",
            "1256/1256 [==============================] - 6s 4ms/step - loss: 0.9022 - accuracy: 0.6982 - val_loss: 0.9509 - val_accuracy: 0.6879\n",
            "Epoch 34/50\n",
            "1256/1256 [==============================] - 6s 4ms/step - loss: 0.8990 - accuracy: 0.6971 - val_loss: 0.9410 - val_accuracy: 0.6929\n",
            "Epoch 35/50\n",
            "1256/1256 [==============================] - 6s 4ms/step - loss: 0.8914 - accuracy: 0.6997 - val_loss: 0.9600 - val_accuracy: 0.6868\n",
            "Epoch 36/50\n",
            "1256/1256 [==============================] - 6s 4ms/step - loss: 0.8843 - accuracy: 0.7038 - val_loss: 0.9529 - val_accuracy: 0.6857\n",
            "Epoch 37/50\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.8820 - accuracy: 0.7043 - val_loss: 0.9515 - val_accuracy: 0.6931\n",
            "Epoch 38/50\n",
            "1256/1256 [==============================] - 6s 4ms/step - loss: 0.8822 - accuracy: 0.7043 - val_loss: 0.9527 - val_accuracy: 0.6927\n",
            "Epoch 39/50\n",
            "1256/1256 [==============================] - 6s 4ms/step - loss: 0.8748 - accuracy: 0.7071 - val_loss: 0.9585 - val_accuracy: 0.6922\n",
            "Epoch 40/50\n",
            "1256/1256 [==============================] - 6s 4ms/step - loss: 0.8735 - accuracy: 0.7058 - val_loss: 0.9515 - val_accuracy: 0.6882\n",
            "Epoch 41/50\n",
            "1256/1256 [==============================] - 6s 4ms/step - loss: 0.8689 - accuracy: 0.7076 - val_loss: 0.9457 - val_accuracy: 0.6886\n",
            "Epoch 42/50\n",
            "1256/1256 [==============================] - 6s 4ms/step - loss: 0.8629 - accuracy: 0.7106 - val_loss: 0.9581 - val_accuracy: 0.6846\n",
            "Epoch 43/50\n",
            "1256/1256 [==============================] - 6s 5ms/step - loss: 0.8567 - accuracy: 0.7133 - val_loss: 0.9560 - val_accuracy: 0.6891\n",
            "Epoch 44/50\n",
            "1256/1256 [==============================] - 6s 4ms/step - loss: 0.8539 - accuracy: 0.7123 - val_loss: 0.9500 - val_accuracy: 0.6906\n",
            "Epoch 45/50\n",
            "1256/1256 [==============================] - 6s 4ms/step - loss: 0.8530 - accuracy: 0.7110 - val_loss: 0.9350 - val_accuracy: 0.6960\n",
            "Epoch 46/50\n",
            "1256/1256 [==============================] - 6s 4ms/step - loss: 0.8489 - accuracy: 0.7136 - val_loss: 0.9437 - val_accuracy: 0.6940\n",
            "Epoch 47/50\n",
            "1256/1256 [==============================] - 6s 4ms/step - loss: 0.8394 - accuracy: 0.7145 - val_loss: 0.9393 - val_accuracy: 0.6938\n",
            "Epoch 48/50\n",
            "1256/1256 [==============================] - 6s 4ms/step - loss: 0.8414 - accuracy: 0.7168 - val_loss: 0.9339 - val_accuracy: 0.7007\n",
            "Epoch 49/50\n",
            "1256/1256 [==============================] - 6s 4ms/step - loss: 0.8340 - accuracy: 0.7204 - val_loss: 0.9391 - val_accuracy: 0.6998\n",
            "Epoch 50/50\n",
            "1256/1256 [==============================] - 5s 4ms/step - loss: 0.8316 - accuracy: 0.7195 - val_loss: 0.9405 - val_accuracy: 0.6987\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "layer_list = [ #520 parameters in input\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Dropout(0.3),\n",
        "          layers.Dense(200,kernel_initializer='lecun_normal'), #let's try dimensionality reduction\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(256,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(128,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(64,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(32,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(16,activation='softmax',kernel_initializer='lecun_normal'),\n",
        "          ]\n",
        "model = Sequential(layer_list)\n",
        "model.compile(optimizer=keras.optimizers.Adam(\n",
        "    learning_rate=keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.001,decay_steps=2000,decay_rate=0.99)),\n",
        "    loss=\"categorical_crossentropy\", metrics=['accuracy'],)\n",
        "history = model.fit(xtrain, ytrain_1hot, batch_size=32, epochs=50, validation_data=(xval, yval_1hot))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fNyfO0u5RhWJ",
        "outputId": "8e8b1fea-56eb-4638-dcaf-f92d0dc3a881"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/50\n",
            "1256/1256 [==============================] - 9s 6ms/step - loss: 1.4246 - accuracy: 0.5682 - val_loss: 1.1508 - val_accuracy: 0.6306\n",
            "Epoch 2/50\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 1.2405 - accuracy: 0.6010 - val_loss: 1.1104 - val_accuracy: 0.6411\n",
            "Epoch 3/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 1.2076 - accuracy: 0.6088 - val_loss: 1.0938 - val_accuracy: 0.6434\n",
            "Epoch 4/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 1.1844 - accuracy: 0.6150 - val_loss: 1.0712 - val_accuracy: 0.6570\n",
            "Epoch 5/50\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 1.1600 - accuracy: 0.6241 - val_loss: 1.0566 - val_accuracy: 0.6640\n",
            "Epoch 6/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 1.1398 - accuracy: 0.6257 - val_loss: 1.0553 - val_accuracy: 0.6622\n",
            "Epoch 7/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 1.1215 - accuracy: 0.6341 - val_loss: 1.0427 - val_accuracy: 0.6640\n",
            "Epoch 8/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 1.1103 - accuracy: 0.6354 - val_loss: 1.0300 - val_accuracy: 0.6651\n",
            "Epoch 9/50\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 1.0918 - accuracy: 0.6450 - val_loss: 1.0219 - val_accuracy: 0.6709\n",
            "Epoch 10/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 1.0879 - accuracy: 0.6443 - val_loss: 1.0048 - val_accuracy: 0.6761\n",
            "Epoch 11/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 1.0752 - accuracy: 0.6505 - val_loss: 1.0076 - val_accuracy: 0.6759\n",
            "Epoch 12/50\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 1.0605 - accuracy: 0.6494 - val_loss: 0.9926 - val_accuracy: 0.6792\n",
            "Epoch 13/50\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 1.0432 - accuracy: 0.6565 - val_loss: 0.9887 - val_accuracy: 0.6767\n",
            "Epoch 14/50\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 1.0375 - accuracy: 0.6593 - val_loss: 0.9852 - val_accuracy: 0.6844\n",
            "Epoch 15/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 1.0226 - accuracy: 0.6646 - val_loss: 0.9796 - val_accuracy: 0.6855\n",
            "Epoch 16/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 1.0032 - accuracy: 0.6681 - val_loss: 0.9751 - val_accuracy: 0.6756\n",
            "Epoch 17/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.9903 - accuracy: 0.6732 - val_loss: 0.9687 - val_accuracy: 0.6848\n",
            "Epoch 18/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.9881 - accuracy: 0.6741 - val_loss: 0.9711 - val_accuracy: 0.6871\n",
            "Epoch 19/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.9711 - accuracy: 0.6782 - val_loss: 0.9542 - val_accuracy: 0.6871\n",
            "Epoch 20/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.9644 - accuracy: 0.6784 - val_loss: 0.9582 - val_accuracy: 0.6895\n",
            "Epoch 21/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.9527 - accuracy: 0.6840 - val_loss: 0.9556 - val_accuracy: 0.6897\n",
            "Epoch 22/50\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.9425 - accuracy: 0.6885 - val_loss: 0.9489 - val_accuracy: 0.6888\n",
            "Epoch 23/50\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.9321 - accuracy: 0.6930 - val_loss: 0.9605 - val_accuracy: 0.6873\n",
            "Epoch 24/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.9266 - accuracy: 0.6944 - val_loss: 0.9395 - val_accuracy: 0.6978\n",
            "Epoch 25/50\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.9166 - accuracy: 0.6961 - val_loss: 0.9412 - val_accuracy: 0.6978\n",
            "Epoch 26/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.9050 - accuracy: 0.6964 - val_loss: 0.9401 - val_accuracy: 0.6935\n",
            "Epoch 27/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.8904 - accuracy: 0.7022 - val_loss: 0.9280 - val_accuracy: 0.6998\n",
            "Epoch 28/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.8894 - accuracy: 0.7040 - val_loss: 0.9442 - val_accuracy: 0.6933\n",
            "Epoch 29/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.8799 - accuracy: 0.7062 - val_loss: 0.9450 - val_accuracy: 0.6974\n",
            "Epoch 30/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.8719 - accuracy: 0.7093 - val_loss: 0.9395 - val_accuracy: 0.7016\n",
            "Epoch 31/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.8614 - accuracy: 0.7122 - val_loss: 0.9564 - val_accuracy: 0.6944\n",
            "Epoch 32/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.8558 - accuracy: 0.7131 - val_loss: 0.9466 - val_accuracy: 0.6920\n",
            "Epoch 33/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.8537 - accuracy: 0.7167 - val_loss: 0.9561 - val_accuracy: 0.7007\n",
            "Epoch 34/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.8382 - accuracy: 0.7186 - val_loss: 0.9483 - val_accuracy: 0.6994\n",
            "Epoch 35/50\n",
            "1256/1256 [==============================] - 8s 6ms/step - loss: 0.8349 - accuracy: 0.7201 - val_loss: 0.9471 - val_accuracy: 0.6994\n",
            "Epoch 36/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.8214 - accuracy: 0.7240 - val_loss: 0.9429 - val_accuracy: 0.7027\n",
            "Epoch 37/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.8142 - accuracy: 0.7272 - val_loss: 0.9456 - val_accuracy: 0.6949\n",
            "Epoch 38/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.8103 - accuracy: 0.7270 - val_loss: 0.9494 - val_accuracy: 0.7005\n",
            "Epoch 39/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.8086 - accuracy: 0.7284 - val_loss: 0.9659 - val_accuracy: 0.6911\n",
            "Epoch 40/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.7949 - accuracy: 0.7321 - val_loss: 0.9559 - val_accuracy: 0.7021\n",
            "Epoch 41/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.7912 - accuracy: 0.7324 - val_loss: 0.9543 - val_accuracy: 0.6994\n",
            "Epoch 42/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.7850 - accuracy: 0.7356 - val_loss: 0.9620 - val_accuracy: 0.6967\n",
            "Epoch 43/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.7817 - accuracy: 0.7366 - val_loss: 0.9594 - val_accuracy: 0.6967\n",
            "Epoch 44/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.7695 - accuracy: 0.7398 - val_loss: 0.9618 - val_accuracy: 0.6920\n",
            "Epoch 45/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.7695 - accuracy: 0.7398 - val_loss: 0.9638 - val_accuracy: 0.6922\n",
            "Epoch 46/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.7605 - accuracy: 0.7422 - val_loss: 0.9491 - val_accuracy: 0.7034\n",
            "Epoch 47/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.7593 - accuracy: 0.7435 - val_loss: 0.9703 - val_accuracy: 0.7000\n",
            "Epoch 48/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.7534 - accuracy: 0.7446 - val_loss: 0.9641 - val_accuracy: 0.6998\n",
            "Epoch 49/50\n",
            "1256/1256 [==============================] - 7s 6ms/step - loss: 0.7427 - accuracy: 0.7482 - val_loss: 0.9692 - val_accuracy: 0.6958\n",
            "Epoch 50/50\n",
            "1256/1256 [==============================] - 7s 5ms/step - loss: 0.7415 - accuracy: 0.7488 - val_loss: 0.9634 - val_accuracy: 0.6994\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "layer_list = [ #520 parameters in input\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Dropout(0.3),\n",
        "          layers.Dense(200,kernel_initializer='lecun_normal'), #let's try dimensionality reduction\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(256,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          #layers.Dense(128,kernel_initializer='lecun_normal'),\n",
        "          #layers.BatchNormalization(),\n",
        "          #layers.Activation('selu'),\n",
        "          #layers.Dense(64,kernel_initializer='lecun_normal'),\n",
        "          #layers.BatchNormalization(),\n",
        "          #layers.Activation('selu'),\n",
        "          layers.Dense(32,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(16,activation='softmax',kernel_initializer='lecun_normal'),\n",
        "          ]\n",
        "model = Sequential(layer_list)\n",
        "model.compile(optimizer=keras.optimizers.Adam(\n",
        "    learning_rate=keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.001,decay_steps=2000,decay_rate=0.99)),\n",
        "    loss=\"categorical_crossentropy\", metrics=['accuracy'],)\n",
        "history = model.fit(xtrain, ytrain_1hot, batch_size=32, epochs=50, validation_data=(xval, yval_1hot))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vSUNH9UCTvz1",
        "outputId": "cfaec877-9ba7-47b3-d1b0-6dfd40547c0b"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/50\n",
            "1256/1256 [==============================] - 13s 7ms/step - loss: 1.4180 - accuracy: 0.5701 - val_loss: 1.1520 - val_accuracy: 0.6279\n",
            "Epoch 2/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 1.2274 - accuracy: 0.6042 - val_loss: 1.1130 - val_accuracy: 0.6443\n",
            "Epoch 3/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.1920 - accuracy: 0.6127 - val_loss: 1.0948 - val_accuracy: 0.6478\n",
            "Epoch 4/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.1682 - accuracy: 0.6206 - val_loss: 1.0765 - val_accuracy: 0.6499\n",
            "Epoch 5/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.1488 - accuracy: 0.6229 - val_loss: 1.0821 - val_accuracy: 0.6447\n",
            "Epoch 6/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 1.1323 - accuracy: 0.6301 - val_loss: 1.0407 - val_accuracy: 0.6599\n",
            "Epoch 7/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.1137 - accuracy: 0.6354 - val_loss: 1.0301 - val_accuracy: 0.6655\n",
            "Epoch 8/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 1.0977 - accuracy: 0.6394 - val_loss: 1.0250 - val_accuracy: 0.6647\n",
            "Epoch 9/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.0756 - accuracy: 0.6479 - val_loss: 1.0190 - val_accuracy: 0.6664\n",
            "Epoch 10/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.0650 - accuracy: 0.6487 - val_loss: 1.0120 - val_accuracy: 0.6676\n",
            "Epoch 11/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.0493 - accuracy: 0.6564 - val_loss: 0.9944 - val_accuracy: 0.6754\n",
            "Epoch 12/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.0371 - accuracy: 0.6578 - val_loss: 1.0013 - val_accuracy: 0.6716\n",
            "Epoch 13/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 1.0226 - accuracy: 0.6635 - val_loss: 0.9789 - val_accuracy: 0.6799\n",
            "Epoch 14/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 1.0102 - accuracy: 0.6658 - val_loss: 0.9685 - val_accuracy: 0.6848\n",
            "Epoch 15/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.9974 - accuracy: 0.6690 - val_loss: 0.9662 - val_accuracy: 0.6794\n",
            "Epoch 16/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.9841 - accuracy: 0.6730 - val_loss: 0.9615 - val_accuracy: 0.6832\n",
            "Epoch 17/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.9706 - accuracy: 0.6787 - val_loss: 0.9584 - val_accuracy: 0.6862\n",
            "Epoch 18/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.9557 - accuracy: 0.6824 - val_loss: 0.9545 - val_accuracy: 0.6900\n",
            "Epoch 19/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.9471 - accuracy: 0.6826 - val_loss: 0.9386 - val_accuracy: 0.6944\n",
            "Epoch 20/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.9362 - accuracy: 0.6901 - val_loss: 0.9512 - val_accuracy: 0.6873\n",
            "Epoch 21/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.9248 - accuracy: 0.6928 - val_loss: 0.9519 - val_accuracy: 0.6906\n",
            "Epoch 22/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.9209 - accuracy: 0.6902 - val_loss: 0.9559 - val_accuracy: 0.6888\n",
            "Epoch 23/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.9059 - accuracy: 0.6962 - val_loss: 0.9434 - val_accuracy: 0.6906\n",
            "Epoch 24/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.8958 - accuracy: 0.7004 - val_loss: 0.9435 - val_accuracy: 0.6951\n",
            "Epoch 25/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8845 - accuracy: 0.7032 - val_loss: 0.9426 - val_accuracy: 0.6978\n",
            "Epoch 26/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.8765 - accuracy: 0.7065 - val_loss: 0.9389 - val_accuracy: 0.6974\n",
            "Epoch 27/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.8666 - accuracy: 0.7088 - val_loss: 0.9435 - val_accuracy: 0.7018\n",
            "Epoch 28/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8617 - accuracy: 0.7093 - val_loss: 0.9345 - val_accuracy: 0.6980\n",
            "Epoch 29/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8499 - accuracy: 0.7139 - val_loss: 0.9306 - val_accuracy: 0.6998\n",
            "Epoch 30/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8449 - accuracy: 0.7176 - val_loss: 0.9375 - val_accuracy: 0.7009\n",
            "Epoch 31/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8374 - accuracy: 0.7161 - val_loss: 0.9460 - val_accuracy: 0.6962\n",
            "Epoch 32/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8248 - accuracy: 0.7222 - val_loss: 0.9483 - val_accuracy: 0.6958\n",
            "Epoch 33/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8238 - accuracy: 0.7222 - val_loss: 0.9473 - val_accuracy: 0.7018\n",
            "Epoch 34/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8197 - accuracy: 0.7221 - val_loss: 0.9412 - val_accuracy: 0.6991\n",
            "Epoch 35/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8064 - accuracy: 0.7279 - val_loss: 0.9503 - val_accuracy: 0.6971\n",
            "Epoch 36/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7990 - accuracy: 0.7278 - val_loss: 0.9535 - val_accuracy: 0.7030\n",
            "Epoch 37/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7942 - accuracy: 0.7304 - val_loss: 0.9613 - val_accuracy: 0.6933\n",
            "Epoch 38/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7973 - accuracy: 0.7293 - val_loss: 0.9571 - val_accuracy: 0.6989\n",
            "Epoch 39/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7821 - accuracy: 0.7319 - val_loss: 0.9689 - val_accuracy: 0.6911\n",
            "Epoch 40/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7773 - accuracy: 0.7338 - val_loss: 0.9399 - val_accuracy: 0.7032\n",
            "Epoch 41/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7682 - accuracy: 0.7378 - val_loss: 0.9487 - val_accuracy: 0.6985\n",
            "Epoch 42/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7734 - accuracy: 0.7373 - val_loss: 0.9520 - val_accuracy: 0.6960\n",
            "Epoch 43/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7607 - accuracy: 0.7415 - val_loss: 0.9603 - val_accuracy: 0.6969\n",
            "Epoch 44/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.7602 - accuracy: 0.7411 - val_loss: 0.9552 - val_accuracy: 0.6991\n",
            "Epoch 45/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7477 - accuracy: 0.7432 - val_loss: 0.9541 - val_accuracy: 0.7036\n",
            "Epoch 46/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7455 - accuracy: 0.7450 - val_loss: 0.9571 - val_accuracy: 0.7027\n",
            "Epoch 47/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.7461 - accuracy: 0.7442 - val_loss: 0.9679 - val_accuracy: 0.6951\n",
            "Epoch 48/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7404 - accuracy: 0.7461 - val_loss: 0.9508 - val_accuracy: 0.7052\n",
            "Epoch 49/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.7334 - accuracy: 0.7481 - val_loss: 0.9467 - val_accuracy: 0.7110\n",
            "Epoch 50/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.7329 - accuracy: 0.7485 - val_loss: 0.9561 - val_accuracy: 0.6989\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "layer_list = [ #520 parameters in input\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Dropout(0.3),\n",
        "          layers.Dense(300,kernel_initializer='lecun_normal'), #let's try dimensionality reduction\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(256,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          #layers.Dense(128,kernel_initializer='lecun_normal'),\n",
        "          #layers.BatchNormalization(),\n",
        "          #layers.Activation('selu'),\n",
        "          #layers.Dense(64,kernel_initializer='lecun_normal'),\n",
        "          #layers.BatchNormalization(),\n",
        "          #layers.Activation('selu'),\n",
        "          layers.Dense(32,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(16,activation='softmax',kernel_initializer='lecun_normal'),\n",
        "          ]\n",
        "model = Sequential(layer_list)\n",
        "model.compile(optimizer=keras.optimizers.Adam(\n",
        "    learning_rate=keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.001,decay_steps=2000,decay_rate=0.99)),\n",
        "    loss=\"categorical_crossentropy\", metrics=['accuracy'],)\n",
        "history = model.fit(xtrain, ytrain_1hot, batch_size=32, epochs=50, validation_data=(xval, yval_1hot))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UaGm-Ih3YqlD",
        "outputId": "98addb7c-9d37-491c-a125-e28ff55db62f"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/50\n",
            "1256/1256 [==============================] - 10s 7ms/step - loss: 1.4079 - accuracy: 0.5711 - val_loss: 1.1681 - val_accuracy: 0.6272\n",
            "Epoch 2/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.2300 - accuracy: 0.6025 - val_loss: 1.1189 - val_accuracy: 0.6431\n",
            "Epoch 3/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.1903 - accuracy: 0.6151 - val_loss: 1.0771 - val_accuracy: 0.6526\n",
            "Epoch 4/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.1714 - accuracy: 0.6209 - val_loss: 1.0693 - val_accuracy: 0.6552\n",
            "Epoch 5/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1490 - accuracy: 0.6281 - val_loss: 1.0596 - val_accuracy: 0.6564\n",
            "Epoch 6/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 1.1248 - accuracy: 0.6343 - val_loss: 1.0339 - val_accuracy: 0.6624\n",
            "Epoch 7/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 1.1117 - accuracy: 0.6366 - val_loss: 1.0361 - val_accuracy: 0.6635\n",
            "Epoch 8/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 1.0923 - accuracy: 0.6447 - val_loss: 1.0167 - val_accuracy: 0.6714\n",
            "Epoch 9/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.0746 - accuracy: 0.6477 - val_loss: 1.0153 - val_accuracy: 0.6707\n",
            "Epoch 10/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.0576 - accuracy: 0.6538 - val_loss: 0.9912 - val_accuracy: 0.6714\n",
            "Epoch 11/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 1.0455 - accuracy: 0.6532 - val_loss: 0.9846 - val_accuracy: 0.6866\n",
            "Epoch 12/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 1.0275 - accuracy: 0.6628 - val_loss: 0.9790 - val_accuracy: 0.6812\n",
            "Epoch 13/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.0140 - accuracy: 0.6669 - val_loss: 0.9929 - val_accuracy: 0.6723\n",
            "Epoch 14/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.9981 - accuracy: 0.6704 - val_loss: 0.9642 - val_accuracy: 0.6850\n",
            "Epoch 15/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.9833 - accuracy: 0.6749 - val_loss: 0.9709 - val_accuracy: 0.6817\n",
            "Epoch 16/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.9713 - accuracy: 0.6790 - val_loss: 0.9588 - val_accuracy: 0.6902\n",
            "Epoch 17/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.9618 - accuracy: 0.6796 - val_loss: 0.9447 - val_accuracy: 0.6942\n",
            "Epoch 18/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.9432 - accuracy: 0.6877 - val_loss: 0.9611 - val_accuracy: 0.6850\n",
            "Epoch 19/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.9318 - accuracy: 0.6903 - val_loss: 0.9397 - val_accuracy: 0.6938\n",
            "Epoch 20/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.9232 - accuracy: 0.6929 - val_loss: 0.9610 - val_accuracy: 0.6857\n",
            "Epoch 21/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.9025 - accuracy: 0.7000 - val_loss: 0.9615 - val_accuracy: 0.6848\n",
            "Epoch 22/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8978 - accuracy: 0.6997 - val_loss: 0.9603 - val_accuracy: 0.6888\n",
            "Epoch 23/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8780 - accuracy: 0.7061 - val_loss: 0.9621 - val_accuracy: 0.6868\n",
            "Epoch 24/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.8759 - accuracy: 0.7060 - val_loss: 0.9574 - val_accuracy: 0.6871\n",
            "Epoch 25/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8587 - accuracy: 0.7127 - val_loss: 0.9444 - val_accuracy: 0.6927\n",
            "Epoch 26/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8529 - accuracy: 0.7122 - val_loss: 0.9338 - val_accuracy: 0.7025\n",
            "Epoch 27/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8389 - accuracy: 0.7159 - val_loss: 0.9361 - val_accuracy: 0.7005\n",
            "Epoch 28/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8291 - accuracy: 0.7209 - val_loss: 0.9626 - val_accuracy: 0.6844\n",
            "Epoch 29/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8215 - accuracy: 0.7243 - val_loss: 0.9274 - val_accuracy: 0.7068\n",
            "Epoch 30/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.8087 - accuracy: 0.7261 - val_loss: 0.9463 - val_accuracy: 0.6969\n",
            "Epoch 31/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.8041 - accuracy: 0.7289 - val_loss: 0.9441 - val_accuracy: 0.6983\n",
            "Epoch 32/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7931 - accuracy: 0.7305 - val_loss: 0.9365 - val_accuracy: 0.6996\n",
            "Epoch 33/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.7807 - accuracy: 0.7373 - val_loss: 0.9474 - val_accuracy: 0.7003\n",
            "Epoch 34/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.7752 - accuracy: 0.7370 - val_loss: 0.9577 - val_accuracy: 0.6962\n",
            "Epoch 35/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7705 - accuracy: 0.7397 - val_loss: 0.9716 - val_accuracy: 0.6958\n",
            "Epoch 36/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7651 - accuracy: 0.7401 - val_loss: 0.9478 - val_accuracy: 0.6991\n",
            "Epoch 37/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7521 - accuracy: 0.7432 - val_loss: 0.9441 - val_accuracy: 0.7050\n",
            "Epoch 38/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7460 - accuracy: 0.7425 - val_loss: 0.9620 - val_accuracy: 0.7025\n",
            "Epoch 39/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7378 - accuracy: 0.7471 - val_loss: 0.9588 - val_accuracy: 0.7079\n",
            "Epoch 40/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7275 - accuracy: 0.7514 - val_loss: 0.9542 - val_accuracy: 0.7012\n",
            "Epoch 41/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7223 - accuracy: 0.7501 - val_loss: 0.9854 - val_accuracy: 0.6938\n",
            "Epoch 42/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7160 - accuracy: 0.7531 - val_loss: 0.9546 - val_accuracy: 0.7021\n",
            "Epoch 43/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.7169 - accuracy: 0.7531 - val_loss: 0.9778 - val_accuracy: 0.6971\n",
            "Epoch 44/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7021 - accuracy: 0.7554 - val_loss: 0.9734 - val_accuracy: 0.6953\n",
            "Epoch 45/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6988 - accuracy: 0.7600 - val_loss: 1.0017 - val_accuracy: 0.6891\n",
            "Epoch 46/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6940 - accuracy: 0.7607 - val_loss: 0.9819 - val_accuracy: 0.6974\n",
            "Epoch 47/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.6817 - accuracy: 0.7648 - val_loss: 0.9931 - val_accuracy: 0.6938\n",
            "Epoch 48/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.6798 - accuracy: 0.7651 - val_loss: 0.9915 - val_accuracy: 0.6960\n",
            "Epoch 49/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6673 - accuracy: 0.7697 - val_loss: 0.9996 - val_accuracy: 0.6924\n",
            "Epoch 50/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6652 - accuracy: 0.7679 - val_loss: 0.9989 - val_accuracy: 0.6969\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "layer_list = [ #520 parameters in input\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Dropout(0.4),\n",
        "          layers.Dense(300,kernel_initializer='lecun_normal'), #let's try dimensionality reduction\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('relu'),\n",
        "          layers.Dense(256,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('relu'),\n",
        "          #layers.Dense(128,kernel_initializer='lecun_normal'),\n",
        "          #layers.BatchNormalization(),\n",
        "          #layers.Activation('selu'),\n",
        "          #layers.Dense(64,kernel_initializer='lecun_normal'),\n",
        "          #layers.BatchNormalization(),\n",
        "          #layers.Activation('selu'),\n",
        "          layers.Dense(32,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(16,activation='softmax',kernel_initializer='lecun_normal'),\n",
        "          ]\n",
        "model = Sequential(layer_list)\n",
        "model.compile(optimizer=keras.optimizers.Adam(\n",
        "    learning_rate=keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.001,decay_steps=2000,decay_rate=0.99)),\n",
        "    loss=\"categorical_crossentropy\", metrics=['accuracy'],)\n",
        "history = model.fit(xtrain, ytrain_1hot, batch_size=32, epochs=50, validation_data=(xval, yval_1hot))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "G0OdmmZ8aevi",
        "outputId": "025871fa-95e0-4aa2-fc19-08b480609dfd"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/50\n",
            "1256/1256 [==============================] - 10s 7ms/step - loss: 1.3996 - accuracy: 0.5724 - val_loss: 1.1249 - val_accuracy: 0.6384\n",
            "Epoch 2/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.2012 - accuracy: 0.6109 - val_loss: 1.0698 - val_accuracy: 0.6577\n",
            "Epoch 3/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 1.1562 - accuracy: 0.6224 - val_loss: 1.0455 - val_accuracy: 0.6591\n",
            "Epoch 4/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 1.1239 - accuracy: 0.6310 - val_loss: 1.0219 - val_accuracy: 0.6725\n",
            "Epoch 5/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.0954 - accuracy: 0.6365 - val_loss: 1.0181 - val_accuracy: 0.6756\n",
            "Epoch 6/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 1.0729 - accuracy: 0.6469 - val_loss: 0.9943 - val_accuracy: 0.6799\n",
            "Epoch 7/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.0469 - accuracy: 0.6530 - val_loss: 0.9719 - val_accuracy: 0.6821\n",
            "Epoch 8/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 1.0284 - accuracy: 0.6573 - val_loss: 0.9867 - val_accuracy: 0.6799\n",
            "Epoch 9/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.0053 - accuracy: 0.6666 - val_loss: 0.9667 - val_accuracy: 0.6841\n",
            "Epoch 10/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.9846 - accuracy: 0.6709 - val_loss: 0.9659 - val_accuracy: 0.6848\n",
            "Epoch 11/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.9640 - accuracy: 0.6784 - val_loss: 0.9791 - val_accuracy: 0.6810\n",
            "Epoch 12/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.9497 - accuracy: 0.6813 - val_loss: 0.9486 - val_accuracy: 0.6929\n",
            "Epoch 13/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.9327 - accuracy: 0.6867 - val_loss: 0.9517 - val_accuracy: 0.6911\n",
            "Epoch 14/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.9197 - accuracy: 0.6903 - val_loss: 0.9550 - val_accuracy: 0.6868\n",
            "Epoch 15/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.9039 - accuracy: 0.6957 - val_loss: 0.9389 - val_accuracy: 0.6927\n",
            "Epoch 16/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8909 - accuracy: 0.6988 - val_loss: 0.9564 - val_accuracy: 0.6893\n",
            "Epoch 17/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.8809 - accuracy: 0.7024 - val_loss: 0.9666 - val_accuracy: 0.6920\n",
            "Epoch 18/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8708 - accuracy: 0.7046 - val_loss: 0.9476 - val_accuracy: 0.6998\n",
            "Epoch 19/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8563 - accuracy: 0.7114 - val_loss: 0.9525 - val_accuracy: 0.6962\n",
            "Epoch 20/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8476 - accuracy: 0.7121 - val_loss: 0.9548 - val_accuracy: 0.6974\n",
            "Epoch 21/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.8410 - accuracy: 0.7150 - val_loss: 0.9512 - val_accuracy: 0.6949\n",
            "Epoch 22/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.8277 - accuracy: 0.7186 - val_loss: 0.9499 - val_accuracy: 0.6989\n",
            "Epoch 23/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8163 - accuracy: 0.7215 - val_loss: 0.9574 - val_accuracy: 0.6911\n",
            "Epoch 24/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8110 - accuracy: 0.7221 - val_loss: 0.9561 - val_accuracy: 0.6944\n",
            "Epoch 25/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.8016 - accuracy: 0.7262 - val_loss: 0.9516 - val_accuracy: 0.7003\n",
            "Epoch 26/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.7971 - accuracy: 0.7275 - val_loss: 0.9615 - val_accuracy: 0.6909\n",
            "Epoch 27/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.7821 - accuracy: 0.7319 - val_loss: 0.9644 - val_accuracy: 0.6980\n",
            "Epoch 28/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7742 - accuracy: 0.7322 - val_loss: 0.9732 - val_accuracy: 0.6940\n",
            "Epoch 29/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7718 - accuracy: 0.7357 - val_loss: 0.9457 - val_accuracy: 0.7036\n",
            "Epoch 30/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7636 - accuracy: 0.7368 - val_loss: 0.9737 - val_accuracy: 0.6983\n",
            "Epoch 31/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7599 - accuracy: 0.7377 - val_loss: 0.9809 - val_accuracy: 0.6924\n",
            "Epoch 32/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7545 - accuracy: 0.7397 - val_loss: 0.9686 - val_accuracy: 0.6942\n",
            "Epoch 33/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7391 - accuracy: 0.7444 - val_loss: 0.9632 - val_accuracy: 0.7032\n",
            "Epoch 34/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7418 - accuracy: 0.7432 - val_loss: 0.9619 - val_accuracy: 0.7032\n",
            "Epoch 35/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7354 - accuracy: 0.7448 - val_loss: 0.9694 - val_accuracy: 0.7061\n",
            "Epoch 36/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.7387 - accuracy: 0.7451 - val_loss: 0.9692 - val_accuracy: 0.7074\n",
            "Epoch 37/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7250 - accuracy: 0.7472 - val_loss: 0.9612 - val_accuracy: 0.6991\n",
            "Epoch 38/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.7231 - accuracy: 0.7480 - val_loss: 0.9712 - val_accuracy: 0.7014\n",
            "Epoch 39/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.7197 - accuracy: 0.7502 - val_loss: 0.9710 - val_accuracy: 0.6983\n",
            "Epoch 40/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7063 - accuracy: 0.7553 - val_loss: 0.9766 - val_accuracy: 0.6913\n",
            "Epoch 41/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7066 - accuracy: 0.7539 - val_loss: 0.9758 - val_accuracy: 0.6967\n",
            "Epoch 42/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7027 - accuracy: 0.7551 - val_loss: 0.9911 - val_accuracy: 0.6929\n",
            "Epoch 43/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6993 - accuracy: 0.7578 - val_loss: 0.9796 - val_accuracy: 0.6953\n",
            "Epoch 44/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6862 - accuracy: 0.7625 - val_loss: 0.9709 - val_accuracy: 0.7039\n",
            "Epoch 45/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6827 - accuracy: 0.7621 - val_loss: 0.9855 - val_accuracy: 0.6987\n",
            "Epoch 46/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6886 - accuracy: 0.7595 - val_loss: 0.9849 - val_accuracy: 0.6971\n",
            "Epoch 47/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6766 - accuracy: 0.7639 - val_loss: 0.9793 - val_accuracy: 0.6983\n",
            "Epoch 48/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6771 - accuracy: 0.7621 - val_loss: 0.9962 - val_accuracy: 0.6893\n",
            "Epoch 49/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6766 - accuracy: 0.7642 - val_loss: 0.9780 - val_accuracy: 0.7005\n",
            "Epoch 50/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6673 - accuracy: 0.7691 - val_loss: 0.9811 - val_accuracy: 0.6978\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "layer_list = [ #520 parameters in input\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Dropout(0.4),\n",
        "          layers.Dense(150,kernel_initializer='lecun_normal'), #let's try dimensionality reduction\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('relu'),\n",
        "          layers.Dense(256,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('relu'),\n",
        "          #layers.Dense(128,kernel_initializer='lecun_normal'),\n",
        "          #layers.BatchNormalization(),\n",
        "          #layers.Activation('selu'),\n",
        "          #layers.Dense(64,kernel_initializer='lecun_normal'),\n",
        "          #layers.BatchNormalization(),\n",
        "          #layers.Activation('selu'),\n",
        "          layers.Dense(32,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(16,activation='softmax',kernel_initializer='lecun_normal'),\n",
        "          ]\n",
        "model = Sequential(layer_list)\n",
        "model.compile(optimizer=keras.optimizers.Adam(\n",
        "    learning_rate=keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.001,decay_steps=2000,decay_rate=0.99)),\n",
        "    loss=\"categorical_crossentropy\", metrics=['accuracy'],)\n",
        "history = model.fit(xtrain, ytrain_1hot, batch_size=32, epochs=50, validation_data=(xval, yval_1hot))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Oy2ZZYLzcTW2",
        "outputId": "194027f6-97e1-4986-d86a-a4eb4b601a52"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/50\n",
            "1256/1256 [==============================] - 10s 7ms/step - loss: 1.4199 - accuracy: 0.5654 - val_loss: 1.1292 - val_accuracy: 0.6358\n",
            "Epoch 2/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.2160 - accuracy: 0.6045 - val_loss: 1.0854 - val_accuracy: 0.6550\n",
            "Epoch 3/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.1690 - accuracy: 0.6188 - val_loss: 1.0432 - val_accuracy: 0.6671\n",
            "Epoch 4/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.1409 - accuracy: 0.6254 - val_loss: 1.0349 - val_accuracy: 0.6673\n",
            "Epoch 5/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.1158 - accuracy: 0.6329 - val_loss: 1.0340 - val_accuracy: 0.6642\n",
            "Epoch 6/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 1.0933 - accuracy: 0.6415 - val_loss: 0.9981 - val_accuracy: 0.6723\n",
            "Epoch 7/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 1.0765 - accuracy: 0.6436 - val_loss: 0.9869 - val_accuracy: 0.6797\n",
            "Epoch 8/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.0598 - accuracy: 0.6486 - val_loss: 0.9825 - val_accuracy: 0.6799\n",
            "Epoch 9/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.0443 - accuracy: 0.6523 - val_loss: 0.9818 - val_accuracy: 0.6808\n",
            "Epoch 10/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.0272 - accuracy: 0.6585 - val_loss: 0.9720 - val_accuracy: 0.6844\n",
            "Epoch 11/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.0103 - accuracy: 0.6625 - val_loss: 0.9738 - val_accuracy: 0.6803\n",
            "Epoch 12/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.0045 - accuracy: 0.6647 - val_loss: 0.9723 - val_accuracy: 0.6837\n",
            "Epoch 13/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.9955 - accuracy: 0.6694 - val_loss: 0.9602 - val_accuracy: 0.6886\n",
            "Epoch 14/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.9837 - accuracy: 0.6722 - val_loss: 0.9649 - val_accuracy: 0.6832\n",
            "Epoch 15/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.9710 - accuracy: 0.6763 - val_loss: 0.9742 - val_accuracy: 0.6803\n",
            "Epoch 16/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.9626 - accuracy: 0.6769 - val_loss: 0.9557 - val_accuracy: 0.6857\n",
            "Epoch 17/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.9492 - accuracy: 0.6825 - val_loss: 0.9647 - val_accuracy: 0.6875\n",
            "Epoch 18/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.9446 - accuracy: 0.6827 - val_loss: 0.9627 - val_accuracy: 0.6855\n",
            "Epoch 19/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.9288 - accuracy: 0.6870 - val_loss: 0.9435 - val_accuracy: 0.6922\n",
            "Epoch 20/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.9286 - accuracy: 0.6874 - val_loss: 0.9490 - val_accuracy: 0.6895\n",
            "Epoch 21/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.9227 - accuracy: 0.6901 - val_loss: 0.9565 - val_accuracy: 0.6900\n",
            "Epoch 22/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.9187 - accuracy: 0.6913 - val_loss: 0.9516 - val_accuracy: 0.6884\n",
            "Epoch 23/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.9091 - accuracy: 0.6944 - val_loss: 0.9515 - val_accuracy: 0.6853\n",
            "Epoch 24/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.9015 - accuracy: 0.6945 - val_loss: 0.9405 - val_accuracy: 0.6935\n",
            "Epoch 25/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8916 - accuracy: 0.6984 - val_loss: 0.9666 - val_accuracy: 0.6866\n",
            "Epoch 26/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8930 - accuracy: 0.6985 - val_loss: 0.9410 - val_accuracy: 0.6929\n",
            "Epoch 27/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8806 - accuracy: 0.6988 - val_loss: 0.9529 - val_accuracy: 0.6904\n",
            "Epoch 28/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.8707 - accuracy: 0.7046 - val_loss: 0.9481 - val_accuracy: 0.6906\n",
            "Epoch 29/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8704 - accuracy: 0.7039 - val_loss: 0.9604 - val_accuracy: 0.6888\n",
            "Epoch 30/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8614 - accuracy: 0.7087 - val_loss: 0.9497 - val_accuracy: 0.6915\n",
            "Epoch 31/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.8673 - accuracy: 0.7078 - val_loss: 0.9593 - val_accuracy: 0.6953\n",
            "Epoch 32/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8613 - accuracy: 0.7068 - val_loss: 0.9550 - val_accuracy: 0.6924\n",
            "Epoch 33/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.8589 - accuracy: 0.7078 - val_loss: 0.9546 - val_accuracy: 0.6935\n",
            "Epoch 34/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8514 - accuracy: 0.7103 - val_loss: 0.9458 - val_accuracy: 0.6931\n",
            "Epoch 35/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8466 - accuracy: 0.7122 - val_loss: 0.9404 - val_accuracy: 0.6967\n",
            "Epoch 36/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.8375 - accuracy: 0.7149 - val_loss: 0.9554 - val_accuracy: 0.6938\n",
            "Epoch 37/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8412 - accuracy: 0.7126 - val_loss: 0.9575 - val_accuracy: 0.6895\n",
            "Epoch 38/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.8358 - accuracy: 0.7175 - val_loss: 0.9617 - val_accuracy: 0.6953\n",
            "Epoch 39/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.8208 - accuracy: 0.7154 - val_loss: 0.9691 - val_accuracy: 0.6915\n",
            "Epoch 40/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8300 - accuracy: 0.7163 - val_loss: 0.9456 - val_accuracy: 0.6949\n",
            "Epoch 41/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.8199 - accuracy: 0.7224 - val_loss: 0.9686 - val_accuracy: 0.6897\n",
            "Epoch 42/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.8170 - accuracy: 0.7217 - val_loss: 0.9626 - val_accuracy: 0.6891\n",
            "Epoch 43/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8140 - accuracy: 0.7232 - val_loss: 0.9665 - val_accuracy: 0.6918\n",
            "Epoch 44/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8155 - accuracy: 0.7203 - val_loss: 0.9661 - val_accuracy: 0.6927\n",
            "Epoch 45/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8127 - accuracy: 0.7212 - val_loss: 0.9601 - val_accuracy: 0.6935\n",
            "Epoch 46/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8096 - accuracy: 0.7229 - val_loss: 0.9695 - val_accuracy: 0.6895\n",
            "Epoch 47/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.8012 - accuracy: 0.7225 - val_loss: 0.9682 - val_accuracy: 0.6933\n",
            "Epoch 48/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8003 - accuracy: 0.7260 - val_loss: 0.9599 - val_accuracy: 0.6969\n",
            "Epoch 49/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.7975 - accuracy: 0.7254 - val_loss: 0.9643 - val_accuracy: 0.6974\n",
            "Epoch 50/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.8004 - accuracy: 0.7235 - val_loss: 0.9684 - val_accuracy: 0.6962\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "layer_list = [ #520 parameters in input\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Dropout(0.4),\n",
        "          layers.Dense(100,kernel_initializer='lecun_normal'), #let's try dimensionality reduction\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('relu'),\n",
        "          #layers.Dense(256,kernel_initializer='lecun_normal'),\n",
        "          #layers.BatchNormalization(),\n",
        "          #layers.Activation('relu'),\n",
        "          layers.Dense(128,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          #layers.Dense(64,kernel_initializer='lecun_normal'),\n",
        "          #layers.BatchNormalization(),\n",
        "          #layers.Activation('selu'),\n",
        "          layers.Dense(32,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(16,activation='softmax',kernel_initializer='lecun_normal'),\n",
        "          ]\n",
        "model = Sequential(layer_list)\n",
        "model.compile(optimizer=keras.optimizers.Adam(\n",
        "    learning_rate=keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.001,decay_steps=2000,decay_rate=0.99)),\n",
        "    loss=\"categorical_crossentropy\", metrics=['accuracy'],)\n",
        "history = model.fit(xtrain, ytrain_1hot, batch_size=32, epochs=50, validation_data=(xval, yval_1hot))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "HuCnDu7meUR-",
        "outputId": "a5140c9f-577b-4da0-91e2-263dac45a6cd"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/50\n",
            "1256/1256 [==============================] - 10s 7ms/step - loss: 1.4371 - accuracy: 0.5599 - val_loss: 1.1534 - val_accuracy: 0.6290\n",
            "Epoch 2/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.2274 - accuracy: 0.6033 - val_loss: 1.1103 - val_accuracy: 0.6458\n",
            "Epoch 3/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.1876 - accuracy: 0.6125 - val_loss: 1.0786 - val_accuracy: 0.6521\n",
            "Epoch 4/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.1611 - accuracy: 0.6229 - val_loss: 1.0680 - val_accuracy: 0.6568\n",
            "Epoch 5/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 1.1452 - accuracy: 0.6255 - val_loss: 1.0554 - val_accuracy: 0.6635\n",
            "Epoch 6/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.1263 - accuracy: 0.6281 - val_loss: 1.0429 - val_accuracy: 0.6658\n",
            "Epoch 7/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.1138 - accuracy: 0.6352 - val_loss: 1.0233 - val_accuracy: 0.6660\n",
            "Epoch 8/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 1.1017 - accuracy: 0.6373 - val_loss: 1.0226 - val_accuracy: 0.6642\n",
            "Epoch 9/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.0937 - accuracy: 0.6399 - val_loss: 1.0104 - val_accuracy: 0.6698\n",
            "Epoch 10/50\n",
            "1256/1256 [==============================] - 9s 8ms/step - loss: 1.0820 - accuracy: 0.6438 - val_loss: 1.0090 - val_accuracy: 0.6716\n",
            "Epoch 11/50\n",
            "1256/1256 [==============================] - 12s 10ms/step - loss: 1.0747 - accuracy: 0.6469 - val_loss: 1.0045 - val_accuracy: 0.6765\n",
            "Epoch 12/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.0671 - accuracy: 0.6494 - val_loss: 1.0161 - val_accuracy: 0.6707\n",
            "Epoch 13/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.0521 - accuracy: 0.6528 - val_loss: 1.0032 - val_accuracy: 0.6745\n",
            "Epoch 14/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 1.0484 - accuracy: 0.6520 - val_loss: 0.9994 - val_accuracy: 0.6767\n",
            "Epoch 15/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0456 - accuracy: 0.6547 - val_loss: 0.9985 - val_accuracy: 0.6747\n",
            "Epoch 16/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0365 - accuracy: 0.6559 - val_loss: 0.9920 - val_accuracy: 0.6815\n",
            "Epoch 17/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0325 - accuracy: 0.6593 - val_loss: 0.9888 - val_accuracy: 0.6803\n",
            "Epoch 18/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 1.0279 - accuracy: 0.6599 - val_loss: 0.9857 - val_accuracy: 0.6810\n",
            "Epoch 19/50\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 1.0179 - accuracy: 0.6639 - val_loss: 0.9905 - val_accuracy: 0.6794\n",
            "Epoch 20/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.0164 - accuracy: 0.6627 - val_loss: 0.9811 - val_accuracy: 0.6828\n",
            "Epoch 21/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.0123 - accuracy: 0.6635 - val_loss: 0.9800 - val_accuracy: 0.6797\n",
            "Epoch 22/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.0071 - accuracy: 0.6662 - val_loss: 0.9721 - val_accuracy: 0.6884\n",
            "Epoch 23/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.0036 - accuracy: 0.6676 - val_loss: 0.9724 - val_accuracy: 0.6808\n",
            "Epoch 24/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.9934 - accuracy: 0.6708 - val_loss: 0.9718 - val_accuracy: 0.6864\n",
            "Epoch 25/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9971 - accuracy: 0.6687 - val_loss: 0.9730 - val_accuracy: 0.6877\n",
            "Epoch 26/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.9925 - accuracy: 0.6713 - val_loss: 0.9701 - val_accuracy: 0.6864\n",
            "Epoch 27/50\n",
            " 972/1256 [======================>.......] - ETA: 1s - loss: 0.9910 - accuracy: 0.6695"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-12-f2be9e9654d4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     23\u001b[0m     learning_rate=keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.001,decay_steps=2000,decay_rate=0.99)),\n\u001b[1;32m     24\u001b[0m     loss=\"categorical_crossentropy\", metrics=['accuracy'],)\n\u001b[0;32m---> 25\u001b[0;31m \u001b[0mhistory\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mytrain_1hot\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m50\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myval_1hot\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/utils/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     65\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint: disable=broad-except\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m       \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1387\u001b[0m               \u001b[0mlogs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtmp_logs\u001b[0m  \u001b[0;31m# No error, now safe to assign to logs.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1388\u001b[0m               \u001b[0mend_step\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstep\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep_increment\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1389\u001b[0;31m               \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mend_step\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1390\u001b[0m               \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstop_training\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1391\u001b[0m                 \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36mon_train_batch_end\u001b[0;34m(self, batch, logs)\u001b[0m\n\u001b[1;32m    436\u001b[0m     \"\"\"\n\u001b[1;32m    437\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_should_call_train_batch_hooks\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 438\u001b[0;31m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_batch_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mModeKeys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTRAIN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'end'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    439\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    440\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mon_test_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36m_call_batch_hook\u001b[0;34m(self, mode, hook, batch, logs)\u001b[0m\n\u001b[1;32m    295\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_batch_begin_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    296\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'end'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 297\u001b[0;31m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_batch_end_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    298\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    299\u001b[0m       raise ValueError(\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36m_call_batch_end_hook\u001b[0;34m(self, mode, batch, logs)\u001b[0m\n\u001b[1;32m    316\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_times\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_time\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    317\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 318\u001b[0;31m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_batch_hook_helper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhook_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    319\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    320\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_times\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_batches_for_timing_check\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36m_call_batch_hook_helper\u001b[0;34m(self, hook_name, batch, logs)\u001b[0m\n\u001b[1;32m    354\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mcallback\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    355\u001b[0m       \u001b[0mhook\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcallback\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 356\u001b[0;31m       \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    357\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    358\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_timing\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36mon_train_batch_end\u001b[0;34m(self, batch, logs)\u001b[0m\n\u001b[1;32m   1032\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1033\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mon_train_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1034\u001b[0;31m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_update_progbar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1035\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1036\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mon_test_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36m_batch_update_progbar\u001b[0;34m(self, batch, logs)\u001b[0m\n\u001b[1;32m   1090\u001b[0m     \u001b[0;34m\"\"\"Updates the progbar.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1091\u001b[0m     \u001b[0mlogs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlogs\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1092\u001b[0;31m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_init_progbar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1093\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muse_steps\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1094\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mseen\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m  \u001b[0;31m# One-indexed.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36m_maybe_init_progbar\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1067\u001b[0m       \u001b[0;31m# step.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1068\u001b[0m       self.stateful_metrics = self.stateful_metrics.union(\n\u001b[0;32m-> 1069\u001b[0;31m           set(m.name for m in self.model.metrics))\n\u001b[0m\u001b[1;32m   1070\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1071\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprogbar\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36m<genexpr>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m   1067\u001b[0m       \u001b[0;31m# step.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1068\u001b[0m       self.stateful_metrics = self.stateful_metrics.union(\n\u001b[0;32m-> 1069\u001b[0;31m           set(m.name for m in self.model.metrics))\n\u001b[0m\u001b[1;32m   1070\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1071\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprogbar\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/engine/base_layer.py\u001b[0m in \u001b[0;36mname\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1258\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1259\u001b[0m     \u001b[0;34m\"\"\"Name of the layer (string), set in the constructor.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1260\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_name\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1261\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1262\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "layer_list = [ #520 parameters in input\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Dropout(0.4),\n",
        "          layers.Dense(100,kernel_initializer='lecun_normal'), #let's try dimensionality reduction\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('relu'),\n",
        "          #layers.Dense(256,kernel_initializer='lecun_normal'),\n",
        "          #layers.BatchNormalization(),\n",
        "          #layers.Activation('relu'),\n",
        "          #layers.Dense(128,kernel_initializer='lecun_normal'),\n",
        "          #layers.BatchNormalization(),\n",
        "          #layers.Activation('selu'),\n",
        "          #layers.Dense(64,kernel_initializer='lecun_normal'),\n",
        "          #layers.BatchNormalization(),\n",
        "          #layers.Activation('selu'),\n",
        "          layers.Dense(32,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(16,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('relu'),\n",
        "          layers.Dense(16,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(16,activation='softmax',kernel_initializer='lecun_normal'),\n",
        "          ]\n",
        "model = Sequential(layer_list)\n",
        "model.compile(optimizer=keras.optimizers.Adam(\n",
        "    learning_rate=keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.001,decay_steps=2000,decay_rate=0.99)),\n",
        "    loss=\"categorical_crossentropy\", metrics=['accuracy'],)\n",
        "history = model.fit(xtrain, ytrain_1hot, batch_size=32, epochs=50, validation_data=(xval, yval_1hot))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gyWLt-rWfPbE",
        "outputId": "aba0f3f3-8ec2-42ad-927b-24c17670552c"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/50\n",
            "1256/1256 [==============================] - 12s 8ms/step - loss: 1.5505 - accuracy: 0.5358 - val_loss: 1.1927 - val_accuracy: 0.6290\n",
            "Epoch 2/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.2757 - accuracy: 0.5922 - val_loss: 1.1415 - val_accuracy: 0.6358\n",
            "Epoch 3/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.2342 - accuracy: 0.6012 - val_loss: 1.1039 - val_accuracy: 0.6445\n",
            "Epoch 4/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.2094 - accuracy: 0.6119 - val_loss: 1.0986 - val_accuracy: 0.6503\n",
            "Epoch 5/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1924 - accuracy: 0.6157 - val_loss: 1.0802 - val_accuracy: 0.6470\n",
            "Epoch 6/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1704 - accuracy: 0.6211 - val_loss: 1.0643 - val_accuracy: 0.6591\n",
            "Epoch 7/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1614 - accuracy: 0.6219 - val_loss: 1.0538 - val_accuracy: 0.6622\n",
            "Epoch 8/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1417 - accuracy: 0.6286 - val_loss: 1.0551 - val_accuracy: 0.6568\n",
            "Epoch 9/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1362 - accuracy: 0.6277 - val_loss: 1.0401 - val_accuracy: 0.6591\n",
            "Epoch 10/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1292 - accuracy: 0.6315 - val_loss: 1.0364 - val_accuracy: 0.6622\n",
            "Epoch 11/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1177 - accuracy: 0.6327 - val_loss: 1.0411 - val_accuracy: 0.6586\n",
            "Epoch 12/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1124 - accuracy: 0.6382 - val_loss: 1.0267 - val_accuracy: 0.6705\n",
            "Epoch 13/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1056 - accuracy: 0.6393 - val_loss: 1.0305 - val_accuracy: 0.6664\n",
            "Epoch 14/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0955 - accuracy: 0.6432 - val_loss: 1.0217 - val_accuracy: 0.6743\n",
            "Epoch 15/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0938 - accuracy: 0.6415 - val_loss: 1.0188 - val_accuracy: 0.6720\n",
            "Epoch 16/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0791 - accuracy: 0.6477 - val_loss: 1.0119 - val_accuracy: 0.6754\n",
            "Epoch 17/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0791 - accuracy: 0.6465 - val_loss: 1.0100 - val_accuracy: 0.6709\n",
            "Epoch 18/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0777 - accuracy: 0.6474 - val_loss: 1.0101 - val_accuracy: 0.6707\n",
            "Epoch 19/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0709 - accuracy: 0.6516 - val_loss: 1.0128 - val_accuracy: 0.6705\n",
            "Epoch 20/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0690 - accuracy: 0.6501 - val_loss: 1.0107 - val_accuracy: 0.6790\n",
            "Epoch 21/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0632 - accuracy: 0.6516 - val_loss: 1.0093 - val_accuracy: 0.6761\n",
            "Epoch 22/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0633 - accuracy: 0.6527 - val_loss: 1.0046 - val_accuracy: 0.6808\n",
            "Epoch 23/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0531 - accuracy: 0.6567 - val_loss: 1.0009 - val_accuracy: 0.6761\n",
            "Epoch 24/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0522 - accuracy: 0.6546 - val_loss: 0.9989 - val_accuracy: 0.6794\n",
            "Epoch 25/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0500 - accuracy: 0.6576 - val_loss: 0.9976 - val_accuracy: 0.6770\n",
            "Epoch 26/50\n",
            "1256/1256 [==============================] - 9s 8ms/step - loss: 1.0408 - accuracy: 0.6593 - val_loss: 1.0017 - val_accuracy: 0.6788\n",
            "Epoch 27/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0475 - accuracy: 0.6570 - val_loss: 0.9979 - val_accuracy: 0.6772\n",
            "Epoch 28/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0400 - accuracy: 0.6589 - val_loss: 0.9949 - val_accuracy: 0.6750\n",
            "Epoch 29/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0386 - accuracy: 0.6588 - val_loss: 0.9941 - val_accuracy: 0.6808\n",
            "Epoch 30/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0304 - accuracy: 0.6614 - val_loss: 0.9976 - val_accuracy: 0.6788\n",
            "Epoch 31/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0346 - accuracy: 0.6602 - val_loss: 1.0005 - val_accuracy: 0.6718\n",
            "Epoch 32/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0331 - accuracy: 0.6615 - val_loss: 0.9943 - val_accuracy: 0.6759\n",
            "Epoch 33/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0279 - accuracy: 0.6647 - val_loss: 1.0020 - val_accuracy: 0.6707\n",
            "Epoch 34/50\n",
            "1256/1256 [==============================] - 14s 11ms/step - loss: 1.0219 - accuracy: 0.6653 - val_loss: 0.9945 - val_accuracy: 0.6736\n",
            "Epoch 35/50\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 1.0241 - accuracy: 0.6631 - val_loss: 0.9941 - val_accuracy: 0.6723\n",
            "Epoch 36/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0215 - accuracy: 0.6650 - val_loss: 0.9899 - val_accuracy: 0.6794\n",
            "Epoch 37/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0211 - accuracy: 0.6641 - val_loss: 0.9939 - val_accuracy: 0.6752\n",
            "Epoch 38/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0219 - accuracy: 0.6656 - val_loss: 0.9874 - val_accuracy: 0.6835\n",
            "Epoch 39/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0237 - accuracy: 0.6642 - val_loss: 0.9925 - val_accuracy: 0.6772\n",
            "Epoch 40/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0124 - accuracy: 0.6682 - val_loss: 0.9880 - val_accuracy: 0.6790\n",
            "Epoch 41/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0184 - accuracy: 0.6650 - val_loss: 0.9887 - val_accuracy: 0.6767\n",
            "Epoch 42/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0130 - accuracy: 0.6667 - val_loss: 0.9902 - val_accuracy: 0.6803\n",
            "Epoch 43/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0110 - accuracy: 0.6677 - val_loss: 0.9925 - val_accuracy: 0.6754\n",
            "Epoch 44/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0087 - accuracy: 0.6661 - val_loss: 0.9877 - val_accuracy: 0.6812\n",
            "Epoch 45/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0096 - accuracy: 0.6676 - val_loss: 0.9947 - val_accuracy: 0.6759\n",
            "Epoch 46/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0104 - accuracy: 0.6677 - val_loss: 0.9844 - val_accuracy: 0.6788\n",
            "Epoch 47/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0013 - accuracy: 0.6692 - val_loss: 0.9848 - val_accuracy: 0.6841\n",
            "Epoch 48/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9989 - accuracy: 0.6707 - val_loss: 0.9800 - val_accuracy: 0.6756\n",
            "Epoch 49/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0044 - accuracy: 0.6685 - val_loss: 0.9777 - val_accuracy: 0.6839\n",
            "Epoch 50/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0068 - accuracy: 0.6678 - val_loss: 0.9774 - val_accuracy: 0.6844\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "layer_list = [ #520 parameters in input\n",
        "          layers.BatchNormalization(),\n",
        "          #layers.Dropout(0.4),\n",
        "          layers.Dense(50,kernel_initializer='lecun_normal'), #let's try dimensionality reduction\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('relu'),\n",
        "          #layers.Dense(256,kernel_initializer='lecun_normal'),\n",
        "          #layers.BatchNormalization(),\n",
        "          #layers.Activation('relu'),\n",
        "          #layers.Dense(128,kernel_initializer='lecun_normal'),\n",
        "          #layers.BatchNormalization(),\n",
        "          #layers.Activation('selu'),\n",
        "          #layers.Dense(64,kernel_initializer='lecun_normal'),\n",
        "          #layers.BatchNormalization(),\n",
        "          #layers.Activation('selu'),\n",
        "          layers.Dense(512,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(16,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('relu'),\n",
        "          layers.Dense(16,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(16,activation='softmax',kernel_initializer='lecun_normal'),\n",
        "          ]\n",
        "model = Sequential(layer_list)\n",
        "model.compile(optimizer=keras.optimizers.Adam(\n",
        "    learning_rate=keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.001,decay_steps=2000,decay_rate=0.99)),\n",
        "    loss=\"categorical_crossentropy\", metrics=['accuracy'],)\n",
        "history = model.fit(xtrain, ytrain_1hot, batch_size=32, epochs=50, validation_data=(xval, yval_1hot))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3KXpgIYIhFYH",
        "outputId": "8a12f1d4-6f13-4962-e649-94c401008be9"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/50\n",
            "1256/1256 [==============================] - 12s 8ms/step - loss: 1.4431 - accuracy: 0.5727 - val_loss: 1.1592 - val_accuracy: 0.6366\n",
            "Epoch 2/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1718 - accuracy: 0.6291 - val_loss: 1.0923 - val_accuracy: 0.6584\n",
            "Epoch 3/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1198 - accuracy: 0.6441 - val_loss: 1.0618 - val_accuracy: 0.6635\n",
            "Epoch 4/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0738 - accuracy: 0.6558 - val_loss: 1.0570 - val_accuracy: 0.6624\n",
            "Epoch 5/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0485 - accuracy: 0.6676 - val_loss: 1.0386 - val_accuracy: 0.6653\n",
            "Epoch 6/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0307 - accuracy: 0.6685 - val_loss: 1.0456 - val_accuracy: 0.6617\n",
            "Epoch 7/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0062 - accuracy: 0.6756 - val_loss: 1.0231 - val_accuracy: 0.6767\n",
            "Epoch 8/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9884 - accuracy: 0.6833 - val_loss: 1.0339 - val_accuracy: 0.6647\n",
            "Epoch 9/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9792 - accuracy: 0.6837 - val_loss: 1.0113 - val_accuracy: 0.6781\n",
            "Epoch 10/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9648 - accuracy: 0.6891 - val_loss: 1.0180 - val_accuracy: 0.6732\n",
            "Epoch 11/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9487 - accuracy: 0.6934 - val_loss: 1.0313 - val_accuracy: 0.6718\n",
            "Epoch 12/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9370 - accuracy: 0.6974 - val_loss: 1.0277 - val_accuracy: 0.6741\n",
            "Epoch 13/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9273 - accuracy: 0.7003 - val_loss: 1.0058 - val_accuracy: 0.6794\n",
            "Epoch 14/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9191 - accuracy: 0.7029 - val_loss: 1.0052 - val_accuracy: 0.6832\n",
            "Epoch 15/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9109 - accuracy: 0.7045 - val_loss: 1.0039 - val_accuracy: 0.6790\n",
            "Epoch 16/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9018 - accuracy: 0.7070 - val_loss: 1.0158 - val_accuracy: 0.6725\n",
            "Epoch 17/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.8937 - accuracy: 0.7080 - val_loss: 1.0084 - val_accuracy: 0.6794\n",
            "Epoch 18/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.8882 - accuracy: 0.7112 - val_loss: 1.0112 - val_accuracy: 0.6812\n",
            "Epoch 19/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.8820 - accuracy: 0.7127 - val_loss: 1.0252 - val_accuracy: 0.6738\n",
            "Epoch 20/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.8740 - accuracy: 0.7157 - val_loss: 0.9965 - val_accuracy: 0.6844\n",
            "Epoch 21/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.8651 - accuracy: 0.7163 - val_loss: 1.0082 - val_accuracy: 0.6806\n",
            "Epoch 22/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.8616 - accuracy: 0.7183 - val_loss: 1.0323 - val_accuracy: 0.6756\n",
            "Epoch 23/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.8522 - accuracy: 0.7224 - val_loss: 1.0081 - val_accuracy: 0.6844\n",
            "Epoch 24/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.8452 - accuracy: 0.7222 - val_loss: 1.0161 - val_accuracy: 0.6835\n",
            "Epoch 25/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.8399 - accuracy: 0.7246 - val_loss: 1.0232 - val_accuracy: 0.6781\n",
            "Epoch 26/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.8339 - accuracy: 0.7272 - val_loss: 1.0252 - val_accuracy: 0.6783\n",
            "Epoch 27/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.8288 - accuracy: 0.7276 - val_loss: 1.0193 - val_accuracy: 0.6835\n",
            "Epoch 28/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.8209 - accuracy: 0.7303 - val_loss: 1.0239 - val_accuracy: 0.6792\n",
            "Epoch 29/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.8184 - accuracy: 0.7303 - val_loss: 1.0172 - val_accuracy: 0.6767\n",
            "Epoch 30/50\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8110 - accuracy: 0.7323 - val_loss: 1.0245 - val_accuracy: 0.6761\n",
            "Epoch 31/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.8049 - accuracy: 0.7323 - val_loss: 1.0213 - val_accuracy: 0.6799\n",
            "Epoch 32/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.7950 - accuracy: 0.7365 - val_loss: 1.0420 - val_accuracy: 0.6734\n",
            "Epoch 33/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.7935 - accuracy: 0.7392 - val_loss: 1.0313 - val_accuracy: 0.6770\n",
            "Epoch 34/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.7853 - accuracy: 0.7423 - val_loss: 1.0357 - val_accuracy: 0.6797\n",
            "Epoch 35/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.7761 - accuracy: 0.7424 - val_loss: 1.0401 - val_accuracy: 0.6759\n",
            "Epoch 36/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.7763 - accuracy: 0.7417 - val_loss: 1.0404 - val_accuracy: 0.6837\n",
            "Epoch 37/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.7721 - accuracy: 0.7437 - val_loss: 1.0524 - val_accuracy: 0.6799\n",
            "Epoch 38/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.7659 - accuracy: 0.7437 - val_loss: 1.0452 - val_accuracy: 0.6736\n",
            "Epoch 39/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.7604 - accuracy: 0.7461 - val_loss: 1.0382 - val_accuracy: 0.6808\n",
            "Epoch 40/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.7517 - accuracy: 0.7518 - val_loss: 1.0435 - val_accuracy: 0.6803\n",
            "Epoch 41/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.7560 - accuracy: 0.7500 - val_loss: 1.0483 - val_accuracy: 0.6736\n",
            "Epoch 42/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.7490 - accuracy: 0.7515 - val_loss: 1.0443 - val_accuracy: 0.6783\n",
            "Epoch 43/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.7403 - accuracy: 0.7557 - val_loss: 1.0554 - val_accuracy: 0.6828\n",
            "Epoch 44/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.7368 - accuracy: 0.7548 - val_loss: 1.0588 - val_accuracy: 0.6792\n",
            "Epoch 45/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.7400 - accuracy: 0.7544 - val_loss: 1.0533 - val_accuracy: 0.6772\n",
            "Epoch 46/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.7295 - accuracy: 0.7588 - val_loss: 1.0672 - val_accuracy: 0.6772\n",
            "Epoch 47/50\n",
            "1256/1256 [==============================] - 9s 8ms/step - loss: 0.7246 - accuracy: 0.7574 - val_loss: 1.0791 - val_accuracy: 0.6698\n",
            "Epoch 48/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.7179 - accuracy: 0.7611 - val_loss: 1.0610 - val_accuracy: 0.6812\n",
            "Epoch 49/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.7178 - accuracy: 0.7609 - val_loss: 1.0860 - val_accuracy: 0.6763\n",
            "Epoch 50/50\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.7186 - accuracy: 0.7623 - val_loss: 1.0571 - val_accuracy: 0.6720\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# this was one of the best performing models from earlier, i'll just let it go\n",
        "layer_list = [ #520 parameters in input\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Dropout(0.3),\n",
        "          layers.Dense(200,kernel_initializer='lecun_normal'), #let's try dimensionality reduction\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(256,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          #layers.Dense(128,kernel_initializer='lecun_normal'),\n",
        "          #layers.BatchNormalization(),\n",
        "          #layers.Activation('selu'),\n",
        "          #layers.Dense(64,kernel_initializer='lecun_normal'),\n",
        "          #layers.BatchNormalization(),\n",
        "          #layers.Activation('selu'),\n",
        "          layers.Dense(32,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(16,activation='softmax',kernel_initializer='lecun_normal'),\n",
        "          ]\n",
        "model = Sequential(layer_list)\n",
        "model.compile(optimizer=keras.optimizers.Adam(\n",
        "    learning_rate=keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.001,decay_steps=2000,decay_rate=0.99)),\n",
        "    loss=\"categorical_crossentropy\", metrics=['accuracy'],)\n",
        "history = model.fit(xtrain, ytrain_1hot, batch_size=32, epochs=500, validation_data=(xval, yval_1hot))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "-fMk9Eyvjto_",
        "outputId": "28fcc5c7-70f8-489e-e549-f6d6196d3e6e"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/500\n",
            "1256/1256 [==============================] - 10s 7ms/step - loss: 1.4226 - accuracy: 0.5664 - val_loss: 1.1653 - val_accuracy: 0.6288\n",
            "Epoch 2/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.2267 - accuracy: 0.6034 - val_loss: 1.1129 - val_accuracy: 0.6546\n",
            "Epoch 3/500\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 1.1940 - accuracy: 0.6137 - val_loss: 1.0987 - val_accuracy: 0.6478\n",
            "Epoch 4/500\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 1.1734 - accuracy: 0.6170 - val_loss: 1.0802 - val_accuracy: 0.6508\n",
            "Epoch 5/500\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 1.1497 - accuracy: 0.6248 - val_loss: 1.0732 - val_accuracy: 0.6546\n",
            "Epoch 6/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.1307 - accuracy: 0.6329 - val_loss: 1.0554 - val_accuracy: 0.6559\n",
            "Epoch 7/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.1108 - accuracy: 0.6350 - val_loss: 1.0343 - val_accuracy: 0.6647\n",
            "Epoch 8/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.0947 - accuracy: 0.6433 - val_loss: 1.0301 - val_accuracy: 0.6698\n",
            "Epoch 9/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.0810 - accuracy: 0.6451 - val_loss: 1.0173 - val_accuracy: 0.6718\n",
            "Epoch 10/500\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 1.0633 - accuracy: 0.6530 - val_loss: 1.0145 - val_accuracy: 0.6631\n",
            "Epoch 11/500\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 1.0556 - accuracy: 0.6531 - val_loss: 0.9999 - val_accuracy: 0.6736\n",
            "Epoch 12/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.0386 - accuracy: 0.6589 - val_loss: 1.0089 - val_accuracy: 0.6759\n",
            "Epoch 13/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.0258 - accuracy: 0.6615 - val_loss: 0.9826 - val_accuracy: 0.6792\n",
            "Epoch 14/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.0118 - accuracy: 0.6650 - val_loss: 0.9840 - val_accuracy: 0.6759\n",
            "Epoch 15/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 1.0048 - accuracy: 0.6675 - val_loss: 0.9817 - val_accuracy: 0.6767\n",
            "Epoch 16/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.9871 - accuracy: 0.6728 - val_loss: 0.9700 - val_accuracy: 0.6797\n",
            "Epoch 17/500\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.9710 - accuracy: 0.6791 - val_loss: 0.9812 - val_accuracy: 0.6767\n",
            "Epoch 18/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.9653 - accuracy: 0.6803 - val_loss: 0.9736 - val_accuracy: 0.6815\n",
            "Epoch 19/500\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.9491 - accuracy: 0.6829 - val_loss: 0.9495 - val_accuracy: 0.6868\n",
            "Epoch 20/500\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.9404 - accuracy: 0.6881 - val_loss: 0.9613 - val_accuracy: 0.6848\n",
            "Epoch 21/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.9278 - accuracy: 0.6864 - val_loss: 0.9543 - val_accuracy: 0.6913\n",
            "Epoch 22/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.9167 - accuracy: 0.6958 - val_loss: 0.9644 - val_accuracy: 0.6857\n",
            "Epoch 23/500\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.9092 - accuracy: 0.6953 - val_loss: 0.9589 - val_accuracy: 0.6884\n",
            "Epoch 24/500\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.8964 - accuracy: 0.6988 - val_loss: 0.9615 - val_accuracy: 0.6915\n",
            "Epoch 25/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8837 - accuracy: 0.7029 - val_loss: 0.9521 - val_accuracy: 0.6911\n",
            "Epoch 26/500\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.8781 - accuracy: 0.7060 - val_loss: 0.9520 - val_accuracy: 0.6931\n",
            "Epoch 27/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8718 - accuracy: 0.7056 - val_loss: 0.9573 - val_accuracy: 0.6927\n",
            "Epoch 28/500\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.8680 - accuracy: 0.7063 - val_loss: 0.9416 - val_accuracy: 0.6985\n",
            "Epoch 29/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8534 - accuracy: 0.7129 - val_loss: 0.9441 - val_accuracy: 0.6987\n",
            "Epoch 30/500\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.8518 - accuracy: 0.7118 - val_loss: 0.9560 - val_accuracy: 0.6920\n",
            "Epoch 31/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8411 - accuracy: 0.7155 - val_loss: 0.9632 - val_accuracy: 0.6902\n",
            "Epoch 32/500\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.8307 - accuracy: 0.7211 - val_loss: 0.9526 - val_accuracy: 0.6931\n",
            "Epoch 33/500\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.8214 - accuracy: 0.7226 - val_loss: 0.9485 - val_accuracy: 0.6951\n",
            "Epoch 34/500\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.8217 - accuracy: 0.7204 - val_loss: 0.9562 - val_accuracy: 0.6888\n",
            "Epoch 35/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8122 - accuracy: 0.7243 - val_loss: 0.9531 - val_accuracy: 0.6949\n",
            "Epoch 36/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.8089 - accuracy: 0.7260 - val_loss: 0.9632 - val_accuracy: 0.6895\n",
            "Epoch 37/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7986 - accuracy: 0.7271 - val_loss: 0.9580 - val_accuracy: 0.7043\n",
            "Epoch 38/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7892 - accuracy: 0.7306 - val_loss: 0.9531 - val_accuracy: 0.6947\n",
            "Epoch 39/500\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.7820 - accuracy: 0.7302 - val_loss: 0.9654 - val_accuracy: 0.6953\n",
            "Epoch 40/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7851 - accuracy: 0.7311 - val_loss: 0.9622 - val_accuracy: 0.6962\n",
            "Epoch 41/500\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.7756 - accuracy: 0.7358 - val_loss: 0.9711 - val_accuracy: 0.6924\n",
            "Epoch 42/500\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.7652 - accuracy: 0.7394 - val_loss: 0.9655 - val_accuracy: 0.6960\n",
            "Epoch 43/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7590 - accuracy: 0.7410 - val_loss: 0.9480 - val_accuracy: 0.7036\n",
            "Epoch 44/500\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.7562 - accuracy: 0.7398 - val_loss: 0.9748 - val_accuracy: 0.6924\n",
            "Epoch 45/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7533 - accuracy: 0.7417 - val_loss: 0.9683 - val_accuracy: 0.6962\n",
            "Epoch 46/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7463 - accuracy: 0.7436 - val_loss: 0.9757 - val_accuracy: 0.6927\n",
            "Epoch 47/500\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.7424 - accuracy: 0.7449 - val_loss: 0.9789 - val_accuracy: 0.6947\n",
            "Epoch 48/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7369 - accuracy: 0.7462 - val_loss: 0.9730 - val_accuracy: 0.6989\n",
            "Epoch 49/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7341 - accuracy: 0.7476 - val_loss: 0.9775 - val_accuracy: 0.6978\n",
            "Epoch 50/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7266 - accuracy: 0.7490 - val_loss: 0.9779 - val_accuracy: 0.6991\n",
            "Epoch 51/500\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.7261 - accuracy: 0.7499 - val_loss: 0.9963 - val_accuracy: 0.6906\n",
            "Epoch 52/500\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.7218 - accuracy: 0.7513 - val_loss: 0.9707 - val_accuracy: 0.6994\n",
            "Epoch 53/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7233 - accuracy: 0.7517 - val_loss: 0.9878 - val_accuracy: 0.6931\n",
            "Epoch 54/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7074 - accuracy: 0.7528 - val_loss: 0.9906 - val_accuracy: 0.6956\n",
            "Epoch 55/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7125 - accuracy: 0.7540 - val_loss: 0.9952 - val_accuracy: 0.6904\n",
            "Epoch 56/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7024 - accuracy: 0.7574 - val_loss: 0.9907 - val_accuracy: 0.6922\n",
            "Epoch 57/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.7004 - accuracy: 0.7551 - val_loss: 0.9805 - val_accuracy: 0.6960\n",
            "Epoch 58/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6957 - accuracy: 0.7601 - val_loss: 0.9875 - val_accuracy: 0.6884\n",
            "Epoch 59/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6956 - accuracy: 0.7591 - val_loss: 0.9828 - val_accuracy: 0.7025\n",
            "Epoch 60/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6859 - accuracy: 0.7627 - val_loss: 0.9822 - val_accuracy: 0.6951\n",
            "Epoch 61/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6868 - accuracy: 0.7638 - val_loss: 0.9818 - val_accuracy: 0.7050\n",
            "Epoch 62/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6801 - accuracy: 0.7612 - val_loss: 0.9731 - val_accuracy: 0.7045\n",
            "Epoch 63/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6751 - accuracy: 0.7647 - val_loss: 1.0035 - val_accuracy: 0.6873\n",
            "Epoch 64/500\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.6824 - accuracy: 0.7632 - val_loss: 0.9803 - val_accuracy: 0.7016\n",
            "Epoch 65/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6718 - accuracy: 0.7675 - val_loss: 0.9884 - val_accuracy: 0.6897\n",
            "Epoch 66/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6694 - accuracy: 0.7680 - val_loss: 0.9909 - val_accuracy: 0.6927\n",
            "Epoch 67/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6599 - accuracy: 0.7709 - val_loss: 1.0123 - val_accuracy: 0.6904\n",
            "Epoch 68/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6703 - accuracy: 0.7672 - val_loss: 0.9928 - val_accuracy: 0.7023\n",
            "Epoch 69/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6605 - accuracy: 0.7710 - val_loss: 0.9917 - val_accuracy: 0.7021\n",
            "Epoch 70/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6563 - accuracy: 0.7713 - val_loss: 1.0232 - val_accuracy: 0.6920\n",
            "Epoch 71/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6554 - accuracy: 0.7712 - val_loss: 1.0083 - val_accuracy: 0.6942\n",
            "Epoch 72/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6487 - accuracy: 0.7741 - val_loss: 1.0032 - val_accuracy: 0.7012\n",
            "Epoch 73/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6458 - accuracy: 0.7743 - val_loss: 1.0155 - val_accuracy: 0.6873\n",
            "Epoch 74/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6401 - accuracy: 0.7766 - val_loss: 1.0284 - val_accuracy: 0.6879\n",
            "Epoch 75/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6412 - accuracy: 0.7759 - val_loss: 1.0126 - val_accuracy: 0.6949\n",
            "Epoch 76/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6387 - accuracy: 0.7771 - val_loss: 1.0227 - val_accuracy: 0.6951\n",
            "Epoch 77/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6365 - accuracy: 0.7766 - val_loss: 1.0110 - val_accuracy: 0.6931\n",
            "Epoch 78/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6302 - accuracy: 0.7813 - val_loss: 1.0268 - val_accuracy: 0.6915\n",
            "Epoch 79/500\n",
            "1256/1256 [==============================] - 8s 7ms/step - loss: 0.6329 - accuracy: 0.7791 - val_loss: 1.0214 - val_accuracy: 0.6875\n",
            "Epoch 80/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6225 - accuracy: 0.7847 - val_loss: 1.0197 - val_accuracy: 0.6974\n",
            "Epoch 81/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6267 - accuracy: 0.7798 - val_loss: 1.0145 - val_accuracy: 0.6940\n",
            "Epoch 82/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6263 - accuracy: 0.7808 - val_loss: 1.0157 - val_accuracy: 0.6904\n",
            "Epoch 83/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6224 - accuracy: 0.7831 - val_loss: 1.0083 - val_accuracy: 0.6944\n",
            "Epoch 84/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6193 - accuracy: 0.7819 - val_loss: 1.0203 - val_accuracy: 0.6951\n",
            "Epoch 85/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6165 - accuracy: 0.7841 - val_loss: 1.0262 - val_accuracy: 0.6962\n",
            "Epoch 86/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6128 - accuracy: 0.7854 - val_loss: 1.0229 - val_accuracy: 0.6956\n",
            "Epoch 87/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6130 - accuracy: 0.7857 - val_loss: 1.0284 - val_accuracy: 0.6904\n",
            "Epoch 88/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6086 - accuracy: 0.7849 - val_loss: 1.0269 - val_accuracy: 0.6920\n",
            "Epoch 89/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6118 - accuracy: 0.7854 - val_loss: 1.0190 - val_accuracy: 0.6897\n",
            "Epoch 90/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6055 - accuracy: 0.7870 - val_loss: 1.0184 - val_accuracy: 0.6920\n",
            "Epoch 91/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.6063 - accuracy: 0.7866 - val_loss: 1.0213 - val_accuracy: 0.6895\n",
            "Epoch 92/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5981 - accuracy: 0.7929 - val_loss: 1.0308 - val_accuracy: 0.6947\n",
            "Epoch 93/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5933 - accuracy: 0.7934 - val_loss: 1.0385 - val_accuracy: 0.6944\n",
            "Epoch 94/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5985 - accuracy: 0.7886 - val_loss: 1.0294 - val_accuracy: 0.6980\n",
            "Epoch 95/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5977 - accuracy: 0.7909 - val_loss: 1.0399 - val_accuracy: 0.6978\n",
            "Epoch 96/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5962 - accuracy: 0.7899 - val_loss: 1.0317 - val_accuracy: 0.6974\n",
            "Epoch 97/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5915 - accuracy: 0.7918 - val_loss: 1.0281 - val_accuracy: 0.6987\n",
            "Epoch 98/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5862 - accuracy: 0.7950 - val_loss: 1.0318 - val_accuracy: 0.6962\n",
            "Epoch 99/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5853 - accuracy: 0.7942 - val_loss: 1.0390 - val_accuracy: 0.6935\n",
            "Epoch 100/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5838 - accuracy: 0.7928 - val_loss: 1.0414 - val_accuracy: 0.6944\n",
            "Epoch 101/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5875 - accuracy: 0.7950 - val_loss: 1.0393 - val_accuracy: 0.6983\n",
            "Epoch 102/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5883 - accuracy: 0.7942 - val_loss: 1.0423 - val_accuracy: 0.6893\n",
            "Epoch 103/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5822 - accuracy: 0.7952 - val_loss: 1.0399 - val_accuracy: 0.6915\n",
            "Epoch 104/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5739 - accuracy: 0.7985 - val_loss: 1.0543 - val_accuracy: 0.6918\n",
            "Epoch 105/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5791 - accuracy: 0.7979 - val_loss: 1.0547 - val_accuracy: 0.6864\n",
            "Epoch 106/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5690 - accuracy: 0.7992 - val_loss: 1.0414 - val_accuracy: 0.6947\n",
            "Epoch 107/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5794 - accuracy: 0.7966 - val_loss: 1.0413 - val_accuracy: 0.6900\n",
            "Epoch 108/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5679 - accuracy: 0.8012 - val_loss: 1.0317 - val_accuracy: 0.6935\n",
            "Epoch 109/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5744 - accuracy: 0.7987 - val_loss: 1.0461 - val_accuracy: 0.6967\n",
            "Epoch 110/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5703 - accuracy: 0.7998 - val_loss: 1.0362 - val_accuracy: 0.6953\n",
            "Epoch 111/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5653 - accuracy: 0.8022 - val_loss: 1.0395 - val_accuracy: 0.6965\n",
            "Epoch 112/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5677 - accuracy: 0.7994 - val_loss: 1.0491 - val_accuracy: 0.6931\n",
            "Epoch 113/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5685 - accuracy: 0.7990 - val_loss: 1.0284 - val_accuracy: 0.7005\n",
            "Epoch 114/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5604 - accuracy: 0.8037 - val_loss: 1.0385 - val_accuracy: 0.6938\n",
            "Epoch 115/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5628 - accuracy: 0.8032 - val_loss: 1.0529 - val_accuracy: 0.6929\n",
            "Epoch 116/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5623 - accuracy: 0.8029 - val_loss: 1.0440 - val_accuracy: 0.6956\n",
            "Epoch 117/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5650 - accuracy: 0.7987 - val_loss: 1.0354 - val_accuracy: 0.6947\n",
            "Epoch 118/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5569 - accuracy: 0.8016 - val_loss: 1.0599 - val_accuracy: 0.6918\n",
            "Epoch 119/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5607 - accuracy: 0.8028 - val_loss: 1.0318 - val_accuracy: 0.7000\n",
            "Epoch 120/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5572 - accuracy: 0.8033 - val_loss: 1.0606 - val_accuracy: 0.6949\n",
            "Epoch 121/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5548 - accuracy: 0.8051 - val_loss: 1.0416 - val_accuracy: 0.7059\n",
            "Epoch 122/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5583 - accuracy: 0.8024 - val_loss: 1.0485 - val_accuracy: 0.6931\n",
            "Epoch 123/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5503 - accuracy: 0.8078 - val_loss: 1.0414 - val_accuracy: 0.6991\n",
            "Epoch 124/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5535 - accuracy: 0.8030 - val_loss: 1.0545 - val_accuracy: 0.6987\n",
            "Epoch 125/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5384 - accuracy: 0.8100 - val_loss: 1.0500 - val_accuracy: 0.6998\n",
            "Epoch 126/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5426 - accuracy: 0.8082 - val_loss: 1.0588 - val_accuracy: 0.6976\n",
            "Epoch 127/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5468 - accuracy: 0.8082 - val_loss: 1.0592 - val_accuracy: 0.6956\n",
            "Epoch 128/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5458 - accuracy: 0.8083 - val_loss: 1.0560 - val_accuracy: 0.6980\n",
            "Epoch 129/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5423 - accuracy: 0.8073 - val_loss: 1.0664 - val_accuracy: 0.6933\n",
            "Epoch 130/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5387 - accuracy: 0.8091 - val_loss: 1.0562 - val_accuracy: 0.6931\n",
            "Epoch 131/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5439 - accuracy: 0.8090 - val_loss: 1.0548 - val_accuracy: 0.6951\n",
            "Epoch 132/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5379 - accuracy: 0.8093 - val_loss: 1.0556 - val_accuracy: 0.6915\n",
            "Epoch 133/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5369 - accuracy: 0.8086 - val_loss: 1.0664 - val_accuracy: 0.6960\n",
            "Epoch 134/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5324 - accuracy: 0.8141 - val_loss: 1.0707 - val_accuracy: 0.6935\n",
            "Epoch 135/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5369 - accuracy: 0.8124 - val_loss: 1.0647 - val_accuracy: 0.6953\n",
            "Epoch 136/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5338 - accuracy: 0.8115 - val_loss: 1.0789 - val_accuracy: 0.6944\n",
            "Epoch 137/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5378 - accuracy: 0.8107 - val_loss: 1.0795 - val_accuracy: 0.6915\n",
            "Epoch 138/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5336 - accuracy: 0.8120 - val_loss: 1.0610 - val_accuracy: 0.6974\n",
            "Epoch 139/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5252 - accuracy: 0.8143 - val_loss: 1.0633 - val_accuracy: 0.6967\n",
            "Epoch 140/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5292 - accuracy: 0.8139 - val_loss: 1.0720 - val_accuracy: 0.6893\n",
            "Epoch 141/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5275 - accuracy: 0.8118 - val_loss: 1.0838 - val_accuracy: 0.6909\n",
            "Epoch 142/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5288 - accuracy: 0.8140 - val_loss: 1.0770 - val_accuracy: 0.6947\n",
            "Epoch 143/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5249 - accuracy: 0.8149 - val_loss: 1.0782 - val_accuracy: 0.6922\n",
            "Epoch 144/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5253 - accuracy: 0.8149 - val_loss: 1.0809 - val_accuracy: 0.6935\n",
            "Epoch 145/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5267 - accuracy: 0.8149 - val_loss: 1.0773 - val_accuracy: 0.6866\n",
            "Epoch 146/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5245 - accuracy: 0.8127 - val_loss: 1.0641 - val_accuracy: 0.6980\n",
            "Epoch 147/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5232 - accuracy: 0.8131 - val_loss: 1.0720 - val_accuracy: 0.6906\n",
            "Epoch 148/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5267 - accuracy: 0.8148 - val_loss: 1.0689 - val_accuracy: 0.6906\n",
            "Epoch 149/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5228 - accuracy: 0.8153 - val_loss: 1.0852 - val_accuracy: 0.6965\n",
            "Epoch 150/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5191 - accuracy: 0.8171 - val_loss: 1.0919 - val_accuracy: 0.6913\n",
            "Epoch 151/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5190 - accuracy: 0.8178 - val_loss: 1.0820 - val_accuracy: 0.6913\n",
            "Epoch 152/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5193 - accuracy: 0.8161 - val_loss: 1.0762 - val_accuracy: 0.6949\n",
            "Epoch 153/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5177 - accuracy: 0.8181 - val_loss: 1.0828 - val_accuracy: 0.7005\n",
            "Epoch 154/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5131 - accuracy: 0.8183 - val_loss: 1.0936 - val_accuracy: 0.6911\n",
            "Epoch 155/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5140 - accuracy: 0.8187 - val_loss: 1.0916 - val_accuracy: 0.6911\n",
            "Epoch 156/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5206 - accuracy: 0.8152 - val_loss: 1.0840 - val_accuracy: 0.6938\n",
            "Epoch 157/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5102 - accuracy: 0.8194 - val_loss: 1.0890 - val_accuracy: 0.6949\n",
            "Epoch 158/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5133 - accuracy: 0.8186 - val_loss: 1.0937 - val_accuracy: 0.6897\n",
            "Epoch 159/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5089 - accuracy: 0.8217 - val_loss: 1.0971 - val_accuracy: 0.6902\n",
            "Epoch 160/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5109 - accuracy: 0.8194 - val_loss: 1.0925 - val_accuracy: 0.6871\n",
            "Epoch 161/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5074 - accuracy: 0.8198 - val_loss: 1.0878 - val_accuracy: 0.6935\n",
            "Epoch 162/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5137 - accuracy: 0.8173 - val_loss: 1.0765 - val_accuracy: 0.6980\n",
            "Epoch 163/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5079 - accuracy: 0.8209 - val_loss: 1.0774 - val_accuracy: 0.6998\n",
            "Epoch 164/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5080 - accuracy: 0.8220 - val_loss: 1.0824 - val_accuracy: 0.6971\n",
            "Epoch 165/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5034 - accuracy: 0.8217 - val_loss: 1.0981 - val_accuracy: 0.6951\n",
            "Epoch 166/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5099 - accuracy: 0.8203 - val_loss: 1.0846 - val_accuracy: 0.6915\n",
            "Epoch 167/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5049 - accuracy: 0.8210 - val_loss: 1.0894 - val_accuracy: 0.6947\n",
            "Epoch 168/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4972 - accuracy: 0.8257 - val_loss: 1.0861 - val_accuracy: 0.6911\n",
            "Epoch 169/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5049 - accuracy: 0.8211 - val_loss: 1.0922 - val_accuracy: 0.6940\n",
            "Epoch 170/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5057 - accuracy: 0.8220 - val_loss: 1.0903 - val_accuracy: 0.6911\n",
            "Epoch 171/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4987 - accuracy: 0.8222 - val_loss: 1.0861 - val_accuracy: 0.6915\n",
            "Epoch 172/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5049 - accuracy: 0.8187 - val_loss: 1.0911 - val_accuracy: 0.6931\n",
            "Epoch 173/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5001 - accuracy: 0.8234 - val_loss: 1.0802 - val_accuracy: 0.6978\n",
            "Epoch 174/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4959 - accuracy: 0.8256 - val_loss: 1.0887 - val_accuracy: 0.6949\n",
            "Epoch 175/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4962 - accuracy: 0.8245 - val_loss: 1.0999 - val_accuracy: 0.6924\n",
            "Epoch 176/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4992 - accuracy: 0.8224 - val_loss: 1.0943 - val_accuracy: 0.6956\n",
            "Epoch 177/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.5000 - accuracy: 0.8238 - val_loss: 1.0946 - val_accuracy: 0.6938\n",
            "Epoch 178/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4976 - accuracy: 0.8241 - val_loss: 1.0862 - val_accuracy: 0.6940\n",
            "Epoch 179/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4950 - accuracy: 0.8251 - val_loss: 1.0834 - val_accuracy: 0.6989\n",
            "Epoch 180/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4925 - accuracy: 0.8249 - val_loss: 1.1052 - val_accuracy: 0.6935\n",
            "Epoch 181/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4928 - accuracy: 0.8260 - val_loss: 1.0904 - val_accuracy: 0.6893\n",
            "Epoch 182/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4918 - accuracy: 0.8259 - val_loss: 1.0977 - val_accuracy: 0.6913\n",
            "Epoch 183/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4907 - accuracy: 0.8262 - val_loss: 1.1109 - val_accuracy: 0.6949\n",
            "Epoch 184/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4938 - accuracy: 0.8261 - val_loss: 1.0982 - val_accuracy: 0.6980\n",
            "Epoch 185/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4903 - accuracy: 0.8250 - val_loss: 1.1013 - val_accuracy: 0.6902\n",
            "Epoch 186/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4875 - accuracy: 0.8290 - val_loss: 1.0997 - val_accuracy: 0.6888\n",
            "Epoch 187/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4841 - accuracy: 0.8267 - val_loss: 1.0960 - val_accuracy: 0.6951\n",
            "Epoch 188/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4851 - accuracy: 0.8260 - val_loss: 1.1041 - val_accuracy: 0.6947\n",
            "Epoch 189/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4858 - accuracy: 0.8282 - val_loss: 1.1037 - val_accuracy: 0.6974\n",
            "Epoch 190/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4870 - accuracy: 0.8261 - val_loss: 1.1055 - val_accuracy: 0.6891\n",
            "Epoch 191/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4878 - accuracy: 0.8285 - val_loss: 1.1121 - val_accuracy: 0.6924\n",
            "Epoch 192/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4877 - accuracy: 0.8276 - val_loss: 1.1126 - val_accuracy: 0.6956\n",
            "Epoch 193/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4807 - accuracy: 0.8303 - val_loss: 1.1042 - val_accuracy: 0.6942\n",
            "Epoch 194/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4815 - accuracy: 0.8295 - val_loss: 1.1021 - val_accuracy: 0.6991\n",
            "Epoch 195/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4892 - accuracy: 0.8285 - val_loss: 1.1000 - val_accuracy: 0.6987\n",
            "Epoch 196/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4834 - accuracy: 0.8289 - val_loss: 1.1095 - val_accuracy: 0.6929\n",
            "Epoch 197/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4834 - accuracy: 0.8296 - val_loss: 1.1038 - val_accuracy: 0.6938\n",
            "Epoch 198/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4808 - accuracy: 0.8303 - val_loss: 1.1021 - val_accuracy: 0.6935\n",
            "Epoch 199/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4732 - accuracy: 0.8323 - val_loss: 1.1147 - val_accuracy: 0.6958\n",
            "Epoch 200/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4798 - accuracy: 0.8321 - val_loss: 1.1100 - val_accuracy: 0.6929\n",
            "Epoch 201/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4747 - accuracy: 0.8316 - val_loss: 1.1135 - val_accuracy: 0.6913\n",
            "Epoch 202/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4730 - accuracy: 0.8312 - val_loss: 1.1015 - val_accuracy: 0.6935\n",
            "Epoch 203/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4786 - accuracy: 0.8287 - val_loss: 1.1235 - val_accuracy: 0.6906\n",
            "Epoch 204/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4755 - accuracy: 0.8295 - val_loss: 1.1216 - val_accuracy: 0.6875\n",
            "Epoch 205/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4791 - accuracy: 0.8294 - val_loss: 1.1172 - val_accuracy: 0.6920\n",
            "Epoch 206/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4759 - accuracy: 0.8307 - val_loss: 1.1150 - val_accuracy: 0.6924\n",
            "Epoch 207/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4723 - accuracy: 0.8319 - val_loss: 1.1124 - val_accuracy: 0.6904\n",
            "Epoch 208/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4679 - accuracy: 0.8340 - val_loss: 1.1252 - val_accuracy: 0.6859\n",
            "Epoch 209/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4678 - accuracy: 0.8326 - val_loss: 1.1222 - val_accuracy: 0.6895\n",
            "Epoch 210/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4758 - accuracy: 0.8309 - val_loss: 1.1159 - val_accuracy: 0.6927\n",
            "Epoch 211/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4650 - accuracy: 0.8336 - val_loss: 1.1145 - val_accuracy: 0.6958\n",
            "Epoch 212/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4668 - accuracy: 0.8344 - val_loss: 1.1228 - val_accuracy: 0.6909\n",
            "Epoch 213/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4697 - accuracy: 0.8353 - val_loss: 1.1212 - val_accuracy: 0.6897\n",
            "Epoch 214/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4616 - accuracy: 0.8367 - val_loss: 1.1205 - val_accuracy: 0.6938\n",
            "Epoch 215/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4663 - accuracy: 0.8333 - val_loss: 1.1180 - val_accuracy: 0.6902\n",
            "Epoch 216/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4767 - accuracy: 0.8293 - val_loss: 1.1095 - val_accuracy: 0.6949\n",
            "Epoch 217/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4710 - accuracy: 0.8330 - val_loss: 1.1203 - val_accuracy: 0.6900\n",
            "Epoch 218/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4699 - accuracy: 0.8321 - val_loss: 1.1135 - val_accuracy: 0.6922\n",
            "Epoch 219/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4657 - accuracy: 0.8345 - val_loss: 1.1124 - val_accuracy: 0.6953\n",
            "Epoch 220/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4696 - accuracy: 0.8328 - val_loss: 1.1266 - val_accuracy: 0.6913\n",
            "Epoch 221/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4644 - accuracy: 0.8333 - val_loss: 1.1167 - val_accuracy: 0.6900\n",
            "Epoch 222/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4730 - accuracy: 0.8317 - val_loss: 1.1113 - val_accuracy: 0.6947\n",
            "Epoch 223/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4619 - accuracy: 0.8381 - val_loss: 1.1094 - val_accuracy: 0.6913\n",
            "Epoch 224/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4584 - accuracy: 0.8381 - val_loss: 1.1075 - val_accuracy: 0.6938\n",
            "Epoch 225/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4678 - accuracy: 0.8345 - val_loss: 1.1041 - val_accuracy: 0.6953\n",
            "Epoch 226/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4624 - accuracy: 0.8345 - val_loss: 1.1301 - val_accuracy: 0.6915\n",
            "Epoch 227/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4630 - accuracy: 0.8351 - val_loss: 1.1155 - val_accuracy: 0.6931\n",
            "Epoch 228/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4608 - accuracy: 0.8349 - val_loss: 1.1230 - val_accuracy: 0.6931\n",
            "Epoch 229/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4672 - accuracy: 0.8368 - val_loss: 1.1225 - val_accuracy: 0.6888\n",
            "Epoch 230/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4642 - accuracy: 0.8367 - val_loss: 1.1085 - val_accuracy: 0.6929\n",
            "Epoch 231/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4613 - accuracy: 0.8356 - val_loss: 1.1210 - val_accuracy: 0.6953\n",
            "Epoch 232/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4623 - accuracy: 0.8343 - val_loss: 1.1069 - val_accuracy: 0.6927\n",
            "Epoch 233/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4511 - accuracy: 0.8385 - val_loss: 1.1090 - val_accuracy: 0.6940\n",
            "Epoch 234/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4575 - accuracy: 0.8387 - val_loss: 1.1219 - val_accuracy: 0.6915\n",
            "Epoch 235/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4595 - accuracy: 0.8371 - val_loss: 1.1302 - val_accuracy: 0.6886\n",
            "Epoch 236/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4633 - accuracy: 0.8343 - val_loss: 1.1224 - val_accuracy: 0.6942\n",
            "Epoch 237/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4621 - accuracy: 0.8369 - val_loss: 1.1227 - val_accuracy: 0.6942\n",
            "Epoch 238/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4605 - accuracy: 0.8385 - val_loss: 1.1302 - val_accuracy: 0.6927\n",
            "Epoch 239/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4514 - accuracy: 0.8380 - val_loss: 1.1274 - val_accuracy: 0.6924\n",
            "Epoch 240/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4647 - accuracy: 0.8359 - val_loss: 1.1308 - val_accuracy: 0.6879\n",
            "Epoch 241/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4582 - accuracy: 0.8380 - val_loss: 1.1359 - val_accuracy: 0.6911\n",
            "Epoch 242/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4594 - accuracy: 0.8373 - val_loss: 1.1330 - val_accuracy: 0.6909\n",
            "Epoch 243/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4537 - accuracy: 0.8370 - val_loss: 1.1291 - val_accuracy: 0.6897\n",
            "Epoch 244/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4487 - accuracy: 0.8399 - val_loss: 1.1244 - val_accuracy: 0.6897\n",
            "Epoch 245/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4509 - accuracy: 0.8390 - val_loss: 1.1214 - val_accuracy: 0.6922\n",
            "Epoch 246/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4511 - accuracy: 0.8388 - val_loss: 1.1361 - val_accuracy: 0.6920\n",
            "Epoch 247/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4527 - accuracy: 0.8395 - val_loss: 1.1315 - val_accuracy: 0.6893\n",
            "Epoch 248/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4489 - accuracy: 0.8395 - val_loss: 1.1408 - val_accuracy: 0.6862\n",
            "Epoch 249/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4523 - accuracy: 0.8398 - val_loss: 1.1396 - val_accuracy: 0.6888\n",
            "Epoch 250/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4548 - accuracy: 0.8373 - val_loss: 1.1225 - val_accuracy: 0.6900\n",
            "Epoch 251/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4568 - accuracy: 0.8374 - val_loss: 1.1364 - val_accuracy: 0.6879\n",
            "Epoch 252/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4503 - accuracy: 0.8415 - val_loss: 1.1321 - val_accuracy: 0.6888\n",
            "Epoch 253/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4503 - accuracy: 0.8394 - val_loss: 1.1323 - val_accuracy: 0.6906\n",
            "Epoch 254/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4513 - accuracy: 0.8414 - val_loss: 1.1426 - val_accuracy: 0.6931\n",
            "Epoch 255/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4501 - accuracy: 0.8386 - val_loss: 1.1327 - val_accuracy: 0.6915\n",
            "Epoch 256/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4537 - accuracy: 0.8383 - val_loss: 1.1281 - val_accuracy: 0.6931\n",
            "Epoch 257/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4476 - accuracy: 0.8405 - val_loss: 1.1312 - val_accuracy: 0.6902\n",
            "Epoch 258/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4458 - accuracy: 0.8409 - val_loss: 1.1390 - val_accuracy: 0.6906\n",
            "Epoch 259/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4479 - accuracy: 0.8417 - val_loss: 1.1384 - val_accuracy: 0.6886\n",
            "Epoch 260/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4486 - accuracy: 0.8418 - val_loss: 1.1355 - val_accuracy: 0.6902\n",
            "Epoch 261/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4419 - accuracy: 0.8416 - val_loss: 1.1293 - val_accuracy: 0.6942\n",
            "Epoch 262/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4447 - accuracy: 0.8431 - val_loss: 1.1350 - val_accuracy: 0.6891\n",
            "Epoch 263/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4497 - accuracy: 0.8387 - val_loss: 1.1297 - val_accuracy: 0.6927\n",
            "Epoch 264/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4452 - accuracy: 0.8427 - val_loss: 1.1361 - val_accuracy: 0.6913\n",
            "Epoch 265/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4463 - accuracy: 0.8425 - val_loss: 1.1302 - val_accuracy: 0.6893\n",
            "Epoch 266/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4461 - accuracy: 0.8410 - val_loss: 1.1311 - val_accuracy: 0.6922\n",
            "Epoch 267/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4409 - accuracy: 0.8441 - val_loss: 1.1244 - val_accuracy: 0.6913\n",
            "Epoch 268/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4412 - accuracy: 0.8430 - val_loss: 1.1363 - val_accuracy: 0.6888\n",
            "Epoch 269/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4485 - accuracy: 0.8401 - val_loss: 1.1273 - val_accuracy: 0.6938\n",
            "Epoch 270/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4473 - accuracy: 0.8389 - val_loss: 1.1362 - val_accuracy: 0.6927\n",
            "Epoch 271/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4545 - accuracy: 0.8372 - val_loss: 1.1315 - val_accuracy: 0.6931\n",
            "Epoch 272/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4450 - accuracy: 0.8423 - val_loss: 1.1351 - val_accuracy: 0.6902\n",
            "Epoch 273/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4432 - accuracy: 0.8434 - val_loss: 1.1274 - val_accuracy: 0.6929\n",
            "Epoch 274/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4488 - accuracy: 0.8400 - val_loss: 1.1238 - val_accuracy: 0.6944\n",
            "Epoch 275/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4412 - accuracy: 0.8432 - val_loss: 1.1380 - val_accuracy: 0.6949\n",
            "Epoch 276/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4402 - accuracy: 0.8430 - val_loss: 1.1330 - val_accuracy: 0.6935\n",
            "Epoch 277/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4429 - accuracy: 0.8440 - val_loss: 1.1370 - val_accuracy: 0.6951\n",
            "Epoch 278/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4407 - accuracy: 0.8427 - val_loss: 1.1295 - val_accuracy: 0.6922\n",
            "Epoch 279/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4388 - accuracy: 0.8450 - val_loss: 1.1496 - val_accuracy: 0.6873\n",
            "Epoch 280/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4434 - accuracy: 0.8440 - val_loss: 1.1423 - val_accuracy: 0.6918\n",
            "Epoch 281/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4457 - accuracy: 0.8400 - val_loss: 1.1372 - val_accuracy: 0.6915\n",
            "Epoch 282/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4436 - accuracy: 0.8425 - val_loss: 1.1338 - val_accuracy: 0.6891\n",
            "Epoch 283/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4470 - accuracy: 0.8398 - val_loss: 1.1384 - val_accuracy: 0.6929\n",
            "Epoch 284/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4361 - accuracy: 0.8449 - val_loss: 1.1380 - val_accuracy: 0.6942\n",
            "Epoch 285/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4392 - accuracy: 0.8438 - val_loss: 1.1353 - val_accuracy: 0.6904\n",
            "Epoch 286/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4397 - accuracy: 0.8434 - val_loss: 1.1458 - val_accuracy: 0.6949\n",
            "Epoch 287/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4401 - accuracy: 0.8448 - val_loss: 1.1251 - val_accuracy: 0.6931\n",
            "Epoch 288/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4392 - accuracy: 0.8444 - val_loss: 1.1353 - val_accuracy: 0.6931\n",
            "Epoch 289/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4327 - accuracy: 0.8464 - val_loss: 1.1507 - val_accuracy: 0.6931\n",
            "Epoch 290/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4403 - accuracy: 0.8428 - val_loss: 1.1458 - val_accuracy: 0.6942\n",
            "Epoch 291/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4397 - accuracy: 0.8456 - val_loss: 1.1451 - val_accuracy: 0.6940\n",
            "Epoch 292/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4416 - accuracy: 0.8421 - val_loss: 1.1346 - val_accuracy: 0.6900\n",
            "Epoch 293/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4461 - accuracy: 0.8428 - val_loss: 1.1446 - val_accuracy: 0.6929\n",
            "Epoch 294/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4381 - accuracy: 0.8443 - val_loss: 1.1456 - val_accuracy: 0.6931\n",
            "Epoch 295/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4352 - accuracy: 0.8445 - val_loss: 1.1437 - val_accuracy: 0.6922\n",
            "Epoch 296/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4355 - accuracy: 0.8456 - val_loss: 1.1404 - val_accuracy: 0.6915\n",
            "Epoch 297/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4305 - accuracy: 0.8475 - val_loss: 1.1454 - val_accuracy: 0.6900\n",
            "Epoch 298/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4367 - accuracy: 0.8446 - val_loss: 1.1425 - val_accuracy: 0.6913\n",
            "Epoch 299/500\n",
            "1256/1256 [==============================] - 9s 8ms/step - loss: 0.4342 - accuracy: 0.8466 - val_loss: 1.1422 - val_accuracy: 0.6904\n",
            "Epoch 300/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4324 - accuracy: 0.8465 - val_loss: 1.1405 - val_accuracy: 0.6922\n",
            "Epoch 301/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4404 - accuracy: 0.8425 - val_loss: 1.1480 - val_accuracy: 0.6958\n",
            "Epoch 302/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4448 - accuracy: 0.8432 - val_loss: 1.1381 - val_accuracy: 0.6944\n",
            "Epoch 303/500\n",
            "1256/1256 [==============================] - 9s 8ms/step - loss: 0.4317 - accuracy: 0.8451 - val_loss: 1.1393 - val_accuracy: 0.6904\n",
            "Epoch 304/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4313 - accuracy: 0.8467 - val_loss: 1.1446 - val_accuracy: 0.6915\n",
            "Epoch 305/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4346 - accuracy: 0.8443 - val_loss: 1.1485 - val_accuracy: 0.6893\n",
            "Epoch 306/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4360 - accuracy: 0.8453 - val_loss: 1.1502 - val_accuracy: 0.6895\n",
            "Epoch 307/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4356 - accuracy: 0.8466 - val_loss: 1.1484 - val_accuracy: 0.6915\n",
            "Epoch 308/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4275 - accuracy: 0.8463 - val_loss: 1.1541 - val_accuracy: 0.6947\n",
            "Epoch 309/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4313 - accuracy: 0.8477 - val_loss: 1.1504 - val_accuracy: 0.6913\n",
            "Epoch 310/500\n",
            "1256/1256 [==============================] - 9s 8ms/step - loss: 0.4306 - accuracy: 0.8464 - val_loss: 1.1449 - val_accuracy: 0.6927\n",
            "Epoch 311/500\n",
            "1256/1256 [==============================] - 9s 8ms/step - loss: 0.4401 - accuracy: 0.8432 - val_loss: 1.1416 - val_accuracy: 0.6956\n",
            "Epoch 312/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4296 - accuracy: 0.8456 - val_loss: 1.1436 - val_accuracy: 0.6918\n",
            "Epoch 313/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4290 - accuracy: 0.8467 - val_loss: 1.1521 - val_accuracy: 0.6920\n",
            "Epoch 314/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4262 - accuracy: 0.8500 - val_loss: 1.1489 - val_accuracy: 0.6931\n",
            "Epoch 315/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4311 - accuracy: 0.8468 - val_loss: 1.1535 - val_accuracy: 0.6924\n",
            "Epoch 316/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4243 - accuracy: 0.8483 - val_loss: 1.1506 - val_accuracy: 0.6938\n",
            "Epoch 317/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4237 - accuracy: 0.8494 - val_loss: 1.1447 - val_accuracy: 0.6944\n",
            "Epoch 318/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4324 - accuracy: 0.8470 - val_loss: 1.1540 - val_accuracy: 0.6913\n",
            "Epoch 319/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4271 - accuracy: 0.8473 - val_loss: 1.1504 - val_accuracy: 0.6974\n",
            "Epoch 320/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4329 - accuracy: 0.8484 - val_loss: 1.1469 - val_accuracy: 0.6913\n",
            "Epoch 321/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4278 - accuracy: 0.8482 - val_loss: 1.1555 - val_accuracy: 0.6927\n",
            "Epoch 322/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4297 - accuracy: 0.8483 - val_loss: 1.1507 - val_accuracy: 0.6938\n",
            "Epoch 323/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4260 - accuracy: 0.8463 - val_loss: 1.1414 - val_accuracy: 0.6888\n",
            "Epoch 324/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4267 - accuracy: 0.8489 - val_loss: 1.1541 - val_accuracy: 0.6931\n",
            "Epoch 325/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4331 - accuracy: 0.8462 - val_loss: 1.1481 - val_accuracy: 0.6935\n",
            "Epoch 326/500\n",
            "1256/1256 [==============================] - 9s 8ms/step - loss: 0.4298 - accuracy: 0.8472 - val_loss: 1.1477 - val_accuracy: 0.6938\n",
            "Epoch 327/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4174 - accuracy: 0.8523 - val_loss: 1.1503 - val_accuracy: 0.6922\n",
            "Epoch 328/500\n",
            "1256/1256 [==============================] - 9s 8ms/step - loss: 0.4269 - accuracy: 0.8474 - val_loss: 1.1556 - val_accuracy: 0.6904\n",
            "Epoch 329/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4281 - accuracy: 0.8463 - val_loss: 1.1584 - val_accuracy: 0.6911\n",
            "Epoch 330/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4302 - accuracy: 0.8470 - val_loss: 1.1471 - val_accuracy: 0.6927\n",
            "Epoch 331/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4266 - accuracy: 0.8486 - val_loss: 1.1513 - val_accuracy: 0.6927\n",
            "Epoch 332/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4259 - accuracy: 0.8475 - val_loss: 1.1457 - val_accuracy: 0.6911\n",
            "Epoch 333/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4219 - accuracy: 0.8496 - val_loss: 1.1536 - val_accuracy: 0.6915\n",
            "Epoch 334/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4251 - accuracy: 0.8511 - val_loss: 1.1531 - val_accuracy: 0.6915\n",
            "Epoch 335/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4232 - accuracy: 0.8476 - val_loss: 1.1503 - val_accuracy: 0.6888\n",
            "Epoch 336/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4232 - accuracy: 0.8495 - val_loss: 1.1546 - val_accuracy: 0.6933\n",
            "Epoch 337/500\n",
            "1256/1256 [==============================] - 9s 8ms/step - loss: 0.4195 - accuracy: 0.8496 - val_loss: 1.1675 - val_accuracy: 0.6909\n",
            "Epoch 338/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4251 - accuracy: 0.8480 - val_loss: 1.1555 - val_accuracy: 0.6886\n",
            "Epoch 339/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4257 - accuracy: 0.8483 - val_loss: 1.1484 - val_accuracy: 0.6891\n",
            "Epoch 340/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4237 - accuracy: 0.8496 - val_loss: 1.1439 - val_accuracy: 0.6891\n",
            "Epoch 341/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4247 - accuracy: 0.8477 - val_loss: 1.1438 - val_accuracy: 0.6944\n",
            "Epoch 342/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4263 - accuracy: 0.8486 - val_loss: 1.1516 - val_accuracy: 0.6911\n",
            "Epoch 343/500\n",
            "1256/1256 [==============================] - 9s 8ms/step - loss: 0.4191 - accuracy: 0.8529 - val_loss: 1.1542 - val_accuracy: 0.6918\n",
            "Epoch 344/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4212 - accuracy: 0.8500 - val_loss: 1.1531 - val_accuracy: 0.6886\n",
            "Epoch 345/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4241 - accuracy: 0.8479 - val_loss: 1.1482 - val_accuracy: 0.6969\n",
            "Epoch 346/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4187 - accuracy: 0.8506 - val_loss: 1.1495 - val_accuracy: 0.6933\n",
            "Epoch 347/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4222 - accuracy: 0.8491 - val_loss: 1.1584 - val_accuracy: 0.6913\n",
            "Epoch 348/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4216 - accuracy: 0.8528 - val_loss: 1.1633 - val_accuracy: 0.6913\n",
            "Epoch 349/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4212 - accuracy: 0.8517 - val_loss: 1.1628 - val_accuracy: 0.6906\n",
            "Epoch 350/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4222 - accuracy: 0.8483 - val_loss: 1.1617 - val_accuracy: 0.6911\n",
            "Epoch 351/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4223 - accuracy: 0.8506 - val_loss: 1.1669 - val_accuracy: 0.6935\n",
            "Epoch 352/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4172 - accuracy: 0.8508 - val_loss: 1.1567 - val_accuracy: 0.6884\n",
            "Epoch 353/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4186 - accuracy: 0.8507 - val_loss: 1.1612 - val_accuracy: 0.6927\n",
            "Epoch 354/500\n",
            "1256/1256 [==============================] - 9s 8ms/step - loss: 0.4206 - accuracy: 0.8493 - val_loss: 1.1689 - val_accuracy: 0.6906\n",
            "Epoch 355/500\n",
            "1256/1256 [==============================] - 9s 8ms/step - loss: 0.4230 - accuracy: 0.8488 - val_loss: 1.1666 - val_accuracy: 0.6918\n",
            "Epoch 356/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4250 - accuracy: 0.8467 - val_loss: 1.1598 - val_accuracy: 0.6938\n",
            "Epoch 357/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4200 - accuracy: 0.8521 - val_loss: 1.1608 - val_accuracy: 0.6922\n",
            "Epoch 358/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4172 - accuracy: 0.8529 - val_loss: 1.1585 - val_accuracy: 0.6911\n",
            "Epoch 359/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4207 - accuracy: 0.8526 - val_loss: 1.1578 - val_accuracy: 0.6949\n",
            "Epoch 360/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4170 - accuracy: 0.8512 - val_loss: 1.1665 - val_accuracy: 0.6904\n",
            "Epoch 361/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4200 - accuracy: 0.8511 - val_loss: 1.1551 - val_accuracy: 0.6913\n",
            "Epoch 362/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4243 - accuracy: 0.8501 - val_loss: 1.1605 - val_accuracy: 0.6900\n",
            "Epoch 363/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4171 - accuracy: 0.8517 - val_loss: 1.1550 - val_accuracy: 0.6951\n",
            "Epoch 364/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4212 - accuracy: 0.8503 - val_loss: 1.1586 - val_accuracy: 0.6922\n",
            "Epoch 365/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4204 - accuracy: 0.8493 - val_loss: 1.1515 - val_accuracy: 0.6904\n",
            "Epoch 366/500\n",
            "1256/1256 [==============================] - 9s 8ms/step - loss: 0.4180 - accuracy: 0.8517 - val_loss: 1.1618 - val_accuracy: 0.6933\n",
            "Epoch 367/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4193 - accuracy: 0.8525 - val_loss: 1.1524 - val_accuracy: 0.6940\n",
            "Epoch 368/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4144 - accuracy: 0.8512 - val_loss: 1.1679 - val_accuracy: 0.6906\n",
            "Epoch 369/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4253 - accuracy: 0.8507 - val_loss: 1.1535 - val_accuracy: 0.6929\n",
            "Epoch 370/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4122 - accuracy: 0.8534 - val_loss: 1.1588 - val_accuracy: 0.6933\n",
            "Epoch 371/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4169 - accuracy: 0.8522 - val_loss: 1.1577 - val_accuracy: 0.6965\n",
            "Epoch 372/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4222 - accuracy: 0.8484 - val_loss: 1.1644 - val_accuracy: 0.6929\n",
            "Epoch 373/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4137 - accuracy: 0.8527 - val_loss: 1.1631 - val_accuracy: 0.6965\n",
            "Epoch 374/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4184 - accuracy: 0.8522 - val_loss: 1.1663 - val_accuracy: 0.6920\n",
            "Epoch 375/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4183 - accuracy: 0.8514 - val_loss: 1.1598 - val_accuracy: 0.6929\n",
            "Epoch 376/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4160 - accuracy: 0.8535 - val_loss: 1.1590 - val_accuracy: 0.6897\n",
            "Epoch 377/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4174 - accuracy: 0.8517 - val_loss: 1.1601 - val_accuracy: 0.6888\n",
            "Epoch 378/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4107 - accuracy: 0.8523 - val_loss: 1.1574 - val_accuracy: 0.6927\n",
            "Epoch 379/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4114 - accuracy: 0.8533 - val_loss: 1.1732 - val_accuracy: 0.6893\n",
            "Epoch 380/500\n",
            "1256/1256 [==============================] - 9s 8ms/step - loss: 0.4078 - accuracy: 0.8546 - val_loss: 1.1585 - val_accuracy: 0.6922\n",
            "Epoch 381/500\n",
            "1256/1256 [==============================] - 9s 8ms/step - loss: 0.4042 - accuracy: 0.8550 - val_loss: 1.1611 - val_accuracy: 0.6900\n",
            "Epoch 382/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4178 - accuracy: 0.8520 - val_loss: 1.1646 - val_accuracy: 0.6922\n",
            "Epoch 383/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4174 - accuracy: 0.8520 - val_loss: 1.1650 - val_accuracy: 0.6895\n",
            "Epoch 384/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4146 - accuracy: 0.8517 - val_loss: 1.1684 - val_accuracy: 0.6927\n",
            "Epoch 385/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4163 - accuracy: 0.8525 - val_loss: 1.1667 - val_accuracy: 0.6944\n",
            "Epoch 386/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4126 - accuracy: 0.8552 - val_loss: 1.1674 - val_accuracy: 0.6918\n",
            "Epoch 387/500\n",
            "1256/1256 [==============================] - 9s 8ms/step - loss: 0.4194 - accuracy: 0.8514 - val_loss: 1.1522 - val_accuracy: 0.6927\n",
            "Epoch 388/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4110 - accuracy: 0.8544 - val_loss: 1.1698 - val_accuracy: 0.6931\n",
            "Epoch 389/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4105 - accuracy: 0.8524 - val_loss: 1.1588 - val_accuracy: 0.6922\n",
            "Epoch 390/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4194 - accuracy: 0.8512 - val_loss: 1.1608 - val_accuracy: 0.6951\n",
            "Epoch 391/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4084 - accuracy: 0.8550 - val_loss: 1.1702 - val_accuracy: 0.6924\n",
            "Epoch 392/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4143 - accuracy: 0.8529 - val_loss: 1.1636 - val_accuracy: 0.6933\n",
            "Epoch 393/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4150 - accuracy: 0.8522 - val_loss: 1.1617 - val_accuracy: 0.6933\n",
            "Epoch 394/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4180 - accuracy: 0.8506 - val_loss: 1.1680 - val_accuracy: 0.6913\n",
            "Epoch 395/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4125 - accuracy: 0.8527 - val_loss: 1.1635 - val_accuracy: 0.6931\n",
            "Epoch 396/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4161 - accuracy: 0.8508 - val_loss: 1.1592 - val_accuracy: 0.6911\n",
            "Epoch 397/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4232 - accuracy: 0.8496 - val_loss: 1.1621 - val_accuracy: 0.6931\n",
            "Epoch 398/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4131 - accuracy: 0.8530 - val_loss: 1.1576 - val_accuracy: 0.6920\n",
            "Epoch 399/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4091 - accuracy: 0.8555 - val_loss: 1.1609 - val_accuracy: 0.6933\n",
            "Epoch 400/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4133 - accuracy: 0.8540 - val_loss: 1.1647 - val_accuracy: 0.6915\n",
            "Epoch 401/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4151 - accuracy: 0.8506 - val_loss: 1.1639 - val_accuracy: 0.6927\n",
            "Epoch 402/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4092 - accuracy: 0.8559 - val_loss: 1.1725 - val_accuracy: 0.6911\n",
            "Epoch 403/500\n",
            "1256/1256 [==============================] - 9s 7ms/step - loss: 0.4195 - accuracy: 0.8490 - val_loss: 1.1616 - val_accuracy: 0.6920\n",
            "Epoch 404/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4136 - accuracy: 0.8539 - val_loss: 1.1667 - val_accuracy: 0.6911\n",
            "Epoch 405/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4174 - accuracy: 0.8516 - val_loss: 1.1711 - val_accuracy: 0.6924\n",
            "Epoch 406/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.4118 - accuracy: 0.8533 - val_loss: 1.1561 - val_accuracy: 0.6951\n",
            "Epoch 407/500\n",
            " 801/1256 [==================>...........] - ETA: 3s - loss: 0.4143 - accuracy: 0.8546"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-15-c3cc9c039a26>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     24\u001b[0m     learning_rate=keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.001,decay_steps=2000,decay_rate=0.99)),\n\u001b[1;32m     25\u001b[0m     loss=\"categorical_crossentropy\", metrics=['accuracy'],)\n\u001b[0;32m---> 26\u001b[0;31m \u001b[0mhistory\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mytrain_1hot\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m500\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myval_1hot\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/utils/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     65\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint: disable=broad-except\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m       \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1382\u001b[0m                 _r=1):\n\u001b[1;32m   1383\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1384\u001b[0;31m               \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1385\u001b[0m               \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1386\u001b[0m                 \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/util/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    148\u001b[0m     \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    149\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 150\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    151\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    152\u001b[0m       \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    913\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    914\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0mOptionalXlaContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jit_compile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 915\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    916\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    917\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    945\u001b[0m       \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    946\u001b[0m       \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 947\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    948\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    949\u001b[0m       \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   2955\u001b[0m        filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[1;32m   2956\u001b[0m     return graph_function._call_flat(\n\u001b[0;32m-> 2957\u001b[0;31m         filtered_flat_args, captured_inputs=graph_function.captured_inputs)  # pylint: disable=protected-access\n\u001b[0m\u001b[1;32m   2958\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2959\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1852\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1853\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1854\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m   1855\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1856\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    502\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    503\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 504\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    505\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    506\u001b[0m           outputs = execute.execute_with_cancellation(\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     53\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0;32m---> 55\u001b[0;31m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[1;32m     56\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "layer_list = [ #520 parameters in input\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Dropout(0.4),\n",
        "          layers.Dense(100,kernel_initializer='lecun_normal'), #let's try dimensionality reduction\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('relu'),\n",
        "          #layers.Dense(256,kernel_initializer='lecun_normal'),\n",
        "          #layers.BatchNormalization(),\n",
        "          #layers.Activation('relu'),\n",
        "          #layers.Dense(128,kernel_initializer='lecun_normal'),\n",
        "          #layers.BatchNormalization(),\n",
        "          #layers.Activation('selu'),\n",
        "          #layers.Dense(64,kernel_initializer='lecun_normal'),\n",
        "          #layers.BatchNormalization(),\n",
        "          #layers.Activation('selu'),\n",
        "          layers.Dense(32,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(16,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('relu'),\n",
        "          layers.Dense(16,kernel_initializer='lecun_normal'),\n",
        "          layers.BatchNormalization(),\n",
        "          layers.Activation('selu'),\n",
        "          layers.Dense(16,activation='softmax',kernel_initializer='lecun_normal'),\n",
        "          ]\n",
        "model = Sequential(layer_list)\n",
        "model.compile(optimizer=keras.optimizers.Adam(\n",
        "    learning_rate=keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.001,decay_steps=2000,decay_rate=0.99)),\n",
        "    loss=\"categorical_crossentropy\", metrics=['accuracy'],)\n",
        "history = model.fit(xtrain, ytrain_1hot, batch_size=32, epochs=500, validation_data=(xval, yval_1hot))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BQYagcb3yITZ",
        "outputId": "4993cf1e-5751-4994-b668-d7128633ee43"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/500\n",
            "1256/1256 [==============================] - 12s 8ms/step - loss: 1.5580 - accuracy: 0.5327 - val_loss: 1.2057 - val_accuracy: 0.6203\n",
            "Epoch 2/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.2761 - accuracy: 0.5926 - val_loss: 1.1402 - val_accuracy: 0.6393\n",
            "Epoch 3/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.2296 - accuracy: 0.6049 - val_loss: 1.1031 - val_accuracy: 0.6532\n",
            "Epoch 4/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.2054 - accuracy: 0.6101 - val_loss: 1.0803 - val_accuracy: 0.6552\n",
            "Epoch 5/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1843 - accuracy: 0.6168 - val_loss: 1.0586 - val_accuracy: 0.6635\n",
            "Epoch 6/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1651 - accuracy: 0.6258 - val_loss: 1.0686 - val_accuracy: 0.6640\n",
            "Epoch 7/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1501 - accuracy: 0.6277 - val_loss: 1.0450 - val_accuracy: 0.6691\n",
            "Epoch 8/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1433 - accuracy: 0.6277 - val_loss: 1.0464 - val_accuracy: 0.6655\n",
            "Epoch 9/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1306 - accuracy: 0.6343 - val_loss: 1.0348 - val_accuracy: 0.6653\n",
            "Epoch 10/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1247 - accuracy: 0.6353 - val_loss: 1.0231 - val_accuracy: 0.6745\n",
            "Epoch 11/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1172 - accuracy: 0.6360 - val_loss: 1.0272 - val_accuracy: 0.6687\n",
            "Epoch 12/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.1093 - accuracy: 0.6384 - val_loss: 1.0170 - val_accuracy: 0.6714\n",
            "Epoch 13/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0980 - accuracy: 0.6418 - val_loss: 1.0105 - val_accuracy: 0.6747\n",
            "Epoch 14/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0923 - accuracy: 0.6433 - val_loss: 1.0049 - val_accuracy: 0.6839\n",
            "Epoch 15/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0882 - accuracy: 0.6463 - val_loss: 1.0015 - val_accuracy: 0.6841\n",
            "Epoch 16/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0827 - accuracy: 0.6466 - val_loss: 1.0039 - val_accuracy: 0.6821\n",
            "Epoch 17/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0763 - accuracy: 0.6476 - val_loss: 0.9998 - val_accuracy: 0.6785\n",
            "Epoch 18/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0692 - accuracy: 0.6523 - val_loss: 0.9992 - val_accuracy: 0.6770\n",
            "Epoch 19/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0772 - accuracy: 0.6488 - val_loss: 1.0016 - val_accuracy: 0.6803\n",
            "Epoch 20/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0630 - accuracy: 0.6508 - val_loss: 0.9990 - val_accuracy: 0.6788\n",
            "Epoch 21/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0663 - accuracy: 0.6518 - val_loss: 1.0019 - val_accuracy: 0.6747\n",
            "Epoch 22/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0557 - accuracy: 0.6547 - val_loss: 1.0005 - val_accuracy: 0.6794\n",
            "Epoch 23/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0547 - accuracy: 0.6575 - val_loss: 0.9911 - val_accuracy: 0.6785\n",
            "Epoch 24/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0527 - accuracy: 0.6543 - val_loss: 0.9935 - val_accuracy: 0.6810\n",
            "Epoch 25/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0451 - accuracy: 0.6578 - val_loss: 0.9834 - val_accuracy: 0.6841\n",
            "Epoch 26/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0514 - accuracy: 0.6544 - val_loss: 0.9853 - val_accuracy: 0.6823\n",
            "Epoch 27/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0387 - accuracy: 0.6609 - val_loss: 0.9881 - val_accuracy: 0.6797\n",
            "Epoch 28/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0425 - accuracy: 0.6577 - val_loss: 0.9888 - val_accuracy: 0.6823\n",
            "Epoch 29/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0343 - accuracy: 0.6602 - val_loss: 0.9954 - val_accuracy: 0.6788\n",
            "Epoch 30/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0291 - accuracy: 0.6634 - val_loss: 0.9831 - val_accuracy: 0.6810\n",
            "Epoch 31/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0333 - accuracy: 0.6593 - val_loss: 0.9744 - val_accuracy: 0.6862\n",
            "Epoch 32/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0334 - accuracy: 0.6625 - val_loss: 0.9753 - val_accuracy: 0.6850\n",
            "Epoch 33/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0319 - accuracy: 0.6610 - val_loss: 0.9735 - val_accuracy: 0.6848\n",
            "Epoch 34/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0243 - accuracy: 0.6641 - val_loss: 0.9748 - val_accuracy: 0.6877\n",
            "Epoch 35/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0196 - accuracy: 0.6670 - val_loss: 0.9763 - val_accuracy: 0.6835\n",
            "Epoch 36/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0225 - accuracy: 0.6641 - val_loss: 0.9713 - val_accuracy: 0.6871\n",
            "Epoch 37/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0242 - accuracy: 0.6614 - val_loss: 0.9758 - val_accuracy: 0.6828\n",
            "Epoch 38/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0188 - accuracy: 0.6662 - val_loss: 0.9739 - val_accuracy: 0.6871\n",
            "Epoch 39/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0079 - accuracy: 0.6698 - val_loss: 0.9748 - val_accuracy: 0.6864\n",
            "Epoch 40/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0169 - accuracy: 0.6656 - val_loss: 0.9689 - val_accuracy: 0.6915\n",
            "Epoch 41/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0146 - accuracy: 0.6682 - val_loss: 0.9801 - val_accuracy: 0.6803\n",
            "Epoch 42/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0186 - accuracy: 0.6641 - val_loss: 0.9675 - val_accuracy: 0.6853\n",
            "Epoch 43/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0037 - accuracy: 0.6705 - val_loss: 0.9726 - val_accuracy: 0.6817\n",
            "Epoch 44/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0127 - accuracy: 0.6682 - val_loss: 0.9681 - val_accuracy: 0.6866\n",
            "Epoch 45/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0013 - accuracy: 0.6743 - val_loss: 0.9718 - val_accuracy: 0.6815\n",
            "Epoch 46/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0045 - accuracy: 0.6703 - val_loss: 0.9742 - val_accuracy: 0.6801\n",
            "Epoch 47/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0029 - accuracy: 0.6711 - val_loss: 0.9678 - val_accuracy: 0.6864\n",
            "Epoch 48/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0009 - accuracy: 0.6686 - val_loss: 0.9676 - val_accuracy: 0.6850\n",
            "Epoch 49/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 1.0017 - accuracy: 0.6696 - val_loss: 0.9694 - val_accuracy: 0.6828\n",
            "Epoch 50/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9970 - accuracy: 0.6706 - val_loss: 0.9656 - val_accuracy: 0.6873\n",
            "Epoch 51/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9943 - accuracy: 0.6736 - val_loss: 0.9654 - val_accuracy: 0.6884\n",
            "Epoch 52/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9972 - accuracy: 0.6720 - val_loss: 0.9629 - val_accuracy: 0.6873\n",
            "Epoch 53/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9954 - accuracy: 0.6713 - val_loss: 0.9694 - val_accuracy: 0.6866\n",
            "Epoch 54/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9967 - accuracy: 0.6742 - val_loss: 0.9639 - val_accuracy: 0.6832\n",
            "Epoch 55/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9911 - accuracy: 0.6745 - val_loss: 0.9636 - val_accuracy: 0.6855\n",
            "Epoch 56/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9895 - accuracy: 0.6724 - val_loss: 0.9712 - val_accuracy: 0.6835\n",
            "Epoch 57/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9880 - accuracy: 0.6747 - val_loss: 0.9666 - val_accuracy: 0.6848\n",
            "Epoch 58/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9885 - accuracy: 0.6744 - val_loss: 0.9665 - val_accuracy: 0.6839\n",
            "Epoch 59/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9900 - accuracy: 0.6751 - val_loss: 0.9640 - val_accuracy: 0.6841\n",
            "Epoch 60/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9870 - accuracy: 0.6757 - val_loss: 0.9594 - val_accuracy: 0.6873\n",
            "Epoch 61/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9865 - accuracy: 0.6755 - val_loss: 0.9671 - val_accuracy: 0.6848\n",
            "Epoch 62/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9827 - accuracy: 0.6753 - val_loss: 0.9675 - val_accuracy: 0.6792\n",
            "Epoch 63/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9766 - accuracy: 0.6772 - val_loss: 0.9576 - val_accuracy: 0.6924\n",
            "Epoch 64/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9874 - accuracy: 0.6745 - val_loss: 0.9661 - val_accuracy: 0.6871\n",
            "Epoch 65/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9836 - accuracy: 0.6754 - val_loss: 0.9597 - val_accuracy: 0.6857\n",
            "Epoch 66/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9841 - accuracy: 0.6741 - val_loss: 0.9608 - val_accuracy: 0.6877\n",
            "Epoch 67/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9813 - accuracy: 0.6772 - val_loss: 0.9674 - val_accuracy: 0.6835\n",
            "Epoch 68/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9791 - accuracy: 0.6770 - val_loss: 0.9600 - val_accuracy: 0.6897\n",
            "Epoch 69/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9817 - accuracy: 0.6759 - val_loss: 0.9649 - val_accuracy: 0.6830\n",
            "Epoch 70/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9833 - accuracy: 0.6770 - val_loss: 0.9607 - val_accuracy: 0.6886\n",
            "Epoch 71/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9793 - accuracy: 0.6781 - val_loss: 0.9586 - val_accuracy: 0.6855\n",
            "Epoch 72/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9770 - accuracy: 0.6793 - val_loss: 0.9622 - val_accuracy: 0.6841\n",
            "Epoch 73/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9800 - accuracy: 0.6773 - val_loss: 0.9587 - val_accuracy: 0.6873\n",
            "Epoch 74/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9757 - accuracy: 0.6768 - val_loss: 0.9583 - val_accuracy: 0.6864\n",
            "Epoch 75/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9804 - accuracy: 0.6783 - val_loss: 0.9561 - val_accuracy: 0.6877\n",
            "Epoch 76/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9761 - accuracy: 0.6778 - val_loss: 0.9653 - val_accuracy: 0.6848\n",
            "Epoch 77/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9754 - accuracy: 0.6783 - val_loss: 0.9563 - val_accuracy: 0.6873\n",
            "Epoch 78/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9728 - accuracy: 0.6765 - val_loss: 0.9649 - val_accuracy: 0.6832\n",
            "Epoch 79/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9681 - accuracy: 0.6780 - val_loss: 0.9592 - val_accuracy: 0.6875\n",
            "Epoch 80/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9725 - accuracy: 0.6780 - val_loss: 0.9593 - val_accuracy: 0.6875\n",
            "Epoch 81/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9677 - accuracy: 0.6815 - val_loss: 0.9664 - val_accuracy: 0.6848\n",
            "Epoch 82/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9711 - accuracy: 0.6779 - val_loss: 0.9601 - val_accuracy: 0.6855\n",
            "Epoch 83/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9736 - accuracy: 0.6788 - val_loss: 0.9640 - val_accuracy: 0.6841\n",
            "Epoch 84/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9683 - accuracy: 0.6787 - val_loss: 0.9665 - val_accuracy: 0.6848\n",
            "Epoch 85/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9658 - accuracy: 0.6814 - val_loss: 0.9652 - val_accuracy: 0.6830\n",
            "Epoch 86/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9678 - accuracy: 0.6807 - val_loss: 0.9619 - val_accuracy: 0.6817\n",
            "Epoch 87/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9650 - accuracy: 0.6839 - val_loss: 0.9609 - val_accuracy: 0.6864\n",
            "Epoch 88/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9631 - accuracy: 0.6820 - val_loss: 0.9546 - val_accuracy: 0.6893\n",
            "Epoch 89/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9593 - accuracy: 0.6815 - val_loss: 0.9598 - val_accuracy: 0.6864\n",
            "Epoch 90/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9641 - accuracy: 0.6796 - val_loss: 0.9628 - val_accuracy: 0.6868\n",
            "Epoch 91/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9637 - accuracy: 0.6794 - val_loss: 0.9596 - val_accuracy: 0.6862\n",
            "Epoch 92/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9645 - accuracy: 0.6835 - val_loss: 0.9550 - val_accuracy: 0.6900\n",
            "Epoch 93/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9566 - accuracy: 0.6841 - val_loss: 0.9601 - val_accuracy: 0.6875\n",
            "Epoch 94/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9582 - accuracy: 0.6825 - val_loss: 0.9593 - val_accuracy: 0.6857\n",
            "Epoch 95/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9567 - accuracy: 0.6837 - val_loss: 0.9578 - val_accuracy: 0.6866\n",
            "Epoch 96/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9563 - accuracy: 0.6846 - val_loss: 0.9619 - val_accuracy: 0.6850\n",
            "Epoch 97/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9602 - accuracy: 0.6825 - val_loss: 0.9503 - val_accuracy: 0.6886\n",
            "Epoch 98/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9581 - accuracy: 0.6827 - val_loss: 0.9548 - val_accuracy: 0.6879\n",
            "Epoch 99/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9580 - accuracy: 0.6814 - val_loss: 0.9593 - val_accuracy: 0.6893\n",
            "Epoch 100/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9609 - accuracy: 0.6824 - val_loss: 0.9573 - val_accuracy: 0.6864\n",
            "Epoch 101/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9588 - accuracy: 0.6816 - val_loss: 0.9602 - val_accuracy: 0.6871\n",
            "Epoch 102/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9608 - accuracy: 0.6827 - val_loss: 0.9530 - val_accuracy: 0.6951\n",
            "Epoch 103/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9546 - accuracy: 0.6854 - val_loss: 0.9565 - val_accuracy: 0.6920\n",
            "Epoch 104/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9524 - accuracy: 0.6833 - val_loss: 0.9594 - val_accuracy: 0.6850\n",
            "Epoch 105/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9555 - accuracy: 0.6856 - val_loss: 0.9653 - val_accuracy: 0.6848\n",
            "Epoch 106/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9571 - accuracy: 0.6828 - val_loss: 0.9624 - val_accuracy: 0.6879\n",
            "Epoch 107/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9514 - accuracy: 0.6853 - val_loss: 0.9557 - val_accuracy: 0.6888\n",
            "Epoch 108/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9463 - accuracy: 0.6878 - val_loss: 0.9576 - val_accuracy: 0.6893\n",
            "Epoch 109/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9482 - accuracy: 0.6864 - val_loss: 0.9554 - val_accuracy: 0.6915\n",
            "Epoch 110/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9581 - accuracy: 0.6811 - val_loss: 0.9505 - val_accuracy: 0.6940\n",
            "Epoch 111/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9517 - accuracy: 0.6867 - val_loss: 0.9576 - val_accuracy: 0.6897\n",
            "Epoch 112/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9589 - accuracy: 0.6839 - val_loss: 0.9540 - val_accuracy: 0.6868\n",
            "Epoch 113/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9523 - accuracy: 0.6834 - val_loss: 0.9596 - val_accuracy: 0.6828\n",
            "Epoch 114/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9562 - accuracy: 0.6844 - val_loss: 0.9552 - val_accuracy: 0.6817\n",
            "Epoch 115/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9486 - accuracy: 0.6876 - val_loss: 0.9506 - val_accuracy: 0.6868\n",
            "Epoch 116/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9539 - accuracy: 0.6831 - val_loss: 0.9574 - val_accuracy: 0.6848\n",
            "Epoch 117/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9493 - accuracy: 0.6855 - val_loss: 0.9611 - val_accuracy: 0.6846\n",
            "Epoch 118/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9522 - accuracy: 0.6831 - val_loss: 0.9579 - val_accuracy: 0.6830\n",
            "Epoch 119/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9476 - accuracy: 0.6860 - val_loss: 0.9554 - val_accuracy: 0.6873\n",
            "Epoch 120/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9482 - accuracy: 0.6871 - val_loss: 0.9552 - val_accuracy: 0.6877\n",
            "Epoch 121/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9587 - accuracy: 0.6815 - val_loss: 0.9571 - val_accuracy: 0.6868\n",
            "Epoch 122/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9522 - accuracy: 0.6837 - val_loss: 0.9509 - val_accuracy: 0.6882\n",
            "Epoch 123/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9487 - accuracy: 0.6843 - val_loss: 0.9499 - val_accuracy: 0.6895\n",
            "Epoch 124/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9505 - accuracy: 0.6836 - val_loss: 0.9569 - val_accuracy: 0.6871\n",
            "Epoch 125/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9531 - accuracy: 0.6836 - val_loss: 0.9544 - val_accuracy: 0.6873\n",
            "Epoch 126/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9442 - accuracy: 0.6848 - val_loss: 0.9528 - val_accuracy: 0.6891\n",
            "Epoch 127/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9458 - accuracy: 0.6874 - val_loss: 0.9517 - val_accuracy: 0.6871\n",
            "Epoch 128/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9512 - accuracy: 0.6832 - val_loss: 0.9556 - val_accuracy: 0.6871\n",
            "Epoch 129/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9505 - accuracy: 0.6848 - val_loss: 0.9575 - val_accuracy: 0.6837\n",
            "Epoch 130/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9436 - accuracy: 0.6875 - val_loss: 0.9566 - val_accuracy: 0.6844\n",
            "Epoch 131/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9481 - accuracy: 0.6855 - val_loss: 0.9538 - val_accuracy: 0.6884\n",
            "Epoch 132/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9455 - accuracy: 0.6865 - val_loss: 0.9564 - val_accuracy: 0.6884\n",
            "Epoch 133/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9469 - accuracy: 0.6860 - val_loss: 0.9631 - val_accuracy: 0.6868\n",
            "Epoch 134/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9474 - accuracy: 0.6854 - val_loss: 0.9532 - val_accuracy: 0.6882\n",
            "Epoch 135/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9457 - accuracy: 0.6873 - val_loss: 0.9545 - val_accuracy: 0.6882\n",
            "Epoch 136/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9427 - accuracy: 0.6870 - val_loss: 0.9565 - val_accuracy: 0.6857\n",
            "Epoch 137/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9408 - accuracy: 0.6894 - val_loss: 0.9511 - val_accuracy: 0.6913\n",
            "Epoch 138/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9467 - accuracy: 0.6866 - val_loss: 0.9603 - val_accuracy: 0.6850\n",
            "Epoch 139/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9423 - accuracy: 0.6896 - val_loss: 0.9562 - val_accuracy: 0.6902\n",
            "Epoch 140/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9416 - accuracy: 0.6889 - val_loss: 0.9606 - val_accuracy: 0.6873\n",
            "Epoch 141/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9389 - accuracy: 0.6863 - val_loss: 0.9565 - val_accuracy: 0.6864\n",
            "Epoch 142/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9414 - accuracy: 0.6882 - val_loss: 0.9530 - val_accuracy: 0.6859\n",
            "Epoch 143/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9398 - accuracy: 0.6878 - val_loss: 0.9599 - val_accuracy: 0.6844\n",
            "Epoch 144/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9417 - accuracy: 0.6873 - val_loss: 0.9543 - val_accuracy: 0.6895\n",
            "Epoch 145/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9447 - accuracy: 0.6865 - val_loss: 0.9559 - val_accuracy: 0.6900\n",
            "Epoch 146/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9436 - accuracy: 0.6866 - val_loss: 0.9568 - val_accuracy: 0.6888\n",
            "Epoch 147/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9396 - accuracy: 0.6873 - val_loss: 0.9452 - val_accuracy: 0.6913\n",
            "Epoch 148/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9354 - accuracy: 0.6892 - val_loss: 0.9570 - val_accuracy: 0.6857\n",
            "Epoch 149/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9425 - accuracy: 0.6884 - val_loss: 0.9526 - val_accuracy: 0.6888\n",
            "Epoch 150/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9397 - accuracy: 0.6883 - val_loss: 0.9613 - val_accuracy: 0.6846\n",
            "Epoch 151/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9394 - accuracy: 0.6880 - val_loss: 0.9579 - val_accuracy: 0.6879\n",
            "Epoch 152/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9333 - accuracy: 0.6889 - val_loss: 0.9549 - val_accuracy: 0.6888\n",
            "Epoch 153/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9349 - accuracy: 0.6907 - val_loss: 0.9526 - val_accuracy: 0.6938\n",
            "Epoch 154/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9397 - accuracy: 0.6857 - val_loss: 0.9555 - val_accuracy: 0.6918\n",
            "Epoch 155/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9342 - accuracy: 0.6893 - val_loss: 0.9553 - val_accuracy: 0.6895\n",
            "Epoch 156/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9356 - accuracy: 0.6885 - val_loss: 0.9606 - val_accuracy: 0.6877\n",
            "Epoch 157/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9328 - accuracy: 0.6890 - val_loss: 0.9601 - val_accuracy: 0.6864\n",
            "Epoch 158/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9414 - accuracy: 0.6894 - val_loss: 0.9560 - val_accuracy: 0.6866\n",
            "Epoch 159/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9394 - accuracy: 0.6898 - val_loss: 0.9585 - val_accuracy: 0.6882\n",
            "Epoch 160/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9288 - accuracy: 0.6916 - val_loss: 0.9590 - val_accuracy: 0.6911\n",
            "Epoch 161/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9343 - accuracy: 0.6898 - val_loss: 0.9575 - val_accuracy: 0.6922\n",
            "Epoch 162/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9337 - accuracy: 0.6908 - val_loss: 0.9611 - val_accuracy: 0.6884\n",
            "Epoch 163/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9341 - accuracy: 0.6886 - val_loss: 0.9622 - val_accuracy: 0.6915\n",
            "Epoch 164/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9301 - accuracy: 0.6908 - val_loss: 0.9591 - val_accuracy: 0.6893\n",
            "Epoch 165/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9316 - accuracy: 0.6892 - val_loss: 0.9610 - val_accuracy: 0.6875\n",
            "Epoch 166/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9311 - accuracy: 0.6912 - val_loss: 0.9567 - val_accuracy: 0.6906\n",
            "Epoch 167/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9338 - accuracy: 0.6890 - val_loss: 0.9600 - val_accuracy: 0.6888\n",
            "Epoch 168/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9335 - accuracy: 0.6910 - val_loss: 0.9589 - val_accuracy: 0.6909\n",
            "Epoch 169/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9311 - accuracy: 0.6916 - val_loss: 0.9607 - val_accuracy: 0.6891\n",
            "Epoch 170/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9330 - accuracy: 0.6902 - val_loss: 0.9629 - val_accuracy: 0.6873\n",
            "Epoch 171/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9336 - accuracy: 0.6915 - val_loss: 0.9566 - val_accuracy: 0.6918\n",
            "Epoch 172/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9302 - accuracy: 0.6891 - val_loss: 0.9600 - val_accuracy: 0.6920\n",
            "Epoch 173/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9393 - accuracy: 0.6870 - val_loss: 0.9560 - val_accuracy: 0.6947\n",
            "Epoch 174/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9306 - accuracy: 0.6902 - val_loss: 0.9551 - val_accuracy: 0.6924\n",
            "Epoch 175/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9296 - accuracy: 0.6910 - val_loss: 0.9550 - val_accuracy: 0.6882\n",
            "Epoch 176/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9399 - accuracy: 0.6891 - val_loss: 0.9574 - val_accuracy: 0.6900\n",
            "Epoch 177/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9328 - accuracy: 0.6908 - val_loss: 0.9599 - val_accuracy: 0.6884\n",
            "Epoch 178/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9337 - accuracy: 0.6899 - val_loss: 0.9615 - val_accuracy: 0.6895\n",
            "Epoch 179/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9352 - accuracy: 0.6893 - val_loss: 0.9559 - val_accuracy: 0.6944\n",
            "Epoch 180/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9332 - accuracy: 0.6908 - val_loss: 0.9566 - val_accuracy: 0.6893\n",
            "Epoch 181/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9315 - accuracy: 0.6899 - val_loss: 0.9557 - val_accuracy: 0.6960\n",
            "Epoch 182/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9281 - accuracy: 0.6920 - val_loss: 0.9538 - val_accuracy: 0.6933\n",
            "Epoch 183/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9369 - accuracy: 0.6904 - val_loss: 0.9524 - val_accuracy: 0.6888\n",
            "Epoch 184/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9324 - accuracy: 0.6899 - val_loss: 0.9511 - val_accuracy: 0.6947\n",
            "Epoch 185/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9297 - accuracy: 0.6897 - val_loss: 0.9547 - val_accuracy: 0.6927\n",
            "Epoch 186/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9293 - accuracy: 0.6953 - val_loss: 0.9578 - val_accuracy: 0.6927\n",
            "Epoch 187/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9308 - accuracy: 0.6902 - val_loss: 0.9519 - val_accuracy: 0.6909\n",
            "Epoch 188/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9253 - accuracy: 0.6917 - val_loss: 0.9560 - val_accuracy: 0.6913\n",
            "Epoch 189/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9300 - accuracy: 0.6914 - val_loss: 0.9539 - val_accuracy: 0.6918\n",
            "Epoch 190/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9282 - accuracy: 0.6895 - val_loss: 0.9550 - val_accuracy: 0.6915\n",
            "Epoch 191/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9348 - accuracy: 0.6896 - val_loss: 0.9555 - val_accuracy: 0.6938\n",
            "Epoch 192/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9298 - accuracy: 0.6919 - val_loss: 0.9558 - val_accuracy: 0.6902\n",
            "Epoch 193/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9277 - accuracy: 0.6921 - val_loss: 0.9560 - val_accuracy: 0.6935\n",
            "Epoch 194/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9274 - accuracy: 0.6894 - val_loss: 0.9560 - val_accuracy: 0.6913\n",
            "Epoch 195/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9299 - accuracy: 0.6904 - val_loss: 0.9532 - val_accuracy: 0.6960\n",
            "Epoch 196/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9237 - accuracy: 0.6922 - val_loss: 0.9540 - val_accuracy: 0.6922\n",
            "Epoch 197/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9288 - accuracy: 0.6903 - val_loss: 0.9557 - val_accuracy: 0.6944\n",
            "Epoch 198/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9322 - accuracy: 0.6910 - val_loss: 0.9569 - val_accuracy: 0.6933\n",
            "Epoch 199/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9266 - accuracy: 0.6924 - val_loss: 0.9553 - val_accuracy: 0.6940\n",
            "Epoch 200/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9238 - accuracy: 0.6936 - val_loss: 0.9525 - val_accuracy: 0.6924\n",
            "Epoch 201/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9286 - accuracy: 0.6906 - val_loss: 0.9568 - val_accuracy: 0.6902\n",
            "Epoch 202/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9194 - accuracy: 0.6942 - val_loss: 0.9602 - val_accuracy: 0.6868\n",
            "Epoch 203/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9248 - accuracy: 0.6925 - val_loss: 0.9554 - val_accuracy: 0.6900\n",
            "Epoch 204/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9310 - accuracy: 0.6888 - val_loss: 0.9585 - val_accuracy: 0.6940\n",
            "Epoch 205/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9274 - accuracy: 0.6897 - val_loss: 0.9574 - val_accuracy: 0.6913\n",
            "Epoch 206/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9264 - accuracy: 0.6926 - val_loss: 0.9575 - val_accuracy: 0.6909\n",
            "Epoch 207/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9237 - accuracy: 0.6954 - val_loss: 0.9577 - val_accuracy: 0.6902\n",
            "Epoch 208/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9160 - accuracy: 0.6936 - val_loss: 0.9613 - val_accuracy: 0.6900\n",
            "Epoch 209/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9259 - accuracy: 0.6913 - val_loss: 0.9577 - val_accuracy: 0.6947\n",
            "Epoch 210/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9172 - accuracy: 0.6943 - val_loss: 0.9592 - val_accuracy: 0.6951\n",
            "Epoch 211/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9250 - accuracy: 0.6916 - val_loss: 0.9601 - val_accuracy: 0.6884\n",
            "Epoch 212/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9255 - accuracy: 0.6918 - val_loss: 0.9578 - val_accuracy: 0.6953\n",
            "Epoch 213/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9277 - accuracy: 0.6907 - val_loss: 0.9586 - val_accuracy: 0.6915\n",
            "Epoch 214/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9255 - accuracy: 0.6922 - val_loss: 0.9590 - val_accuracy: 0.6902\n",
            "Epoch 215/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9222 - accuracy: 0.6925 - val_loss: 0.9591 - val_accuracy: 0.6906\n",
            "Epoch 216/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9244 - accuracy: 0.6913 - val_loss: 0.9541 - val_accuracy: 0.6967\n",
            "Epoch 217/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9251 - accuracy: 0.6929 - val_loss: 0.9529 - val_accuracy: 0.6922\n",
            "Epoch 218/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9220 - accuracy: 0.6910 - val_loss: 0.9566 - val_accuracy: 0.6935\n",
            "Epoch 219/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9271 - accuracy: 0.6925 - val_loss: 0.9598 - val_accuracy: 0.6902\n",
            "Epoch 220/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9291 - accuracy: 0.6894 - val_loss: 0.9534 - val_accuracy: 0.6960\n",
            "Epoch 221/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9268 - accuracy: 0.6919 - val_loss: 0.9561 - val_accuracy: 0.6913\n",
            "Epoch 222/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9172 - accuracy: 0.6944 - val_loss: 0.9613 - val_accuracy: 0.6915\n",
            "Epoch 223/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9218 - accuracy: 0.6939 - val_loss: 0.9541 - val_accuracy: 0.6924\n",
            "Epoch 224/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9205 - accuracy: 0.6917 - val_loss: 0.9568 - val_accuracy: 0.6906\n",
            "Epoch 225/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9205 - accuracy: 0.6940 - val_loss: 0.9532 - val_accuracy: 0.6942\n",
            "Epoch 226/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9175 - accuracy: 0.6960 - val_loss: 0.9553 - val_accuracy: 0.6909\n",
            "Epoch 227/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9207 - accuracy: 0.6907 - val_loss: 0.9544 - val_accuracy: 0.6906\n",
            "Epoch 228/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9239 - accuracy: 0.6931 - val_loss: 0.9559 - val_accuracy: 0.6900\n",
            "Epoch 229/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9172 - accuracy: 0.6921 - val_loss: 0.9544 - val_accuracy: 0.6938\n",
            "Epoch 230/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9182 - accuracy: 0.6952 - val_loss: 0.9547 - val_accuracy: 0.6953\n",
            "Epoch 231/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9235 - accuracy: 0.6921 - val_loss: 0.9578 - val_accuracy: 0.6909\n",
            "Epoch 232/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9214 - accuracy: 0.6939 - val_loss: 0.9550 - val_accuracy: 0.6902\n",
            "Epoch 233/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9196 - accuracy: 0.6934 - val_loss: 0.9588 - val_accuracy: 0.6920\n",
            "Epoch 234/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9181 - accuracy: 0.6937 - val_loss: 0.9573 - val_accuracy: 0.6915\n",
            "Epoch 235/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9231 - accuracy: 0.6917 - val_loss: 0.9578 - val_accuracy: 0.6929\n",
            "Epoch 236/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9206 - accuracy: 0.6973 - val_loss: 0.9645 - val_accuracy: 0.6879\n",
            "Epoch 237/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9200 - accuracy: 0.6926 - val_loss: 0.9593 - val_accuracy: 0.6922\n",
            "Epoch 238/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9225 - accuracy: 0.6956 - val_loss: 0.9579 - val_accuracy: 0.6915\n",
            "Epoch 239/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9220 - accuracy: 0.6939 - val_loss: 0.9596 - val_accuracy: 0.6938\n",
            "Epoch 240/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9228 - accuracy: 0.6939 - val_loss: 0.9577 - val_accuracy: 0.6915\n",
            "Epoch 241/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9197 - accuracy: 0.6909 - val_loss: 0.9586 - val_accuracy: 0.6927\n",
            "Epoch 242/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9245 - accuracy: 0.6865 - val_loss: 0.9571 - val_accuracy: 0.6922\n",
            "Epoch 243/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9174 - accuracy: 0.6975 - val_loss: 0.9612 - val_accuracy: 0.6891\n",
            "Epoch 244/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9191 - accuracy: 0.6942 - val_loss: 0.9551 - val_accuracy: 0.6931\n",
            "Epoch 245/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9198 - accuracy: 0.6918 - val_loss: 0.9581 - val_accuracy: 0.6906\n",
            "Epoch 246/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9186 - accuracy: 0.6926 - val_loss: 0.9596 - val_accuracy: 0.6913\n",
            "Epoch 247/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9172 - accuracy: 0.6955 - val_loss: 0.9573 - val_accuracy: 0.6900\n",
            "Epoch 248/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9139 - accuracy: 0.6934 - val_loss: 0.9533 - val_accuracy: 0.6935\n",
            "Epoch 249/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9193 - accuracy: 0.6942 - val_loss: 0.9543 - val_accuracy: 0.6942\n",
            "Epoch 250/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9175 - accuracy: 0.6917 - val_loss: 0.9543 - val_accuracy: 0.6944\n",
            "Epoch 251/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9099 - accuracy: 0.6964 - val_loss: 0.9580 - val_accuracy: 0.6924\n",
            "Epoch 252/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9183 - accuracy: 0.6940 - val_loss: 0.9589 - val_accuracy: 0.6891\n",
            "Epoch 253/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9198 - accuracy: 0.6912 - val_loss: 0.9548 - val_accuracy: 0.6929\n",
            "Epoch 254/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9185 - accuracy: 0.6937 - val_loss: 0.9561 - val_accuracy: 0.6940\n",
            "Epoch 255/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9210 - accuracy: 0.6960 - val_loss: 0.9570 - val_accuracy: 0.6893\n",
            "Epoch 256/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9198 - accuracy: 0.6938 - val_loss: 0.9556 - val_accuracy: 0.6918\n",
            "Epoch 257/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9155 - accuracy: 0.6956 - val_loss: 0.9573 - val_accuracy: 0.6915\n",
            "Epoch 258/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9197 - accuracy: 0.6960 - val_loss: 0.9541 - val_accuracy: 0.6967\n",
            "Epoch 259/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9171 - accuracy: 0.6962 - val_loss: 0.9586 - val_accuracy: 0.6904\n",
            "Epoch 260/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9187 - accuracy: 0.6949 - val_loss: 0.9554 - val_accuracy: 0.6935\n",
            "Epoch 261/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9188 - accuracy: 0.6915 - val_loss: 0.9542 - val_accuracy: 0.6940\n",
            "Epoch 262/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9144 - accuracy: 0.6954 - val_loss: 0.9519 - val_accuracy: 0.6942\n",
            "Epoch 263/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9145 - accuracy: 0.6946 - val_loss: 0.9547 - val_accuracy: 0.6935\n",
            "Epoch 264/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9119 - accuracy: 0.6974 - val_loss: 0.9575 - val_accuracy: 0.6929\n",
            "Epoch 265/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9182 - accuracy: 0.6923 - val_loss: 0.9549 - val_accuracy: 0.6927\n",
            "Epoch 266/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9171 - accuracy: 0.6929 - val_loss: 0.9567 - val_accuracy: 0.6935\n",
            "Epoch 267/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9176 - accuracy: 0.6918 - val_loss: 0.9561 - val_accuracy: 0.6906\n",
            "Epoch 268/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9097 - accuracy: 0.6948 - val_loss: 0.9564 - val_accuracy: 0.6920\n",
            "Epoch 269/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9237 - accuracy: 0.6908 - val_loss: 0.9575 - val_accuracy: 0.6922\n",
            "Epoch 270/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9210 - accuracy: 0.6911 - val_loss: 0.9546 - val_accuracy: 0.6920\n",
            "Epoch 271/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9125 - accuracy: 0.6953 - val_loss: 0.9575 - val_accuracy: 0.6909\n",
            "Epoch 272/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9129 - accuracy: 0.6957 - val_loss: 0.9577 - val_accuracy: 0.6891\n",
            "Epoch 273/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9106 - accuracy: 0.6965 - val_loss: 0.9552 - val_accuracy: 0.6918\n",
            "Epoch 274/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9169 - accuracy: 0.6939 - val_loss: 0.9573 - val_accuracy: 0.6915\n",
            "Epoch 275/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9171 - accuracy: 0.6963 - val_loss: 0.9563 - val_accuracy: 0.6909\n",
            "Epoch 276/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9164 - accuracy: 0.6931 - val_loss: 0.9559 - val_accuracy: 0.6906\n",
            "Epoch 277/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9182 - accuracy: 0.6950 - val_loss: 0.9544 - val_accuracy: 0.6935\n",
            "Epoch 278/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9097 - accuracy: 0.6966 - val_loss: 0.9585 - val_accuracy: 0.6911\n",
            "Epoch 279/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9152 - accuracy: 0.6958 - val_loss: 0.9603 - val_accuracy: 0.6875\n",
            "Epoch 280/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9139 - accuracy: 0.6948 - val_loss: 0.9602 - val_accuracy: 0.6913\n",
            "Epoch 281/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9161 - accuracy: 0.6966 - val_loss: 0.9592 - val_accuracy: 0.6929\n",
            "Epoch 282/500\n",
            "1256/1256 [==============================] - 10s 8ms/step - loss: 0.9109 - accuracy: 0.6957 - val_loss: 0.9555 - val_accuracy: 0.6913\n",
            "Epoch 283/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9102 - accuracy: 0.6963 - val_loss: 0.9542 - val_accuracy: 0.6915\n",
            "Epoch 284/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9175 - accuracy: 0.6954 - val_loss: 0.9569 - val_accuracy: 0.6938\n",
            "Epoch 285/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9142 - accuracy: 0.6957 - val_loss: 0.9577 - val_accuracy: 0.6920\n",
            "Epoch 286/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9150 - accuracy: 0.6941 - val_loss: 0.9587 - val_accuracy: 0.6931\n",
            "Epoch 287/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9175 - accuracy: 0.6932 - val_loss: 0.9586 - val_accuracy: 0.6879\n",
            "Epoch 288/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9147 - accuracy: 0.6950 - val_loss: 0.9586 - val_accuracy: 0.6891\n",
            "Epoch 289/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9116 - accuracy: 0.6972 - val_loss: 0.9591 - val_accuracy: 0.6888\n",
            "Epoch 290/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9187 - accuracy: 0.6925 - val_loss: 0.9574 - val_accuracy: 0.6909\n",
            "Epoch 291/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9131 - accuracy: 0.6980 - val_loss: 0.9579 - val_accuracy: 0.6902\n",
            "Epoch 292/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9145 - accuracy: 0.6937 - val_loss: 0.9591 - val_accuracy: 0.6938\n",
            "Epoch 293/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9171 - accuracy: 0.6953 - val_loss: 0.9577 - val_accuracy: 0.6944\n",
            "Epoch 294/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9129 - accuracy: 0.6944 - val_loss: 0.9549 - val_accuracy: 0.6909\n",
            "Epoch 295/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9119 - accuracy: 0.6961 - val_loss: 0.9573 - val_accuracy: 0.6906\n",
            "Epoch 296/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9097 - accuracy: 0.6972 - val_loss: 0.9594 - val_accuracy: 0.6897\n",
            "Epoch 297/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9091 - accuracy: 0.6953 - val_loss: 0.9562 - val_accuracy: 0.6906\n",
            "Epoch 298/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9162 - accuracy: 0.6936 - val_loss: 0.9543 - val_accuracy: 0.6906\n",
            "Epoch 299/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9073 - accuracy: 0.6961 - val_loss: 0.9604 - val_accuracy: 0.6931\n",
            "Epoch 300/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9112 - accuracy: 0.6954 - val_loss: 0.9561 - val_accuracy: 0.6888\n",
            "Epoch 301/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9140 - accuracy: 0.6932 - val_loss: 0.9593 - val_accuracy: 0.6904\n",
            "Epoch 302/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9137 - accuracy: 0.6947 - val_loss: 0.9556 - val_accuracy: 0.6931\n",
            "Epoch 303/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9129 - accuracy: 0.6954 - val_loss: 0.9555 - val_accuracy: 0.6909\n",
            "Epoch 304/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9097 - accuracy: 0.6952 - val_loss: 0.9530 - val_accuracy: 0.6929\n",
            "Epoch 305/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9090 - accuracy: 0.6970 - val_loss: 0.9570 - val_accuracy: 0.6931\n",
            "Epoch 306/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9142 - accuracy: 0.6929 - val_loss: 0.9566 - val_accuracy: 0.6931\n",
            "Epoch 307/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9147 - accuracy: 0.6938 - val_loss: 0.9564 - val_accuracy: 0.6944\n",
            "Epoch 308/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9141 - accuracy: 0.6945 - val_loss: 0.9586 - val_accuracy: 0.6915\n",
            "Epoch 309/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9086 - accuracy: 0.6973 - val_loss: 0.9542 - val_accuracy: 0.6893\n",
            "Epoch 310/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9080 - accuracy: 0.6984 - val_loss: 0.9559 - val_accuracy: 0.6935\n",
            "Epoch 311/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9160 - accuracy: 0.6921 - val_loss: 0.9614 - val_accuracy: 0.6918\n",
            "Epoch 312/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9145 - accuracy: 0.6956 - val_loss: 0.9552 - val_accuracy: 0.6965\n",
            "Epoch 313/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9155 - accuracy: 0.6944 - val_loss: 0.9591 - val_accuracy: 0.6933\n",
            "Epoch 314/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9097 - accuracy: 0.6966 - val_loss: 0.9563 - val_accuracy: 0.6927\n",
            "Epoch 315/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9127 - accuracy: 0.6943 - val_loss: 0.9570 - val_accuracy: 0.6900\n",
            "Epoch 316/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9132 - accuracy: 0.6956 - val_loss: 0.9590 - val_accuracy: 0.6902\n",
            "Epoch 317/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9107 - accuracy: 0.7004 - val_loss: 0.9571 - val_accuracy: 0.6929\n",
            "Epoch 318/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9120 - accuracy: 0.6954 - val_loss: 0.9537 - val_accuracy: 0.6956\n",
            "Epoch 319/500\n",
            "1256/1256 [==============================] - 11s 8ms/step - loss: 0.9094 - accuracy: 0.6950 - val_loss: 0.9538 - val_accuracy: 0.6938\n",
            "Epoch 320/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9071 - accuracy: 0.6971 - val_loss: 0.9556 - val_accuracy: 0.6933\n",
            "Epoch 321/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9102 - accuracy: 0.6959 - val_loss: 0.9578 - val_accuracy: 0.6956\n",
            "Epoch 322/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9117 - accuracy: 0.6957 - val_loss: 0.9566 - val_accuracy: 0.6956\n",
            "Epoch 323/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9034 - accuracy: 0.6987 - val_loss: 0.9566 - val_accuracy: 0.6913\n",
            "Epoch 324/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9124 - accuracy: 0.6963 - val_loss: 0.9587 - val_accuracy: 0.6958\n",
            "Epoch 325/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9061 - accuracy: 0.6983 - val_loss: 0.9577 - val_accuracy: 0.6944\n",
            "Epoch 326/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9114 - accuracy: 0.6982 - val_loss: 0.9550 - val_accuracy: 0.6949\n",
            "Epoch 327/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9067 - accuracy: 0.6979 - val_loss: 0.9573 - val_accuracy: 0.6953\n",
            "Epoch 328/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9115 - accuracy: 0.6983 - val_loss: 0.9552 - val_accuracy: 0.6953\n",
            "Epoch 329/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9177 - accuracy: 0.6966 - val_loss: 0.9558 - val_accuracy: 0.6944\n",
            "Epoch 330/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9104 - accuracy: 0.6957 - val_loss: 0.9540 - val_accuracy: 0.6924\n",
            "Epoch 331/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9044 - accuracy: 0.6994 - val_loss: 0.9605 - val_accuracy: 0.6951\n",
            "Epoch 332/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9049 - accuracy: 0.6978 - val_loss: 0.9576 - val_accuracy: 0.6960\n",
            "Epoch 333/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9101 - accuracy: 0.6977 - val_loss: 0.9561 - val_accuracy: 0.6951\n",
            "Epoch 334/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9097 - accuracy: 0.6946 - val_loss: 0.9580 - val_accuracy: 0.6913\n",
            "Epoch 335/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9075 - accuracy: 0.6958 - val_loss: 0.9568 - val_accuracy: 0.6938\n",
            "Epoch 336/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9102 - accuracy: 0.6952 - val_loss: 0.9584 - val_accuracy: 0.6933\n",
            "Epoch 337/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9105 - accuracy: 0.6963 - val_loss: 0.9584 - val_accuracy: 0.6947\n",
            "Epoch 338/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9058 - accuracy: 0.6970 - val_loss: 0.9559 - val_accuracy: 0.6918\n",
            "Epoch 339/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9161 - accuracy: 0.6925 - val_loss: 0.9552 - val_accuracy: 0.6929\n",
            "Epoch 340/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9038 - accuracy: 0.7009 - val_loss: 0.9564 - val_accuracy: 0.6942\n",
            "Epoch 341/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9087 - accuracy: 0.6972 - val_loss: 0.9573 - val_accuracy: 0.6920\n",
            "Epoch 342/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9116 - accuracy: 0.6950 - val_loss: 0.9577 - val_accuracy: 0.6922\n",
            "Epoch 343/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9140 - accuracy: 0.6943 - val_loss: 0.9566 - val_accuracy: 0.6962\n",
            "Epoch 344/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9146 - accuracy: 0.6958 - val_loss: 0.9556 - val_accuracy: 0.6924\n",
            "Epoch 345/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9144 - accuracy: 0.6947 - val_loss: 0.9585 - val_accuracy: 0.6911\n",
            "Epoch 346/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9079 - accuracy: 0.6967 - val_loss: 0.9561 - val_accuracy: 0.6933\n",
            "Epoch 347/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9030 - accuracy: 0.6987 - val_loss: 0.9593 - val_accuracy: 0.6902\n",
            "Epoch 348/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9048 - accuracy: 0.7001 - val_loss: 0.9574 - val_accuracy: 0.6904\n",
            "Epoch 349/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9063 - accuracy: 0.6983 - val_loss: 0.9609 - val_accuracy: 0.6915\n",
            "Epoch 350/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9062 - accuracy: 0.7002 - val_loss: 0.9559 - val_accuracy: 0.6944\n",
            "Epoch 351/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9058 - accuracy: 0.6974 - val_loss: 0.9589 - val_accuracy: 0.6922\n",
            "Epoch 352/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9117 - accuracy: 0.6990 - val_loss: 0.9606 - val_accuracy: 0.6918\n",
            "Epoch 353/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9084 - accuracy: 0.6981 - val_loss: 0.9530 - val_accuracy: 0.6924\n",
            "Epoch 354/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9070 - accuracy: 0.6996 - val_loss: 0.9592 - val_accuracy: 0.6942\n",
            "Epoch 355/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9051 - accuracy: 0.7000 - val_loss: 0.9591 - val_accuracy: 0.6947\n",
            "Epoch 356/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9118 - accuracy: 0.6957 - val_loss: 0.9563 - val_accuracy: 0.6935\n",
            "Epoch 357/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9067 - accuracy: 0.6978 - val_loss: 0.9558 - val_accuracy: 0.6929\n",
            "Epoch 358/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9083 - accuracy: 0.6965 - val_loss: 0.9584 - val_accuracy: 0.6906\n",
            "Epoch 359/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9113 - accuracy: 0.6968 - val_loss: 0.9576 - val_accuracy: 0.6958\n",
            "Epoch 360/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9097 - accuracy: 0.6943 - val_loss: 0.9564 - val_accuracy: 0.6922\n",
            "Epoch 361/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9042 - accuracy: 0.6969 - val_loss: 0.9584 - val_accuracy: 0.6920\n",
            "Epoch 362/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9046 - accuracy: 0.6964 - val_loss: 0.9576 - val_accuracy: 0.6940\n",
            "Epoch 363/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9084 - accuracy: 0.6966 - val_loss: 0.9598 - val_accuracy: 0.6947\n",
            "Epoch 364/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9103 - accuracy: 0.6949 - val_loss: 0.9520 - val_accuracy: 0.6938\n",
            "Epoch 365/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9080 - accuracy: 0.6974 - val_loss: 0.9545 - val_accuracy: 0.6927\n",
            "Epoch 366/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9098 - accuracy: 0.6952 - val_loss: 0.9568 - val_accuracy: 0.6935\n",
            "Epoch 367/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9068 - accuracy: 0.6986 - val_loss: 0.9580 - val_accuracy: 0.6909\n",
            "Epoch 368/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9114 - accuracy: 0.6933 - val_loss: 0.9546 - val_accuracy: 0.6958\n",
            "Epoch 369/500\n",
            "1256/1256 [==============================] - 11s 9ms/step - loss: 0.9058 - accuracy: 0.6959 - val_loss: 0.9564 - val_accuracy: 0.6942\n",
            "Epoch 370/500\n",
            " 272/1256 [=====>........................] - ETA: 8s - loss: 0.9042 - accuracy: 0.7022"
          ]
        }
      ]
    }
  ]
}