OurSolution.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-15T09:35:37.893780Z",
     "start_time": "2019-08-15T09:35:37.285220Z"
    }
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import numpy as np\n",
    "import torchvision.transforms as transforms\n",
    "import torch.utils.data as data\n",
    "import matplotlib.pyplot as plt\n",
    "import IPython.display\n",
    "from IPython.display import Audio\n",
    "import torch.optim as optim\n",
    "from types import SimpleNamespace\n",
    "import scipy.signal as sc\n",
    "import time\n",
    "from sklearn.metrics import f1_score\n",
    "import pickle\n",
    "\n",
    "from trainDataset import TrainDataset\n",
    "from testDataset import TestDataset\n",
    "#from trainDatasetNew import TrainDatasetNew\n",
    "#from testDatasetNew import TestDatasetNew\n",
    "from validation_split import get_dataloaders\n",
    "from math_utils import logMagStft, ffts\n",
    "from SpectrogramCNN import SpectrogramCNN\n",
    "from train_utils import train, test, save_output, save_geometric_mean_predictions\n",
    "from evaluation_utils import get_mean_F1, get_loss\n",
    "from MulitScale1DResNet import MSResNet\n",
    "from LSTM import LSTM\n",
    "from FFTMulitScale1DResNet import FFTMSResNet\n",
    "from SpectralResNet import SpectralResNet18, SpectralResNet34\n",
    "from scipy.signal import hilbert\n",
    "from FeatureFNN import FeatureFNN\n",
    "from MelRawCombined import CNN1D, CNN2D"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-16T08:03:32.269198Z",
     "start_time": "2019-08-16T08:03:32.257502Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "with cuda\n"
     ]
    }
   ],
   "source": [
    "validation_split = .1\n",
    "do_plots = False\n",
    "load_model = True\n",
    "args = SimpleNamespace(batch_size=4, test_batch_size=4, epochs=30,\n",
    "                       lr=0.0001, momentum=0.9, seed=1, log_interval=200, weight_decay=0.0001,\n",
    "                      net = SpectralResNet34) #SpectrogramCNN, MSResNet, SpectralResNet34, FFTMSResNet, FeatureFNN, CNN1D, CNN2D \n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "if not torch.cuda.is_available(): # adapt those paths on other machine\n",
    "    print('no cuda')\n",
    "    path_train = './../data/train-small/'\n",
    "    path_test =  './../data/test/kaggle-test/'\n",
    "else:\n",
    "    print('with cuda')\n",
    "    path_train = './../data/kaggle-train/'\n",
    "    path_test =  './../data/kaggle-test/'\n",
    "    \n",
    "path_model = 'models/'\n",
    "path_submission = 'submissions/'\n",
    "    \n",
    "sample_rate = 16000\n",
    "nmbr_classes = 10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Original Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-15T09:35:39.140799Z",
     "start_time": "2019-08-15T09:35:37.925836Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "269776\n",
      "4096\n"
     ]
    }
   ],
   "source": [
    "# todo add in the classes the features and the fft data\n",
    "\n",
    "toFloat = transforms.Lambda(lambda x: x / np.iinfo(np.int16).max)\n",
    "\n",
    "trainDataset = TrainDataset(path_train, transform=toFloat)\n",
    "print(len(trainDataset))\n",
    "\n",
    "testDataset = TestDataset(path_test, transform=toFloat)\n",
    "print(len(testDataset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-15T09:35:39.145857Z",
     "start_time": "2019-08-15T09:35:39.142411Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input size:  64000\n"
     ]
    }
   ],
   "source": [
    "input_size = len(trainDataset[0][0])\n",
    "print('input size: ',input_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Look at Original Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-15T09:35:39.196234Z",
     "start_time": "2019-08-15T09:35:39.147069Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "if do_plots:\n",
    "    # how many instruments are there?\n",
    "    dummy_count = np.zeros(20)\n",
    "\n",
    "    for sample in trainDataset:\n",
    "        dummy_count[sample[1]] += 1\n",
    "\n",
    "    labels_count = []\n",
    "    for elem in dummy_count:\n",
    "        if elem != 0:\n",
    "            labels_count.append(elem)\n",
    "\n",
    "    print(labels_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-15T09:35:39.200291Z",
     "start_time": "2019-08-15T09:35:39.197602Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "if do_plots:\n",
    "    nmbr_classes = len(labels_count)\n",
    "    print('nmbr_classes: ', nmbr_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-15T09:35:39.204858Z",
     "start_time": "2019-08-15T09:35:39.202100Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "if do_plots:\n",
    "    plt.plot(labels_count, '*')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-15T09:35:39.211063Z",
     "start_time": "2019-08-15T09:35:39.207579Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "if do_plots:\n",
    "    # plot one of each\n",
    "\n",
    "    done = np.zeros(nmbr_classes)\n",
    "    examples = []\n",
    "\n",
    "    for sample in trainDataset:\n",
    "        if done[sample[1]] == 0:\n",
    "            examples.append(sample)\n",
    "            done[sample[1]] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-15T09:35:39.216325Z",
     "start_time": "2019-08-15T09:35:39.213244Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "plt.rcParams[\"figure.figsize\"] = (20,10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-15T09:35:39.224741Z",
     "start_time": "2019-08-15T09:35:39.220091Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "#time series\n",
    "\n",
    "if do_plots:\n",
    "\n",
    "    for i in range(len(examples)):\n",
    "        plt.subplot(4,3,i+1)\n",
    "        plt.plot(examples[i][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-15T09:35:39.229542Z",
     "start_time": "2019-08-15T09:35:39.226060Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "#envelope\n",
    "\n",
    "if do_plots:\n",
    "    \n",
    "    m = nn.MaxPool1d(50)\n",
    "\n",
    "    for i in range(len(examples)):\n",
    "        plt.subplot(4,3,i+1)\n",
    "        env = torch.from_numpy(np.abs(hilbert(examples[i][0]))).view(1,1,-1)\n",
    "        env = m(env)\n",
    "        plt.plot(env[0,0].numpy())\n",
    "        \n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-15T09:35:39.233990Z",
     "start_time": "2019-08-15T09:35:39.230722Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# fft\n",
    "\n",
    "if do_plots:\n",
    "\n",
    "    m = nn.MaxPool1d(50)\n",
    "\n",
    "    for i in range(len(examples)):\n",
    "        plt.subplot(4,3,i+1)\n",
    "        fft = torch.from_numpy(ffts(examples[i][0])).view(1,1,-1)\n",
    "        fft = m(fft)\n",
    "        plt.plot(fft[0,0].numpy())\n",
    "        \n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-15T09:35:39.237657Z",
     "start_time": "2019-08-15T09:35:39.235032Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# melspec\n",
    "\n",
    "if do_plots:\n",
    "    n_fft = 510\n",
    "    for i in range(len(examples)):\n",
    "        plt.subplot(4,3,i+1)\n",
    "        spectrogram = logMagStft(examples[i][0], 16000, n_fft)\n",
    "        plt.imshow(spectrogram)\n",
    "        \n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-15T09:35:39.240637Z",
     "start_time": "2019-08-15T09:35:39.238661Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "if do_plots:\n",
    "\n",
    "    for sample in examples:\n",
    "        display(Audio(sample[0], rate=sample_rate))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dataloaders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-15T09:35:39.321792Z",
     "start_time": "2019-08-15T09:35:39.242294Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([4, 64000]) torch.Size([4]) tensor([3, 4, 5, 0])\n",
      "tensor(-0.9380, dtype=torch.float64) tensor(0.9464, dtype=torch.float64)\n",
      "['guitar' 'keyboard' 'mallet' 'bass']\n",
      "torch.Size([4, 64000]) torch.Size([4]) tensor([2, 8, 9, 0])\n",
      "tensor(-0.9466, dtype=torch.float64) tensor(0.9466, dtype=torch.float64)\n",
      "['flute' 'string' 'vocal' 'bass']\n"
     ]
    }
   ],
   "source": [
    "# validation split is done here\n",
    "\n",
    "train_loader, validation_loader, indices = get_dataloaders(trainDataset, \n",
    "                                                  batch_size = args.batch_size, \n",
    "                                                  validation_split = validation_split, \n",
    "                                                  shuffle_dataset = True, \n",
    "                                                  random_seed = None)#,\n",
    "                                                  #indices = indices[0])\n",
    "\n",
    "for samples, instrument_family_target in train_loader:\n",
    "        print(samples.shape, instrument_family_target.shape,\n",
    "              instrument_family_target.data)\n",
    "        print(torch.min(samples), torch.max(samples))\n",
    "        print(trainDataset.transformInstrumentsFamilyToString(instrument_family_target.data))\n",
    "        break\n",
    "        \n",
    "for samples, instrument_family_target in validation_loader:\n",
    "        print(samples.shape, instrument_family_target.shape,\n",
    "              instrument_family_target.data)\n",
    "        print(torch.min(samples), torch.max(samples))\n",
    "        print(trainDataset.transformInstrumentsFamilyToString(instrument_family_target.data))\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-16T08:02:59.894242Z",
     "start_time": "2019-08-16T08:02:59.864390Z"
    }
   },
   "outputs": [],
   "source": [
    "# save indices\n",
    "with open(path_submission + \"indices.txt\", \"wb\") as fp:\n",
    "    pickle.dump([indices], fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-15T09:35:39.355557Z",
     "start_time": "2019-08-15T09:35:39.348106Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([4, 64000])\n",
      "tensor(-0.8568, dtype=torch.float64) tensor(0.9467, dtype=torch.float64)\n"
     ]
    }
   ],
   "source": [
    "test_loader = data.DataLoader(testDataset, batch_size=args.batch_size, shuffle=False) #!!! shuffle should be false\n",
    "for samples in test_loader:\n",
    "        print(samples[0].shape)\n",
    "        print(torch.min(samples[0]), torch.max(samples[0]))\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Main"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-15T09:35:41.408381Z",
     "start_time": "2019-08-15T09:35:39.356916Z"
    }
   },
   "outputs": [],
   "source": [
    "if (args.net == SpectrogramCNN) or (args.net == SpectralResNet18) \\\n",
    "    or (args.net == FeatureFNN) or (args.net == CNN1D) \\\n",
    "    or (args.net == CNN2D) or (args.net == SpectralResNet34):\n",
    "    model = args.net(device).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-15T09:35:41.412333Z",
     "start_time": "2019-08-15T09:35:41.409898Z"
    }
   },
   "outputs": [],
   "source": [
    "if (args.net == MSResNet) or (args.net == FFTMSResNet):\n",
    "    model = args.net(1, device).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-15T09:35:41.416826Z",
     "start_time": "2019-08-15T09:35:41.413444Z"
    }
   },
   "outputs": [],
   "source": [
    "if args.net == LSTM:\n",
    "    model = args.net(device, input_size = 252, hidden_size = 512, num_layers = 1, num_classes = 10).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-15T09:35:41.428704Z",
     "start_time": "2019-08-15T09:35:41.419859Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SpectralResNet(\n",
      "  (conv1): Conv2d(1, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "  (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "  (layer1): Sequential(\n",
      "    (0): BasicBlock(\n",
      "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (shortcut): Sequential()\n",
      "    )\n",
      "    (1): BasicBlock(\n",
      "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (shortcut): Sequential()\n",
      "    )\n",
      "    (2): BasicBlock(\n",
      "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (shortcut): Sequential()\n",
      "    )\n",
      "  )\n",
      "  (layer2): Sequential(\n",
      "    (0): BasicBlock(\n",
      "      (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (shortcut): Sequential(\n",
      "        (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
      "        (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "    )\n",
      "    (1): BasicBlock(\n",
      "      (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (shortcut): Sequential()\n",
      "    )\n",
      "    (2): BasicBlock(\n",
      "      (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (shortcut): Sequential()\n",
      "    )\n",
      "    (3): BasicBlock(\n",
      "      (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (shortcut): Sequential()\n",
      "    )\n",
      "  )\n",
      "  (layer3): Sequential(\n",
      "    (0): BasicBlock(\n",
      "      (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (shortcut): Sequential(\n",
      "        (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
      "        (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "    )\n",
      "    (1): BasicBlock(\n",
      "      (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (shortcut): Sequential()\n",
      "    )\n",
      "    (2): BasicBlock(\n",
      "      (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (shortcut): Sequential()\n",
      "    )\n",
      "    (3): BasicBlock(\n",
      "      (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (shortcut): Sequential()\n",
      "    )\n",
      "    (4): BasicBlock(\n",
      "      (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (shortcut): Sequential()\n",
      "    )\n",
      "    (5): BasicBlock(\n",
      "      (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (shortcut): Sequential()\n",
      "    )\n",
      "  )\n",
      "  (layer4): Sequential(\n",
      "    (0): BasicBlock(\n",
      "      (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (shortcut): Sequential(\n",
      "        (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
      "        (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "    )\n",
      "    (1): BasicBlock(\n",
      "      (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (shortcut): Sequential()\n",
      "    )\n",
      "    (2): BasicBlock(\n",
      "      (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (shortcut): Sequential()\n",
      "    )\n",
      "  )\n",
      "  (linear): Linear(in_features=32768, out_features=10, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-15T09:35:41.433524Z",
     "start_time": "2019-08-15T09:35:41.430192Z"
    }
   },
   "outputs": [],
   "source": [
    "# Main\n",
    "#optimizer = optim.Adam(model.parameters(), lr=args.lr)\n",
    "\n",
    "optimizer = optim.SGD(model.parameters(), lr= args.lr, momentum= args.momentum, weight_decay= args.weight_decay)\n",
    "\n",
    "info = {'highest F1' : 0,\n",
    "        'lowest loss' : 100,\n",
    "        'saved epoch' : 10}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-16T14:33:16.120221Z",
     "start_time": "2019-08-16T08:06:36.647333Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 8 [0/269776 (0%)]\tLoss: 0.000048\tF1: 1.0000\tRuntime: 0.2\n",
      "Train Epoch: 8 [800/269776 (0%)]\tLoss: 0.000707\tF1: 1.0000\tRuntime: 35.4\n",
      "Train Epoch: 8 [1600/269776 (1%)]\tLoss: 0.000004\tF1: 1.0000\tRuntime: 70.6\n",
      "Train Epoch: 8 [2400/269776 (1%)]\tLoss: 0.000020\tF1: 1.0000\tRuntime: 105.9\n",
      "Train Epoch: 8 [3200/269776 (1%)]\tLoss: 0.001192\tF1: 1.0000\tRuntime: 141.2\n",
      "Train Epoch: 8 [4000/269776 (2%)]\tLoss: 0.000005\tF1: 1.0000\tRuntime: 176.7\n",
      "Train Epoch: 8 [4800/269776 (2%)]\tLoss: 0.020992\tF1: 1.0000\tRuntime: 212.3\n",
      "Train Epoch: 8 [5600/269776 (2%)]\tLoss: 0.000195\tF1: 1.0000\tRuntime: 247.7\n",
      "Train Epoch: 8 [6400/269776 (3%)]\tLoss: 0.181952\tF1: 0.7500\tRuntime: 283.2\n",
      "Train Epoch: 8 [7200/269776 (3%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 318.8\n",
      "Train Epoch: 8 [8000/269776 (3%)]\tLoss: 0.000037\tF1: 1.0000\tRuntime: 354.3\n",
      "Train Epoch: 8 [8800/269776 (4%)]\tLoss: 0.000092\tF1: 1.0000\tRuntime: 389.7\n",
      "Train Epoch: 8 [9600/269776 (4%)]\tLoss: 0.000064\tF1: 1.0000\tRuntime: 425.2\n",
      "Train Epoch: 8 [10400/269776 (4%)]\tLoss: 0.000061\tF1: 1.0000\tRuntime: 460.8\n",
      "Train Epoch: 8 [11200/269776 (5%)]\tLoss: 0.000018\tF1: 1.0000\tRuntime: 496.3\n",
      "Train Epoch: 8 [12000/269776 (5%)]\tLoss: 0.000003\tF1: 1.0000\tRuntime: 531.8\n",
      "Train Epoch: 8 [12800/269776 (5%)]\tLoss: 0.000079\tF1: 1.0000\tRuntime: 567.2\n",
      "Train Epoch: 8 [13600/269776 (6%)]\tLoss: 0.000011\tF1: 1.0000\tRuntime: 602.7\n",
      "Train Epoch: 8 [14400/269776 (6%)]\tLoss: 0.000006\tF1: 1.0000\tRuntime: 638.2\n",
      "Train Epoch: 8 [15200/269776 (6%)]\tLoss: 0.000024\tF1: 1.0000\tRuntime: 673.6\n",
      "Train Epoch: 8 [16000/269776 (7%)]\tLoss: 0.002929\tF1: 1.0000\tRuntime: 709.0\n",
      "Train Epoch: 8 [16800/269776 (7%)]\tLoss: 0.000019\tF1: 1.0000\tRuntime: 744.4\n",
      "Train Epoch: 8 [17600/269776 (7%)]\tLoss: 0.000151\tF1: 1.0000\tRuntime: 780.0\n",
      "Train Epoch: 8 [18400/269776 (8%)]\tLoss: 0.000302\tF1: 1.0000\tRuntime: 815.4\n",
      "Train Epoch: 8 [19200/269776 (8%)]\tLoss: 0.000373\tF1: 1.0000\tRuntime: 850.7\n",
      "Train Epoch: 8 [20000/269776 (8%)]\tLoss: 0.000004\tF1: 1.0000\tRuntime: 886.1\n",
      "Train Epoch: 8 [20800/269776 (9%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 921.5\n",
      "Train Epoch: 8 [21600/269776 (9%)]\tLoss: 0.000002\tF1: 1.0000\tRuntime: 956.9\n",
      "Train Epoch: 8 [22400/269776 (9%)]\tLoss: 0.000028\tF1: 1.0000\tRuntime: 992.3\n",
      "Train Epoch: 8 [23200/269776 (10%)]\tLoss: 0.000038\tF1: 1.0000\tRuntime: 1027.7\n",
      "Train Epoch: 8 [24000/269776 (10%)]\tLoss: 0.001318\tF1: 1.0000\tRuntime: 1063.3\n",
      "Train Epoch: 8 [24800/269776 (10%)]\tLoss: 0.000091\tF1: 1.0000\tRuntime: 1098.7\n",
      "Train Epoch: 8 [25600/269776 (11%)]\tLoss: 0.000003\tF1: 1.0000\tRuntime: 1134.1\n",
      "Train Epoch: 8 [26400/269776 (11%)]\tLoss: 0.000006\tF1: 1.0000\tRuntime: 1169.5\n",
      "Train Epoch: 8 [27200/269776 (11%)]\tLoss: 0.000493\tF1: 1.0000\tRuntime: 1204.9\n",
      "Train Epoch: 8 [28000/269776 (12%)]\tLoss: 0.000083\tF1: 1.0000\tRuntime: 1240.2\n",
      "Train Epoch: 8 [28800/269776 (12%)]\tLoss: 0.000029\tF1: 1.0000\tRuntime: 1275.6\n",
      "Train Epoch: 8 [29600/269776 (12%)]\tLoss: 0.000006\tF1: 1.0000\tRuntime: 1311.0\n",
      "Train Epoch: 8 [30400/269776 (13%)]\tLoss: 0.001063\tF1: 1.0000\tRuntime: 1346.4\n",
      "Train Epoch: 8 [31200/269776 (13%)]\tLoss: 0.000002\tF1: 1.0000\tRuntime: 1381.8\n",
      "Train Epoch: 8 [32000/269776 (13%)]\tLoss: 0.000011\tF1: 1.0000\tRuntime: 1417.2\n",
      "Train Epoch: 8 [32800/269776 (14%)]\tLoss: 0.000002\tF1: 1.0000\tRuntime: 1452.5\n",
      "Train Epoch: 8 [33600/269776 (14%)]\tLoss: 0.000273\tF1: 1.0000\tRuntime: 1487.9\n",
      "Train Epoch: 8 [34400/269776 (14%)]\tLoss: 0.000051\tF1: 1.0000\tRuntime: 1523.3\n",
      "Train Epoch: 8 [35200/269776 (14%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 1558.7\n",
      "Train Epoch: 8 [36000/269776 (15%)]\tLoss: 0.000001\tF1: 1.0000\tRuntime: 1594.1\n",
      "Train Epoch: 8 [36800/269776 (15%)]\tLoss: 0.009872\tF1: 1.0000\tRuntime: 1629.5\n",
      "Train Epoch: 8 [37600/269776 (15%)]\tLoss: 0.000561\tF1: 1.0000\tRuntime: 1664.8\n",
      "Train Epoch: 8 [38400/269776 (16%)]\tLoss: 0.000052\tF1: 1.0000\tRuntime: 1700.2\n",
      "Train Epoch: 8 [39200/269776 (16%)]\tLoss: 0.000059\tF1: 1.0000\tRuntime: 1735.6\n",
      "Train Epoch: 8 [40000/269776 (16%)]\tLoss: 0.010491\tF1: 1.0000\tRuntime: 1771.0\n",
      "Train Epoch: 8 [40800/269776 (17%)]\tLoss: 0.000001\tF1: 1.0000\tRuntime: 1806.4\n",
      "Train Epoch: 8 [41600/269776 (17%)]\tLoss: 0.000112\tF1: 1.0000\tRuntime: 1841.8\n",
      "Train Epoch: 8 [42400/269776 (17%)]\tLoss: 0.000097\tF1: 1.0000\tRuntime: 1877.1\n",
      "Train Epoch: 8 [43200/269776 (18%)]\tLoss: 0.000003\tF1: 1.0000\tRuntime: 1912.5\n",
      "Train Epoch: 8 [44000/269776 (18%)]\tLoss: 0.000001\tF1: 1.0000\tRuntime: 1947.9\n",
      "Train Epoch: 8 [44800/269776 (18%)]\tLoss: 0.001998\tF1: 1.0000\tRuntime: 1983.3\n",
      "Train Epoch: 8 [45600/269776 (19%)]\tLoss: 0.000058\tF1: 1.0000\tRuntime: 2018.6\n",
      "Train Epoch: 8 [46400/269776 (19%)]\tLoss: 0.000034\tF1: 1.0000\tRuntime: 2054.0\n",
      "Train Epoch: 8 [47200/269776 (19%)]\tLoss: 0.000034\tF1: 1.0000\tRuntime: 2089.4\n",
      "Train Epoch: 8 [48000/269776 (20%)]\tLoss: 0.000001\tF1: 1.0000\tRuntime: 2124.8\n",
      "Train Epoch: 8 [48800/269776 (20%)]\tLoss: 0.000018\tF1: 1.0000\tRuntime: 2160.1\n",
      "Train Epoch: 8 [49600/269776 (20%)]\tLoss: 0.000480\tF1: 1.0000\tRuntime: 2195.5\n",
      "Train Epoch: 8 [50400/269776 (21%)]\tLoss: 0.000242\tF1: 1.0000\tRuntime: 2230.9\n",
      "Train Epoch: 8 [51200/269776 (21%)]\tLoss: 0.000007\tF1: 1.0000\tRuntime: 2266.2\n",
      "Train Epoch: 8 [52000/269776 (21%)]\tLoss: 0.000022\tF1: 1.0000\tRuntime: 2301.6\n",
      "Train Epoch: 8 [52800/269776 (22%)]\tLoss: 0.000106\tF1: 1.0000\tRuntime: 2337.0\n",
      "Train Epoch: 8 [53600/269776 (22%)]\tLoss: 0.001182\tF1: 1.0000\tRuntime: 2372.3\n",
      "Train Epoch: 8 [54400/269776 (22%)]\tLoss: 0.000132\tF1: 1.0000\tRuntime: 2407.7\n",
      "Train Epoch: 8 [55200/269776 (23%)]\tLoss: 0.003029\tF1: 1.0000\tRuntime: 2443.1\n",
      "Train Epoch: 8 [56000/269776 (23%)]\tLoss: 0.005154\tF1: 1.0000\tRuntime: 2478.5\n",
      "Train Epoch: 8 [56800/269776 (23%)]\tLoss: 0.000005\tF1: 1.0000\tRuntime: 2513.8\n",
      "Train Epoch: 8 [57600/269776 (24%)]\tLoss: 0.000004\tF1: 1.0000\tRuntime: 2549.2\n",
      "Train Epoch: 8 [58400/269776 (24%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 2584.6\n",
      "Train Epoch: 8 [59200/269776 (24%)]\tLoss: 0.050117\tF1: 1.0000\tRuntime: 2619.9\n",
      "Train Epoch: 8 [60000/269776 (25%)]\tLoss: 0.000015\tF1: 1.0000\tRuntime: 2655.3\n",
      "Train Epoch: 8 [60800/269776 (25%)]\tLoss: 0.000053\tF1: 1.0000\tRuntime: 2690.7\n",
      "Train Epoch: 8 [61600/269776 (25%)]\tLoss: 0.000183\tF1: 1.0000\tRuntime: 2726.1\n",
      "Train Epoch: 8 [62400/269776 (26%)]\tLoss: 0.000038\tF1: 1.0000\tRuntime: 2761.5\n",
      "Train Epoch: 8 [63200/269776 (26%)]\tLoss: 0.000024\tF1: 1.0000\tRuntime: 2796.8\n",
      "Train Epoch: 8 [64000/269776 (26%)]\tLoss: 0.000013\tF1: 1.0000\tRuntime: 2832.2\n",
      "Train Epoch: 8 [64800/269776 (27%)]\tLoss: 0.000009\tF1: 1.0000\tRuntime: 2867.6\n",
      "Train Epoch: 8 [65600/269776 (27%)]\tLoss: 0.000011\tF1: 1.0000\tRuntime: 2903.0\n",
      "Train Epoch: 8 [66400/269776 (27%)]\tLoss: 0.000030\tF1: 1.0000\tRuntime: 2938.4\n",
      "Train Epoch: 8 [67200/269776 (28%)]\tLoss: 0.000001\tF1: 1.0000\tRuntime: 2973.8\n",
      "Train Epoch: 8 [68000/269776 (28%)]\tLoss: 0.000154\tF1: 1.0000\tRuntime: 3009.1\n",
      "Train Epoch: 8 [68800/269776 (28%)]\tLoss: 0.000018\tF1: 1.0000\tRuntime: 3044.5\n",
      "Train Epoch: 8 [69600/269776 (29%)]\tLoss: 0.000017\tF1: 1.0000\tRuntime: 3079.9\n",
      "Train Epoch: 8 [70400/269776 (29%)]\tLoss: 0.000176\tF1: 1.0000\tRuntime: 3115.3\n",
      "Train Epoch: 8 [71200/269776 (29%)]\tLoss: 0.003053\tF1: 1.0000\tRuntime: 3150.7\n",
      "Train Epoch: 8 [72000/269776 (30%)]\tLoss: 0.000002\tF1: 1.0000\tRuntime: 3186.0\n",
      "Train Epoch: 8 [72800/269776 (30%)]\tLoss: 0.000015\tF1: 1.0000\tRuntime: 3221.4\n",
      "Train Epoch: 8 [73600/269776 (30%)]\tLoss: 0.000102\tF1: 1.0000\tRuntime: 3256.8\n",
      "Train Epoch: 8 [74400/269776 (31%)]\tLoss: 0.000027\tF1: 1.0000\tRuntime: 3292.2\n",
      "Train Epoch: 8 [75200/269776 (31%)]\tLoss: 0.000012\tF1: 1.0000\tRuntime: 3327.5\n",
      "Train Epoch: 8 [76000/269776 (31%)]\tLoss: 0.000403\tF1: 1.0000\tRuntime: 3362.9\n",
      "Train Epoch: 8 [76800/269776 (32%)]\tLoss: 0.000118\tF1: 1.0000\tRuntime: 3398.3\n",
      "Train Epoch: 8 [77600/269776 (32%)]\tLoss: 0.000010\tF1: 1.0000\tRuntime: 3433.7\n",
      "Train Epoch: 8 [78400/269776 (32%)]\tLoss: 0.000066\tF1: 1.0000\tRuntime: 3469.1\n",
      "Train Epoch: 8 [79200/269776 (33%)]\tLoss: 0.000606\tF1: 1.0000\tRuntime: 3504.5\n",
      "Train Epoch: 8 [80000/269776 (33%)]\tLoss: 0.000003\tF1: 1.0000\tRuntime: 3539.9\n",
      "Train Epoch: 8 [80800/269776 (33%)]\tLoss: 0.000278\tF1: 1.0000\tRuntime: 3575.2\n",
      "Train Epoch: 8 [81600/269776 (34%)]\tLoss: 0.000015\tF1: 1.0000\tRuntime: 3610.6\n",
      "Train Epoch: 8 [82400/269776 (34%)]\tLoss: 0.001921\tF1: 1.0000\tRuntime: 3646.0\n",
      "Train Epoch: 8 [83200/269776 (34%)]\tLoss: 0.000091\tF1: 1.0000\tRuntime: 3681.4\n",
      "Train Epoch: 8 [84000/269776 (35%)]\tLoss: 0.000173\tF1: 1.0000\tRuntime: 3716.8\n",
      "Train Epoch: 8 [84800/269776 (35%)]\tLoss: 0.001201\tF1: 1.0000\tRuntime: 3752.1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 8 [85600/269776 (35%)]\tLoss: 0.000003\tF1: 1.0000\tRuntime: 3787.5\n",
      "Train Epoch: 8 [86400/269776 (36%)]\tLoss: 0.000004\tF1: 1.0000\tRuntime: 3822.9\n",
      "Train Epoch: 8 [87200/269776 (36%)]\tLoss: 0.000993\tF1: 1.0000\tRuntime: 3858.2\n",
      "Train Epoch: 8 [88000/269776 (36%)]\tLoss: 0.001828\tF1: 1.0000\tRuntime: 3893.6\n",
      "Train Epoch: 8 [88800/269776 (37%)]\tLoss: 0.000002\tF1: 1.0000\tRuntime: 3929.0\n",
      "Train Epoch: 8 [89600/269776 (37%)]\tLoss: 0.000530\tF1: 1.0000\tRuntime: 3964.4\n",
      "Train Epoch: 8 [90400/269776 (37%)]\tLoss: 0.000001\tF1: 1.0000\tRuntime: 3999.7\n",
      "Train Epoch: 8 [91200/269776 (38%)]\tLoss: 0.000035\tF1: 1.0000\tRuntime: 4035.1\n",
      "Train Epoch: 8 [92000/269776 (38%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 4070.4\n",
      "Train Epoch: 8 [92800/269776 (38%)]\tLoss: 0.000101\tF1: 1.0000\tRuntime: 4105.8\n",
      "Train Epoch: 8 [93600/269776 (39%)]\tLoss: 0.000001\tF1: 1.0000\tRuntime: 4141.2\n",
      "Train Epoch: 8 [94400/269776 (39%)]\tLoss: 0.000010\tF1: 1.0000\tRuntime: 4176.6\n",
      "Train Epoch: 8 [95200/269776 (39%)]\tLoss: 0.000107\tF1: 1.0000\tRuntime: 4211.9\n",
      "Train Epoch: 8 [96000/269776 (40%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 4247.3\n",
      "Train Epoch: 8 [96800/269776 (40%)]\tLoss: 0.002914\tF1: 1.0000\tRuntime: 4282.6\n",
      "Train Epoch: 8 [97600/269776 (40%)]\tLoss: 0.000003\tF1: 1.0000\tRuntime: 4318.0\n",
      "Train Epoch: 8 [98400/269776 (41%)]\tLoss: 0.000435\tF1: 1.0000\tRuntime: 4353.3\n",
      "Train Epoch: 8 [99200/269776 (41%)]\tLoss: 0.000012\tF1: 1.0000\tRuntime: 4388.7\n",
      "Train Epoch: 8 [100000/269776 (41%)]\tLoss: 0.000247\tF1: 1.0000\tRuntime: 4424.1\n",
      "Train Epoch: 8 [100800/269776 (42%)]\tLoss: 0.002921\tF1: 1.0000\tRuntime: 4459.4\n",
      "Train Epoch: 8 [101600/269776 (42%)]\tLoss: 0.000929\tF1: 1.0000\tRuntime: 4494.8\n",
      "Train Epoch: 8 [102400/269776 (42%)]\tLoss: 0.000011\tF1: 1.0000\tRuntime: 4530.2\n",
      "Train Epoch: 8 [103200/269776 (43%)]\tLoss: 0.000068\tF1: 1.0000\tRuntime: 4565.5\n",
      "Train Epoch: 8 [104000/269776 (43%)]\tLoss: 0.000007\tF1: 1.0000\tRuntime: 4600.8\n",
      "Train Epoch: 8 [104800/269776 (43%)]\tLoss: 0.000006\tF1: 1.0000\tRuntime: 4636.2\n",
      "Train Epoch: 8 [105600/269776 (43%)]\tLoss: 0.000017\tF1: 1.0000\tRuntime: 4671.6\n",
      "Train Epoch: 8 [106400/269776 (44%)]\tLoss: 0.006996\tF1: 1.0000\tRuntime: 4707.0\n",
      "Train Epoch: 8 [107200/269776 (44%)]\tLoss: 0.004128\tF1: 1.0000\tRuntime: 4742.3\n",
      "Train Epoch: 8 [108000/269776 (44%)]\tLoss: 0.000408\tF1: 1.0000\tRuntime: 4777.7\n",
      "Train Epoch: 8 [108800/269776 (45%)]\tLoss: 0.000245\tF1: 1.0000\tRuntime: 4813.1\n",
      "Train Epoch: 8 [109600/269776 (45%)]\tLoss: 0.000088\tF1: 1.0000\tRuntime: 4848.4\n",
      "Train Epoch: 8 [110400/269776 (45%)]\tLoss: 0.000100\tF1: 1.0000\tRuntime: 4883.7\n",
      "Train Epoch: 8 [111200/269776 (46%)]\tLoss: 0.000340\tF1: 1.0000\tRuntime: 4919.1\n",
      "Train Epoch: 8 [112000/269776 (46%)]\tLoss: 0.000004\tF1: 1.0000\tRuntime: 4954.5\n",
      "Train Epoch: 8 [112800/269776 (46%)]\tLoss: 0.002100\tF1: 1.0000\tRuntime: 4989.8\n",
      "Train Epoch: 8 [113600/269776 (47%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 5025.2\n",
      "Train Epoch: 8 [114400/269776 (47%)]\tLoss: 0.000001\tF1: 1.0000\tRuntime: 5060.5\n",
      "Train Epoch: 8 [115200/269776 (47%)]\tLoss: 0.000009\tF1: 1.0000\tRuntime: 5095.9\n",
      "Train Epoch: 8 [116000/269776 (48%)]\tLoss: 0.000046\tF1: 1.0000\tRuntime: 5131.2\n",
      "Train Epoch: 8 [116800/269776 (48%)]\tLoss: 0.001383\tF1: 1.0000\tRuntime: 5166.6\n",
      "Train Epoch: 8 [117600/269776 (48%)]\tLoss: 0.000013\tF1: 1.0000\tRuntime: 5201.9\n",
      "Train Epoch: 8 [118400/269776 (49%)]\tLoss: 0.001850\tF1: 1.0000\tRuntime: 5237.3\n",
      "Train Epoch: 8 [119200/269776 (49%)]\tLoss: 0.000019\tF1: 1.0000\tRuntime: 5272.6\n",
      "Train Epoch: 8 [120000/269776 (49%)]\tLoss: 0.000016\tF1: 1.0000\tRuntime: 5308.0\n",
      "Train Epoch: 8 [120800/269776 (50%)]\tLoss: 0.002244\tF1: 1.0000\tRuntime: 5343.3\n",
      "Train Epoch: 8 [121600/269776 (50%)]\tLoss: 0.000001\tF1: 1.0000\tRuntime: 5378.7\n",
      "Train Epoch: 8 [122400/269776 (50%)]\tLoss: 0.000008\tF1: 1.0000\tRuntime: 5414.0\n",
      "Train Epoch: 8 [123200/269776 (51%)]\tLoss: 0.000001\tF1: 1.0000\tRuntime: 5449.3\n",
      "Train Epoch: 8 [124000/269776 (51%)]\tLoss: 0.000001\tF1: 1.0000\tRuntime: 5484.7\n",
      "Train Epoch: 8 [124800/269776 (51%)]\tLoss: 0.000145\tF1: 1.0000\tRuntime: 5520.0\n",
      "Train Epoch: 8 [125600/269776 (52%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 5555.4\n",
      "Train Epoch: 8 [126400/269776 (52%)]\tLoss: 0.005136\tF1: 1.0000\tRuntime: 5590.7\n",
      "Train Epoch: 8 [127200/269776 (52%)]\tLoss: 0.013041\tF1: 1.0000\tRuntime: 5626.1\n",
      "Train Epoch: 8 [128000/269776 (53%)]\tLoss: 0.000004\tF1: 1.0000\tRuntime: 5661.4\n",
      "Train Epoch: 8 [128800/269776 (53%)]\tLoss: 0.000168\tF1: 1.0000\tRuntime: 5696.8\n",
      "Train Epoch: 8 [129600/269776 (53%)]\tLoss: 0.000894\tF1: 1.0000\tRuntime: 5732.2\n",
      "Train Epoch: 8 [130400/269776 (54%)]\tLoss: 0.000006\tF1: 1.0000\tRuntime: 5767.5\n",
      "Train Epoch: 8 [131200/269776 (54%)]\tLoss: 0.000009\tF1: 1.0000\tRuntime: 5802.9\n",
      "Train Epoch: 8 [132000/269776 (54%)]\tLoss: 0.000013\tF1: 1.0000\tRuntime: 5838.2\n",
      "Train Epoch: 8 [132800/269776 (55%)]\tLoss: 0.000003\tF1: 1.0000\tRuntime: 5873.6\n",
      "Train Epoch: 8 [133600/269776 (55%)]\tLoss: 0.000005\tF1: 1.0000\tRuntime: 5908.9\n",
      "Train Epoch: 8 [134400/269776 (55%)]\tLoss: 0.002697\tF1: 1.0000\tRuntime: 5944.3\n",
      "Train Epoch: 8 [135200/269776 (56%)]\tLoss: 0.000002\tF1: 1.0000\tRuntime: 5979.6\n",
      "Train Epoch: 8 [136000/269776 (56%)]\tLoss: 0.007494\tF1: 1.0000\tRuntime: 6015.0\n",
      "Train Epoch: 8 [136800/269776 (56%)]\tLoss: 0.000004\tF1: 1.0000\tRuntime: 6050.3\n",
      "Train Epoch: 8 [137600/269776 (57%)]\tLoss: 0.000639\tF1: 1.0000\tRuntime: 6085.7\n",
      "Train Epoch: 8 [138400/269776 (57%)]\tLoss: 0.000003\tF1: 1.0000\tRuntime: 6121.0\n",
      "Train Epoch: 8 [139200/269776 (57%)]\tLoss: 0.000010\tF1: 1.0000\tRuntime: 6156.3\n",
      "Train Epoch: 8 [140000/269776 (58%)]\tLoss: 0.001683\tF1: 1.0000\tRuntime: 6191.9\n",
      "Train Epoch: 8 [140800/269776 (58%)]\tLoss: 0.000015\tF1: 1.0000\tRuntime: 6227.9\n",
      "Train Epoch: 8 [141600/269776 (58%)]\tLoss: 0.000432\tF1: 1.0000\tRuntime: 6263.3\n",
      "Train Epoch: 8 [142400/269776 (59%)]\tLoss: 0.000004\tF1: 1.0000\tRuntime: 6298.7\n",
      "Train Epoch: 8 [143200/269776 (59%)]\tLoss: 0.000001\tF1: 1.0000\tRuntime: 6334.0\n",
      "Train Epoch: 8 [144000/269776 (59%)]\tLoss: 0.130351\tF1: 1.0000\tRuntime: 6369.4\n",
      "Train Epoch: 8 [144800/269776 (60%)]\tLoss: 0.000003\tF1: 1.0000\tRuntime: 6404.8\n",
      "Train Epoch: 8 [145600/269776 (60%)]\tLoss: 0.000041\tF1: 1.0000\tRuntime: 6440.2\n",
      "Train Epoch: 8 [146400/269776 (60%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 6475.6\n",
      "Train Epoch: 8 [147200/269776 (61%)]\tLoss: 0.000025\tF1: 1.0000\tRuntime: 6511.1\n",
      "Train Epoch: 8 [148000/269776 (61%)]\tLoss: 0.000939\tF1: 1.0000\tRuntime: 6546.5\n",
      "Train Epoch: 8 [148800/269776 (61%)]\tLoss: 0.000314\tF1: 1.0000\tRuntime: 6581.9\n",
      "Train Epoch: 8 [149600/269776 (62%)]\tLoss: 0.000010\tF1: 1.0000\tRuntime: 6617.3\n",
      "Train Epoch: 8 [150400/269776 (62%)]\tLoss: 0.000001\tF1: 1.0000\tRuntime: 6652.6\n",
      "Train Epoch: 8 [151200/269776 (62%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 6688.0\n",
      "Train Epoch: 8 [152000/269776 (63%)]\tLoss: 0.000029\tF1: 1.0000\tRuntime: 6723.4\n",
      "Train Epoch: 8 [152800/269776 (63%)]\tLoss: 0.000219\tF1: 1.0000\tRuntime: 6758.7\n",
      "Train Epoch: 8 [153600/269776 (63%)]\tLoss: 0.000024\tF1: 1.0000\tRuntime: 6794.1\n",
      "Train Epoch: 8 [154400/269776 (64%)]\tLoss: 0.000009\tF1: 1.0000\tRuntime: 6829.4\n",
      "Train Epoch: 8 [155200/269776 (64%)]\tLoss: 0.000003\tF1: 1.0000\tRuntime: 6864.8\n",
      "Train Epoch: 8 [156000/269776 (64%)]\tLoss: 0.000051\tF1: 1.0000\tRuntime: 6900.2\n",
      "Train Epoch: 8 [156800/269776 (65%)]\tLoss: 0.000010\tF1: 1.0000\tRuntime: 6935.5\n",
      "Train Epoch: 8 [157600/269776 (65%)]\tLoss: 0.004451\tF1: 1.0000\tRuntime: 6970.8\n",
      "Train Epoch: 8 [158400/269776 (65%)]\tLoss: 0.000008\tF1: 1.0000\tRuntime: 7006.2\n",
      "Train Epoch: 8 [159200/269776 (66%)]\tLoss: 0.000018\tF1: 1.0000\tRuntime: 7041.6\n",
      "Train Epoch: 8 [160000/269776 (66%)]\tLoss: 0.000007\tF1: 1.0000\tRuntime: 7076.9\n",
      "Train Epoch: 8 [160800/269776 (66%)]\tLoss: 0.000278\tF1: 1.0000\tRuntime: 7112.3\n",
      "Train Epoch: 8 [161600/269776 (67%)]\tLoss: 0.000005\tF1: 1.0000\tRuntime: 7147.6\n",
      "Train Epoch: 8 [162400/269776 (67%)]\tLoss: 0.000015\tF1: 1.0000\tRuntime: 7183.0\n",
      "Train Epoch: 8 [163200/269776 (67%)]\tLoss: 0.000015\tF1: 1.0000\tRuntime: 7218.3\n",
      "Train Epoch: 8 [164000/269776 (68%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 7253.7\n",
      "Train Epoch: 8 [164800/269776 (68%)]\tLoss: 0.000937\tF1: 1.0000\tRuntime: 7289.0\n",
      "Train Epoch: 8 [165600/269776 (68%)]\tLoss: 0.000052\tF1: 1.0000\tRuntime: 7324.4\n",
      "Train Epoch: 8 [166400/269776 (69%)]\tLoss: 0.000018\tF1: 1.0000\tRuntime: 7359.8\n",
      "Train Epoch: 8 [167200/269776 (69%)]\tLoss: 0.000002\tF1: 1.0000\tRuntime: 7395.1\n",
      "Train Epoch: 8 [168000/269776 (69%)]\tLoss: 0.000058\tF1: 1.0000\tRuntime: 7430.5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 8 [168800/269776 (70%)]\tLoss: 0.000010\tF1: 1.0000\tRuntime: 7465.8\n",
      "Train Epoch: 8 [169600/269776 (70%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 7501.2\n",
      "Train Epoch: 8 [170400/269776 (70%)]\tLoss: 0.000077\tF1: 1.0000\tRuntime: 7536.6\n",
      "Train Epoch: 8 [171200/269776 (71%)]\tLoss: 0.000080\tF1: 1.0000\tRuntime: 7571.9\n",
      "Train Epoch: 8 [172000/269776 (71%)]\tLoss: 0.000017\tF1: 1.0000\tRuntime: 7608.5\n",
      "Train Epoch: 8 [172800/269776 (71%)]\tLoss: 0.000005\tF1: 1.0000\tRuntime: 7644.8\n",
      "Train Epoch: 8 [173600/269776 (71%)]\tLoss: 0.000046\tF1: 1.0000\tRuntime: 7680.3\n",
      "Train Epoch: 8 [174400/269776 (72%)]\tLoss: 0.000141\tF1: 1.0000\tRuntime: 7715.9\n",
      "Train Epoch: 8 [175200/269776 (72%)]\tLoss: 0.001842\tF1: 1.0000\tRuntime: 7751.3\n",
      "Train Epoch: 8 [176000/269776 (72%)]\tLoss: 0.000035\tF1: 1.0000\tRuntime: 7786.7\n",
      "Train Epoch: 8 [176800/269776 (73%)]\tLoss: 0.000004\tF1: 1.0000\tRuntime: 7822.1\n",
      "Train Epoch: 8 [177600/269776 (73%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 7857.5\n",
      "Train Epoch: 8 [178400/269776 (73%)]\tLoss: 0.000473\tF1: 1.0000\tRuntime: 7892.8\n",
      "Train Epoch: 8 [179200/269776 (74%)]\tLoss: 0.000376\tF1: 1.0000\tRuntime: 7928.2\n",
      "Train Epoch: 8 [180000/269776 (74%)]\tLoss: 0.000001\tF1: 1.0000\tRuntime: 7963.6\n",
      "Train Epoch: 8 [180800/269776 (74%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 7999.0\n",
      "Train Epoch: 8 [181600/269776 (75%)]\tLoss: 0.000100\tF1: 1.0000\tRuntime: 8034.5\n",
      "Train Epoch: 8 [182400/269776 (75%)]\tLoss: 0.000471\tF1: 1.0000\tRuntime: 8069.9\n",
      "Train Epoch: 8 [183200/269776 (75%)]\tLoss: 0.000061\tF1: 1.0000\tRuntime: 8105.3\n",
      "Train Epoch: 8 [184000/269776 (76%)]\tLoss: 0.000010\tF1: 1.0000\tRuntime: 8140.6\n",
      "Train Epoch: 8 [184800/269776 (76%)]\tLoss: 0.000235\tF1: 1.0000\tRuntime: 8175.9\n",
      "Train Epoch: 8 [185600/269776 (76%)]\tLoss: 0.000055\tF1: 1.0000\tRuntime: 8211.3\n",
      "Train Epoch: 8 [186400/269776 (77%)]\tLoss: 0.000059\tF1: 1.0000\tRuntime: 8246.7\n",
      "Train Epoch: 8 [187200/269776 (77%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 8282.1\n",
      "Train Epoch: 8 [188000/269776 (77%)]\tLoss: 0.000023\tF1: 1.0000\tRuntime: 8317.4\n",
      "Train Epoch: 8 [188800/269776 (78%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 8352.8\n",
      "Train Epoch: 8 [189600/269776 (78%)]\tLoss: 0.000799\tF1: 1.0000\tRuntime: 8388.2\n",
      "Train Epoch: 8 [190400/269776 (78%)]\tLoss: 0.000020\tF1: 1.0000\tRuntime: 8423.5\n",
      "Train Epoch: 8 [191200/269776 (79%)]\tLoss: 0.000013\tF1: 1.0000\tRuntime: 8458.9\n",
      "Train Epoch: 8 [192000/269776 (79%)]\tLoss: 0.000004\tF1: 1.0000\tRuntime: 8494.3\n",
      "Train Epoch: 8 [192800/269776 (79%)]\tLoss: 0.000001\tF1: 1.0000\tRuntime: 8529.6\n",
      "Train Epoch: 8 [193600/269776 (80%)]\tLoss: 0.000050\tF1: 1.0000\tRuntime: 8565.0\n",
      "Train Epoch: 8 [194400/269776 (80%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 8600.4\n",
      "Train Epoch: 8 [195200/269776 (80%)]\tLoss: 0.000015\tF1: 1.0000\tRuntime: 8635.7\n",
      "Train Epoch: 8 [196000/269776 (81%)]\tLoss: 0.000021\tF1: 1.0000\tRuntime: 8671.1\n",
      "Train Epoch: 8 [196800/269776 (81%)]\tLoss: 0.000477\tF1: 1.0000\tRuntime: 8706.5\n",
      "Train Epoch: 8 [197600/269776 (81%)]\tLoss: 0.000039\tF1: 1.0000\tRuntime: 8741.9\n",
      "Train Epoch: 8 [198400/269776 (82%)]\tLoss: 0.000216\tF1: 1.0000\tRuntime: 8777.3\n",
      "Train Epoch: 8 [199200/269776 (82%)]\tLoss: 0.000003\tF1: 1.0000\tRuntime: 8812.7\n",
      "Train Epoch: 8 [200000/269776 (82%)]\tLoss: 0.000040\tF1: 1.0000\tRuntime: 8848.0\n",
      "Train Epoch: 8 [200800/269776 (83%)]\tLoss: 0.001772\tF1: 1.0000\tRuntime: 8883.4\n",
      "Train Epoch: 8 [201600/269776 (83%)]\tLoss: 0.000009\tF1: 1.0000\tRuntime: 8918.7\n",
      "Train Epoch: 8 [202400/269776 (83%)]\tLoss: 0.000026\tF1: 1.0000\tRuntime: 8954.1\n",
      "Train Epoch: 8 [203200/269776 (84%)]\tLoss: 0.000688\tF1: 1.0000\tRuntime: 8989.5\n",
      "Train Epoch: 8 [204000/269776 (84%)]\tLoss: 0.000007\tF1: 1.0000\tRuntime: 9024.9\n",
      "Train Epoch: 8 [204800/269776 (84%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 9060.3\n",
      "Train Epoch: 8 [205600/269776 (85%)]\tLoss: 0.001613\tF1: 1.0000\tRuntime: 9095.7\n",
      "Train Epoch: 8 [206400/269776 (85%)]\tLoss: 0.000008\tF1: 1.0000\tRuntime: 9131.0\n",
      "Train Epoch: 8 [207200/269776 (85%)]\tLoss: 0.000001\tF1: 1.0000\tRuntime: 9166.4\n",
      "Train Epoch: 8 [208000/269776 (86%)]\tLoss: 0.000004\tF1: 1.0000\tRuntime: 9201.8\n",
      "Train Epoch: 8 [208800/269776 (86%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 9237.2\n",
      "Train Epoch: 8 [209600/269776 (86%)]\tLoss: 0.000091\tF1: 1.0000\tRuntime: 9272.6\n",
      "Train Epoch: 8 [210400/269776 (87%)]\tLoss: 0.000076\tF1: 1.0000\tRuntime: 9308.0\n",
      "Train Epoch: 8 [211200/269776 (87%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 9343.3\n",
      "Train Epoch: 8 [212000/269776 (87%)]\tLoss: 0.000005\tF1: 1.0000\tRuntime: 9378.7\n",
      "Train Epoch: 8 [212800/269776 (88%)]\tLoss: 0.000114\tF1: 1.0000\tRuntime: 9414.1\n",
      "Train Epoch: 8 [213600/269776 (88%)]\tLoss: 0.004302\tF1: 1.0000\tRuntime: 9449.4\n",
      "Train Epoch: 8 [214400/269776 (88%)]\tLoss: 0.000082\tF1: 1.0000\tRuntime: 9484.8\n",
      "Train Epoch: 8 [215200/269776 (89%)]\tLoss: 0.000193\tF1: 1.0000\tRuntime: 9520.2\n",
      "Train Epoch: 8 [216000/269776 (89%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 9555.6\n",
      "Train Epoch: 8 [216800/269776 (89%)]\tLoss: 0.000175\tF1: 1.0000\tRuntime: 9590.9\n",
      "Train Epoch: 8 [217600/269776 (90%)]\tLoss: 0.000031\tF1: 1.0000\tRuntime: 9626.3\n",
      "Train Epoch: 8 [218400/269776 (90%)]\tLoss: 0.001348\tF1: 1.0000\tRuntime: 9661.7\n",
      "Train Epoch: 8 [219200/269776 (90%)]\tLoss: 0.000577\tF1: 1.0000\tRuntime: 9697.1\n",
      "Train Epoch: 8 [220000/269776 (91%)]\tLoss: 0.000284\tF1: 1.0000\tRuntime: 9732.5\n",
      "Train Epoch: 8 [220800/269776 (91%)]\tLoss: 0.000009\tF1: 1.0000\tRuntime: 9767.9\n",
      "Train Epoch: 8 [221600/269776 (91%)]\tLoss: 0.000001\tF1: 1.0000\tRuntime: 9803.3\n",
      "Train Epoch: 8 [222400/269776 (92%)]\tLoss: 0.000025\tF1: 1.0000\tRuntime: 9838.7\n",
      "Train Epoch: 8 [223200/269776 (92%)]\tLoss: 0.000213\tF1: 1.0000\tRuntime: 9874.1\n",
      "Train Epoch: 8 [224000/269776 (92%)]\tLoss: 0.000013\tF1: 1.0000\tRuntime: 9909.5\n",
      "Train Epoch: 8 [224800/269776 (93%)]\tLoss: 0.000031\tF1: 1.0000\tRuntime: 9944.9\n",
      "Train Epoch: 8 [225600/269776 (93%)]\tLoss: 0.003273\tF1: 1.0000\tRuntime: 9980.3\n",
      "Train Epoch: 8 [226400/269776 (93%)]\tLoss: 0.000023\tF1: 1.0000\tRuntime: 10015.6\n",
      "Train Epoch: 8 [227200/269776 (94%)]\tLoss: 0.000003\tF1: 1.0000\tRuntime: 10051.0\n",
      "Train Epoch: 8 [228000/269776 (94%)]\tLoss: 0.000002\tF1: 1.0000\tRuntime: 10086.4\n",
      "Train Epoch: 8 [228800/269776 (94%)]\tLoss: 0.000026\tF1: 1.0000\tRuntime: 10121.8\n",
      "Train Epoch: 8 [229600/269776 (95%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 10157.2\n",
      "Train Epoch: 8 [230400/269776 (95%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 10192.5\n",
      "Train Epoch: 8 [231200/269776 (95%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 10227.9\n",
      "Train Epoch: 8 [232000/269776 (96%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 10263.3\n",
      "Train Epoch: 8 [232800/269776 (96%)]\tLoss: 0.000001\tF1: 1.0000\tRuntime: 10298.7\n",
      "Train Epoch: 8 [233600/269776 (96%)]\tLoss: 0.000005\tF1: 1.0000\tRuntime: 10334.1\n",
      "Train Epoch: 8 [234400/269776 (97%)]\tLoss: 0.000035\tF1: 1.0000\tRuntime: 10369.4\n",
      "Train Epoch: 8 [235200/269776 (97%)]\tLoss: 0.001066\tF1: 1.0000\tRuntime: 10404.8\n",
      "Train Epoch: 8 [236000/269776 (97%)]\tLoss: 0.000016\tF1: 1.0000\tRuntime: 10440.2\n",
      "Train Epoch: 8 [236800/269776 (98%)]\tLoss: 0.000131\tF1: 1.0000\tRuntime: 10475.5\n",
      "Train Epoch: 8 [237600/269776 (98%)]\tLoss: 0.001464\tF1: 1.0000\tRuntime: 10510.9\n",
      "Train Epoch: 8 [238400/269776 (98%)]\tLoss: 0.000002\tF1: 1.0000\tRuntime: 10546.3\n",
      "Train Epoch: 8 [239200/269776 (99%)]\tLoss: 0.028966\tF1: 1.0000\tRuntime: 10581.6\n",
      "Train Epoch: 8 [240000/269776 (99%)]\tLoss: 0.000036\tF1: 1.0000\tRuntime: 10617.1\n",
      "Train Epoch: 8 [240800/269776 (99%)]\tLoss: 0.000013\tF1: 1.0000\tRuntime: 10652.4\n",
      "Train Epoch: 8 [241600/269776 (100%)]\tLoss: 0.000073\tF1: 1.0000\tRuntime: 10687.8\n",
      "Train Epoch: 8 [242400/269776 (100%)]\tLoss: 0.000134\tF1: 1.0000\tRuntime: 10723.2\n",
      "after epoch 8 got loss of 0.005419029493169288 and f1 of 0.9983691623425607\n",
      "saved predictions\n",
      "currently best model --> saved\n",
      "Train Epoch: 9 [0/269776 (0%)]\tLoss: 0.000131\tF1: 1.0000\tRuntime: 0.2\n",
      "Train Epoch: 9 [800/269776 (0%)]\tLoss: 0.000189\tF1: 1.0000\tRuntime: 35.6\n",
      "Train Epoch: 9 [1600/269776 (1%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 70.9\n",
      "Train Epoch: 9 [2400/269776 (1%)]\tLoss: 0.000063\tF1: 1.0000\tRuntime: 106.3\n",
      "Train Epoch: 9 [3200/269776 (1%)]\tLoss: 0.000014\tF1: 1.0000\tRuntime: 141.7\n",
      "Train Epoch: 9 [4000/269776 (2%)]\tLoss: 0.000002\tF1: 1.0000\tRuntime: 177.0\n",
      "Train Epoch: 9 [4800/269776 (2%)]\tLoss: 0.000062\tF1: 1.0000\tRuntime: 212.4\n",
      "Train Epoch: 9 [5600/269776 (2%)]\tLoss: 0.001150\tF1: 1.0000\tRuntime: 247.8\n",
      "Train Epoch: 9 [6400/269776 (3%)]\tLoss: 0.000019\tF1: 1.0000\tRuntime: 283.2\n",
      "Train Epoch: 9 [7200/269776 (3%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 318.6\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 9 [8000/269776 (3%)]\tLoss: 0.000128\tF1: 1.0000\tRuntime: 354.0\n",
      "Train Epoch: 9 [8800/269776 (4%)]\tLoss: 0.000016\tF1: 1.0000\tRuntime: 389.3\n",
      "Train Epoch: 9 [9600/269776 (4%)]\tLoss: 0.000001\tF1: 1.0000\tRuntime: 424.7\n",
      "Train Epoch: 9 [10400/269776 (4%)]\tLoss: 0.000004\tF1: 1.0000\tRuntime: 460.1\n",
      "Train Epoch: 9 [11200/269776 (5%)]\tLoss: 0.000023\tF1: 1.0000\tRuntime: 495.5\n",
      "Train Epoch: 9 [12000/269776 (5%)]\tLoss: 0.000858\tF1: 1.0000\tRuntime: 530.8\n",
      "Train Epoch: 9 [12800/269776 (5%)]\tLoss: 0.000003\tF1: 1.0000\tRuntime: 566.2\n",
      "Train Epoch: 9 [13600/269776 (6%)]\tLoss: 0.000126\tF1: 1.0000\tRuntime: 601.6\n",
      "Train Epoch: 9 [14400/269776 (6%)]\tLoss: 0.000009\tF1: 1.0000\tRuntime: 637.0\n",
      "Train Epoch: 9 [15200/269776 (6%)]\tLoss: 0.000110\tF1: 1.0000\tRuntime: 672.3\n",
      "Train Epoch: 9 [16000/269776 (7%)]\tLoss: 0.000174\tF1: 1.0000\tRuntime: 707.7\n",
      "Train Epoch: 9 [16800/269776 (7%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 743.1\n",
      "Train Epoch: 9 [17600/269776 (7%)]\tLoss: 0.000096\tF1: 1.0000\tRuntime: 778.5\n",
      "Train Epoch: 9 [18400/269776 (8%)]\tLoss: 0.003264\tF1: 1.0000\tRuntime: 813.9\n",
      "Train Epoch: 9 [19200/269776 (8%)]\tLoss: 0.000423\tF1: 1.0000\tRuntime: 849.3\n",
      "Train Epoch: 9 [20000/269776 (8%)]\tLoss: 0.000153\tF1: 1.0000\tRuntime: 884.7\n",
      "Train Epoch: 9 [20800/269776 (9%)]\tLoss: 0.000019\tF1: 1.0000\tRuntime: 920.1\n",
      "Train Epoch: 9 [21600/269776 (9%)]\tLoss: 0.000004\tF1: 1.0000\tRuntime: 955.4\n",
      "Train Epoch: 9 [22400/269776 (9%)]\tLoss: 0.000001\tF1: 1.0000\tRuntime: 990.8\n",
      "Train Epoch: 9 [23200/269776 (10%)]\tLoss: 0.001019\tF1: 1.0000\tRuntime: 1026.1\n",
      "Train Epoch: 9 [24000/269776 (10%)]\tLoss: 0.000505\tF1: 1.0000\tRuntime: 1061.5\n",
      "Train Epoch: 9 [24800/269776 (10%)]\tLoss: 0.000005\tF1: 1.0000\tRuntime: 1096.9\n",
      "Train Epoch: 9 [25600/269776 (11%)]\tLoss: 0.000109\tF1: 1.0000\tRuntime: 1132.3\n",
      "Train Epoch: 9 [26400/269776 (11%)]\tLoss: 0.000036\tF1: 1.0000\tRuntime: 1167.6\n",
      "Train Epoch: 9 [27200/269776 (11%)]\tLoss: 0.000035\tF1: 1.0000\tRuntime: 1203.0\n",
      "Train Epoch: 9 [28000/269776 (12%)]\tLoss: 0.000156\tF1: 1.0000\tRuntime: 1238.4\n",
      "Train Epoch: 9 [28800/269776 (12%)]\tLoss: 0.000523\tF1: 1.0000\tRuntime: 1273.7\n",
      "Train Epoch: 9 [29600/269776 (12%)]\tLoss: 0.000139\tF1: 1.0000\tRuntime: 1309.1\n",
      "Train Epoch: 9 [30400/269776 (13%)]\tLoss: 0.000001\tF1: 1.0000\tRuntime: 1344.5\n",
      "Train Epoch: 9 [31200/269776 (13%)]\tLoss: 0.000182\tF1: 1.0000\tRuntime: 1379.8\n",
      "Train Epoch: 9 [32000/269776 (13%)]\tLoss: 0.000294\tF1: 1.0000\tRuntime: 1415.2\n",
      "Train Epoch: 9 [32800/269776 (14%)]\tLoss: 0.000089\tF1: 1.0000\tRuntime: 1450.6\n",
      "Train Epoch: 9 [33600/269776 (14%)]\tLoss: 0.000005\tF1: 1.0000\tRuntime: 1486.0\n",
      "Train Epoch: 9 [34400/269776 (14%)]\tLoss: 0.000229\tF1: 1.0000\tRuntime: 1521.4\n",
      "Train Epoch: 9 [35200/269776 (14%)]\tLoss: 0.000012\tF1: 1.0000\tRuntime: 1556.8\n",
      "Train Epoch: 9 [36000/269776 (15%)]\tLoss: 0.000068\tF1: 1.0000\tRuntime: 1592.1\n",
      "Train Epoch: 9 [36800/269776 (15%)]\tLoss: 0.000229\tF1: 1.0000\tRuntime: 1627.5\n",
      "Train Epoch: 9 [37600/269776 (15%)]\tLoss: 0.000182\tF1: 1.0000\tRuntime: 1662.9\n",
      "Train Epoch: 9 [38400/269776 (16%)]\tLoss: 0.000006\tF1: 1.0000\tRuntime: 1698.3\n",
      "Train Epoch: 9 [39200/269776 (16%)]\tLoss: 0.000029\tF1: 1.0000\tRuntime: 1733.7\n",
      "Train Epoch: 9 [40000/269776 (16%)]\tLoss: 0.000039\tF1: 1.0000\tRuntime: 1769.1\n",
      "Train Epoch: 9 [40800/269776 (17%)]\tLoss: 0.000037\tF1: 1.0000\tRuntime: 1804.4\n",
      "Train Epoch: 9 [41600/269776 (17%)]\tLoss: 0.000318\tF1: 1.0000\tRuntime: 1839.8\n",
      "Train Epoch: 9 [42400/269776 (17%)]\tLoss: 0.000001\tF1: 1.0000\tRuntime: 1875.2\n",
      "Train Epoch: 9 [43200/269776 (18%)]\tLoss: 0.000024\tF1: 1.0000\tRuntime: 1910.5\n",
      "Train Epoch: 9 [44000/269776 (18%)]\tLoss: 0.000012\tF1: 1.0000\tRuntime: 1945.9\n",
      "Train Epoch: 9 [44800/269776 (18%)]\tLoss: 0.000013\tF1: 1.0000\tRuntime: 1981.3\n",
      "Train Epoch: 9 [45600/269776 (19%)]\tLoss: 0.000053\tF1: 1.0000\tRuntime: 2016.7\n",
      "Train Epoch: 9 [46400/269776 (19%)]\tLoss: 0.000173\tF1: 1.0000\tRuntime: 2052.1\n",
      "Train Epoch: 9 [47200/269776 (19%)]\tLoss: 0.000612\tF1: 1.0000\tRuntime: 2087.5\n",
      "Train Epoch: 9 [48000/269776 (20%)]\tLoss: 0.000006\tF1: 1.0000\tRuntime: 2122.9\n",
      "Train Epoch: 9 [48800/269776 (20%)]\tLoss: 0.000002\tF1: 1.0000\tRuntime: 2158.3\n",
      "Train Epoch: 9 [49600/269776 (20%)]\tLoss: 0.000036\tF1: 1.0000\tRuntime: 2193.6\n",
      "Train Epoch: 9 [50400/269776 (21%)]\tLoss: 0.046616\tF1: 1.0000\tRuntime: 2229.0\n",
      "Train Epoch: 9 [51200/269776 (21%)]\tLoss: 0.000007\tF1: 1.0000\tRuntime: 2264.4\n",
      "Train Epoch: 9 [52000/269776 (21%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 2299.7\n",
      "Train Epoch: 9 [52800/269776 (22%)]\tLoss: 0.000125\tF1: 1.0000\tRuntime: 2335.1\n",
      "Train Epoch: 9 [53600/269776 (22%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 2370.4\n",
      "Train Epoch: 9 [54400/269776 (22%)]\tLoss: 0.000066\tF1: 1.0000\tRuntime: 2405.8\n",
      "Train Epoch: 9 [55200/269776 (23%)]\tLoss: 0.006752\tF1: 1.0000\tRuntime: 2441.2\n",
      "Train Epoch: 9 [56000/269776 (23%)]\tLoss: 0.000002\tF1: 1.0000\tRuntime: 2476.5\n",
      "Train Epoch: 9 [56800/269776 (23%)]\tLoss: 0.000585\tF1: 1.0000\tRuntime: 2512.0\n",
      "Train Epoch: 9 [57600/269776 (24%)]\tLoss: 0.000044\tF1: 1.0000\tRuntime: 2547.3\n",
      "Train Epoch: 9 [58400/269776 (24%)]\tLoss: 0.000041\tF1: 1.0000\tRuntime: 2582.7\n",
      "Train Epoch: 9 [59200/269776 (24%)]\tLoss: 0.000236\tF1: 1.0000\tRuntime: 2618.1\n",
      "Train Epoch: 9 [60000/269776 (25%)]\tLoss: 0.000646\tF1: 1.0000\tRuntime: 2653.4\n",
      "Train Epoch: 9 [60800/269776 (25%)]\tLoss: 0.000092\tF1: 1.0000\tRuntime: 2688.8\n",
      "Train Epoch: 9 [61600/269776 (25%)]\tLoss: 0.000002\tF1: 1.0000\tRuntime: 2724.2\n",
      "Train Epoch: 9 [62400/269776 (26%)]\tLoss: 0.002168\tF1: 1.0000\tRuntime: 2759.6\n",
      "Train Epoch: 9 [63200/269776 (26%)]\tLoss: 0.000088\tF1: 1.0000\tRuntime: 2795.0\n",
      "Train Epoch: 9 [64000/269776 (26%)]\tLoss: 0.000240\tF1: 1.0000\tRuntime: 2830.4\n",
      "Train Epoch: 9 [64800/269776 (27%)]\tLoss: 0.000011\tF1: 1.0000\tRuntime: 2865.7\n",
      "Train Epoch: 9 [65600/269776 (27%)]\tLoss: 0.000003\tF1: 1.0000\tRuntime: 2901.1\n",
      "Train Epoch: 9 [66400/269776 (27%)]\tLoss: 0.000122\tF1: 1.0000\tRuntime: 2936.4\n",
      "Train Epoch: 9 [67200/269776 (28%)]\tLoss: 0.000015\tF1: 1.0000\tRuntime: 2971.8\n",
      "Train Epoch: 9 [68000/269776 (28%)]\tLoss: 0.000001\tF1: 1.0000\tRuntime: 3007.2\n",
      "Train Epoch: 9 [68800/269776 (28%)]\tLoss: 0.000002\tF1: 1.0000\tRuntime: 3042.6\n",
      "Train Epoch: 9 [69600/269776 (29%)]\tLoss: 0.000004\tF1: 1.0000\tRuntime: 3078.0\n",
      "Train Epoch: 9 [70400/269776 (29%)]\tLoss: 0.000010\tF1: 1.0000\tRuntime: 3113.3\n",
      "Train Epoch: 9 [71200/269776 (29%)]\tLoss: 0.000022\tF1: 1.0000\tRuntime: 3148.7\n",
      "Train Epoch: 9 [72000/269776 (30%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 3184.1\n",
      "Train Epoch: 9 [72800/269776 (30%)]\tLoss: 0.000031\tF1: 1.0000\tRuntime: 3219.5\n",
      "Train Epoch: 9 [73600/269776 (30%)]\tLoss: 0.000007\tF1: 1.0000\tRuntime: 3254.8\n",
      "Train Epoch: 9 [74400/269776 (31%)]\tLoss: 0.000042\tF1: 1.0000\tRuntime: 3290.2\n",
      "Train Epoch: 9 [75200/269776 (31%)]\tLoss: 0.000010\tF1: 1.0000\tRuntime: 3325.6\n",
      "Train Epoch: 9 [76000/269776 (31%)]\tLoss: 0.000339\tF1: 1.0000\tRuntime: 3361.0\n",
      "Train Epoch: 9 [76800/269776 (32%)]\tLoss: 0.000105\tF1: 1.0000\tRuntime: 3396.3\n",
      "Train Epoch: 9 [77600/269776 (32%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 3431.7\n",
      "Train Epoch: 9 [78400/269776 (32%)]\tLoss: 0.000061\tF1: 1.0000\tRuntime: 3467.1\n",
      "Train Epoch: 9 [79200/269776 (33%)]\tLoss: 0.000329\tF1: 1.0000\tRuntime: 3502.5\n",
      "Train Epoch: 9 [80000/269776 (33%)]\tLoss: 0.000063\tF1: 1.0000\tRuntime: 3537.9\n",
      "Train Epoch: 9 [80800/269776 (33%)]\tLoss: 0.000012\tF1: 1.0000\tRuntime: 3573.3\n",
      "Train Epoch: 9 [81600/269776 (34%)]\tLoss: 0.000023\tF1: 1.0000\tRuntime: 3608.7\n",
      "Train Epoch: 9 [82400/269776 (34%)]\tLoss: 0.000004\tF1: 1.0000\tRuntime: 3644.1\n",
      "Train Epoch: 9 [83200/269776 (34%)]\tLoss: 0.000661\tF1: 1.0000\tRuntime: 3679.4\n",
      "Train Epoch: 9 [84000/269776 (35%)]\tLoss: 0.000012\tF1: 1.0000\tRuntime: 3714.8\n",
      "Train Epoch: 9 [84800/269776 (35%)]\tLoss: 0.000035\tF1: 1.0000\tRuntime: 3750.2\n",
      "Train Epoch: 9 [85600/269776 (35%)]\tLoss: 0.000206\tF1: 1.0000\tRuntime: 3785.6\n",
      "Train Epoch: 9 [86400/269776 (36%)]\tLoss: 0.000021\tF1: 1.0000\tRuntime: 3821.0\n",
      "Train Epoch: 9 [87200/269776 (36%)]\tLoss: 0.000001\tF1: 1.0000\tRuntime: 3856.4\n",
      "Train Epoch: 9 [88000/269776 (36%)]\tLoss: 0.000006\tF1: 1.0000\tRuntime: 3891.8\n",
      "Train Epoch: 9 [88800/269776 (37%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 3927.1\n",
      "Train Epoch: 9 [89600/269776 (37%)]\tLoss: 0.001424\tF1: 1.0000\tRuntime: 3962.5\n",
      "Train Epoch: 9 [90400/269776 (37%)]\tLoss: 0.000010\tF1: 1.0000\tRuntime: 3997.9\n",
      "Train Epoch: 9 [91200/269776 (38%)]\tLoss: 0.002272\tF1: 1.0000\tRuntime: 4033.2\n",
      "Train Epoch: 9 [92000/269776 (38%)]\tLoss: 0.000018\tF1: 1.0000\tRuntime: 4068.6\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 9 [92800/269776 (38%)]\tLoss: 0.000016\tF1: 1.0000\tRuntime: 4104.0\n",
      "Train Epoch: 9 [93600/269776 (39%)]\tLoss: 0.000022\tF1: 1.0000\tRuntime: 4139.4\n",
      "Train Epoch: 9 [94400/269776 (39%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 4174.8\n",
      "Train Epoch: 9 [95200/269776 (39%)]\tLoss: 0.000001\tF1: 1.0000\tRuntime: 4210.1\n",
      "Train Epoch: 9 [96000/269776 (40%)]\tLoss: 0.000021\tF1: 1.0000\tRuntime: 4245.5\n",
      "Train Epoch: 9 [96800/269776 (40%)]\tLoss: 0.000041\tF1: 1.0000\tRuntime: 4280.9\n",
      "Train Epoch: 9 [97600/269776 (40%)]\tLoss: 0.000022\tF1: 1.0000\tRuntime: 4316.3\n",
      "Train Epoch: 9 [98400/269776 (41%)]\tLoss: 0.000001\tF1: 1.0000\tRuntime: 4351.6\n",
      "Train Epoch: 9 [99200/269776 (41%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 4387.0\n",
      "Train Epoch: 9 [100000/269776 (41%)]\tLoss: 0.000001\tF1: 1.0000\tRuntime: 4422.4\n",
      "Train Epoch: 9 [100800/269776 (42%)]\tLoss: 0.000023\tF1: 1.0000\tRuntime: 4457.8\n",
      "Train Epoch: 9 [101600/269776 (42%)]\tLoss: 0.000028\tF1: 1.0000\tRuntime: 4493.2\n",
      "Train Epoch: 9 [102400/269776 (42%)]\tLoss: 0.000023\tF1: 1.0000\tRuntime: 4528.6\n",
      "Train Epoch: 9 [103200/269776 (43%)]\tLoss: 0.000270\tF1: 1.0000\tRuntime: 4563.9\n",
      "Train Epoch: 9 [104000/269776 (43%)]\tLoss: 0.000022\tF1: 1.0000\tRuntime: 4599.3\n",
      "Train Epoch: 9 [104800/269776 (43%)]\tLoss: 0.000007\tF1: 1.0000\tRuntime: 4634.7\n",
      "Train Epoch: 9 [105600/269776 (43%)]\tLoss: 0.000090\tF1: 1.0000\tRuntime: 4670.1\n",
      "Train Epoch: 9 [106400/269776 (44%)]\tLoss: 0.000113\tF1: 1.0000\tRuntime: 4705.5\n",
      "Train Epoch: 9 [107200/269776 (44%)]\tLoss: 0.022115\tF1: 1.0000\tRuntime: 4740.9\n",
      "Train Epoch: 9 [108000/269776 (44%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 4776.3\n",
      "Train Epoch: 9 [108800/269776 (45%)]\tLoss: 0.000013\tF1: 1.0000\tRuntime: 4811.7\n",
      "Train Epoch: 9 [109600/269776 (45%)]\tLoss: 0.000004\tF1: 1.0000\tRuntime: 4847.1\n",
      "Train Epoch: 9 [110400/269776 (45%)]\tLoss: 0.000283\tF1: 1.0000\tRuntime: 4882.4\n",
      "Train Epoch: 9 [111200/269776 (46%)]\tLoss: 0.000009\tF1: 1.0000\tRuntime: 4917.8\n",
      "Train Epoch: 9 [112000/269776 (46%)]\tLoss: 0.012727\tF1: 1.0000\tRuntime: 4953.2\n",
      "Train Epoch: 9 [112800/269776 (46%)]\tLoss: 0.004025\tF1: 1.0000\tRuntime: 4988.6\n",
      "Train Epoch: 9 [113600/269776 (47%)]\tLoss: 0.000008\tF1: 1.0000\tRuntime: 5024.0\n",
      "Train Epoch: 9 [114400/269776 (47%)]\tLoss: 0.000003\tF1: 1.0000\tRuntime: 5059.4\n",
      "Train Epoch: 9 [115200/269776 (47%)]\tLoss: 0.000166\tF1: 1.0000\tRuntime: 5094.8\n",
      "Train Epoch: 9 [116000/269776 (48%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 5130.1\n",
      "Train Epoch: 9 [116800/269776 (48%)]\tLoss: 0.000029\tF1: 1.0000\tRuntime: 5165.5\n",
      "Train Epoch: 9 [117600/269776 (48%)]\tLoss: 0.000204\tF1: 1.0000\tRuntime: 5200.9\n",
      "Train Epoch: 9 [118400/269776 (49%)]\tLoss: 0.000043\tF1: 1.0000\tRuntime: 5236.3\n",
      "Train Epoch: 9 [119200/269776 (49%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 5271.7\n",
      "Train Epoch: 9 [120000/269776 (49%)]\tLoss: 0.009171\tF1: 1.0000\tRuntime: 5307.0\n",
      "Train Epoch: 9 [120800/269776 (50%)]\tLoss: 0.000056\tF1: 1.0000\tRuntime: 5342.4\n",
      "Train Epoch: 9 [121600/269776 (50%)]\tLoss: 0.000009\tF1: 1.0000\tRuntime: 5377.8\n",
      "Train Epoch: 9 [122400/269776 (50%)]\tLoss: 0.000438\tF1: 1.0000\tRuntime: 5413.2\n",
      "Train Epoch: 9 [123200/269776 (51%)]\tLoss: 0.000025\tF1: 1.0000\tRuntime: 5448.5\n",
      "Train Epoch: 9 [124000/269776 (51%)]\tLoss: 0.000022\tF1: 1.0000\tRuntime: 5483.9\n",
      "Train Epoch: 9 [124800/269776 (51%)]\tLoss: 0.000024\tF1: 1.0000\tRuntime: 5519.3\n",
      "Train Epoch: 9 [125600/269776 (52%)]\tLoss: 0.000012\tF1: 1.0000\tRuntime: 5554.7\n",
      "Train Epoch: 9 [126400/269776 (52%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 5590.1\n",
      "Train Epoch: 9 [127200/269776 (52%)]\tLoss: 0.003085\tF1: 1.0000\tRuntime: 5625.4\n",
      "Train Epoch: 9 [128000/269776 (53%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 5660.8\n",
      "Train Epoch: 9 [128800/269776 (53%)]\tLoss: 0.000002\tF1: 1.0000\tRuntime: 5696.2\n",
      "Train Epoch: 9 [129600/269776 (53%)]\tLoss: 0.000788\tF1: 1.0000\tRuntime: 5731.6\n",
      "Train Epoch: 9 [130400/269776 (54%)]\tLoss: 0.000002\tF1: 1.0000\tRuntime: 5766.9\n",
      "Train Epoch: 9 [131200/269776 (54%)]\tLoss: 0.000083\tF1: 1.0000\tRuntime: 5802.3\n",
      "Train Epoch: 9 [132000/269776 (54%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 5837.7\n",
      "Train Epoch: 9 [132800/269776 (55%)]\tLoss: 0.002751\tF1: 1.0000\tRuntime: 5873.1\n",
      "Train Epoch: 9 [133600/269776 (55%)]\tLoss: 0.000026\tF1: 1.0000\tRuntime: 5908.5\n",
      "Train Epoch: 9 [134400/269776 (55%)]\tLoss: 0.000110\tF1: 1.0000\tRuntime: 5943.9\n",
      "Train Epoch: 9 [135200/269776 (56%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 5979.3\n",
      "Train Epoch: 9 [136000/269776 (56%)]\tLoss: 0.000002\tF1: 1.0000\tRuntime: 6014.7\n",
      "Train Epoch: 9 [136800/269776 (56%)]\tLoss: 0.000026\tF1: 1.0000\tRuntime: 6050.0\n",
      "Train Epoch: 9 [137600/269776 (57%)]\tLoss: 0.000070\tF1: 1.0000\tRuntime: 6085.4\n",
      "Train Epoch: 9 [138400/269776 (57%)]\tLoss: 0.000184\tF1: 1.0000\tRuntime: 6120.7\n",
      "Train Epoch: 9 [139200/269776 (57%)]\tLoss: 0.000778\tF1: 1.0000\tRuntime: 6156.1\n",
      "Train Epoch: 9 [140000/269776 (58%)]\tLoss: 0.000006\tF1: 1.0000\tRuntime: 6191.5\n",
      "Train Epoch: 9 [140800/269776 (58%)]\tLoss: 0.000019\tF1: 1.0000\tRuntime: 6226.9\n",
      "Train Epoch: 9 [141600/269776 (58%)]\tLoss: 0.000205\tF1: 1.0000\tRuntime: 6262.3\n",
      "Train Epoch: 9 [142400/269776 (59%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 6297.6\n",
      "Train Epoch: 9 [143200/269776 (59%)]\tLoss: 0.000191\tF1: 1.0000\tRuntime: 6333.0\n",
      "Train Epoch: 9 [144000/269776 (59%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 6368.4\n",
      "Train Epoch: 9 [144800/269776 (60%)]\tLoss: 0.000082\tF1: 1.0000\tRuntime: 6403.8\n",
      "Train Epoch: 9 [145600/269776 (60%)]\tLoss: 0.000288\tF1: 1.0000\tRuntime: 6439.2\n",
      "Train Epoch: 9 [146400/269776 (60%)]\tLoss: 0.000010\tF1: 1.0000\tRuntime: 6474.5\n",
      "Train Epoch: 9 [147200/269776 (61%)]\tLoss: 0.000005\tF1: 1.0000\tRuntime: 6509.9\n",
      "Train Epoch: 9 [148000/269776 (61%)]\tLoss: 0.000448\tF1: 1.0000\tRuntime: 6545.3\n",
      "Train Epoch: 9 [148800/269776 (61%)]\tLoss: 0.000008\tF1: 1.0000\tRuntime: 6580.7\n",
      "Train Epoch: 9 [149600/269776 (62%)]\tLoss: 0.000188\tF1: 1.0000\tRuntime: 6616.1\n",
      "Train Epoch: 9 [150400/269776 (62%)]\tLoss: 0.000006\tF1: 1.0000\tRuntime: 6651.5\n",
      "Train Epoch: 9 [151200/269776 (62%)]\tLoss: 0.000431\tF1: 1.0000\tRuntime: 6686.9\n",
      "Train Epoch: 9 [152000/269776 (63%)]\tLoss: 0.000005\tF1: 1.0000\tRuntime: 6722.2\n",
      "Train Epoch: 9 [152800/269776 (63%)]\tLoss: 0.000005\tF1: 1.0000\tRuntime: 6757.6\n",
      "Train Epoch: 9 [153600/269776 (63%)]\tLoss: 0.000158\tF1: 1.0000\tRuntime: 6793.0\n",
      "Train Epoch: 9 [154400/269776 (64%)]\tLoss: 0.000459\tF1: 1.0000\tRuntime: 6828.4\n",
      "Train Epoch: 9 [155200/269776 (64%)]\tLoss: 0.000037\tF1: 1.0000\tRuntime: 6863.8\n",
      "Train Epoch: 9 [156000/269776 (64%)]\tLoss: 0.000057\tF1: 1.0000\tRuntime: 6899.2\n",
      "Train Epoch: 9 [156800/269776 (65%)]\tLoss: 0.022277\tF1: 1.0000\tRuntime: 6934.5\n",
      "Train Epoch: 9 [157600/269776 (65%)]\tLoss: 0.000128\tF1: 1.0000\tRuntime: 6969.9\n",
      "Train Epoch: 9 [158400/269776 (65%)]\tLoss: 0.001734\tF1: 1.0000\tRuntime: 7005.3\n",
      "Train Epoch: 9 [159200/269776 (66%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 7040.7\n",
      "Train Epoch: 9 [160000/269776 (66%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 7076.1\n",
      "Train Epoch: 9 [160800/269776 (66%)]\tLoss: 0.000002\tF1: 1.0000\tRuntime: 7111.4\n",
      "Train Epoch: 9 [161600/269776 (67%)]\tLoss: 0.000028\tF1: 1.0000\tRuntime: 7146.8\n",
      "Train Epoch: 9 [162400/269776 (67%)]\tLoss: 0.000142\tF1: 1.0000\tRuntime: 7182.2\n",
      "Train Epoch: 9 [163200/269776 (67%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 7217.6\n",
      "Train Epoch: 9 [164000/269776 (68%)]\tLoss: 0.000006\tF1: 1.0000\tRuntime: 7253.0\n",
      "Train Epoch: 9 [164800/269776 (68%)]\tLoss: 0.000242\tF1: 1.0000\tRuntime: 7288.4\n",
      "Train Epoch: 9 [165600/269776 (68%)]\tLoss: 0.000020\tF1: 1.0000\tRuntime: 7323.7\n",
      "Train Epoch: 9 [166400/269776 (69%)]\tLoss: 0.000065\tF1: 1.0000\tRuntime: 7359.1\n",
      "Train Epoch: 9 [167200/269776 (69%)]\tLoss: 0.000029\tF1: 1.0000\tRuntime: 7394.5\n",
      "Train Epoch: 9 [168000/269776 (69%)]\tLoss: 0.000193\tF1: 1.0000\tRuntime: 7429.9\n",
      "Train Epoch: 9 [168800/269776 (70%)]\tLoss: 0.000089\tF1: 1.0000\tRuntime: 7465.3\n",
      "Train Epoch: 9 [169600/269776 (70%)]\tLoss: 0.000032\tF1: 1.0000\tRuntime: 7500.6\n",
      "Train Epoch: 9 [170400/269776 (70%)]\tLoss: 0.000114\tF1: 1.0000\tRuntime: 7536.0\n",
      "Train Epoch: 9 [171200/269776 (71%)]\tLoss: 0.000001\tF1: 1.0000\tRuntime: 7571.4\n",
      "Train Epoch: 9 [172000/269776 (71%)]\tLoss: 0.012551\tF1: 1.0000\tRuntime: 7606.8\n",
      "Train Epoch: 9 [172800/269776 (71%)]\tLoss: 0.000002\tF1: 1.0000\tRuntime: 7642.1\n",
      "Train Epoch: 9 [173600/269776 (71%)]\tLoss: 0.000005\tF1: 1.0000\tRuntime: 7677.5\n",
      "Train Epoch: 9 [174400/269776 (72%)]\tLoss: 0.000099\tF1: 1.0000\tRuntime: 7712.9\n",
      "Train Epoch: 9 [175200/269776 (72%)]\tLoss: 0.000051\tF1: 1.0000\tRuntime: 7748.3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 9 [176000/269776 (72%)]\tLoss: 0.000227\tF1: 1.0000\tRuntime: 7783.7\n",
      "Train Epoch: 9 [176800/269776 (73%)]\tLoss: 0.000021\tF1: 1.0000\tRuntime: 7819.0\n",
      "Train Epoch: 9 [177600/269776 (73%)]\tLoss: 0.000017\tF1: 1.0000\tRuntime: 7854.4\n",
      "Train Epoch: 9 [178400/269776 (73%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 7889.8\n",
      "Train Epoch: 9 [179200/269776 (74%)]\tLoss: 0.000001\tF1: 1.0000\tRuntime: 7925.2\n",
      "Train Epoch: 9 [180000/269776 (74%)]\tLoss: 0.000004\tF1: 1.0000\tRuntime: 7960.5\n",
      "Train Epoch: 9 [180800/269776 (74%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 7995.9\n",
      "Train Epoch: 9 [181600/269776 (75%)]\tLoss: 0.000010\tF1: 1.0000\tRuntime: 8031.3\n",
      "Train Epoch: 9 [182400/269776 (75%)]\tLoss: 0.008852\tF1: 1.0000\tRuntime: 8066.7\n",
      "Train Epoch: 9 [183200/269776 (75%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 8102.1\n",
      "Train Epoch: 9 [184000/269776 (76%)]\tLoss: 0.000640\tF1: 1.0000\tRuntime: 8137.4\n",
      "Train Epoch: 9 [184800/269776 (76%)]\tLoss: 0.000007\tF1: 1.0000\tRuntime: 8172.8\n",
      "Train Epoch: 9 [185600/269776 (76%)]\tLoss: 0.000003\tF1: 1.0000\tRuntime: 8208.1\n",
      "Train Epoch: 9 [186400/269776 (77%)]\tLoss: 0.000031\tF1: 1.0000\tRuntime: 8243.5\n",
      "Train Epoch: 9 [187200/269776 (77%)]\tLoss: 0.000024\tF1: 1.0000\tRuntime: 8278.9\n",
      "Train Epoch: 9 [188000/269776 (77%)]\tLoss: 0.008870\tF1: 1.0000\tRuntime: 8314.2\n",
      "Train Epoch: 9 [188800/269776 (78%)]\tLoss: 0.000045\tF1: 1.0000\tRuntime: 8349.6\n",
      "Train Epoch: 9 [189600/269776 (78%)]\tLoss: 0.000015\tF1: 1.0000\tRuntime: 8385.0\n",
      "Train Epoch: 9 [190400/269776 (78%)]\tLoss: 0.001167\tF1: 1.0000\tRuntime: 8420.3\n",
      "Train Epoch: 9 [191200/269776 (79%)]\tLoss: 0.000004\tF1: 1.0000\tRuntime: 8455.7\n",
      "Train Epoch: 9 [192000/269776 (79%)]\tLoss: 0.000033\tF1: 1.0000\tRuntime: 8491.1\n",
      "Train Epoch: 9 [192800/269776 (79%)]\tLoss: 0.000022\tF1: 1.0000\tRuntime: 8526.5\n",
      "Train Epoch: 9 [193600/269776 (80%)]\tLoss: 0.000006\tF1: 1.0000\tRuntime: 8561.8\n",
      "Train Epoch: 9 [194400/269776 (80%)]\tLoss: 0.000001\tF1: 1.0000\tRuntime: 8597.2\n",
      "Train Epoch: 9 [195200/269776 (80%)]\tLoss: 0.000002\tF1: 1.0000\tRuntime: 8632.5\n",
      "Train Epoch: 9 [196000/269776 (81%)]\tLoss: 0.000025\tF1: 1.0000\tRuntime: 8667.9\n",
      "Train Epoch: 9 [196800/269776 (81%)]\tLoss: 0.000409\tF1: 1.0000\tRuntime: 8703.3\n",
      "Train Epoch: 9 [197600/269776 (81%)]\tLoss: 0.000003\tF1: 1.0000\tRuntime: 8738.7\n",
      "Train Epoch: 9 [198400/269776 (82%)]\tLoss: 0.000041\tF1: 1.0000\tRuntime: 8774.1\n",
      "Train Epoch: 9 [199200/269776 (82%)]\tLoss: 0.000639\tF1: 1.0000\tRuntime: 8809.4\n",
      "Train Epoch: 9 [200000/269776 (82%)]\tLoss: 0.000052\tF1: 1.0000\tRuntime: 8844.8\n",
      "Train Epoch: 9 [200800/269776 (83%)]\tLoss: 0.000008\tF1: 1.0000\tRuntime: 8880.2\n",
      "Train Epoch: 9 [201600/269776 (83%)]\tLoss: 0.000508\tF1: 1.0000\tRuntime: 8915.6\n",
      "Train Epoch: 9 [202400/269776 (83%)]\tLoss: 0.000009\tF1: 1.0000\tRuntime: 8950.9\n",
      "Train Epoch: 9 [203200/269776 (84%)]\tLoss: 0.000328\tF1: 1.0000\tRuntime: 8986.3\n",
      "Train Epoch: 9 [204000/269776 (84%)]\tLoss: 0.000008\tF1: 1.0000\tRuntime: 9021.7\n",
      "Train Epoch: 9 [204800/269776 (84%)]\tLoss: 0.000041\tF1: 1.0000\tRuntime: 9057.1\n",
      "Train Epoch: 9 [205600/269776 (85%)]\tLoss: 0.000165\tF1: 1.0000\tRuntime: 9092.4\n",
      "Train Epoch: 9 [206400/269776 (85%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 9127.8\n",
      "Train Epoch: 9 [207200/269776 (85%)]\tLoss: 0.000010\tF1: 1.0000\tRuntime: 9163.2\n",
      "Train Epoch: 9 [208000/269776 (86%)]\tLoss: 0.000010\tF1: 1.0000\tRuntime: 9198.6\n",
      "Train Epoch: 9 [208800/269776 (86%)]\tLoss: 0.000648\tF1: 1.0000\tRuntime: 9234.0\n",
      "Train Epoch: 9 [209600/269776 (86%)]\tLoss: 0.000002\tF1: 1.0000\tRuntime: 9269.4\n",
      "Train Epoch: 9 [210400/269776 (87%)]\tLoss: 0.000004\tF1: 1.0000\tRuntime: 9304.7\n",
      "Train Epoch: 9 [211200/269776 (87%)]\tLoss: 0.000003\tF1: 1.0000\tRuntime: 9340.1\n",
      "Train Epoch: 9 [212000/269776 (87%)]\tLoss: 0.000029\tF1: 1.0000\tRuntime: 9375.5\n",
      "Train Epoch: 9 [212800/269776 (88%)]\tLoss: 0.000053\tF1: 1.0000\tRuntime: 9410.9\n",
      "Train Epoch: 9 [213600/269776 (88%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 9446.3\n",
      "Train Epoch: 9 [214400/269776 (88%)]\tLoss: 0.000003\tF1: 1.0000\tRuntime: 9481.7\n",
      "Train Epoch: 9 [215200/269776 (89%)]\tLoss: 0.000068\tF1: 1.0000\tRuntime: 9517.0\n",
      "Train Epoch: 9 [216000/269776 (89%)]\tLoss: 0.000015\tF1: 1.0000\tRuntime: 9552.4\n",
      "Train Epoch: 9 [216800/269776 (89%)]\tLoss: 0.000178\tF1: 1.0000\tRuntime: 9587.8\n",
      "Train Epoch: 9 [217600/269776 (90%)]\tLoss: 0.029832\tF1: 1.0000\tRuntime: 9623.2\n",
      "Train Epoch: 9 [218400/269776 (90%)]\tLoss: 0.000002\tF1: 1.0000\tRuntime: 9658.6\n",
      "Train Epoch: 9 [219200/269776 (90%)]\tLoss: 0.000126\tF1: 1.0000\tRuntime: 9693.9\n",
      "Train Epoch: 9 [220000/269776 (91%)]\tLoss: 0.000031\tF1: 1.0000\tRuntime: 9729.3\n",
      "Train Epoch: 9 [220800/269776 (91%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 9764.7\n",
      "Train Epoch: 9 [221600/269776 (91%)]\tLoss: 0.000037\tF1: 1.0000\tRuntime: 9800.1\n",
      "Train Epoch: 9 [222400/269776 (92%)]\tLoss: 0.000002\tF1: 1.0000\tRuntime: 9835.4\n",
      "Train Epoch: 9 [223200/269776 (92%)]\tLoss: 0.000088\tF1: 1.0000\tRuntime: 9870.8\n",
      "Train Epoch: 9 [224000/269776 (92%)]\tLoss: 0.000182\tF1: 1.0000\tRuntime: 9906.2\n",
      "Train Epoch: 9 [224800/269776 (93%)]\tLoss: 0.000003\tF1: 1.0000\tRuntime: 9941.6\n",
      "Train Epoch: 9 [225600/269776 (93%)]\tLoss: 0.000226\tF1: 1.0000\tRuntime: 9977.0\n",
      "Train Epoch: 9 [226400/269776 (93%)]\tLoss: 0.000003\tF1: 1.0000\tRuntime: 10012.3\n",
      "Train Epoch: 9 [227200/269776 (94%)]\tLoss: 0.000179\tF1: 1.0000\tRuntime: 10047.7\n",
      "Train Epoch: 9 [228000/269776 (94%)]\tLoss: 0.001007\tF1: 1.0000\tRuntime: 10083.1\n",
      "Train Epoch: 9 [228800/269776 (94%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 10118.5\n",
      "Train Epoch: 9 [229600/269776 (95%)]\tLoss: 0.000014\tF1: 1.0000\tRuntime: 10153.9\n",
      "Train Epoch: 9 [230400/269776 (95%)]\tLoss: 0.000674\tF1: 1.0000\tRuntime: 10189.2\n",
      "Train Epoch: 9 [231200/269776 (95%)]\tLoss: 0.000001\tF1: 1.0000\tRuntime: 10224.6\n",
      "Train Epoch: 9 [232000/269776 (96%)]\tLoss: 0.017949\tF1: 1.0000\tRuntime: 10260.0\n",
      "Train Epoch: 9 [232800/269776 (96%)]\tLoss: 0.000061\tF1: 1.0000\tRuntime: 10295.4\n",
      "Train Epoch: 9 [233600/269776 (96%)]\tLoss: 0.000003\tF1: 1.0000\tRuntime: 10330.7\n",
      "Train Epoch: 9 [234400/269776 (97%)]\tLoss: 0.000874\tF1: 1.0000\tRuntime: 10366.1\n",
      "Train Epoch: 9 [235200/269776 (97%)]\tLoss: 0.000070\tF1: 1.0000\tRuntime: 10401.5\n",
      "Train Epoch: 9 [236000/269776 (97%)]\tLoss: 0.007964\tF1: 1.0000\tRuntime: 10436.8\n",
      "Train Epoch: 9 [236800/269776 (98%)]\tLoss: 0.000171\tF1: 1.0000\tRuntime: 10472.2\n",
      "Train Epoch: 9 [237600/269776 (98%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 10507.6\n",
      "Train Epoch: 9 [238400/269776 (98%)]\tLoss: 0.000019\tF1: 1.0000\tRuntime: 10542.9\n",
      "Train Epoch: 9 [239200/269776 (99%)]\tLoss: 0.000008\tF1: 1.0000\tRuntime: 10578.3\n",
      "Train Epoch: 9 [240000/269776 (99%)]\tLoss: 0.000064\tF1: 1.0000\tRuntime: 10613.7\n",
      "Train Epoch: 9 [240800/269776 (99%)]\tLoss: 0.000084\tF1: 1.0000\tRuntime: 10649.1\n",
      "Train Epoch: 9 [241600/269776 (100%)]\tLoss: 0.001977\tF1: 1.0000\tRuntime: 10684.4\n",
      "Train Epoch: 9 [242400/269776 (100%)]\tLoss: 0.000021\tF1: 1.0000\tRuntime: 10719.8\n",
      "after epoch 9 got loss of 0.004954750569684993 and f1 of 0.9985915492958595\n",
      "saved predictions\n",
      "currently best model --> saved\n",
      "Train Epoch: 10 [0/269776 (0%)]\tLoss: 0.000004\tF1: 1.0000\tRuntime: 0.2\n",
      "Train Epoch: 10 [800/269776 (0%)]\tLoss: 0.000391\tF1: 1.0000\tRuntime: 35.6\n",
      "Train Epoch: 10 [1600/269776 (1%)]\tLoss: 0.000032\tF1: 1.0000\tRuntime: 70.9\n",
      "Train Epoch: 10 [2400/269776 (1%)]\tLoss: 0.000025\tF1: 1.0000\tRuntime: 106.3\n",
      "Train Epoch: 10 [3200/269776 (1%)]\tLoss: 0.000024\tF1: 1.0000\tRuntime: 141.7\n",
      "Train Epoch: 10 [4000/269776 (2%)]\tLoss: 0.000042\tF1: 1.0000\tRuntime: 177.0\n",
      "Train Epoch: 10 [4800/269776 (2%)]\tLoss: 0.000234\tF1: 1.0000\tRuntime: 212.4\n",
      "Train Epoch: 10 [5600/269776 (2%)]\tLoss: 0.000585\tF1: 1.0000\tRuntime: 247.8\n",
      "Train Epoch: 10 [6400/269776 (3%)]\tLoss: 0.000072\tF1: 1.0000\tRuntime: 283.2\n",
      "Train Epoch: 10 [7200/269776 (3%)]\tLoss: 0.000001\tF1: 1.0000\tRuntime: 319.1\n",
      "Train Epoch: 10 [8000/269776 (3%)]\tLoss: 0.000354\tF1: 1.0000\tRuntime: 356.5\n",
      "Train Epoch: 10 [8800/269776 (4%)]\tLoss: 0.000034\tF1: 1.0000\tRuntime: 392.3\n",
      "Train Epoch: 10 [9600/269776 (4%)]\tLoss: 0.000029\tF1: 1.0000\tRuntime: 427.7\n",
      "Train Epoch: 10 [10400/269776 (4%)]\tLoss: 0.000004\tF1: 1.0000\tRuntime: 463.1\n",
      "Train Epoch: 10 [11200/269776 (5%)]\tLoss: 0.000127\tF1: 1.0000\tRuntime: 498.6\n",
      "Train Epoch: 10 [12000/269776 (5%)]\tLoss: 0.000066\tF1: 1.0000\tRuntime: 534.2\n",
      "Train Epoch: 10 [12800/269776 (5%)]\tLoss: 0.000126\tF1: 1.0000\tRuntime: 569.7\n",
      "Train Epoch: 10 [13600/269776 (6%)]\tLoss: 0.000016\tF1: 1.0000\tRuntime: 605.1\n",
      "Train Epoch: 10 [14400/269776 (6%)]\tLoss: 0.000012\tF1: 1.0000\tRuntime: 640.5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 10 [15200/269776 (6%)]\tLoss: 0.000078\tF1: 1.0000\tRuntime: 675.9\n",
      "Train Epoch: 10 [16000/269776 (7%)]\tLoss: 0.000000\tF1: 1.0000\tRuntime: 711.6\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-31-e07eff9b7170>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0;31m#        print('New Learning Rate: ', param_group['lr'])\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m     \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstart_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m     \u001b[0;31m#f1 = get_mean_F1(model, validation_loader)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m     \u001b[0mloss_epoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmean_f1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_loss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/challenge/oeawai_challenge/train_utils.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(args, model, device, train_loader, optimizer, epoch, start_time)\u001b[0m\n\u001b[1;32m     30\u001b[0m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnll_loss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     31\u001b[0m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 32\u001b[0;31m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     33\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mbatch_idx\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog_interval\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     34\u001b[0m             print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}\\tF1: {:.4f}\\tRuntime: {:.1f}'.format(\n",
      "\u001b[0;32m~/miniconda3/envs/felix_ml/lib/python3.7/site-packages/torch/optim/sgd.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m     89\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrad\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m                     \u001b[0;32mcontinue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 91\u001b[0;31m                 \u001b[0md_p\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrad\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     92\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mweight_decay\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m                     \u001b[0md_p\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mweight_decay\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for epoch in range(1 +7, args.epochs + 1 +7):\n",
    "    \n",
    "    #if epoch%7 == 0:\n",
    "    #    for param_group in optimizer.param_groups:\n",
    "    #        param_group['lr'] = param_group['lr'] / 10\n",
    "    #        print('New Learning Rate: ', param_group['lr'])\n",
    "    \n",
    "    train(args, model, device, train_loader, optimizer, epoch, start_time = time.time())\n",
    "    #f1 = get_mean_F1(model, validation_loader)\n",
    "    loss_epoch, mean_f1 = get_loss(model, validation_loader, device)\n",
    "    #print('after epoch {} got f1 score of {}'.format(epoch , f1))\n",
    "    print('after epoch {} got loss of {} and f1 of {}'.format(epoch , loss_epoch, mean_f1))\n",
    "    #if f1 > info['highest F1']:\n",
    "    #    info['highest F1'] = np.copy(f1)\n",
    "    #    info['saved epoch'] = epoch \n",
    "    #    test(args, model, device, test_loader, epoch, trainDataset, testDataset, path_submission)\n",
    "    #    torch.save(model, path_model)\n",
    "    #    print('currently best model --> saved')\n",
    "    if loss_epoch < info['lowest loss']:\n",
    "        info['lowest loss'] = np.copy(loss_epoch)\n",
    "        info['highest F1'] = np.copy(mean_f1)\n",
    "        info['saved epoch'] = epoch \n",
    "        test(args, model, device, test_loader, epoch, trainDataset, testDataset, path_submission)\n",
    "        torch.save(model, path_model + 'model_' + str(epoch) + 'epoch.pt')\n",
    "        print('currently best model --> saved')    \n",
    "        \n",
    "print(info)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### save or load outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-16T14:37:48.194058Z",
     "start_time": "2019-08-16T14:36:47.890141Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saved outputs\n"
     ]
    }
   ],
   "source": [
    "save_output(args, model, device, test_loader, 'SpectralResNet34-score776', \n",
    "            trainDataset, testDataset, path_save = path_submission)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-16T14:39:35.879854Z",
     "start_time": "2019-08-16T14:39:35.649837Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saved predictions\n"
     ]
    }
   ],
   "source": [
    "save_geometric_mean_predictions(path_submission + 'output-MSResNet-score701.txt', \n",
    "                                path_submission + 'output-SpectralResNet34-score776.txt', \n",
    "                                path_submission, \n",
    "                                trainDataset, testDataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-16T14:36:33.461849Z",
     "start_time": "2019-08-16T14:36:33.299547Z"
    }
   },
   "outputs": [],
   "source": [
    "if load_model:\n",
    "    #model = torch.load(path_model)\n",
    "    model = torch.load('models/SpectralResNet34_7epoch_score776.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-14T20:47:16.041071Z",
     "start_time": "2019-08-14T20:47:16.015268Z"
    }
   },
   "outputs": [],
   "source": [
    "# load indices\n",
    "with open(path_submission + \"indices_MSResNet_score701.pt\", \"rb\") as fp:\n",
    "    indices = pickle.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-15T09:43:13.452665Z",
     "start_time": "2019-08-15T09:35:57.796051Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2.308869604184242, 0.03617494440326128)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#get_mean_F1(model, validation_loader)\n",
    "get_loss(model, validation_loader,device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-16T08:06:18.293448Z",
     "start_time": "2019-08-16T08:06:18.290095Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New Learning Rate:  1e-05\n"
     ]
    }
   ],
   "source": [
    "#epoch=10\n",
    "#test(args, model, device, test_loader, epoch, trainDataset, testDataset, path_submission)\n",
    "\n",
    "for param_group in optimizer.param_groups:\n",
    "    param_group['lr'] = param_group['lr'] / 10\n",
    "    print('New Learning Rate: ', param_group['lr'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}