Tokenization, N-grams, Neural Networks.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0010030022202183711"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = 600410177268\n",
    "y = 601012994636\n",
    "(abs(x - y)) / y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "from spacy import displacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\vas\\AppData\\Local\\Continuum\\anaconda3\\lib\\importlib\\_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n",
      "C:\\Users\\vas\\AppData\\Local\\Continuum\\anaconda3\\lib\\importlib\\_bootstrap.py:219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192, got 176\n",
      "  return f(*args, **kwds)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['mary', ',', 'do', \"n't\", 'slap', 'the', 'green', 'witch']\n"
     ]
    }
   ],
   "source": [
    "nlp = spacy.load('en')\n",
    "text = \"Mary, don't slap the green witch\"\n",
    "print([str(token) for token in nlp(text.lower())])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['snow', 'white', 'and', 'the', 'seven', 'degrees', '#makeamoviecold', '@midnight', ':-)']\n"
     ]
    }
   ],
   "source": [
    "from nltk.tokenize import TweetTokenizer\n",
    "tweet = u\"Snow White and the Seven Degrees \\\n",
    "    #MakeAMovieCold@midnight:-)\"\n",
    "tokenizer = TweetTokenizer()\n",
    "print(tokenizer.tokenize(tweet.lower()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div class=\"entities\" style=\"line-height: 2.5\">He is Mr. \n",
       "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    Vahid\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
       "</mark>\n",
       " and they are going to \n",
       "<mark class=\"entity\" style=\"background: #feca74; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    Orage County\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">GPE</span>\n",
       "</mark>\n",
       " in \n",
       "<mark class=\"entity\" style=\"background: #feca74; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    U.S.\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">GPE</span>\n",
       "</mark>\n",
       "</div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "text = \"He is Mr. Vahid and they are going to Orage County in U.S.\"\n",
    "doc = nlp(text)\n",
    "displacy.render(doc, style='ent', jupyter=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## N-grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('he', 'was', \"n't\")\n",
      "('was', \"n't\", 'found')\n",
      "(\"n't\", 'found', 'alive')\n"
     ]
    }
   ],
   "source": [
    "text = \"he wasn't found alive\"\n",
    "doc = nlp(text.lower())\n",
    "tokens = [str(token) for token in doc]\n",
    "for grams in nltk.ngrams(tokens, 3):\n",
    "    print(grams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "he --> -PRON-\n",
      "was --> be\n",
      "n't --> not\n",
      "found --> find\n",
      "alive --> alive\n"
     ]
    }
   ],
   "source": [
    "for token in doc:\n",
    "    print('{} --> {}'.format(token, token.lemma_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "he --> PRON\n",
      "was --> VERB\n",
      "n't --> ADV\n",
      "found --> VERB\n",
      "alive --> ADJ\n"
     ]
    }
   ],
   "source": [
    "for token in doc:\n",
    "    print('{} --> {}'.format(token, token.pos_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mary --> NP\n",
      "the green witch --> NP\n"
     ]
    }
   ],
   "source": [
    "doc  = nlp(u\"Mary slapped the green witch.\")\n",
    "for chunk in doc.noun_chunks:\n",
    "    print('{} --> {}'.format(chunk, chunk.label_))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Neural Networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch\n",
    "class perceptron:\n",
    "    def __init__(self, input_dim):\n",
    "        self.fc1 = nn.Linear(input_dim, 1)\n",
    "    def forward(X):\n",
    "        return torch.sigmoid(self.fc(X)).squeeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x256ae72c2e8>]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl01fWd//HnO/seliRsSVgk7LgRWXSqtm6IVrs4KopLXbCntbVTq3Xpz+lop63aqXU6thZx6o5FrZZaKlVrRSsgIPsOYUkIkI3s600+80eiv4jBXMJNvnd5Pc7h5C5fbl73JHmdTz75fD9fc84hIiLhJcrrACIiEngqdxGRMKRyFxEJQyp3EZEwpHIXEQlDKncRkTCkchcRCUMqdxGRMKRyFxEJQzFefeKMjAw3YsQIrz69iEhIWr16dZlzLrO74zwr9xEjRrBq1SqvPr2ISEgys73+HKdpGRGRMKRyFxEJQyp3EZEw1G25m9n/mlmJmW08yvNmZv9tZjvNbL2ZnRr4mCIiciz8Gbk/Bcz8nOcvBPI6/s0Ffnv8sURE5Hh0W+7OuaVAxecccinwjGu3HOhnZkMCFVBERI5dIObchwGFne4XdTwmIiIeCcQ6d+visS6v3Wdmc2mfuiE3NzcAn1pEJHi0tTlqmnxUN7RQ0+ijtslHTWMLtU3tt+uafNQ2tXLOuCxOyunXq1kCUe5FQE6n+9lAcVcHOufmAfMA8vPzdfFWEQlazjmqG3yU1jZRVttEeW0z5XXtHw/XN3O4voXK+vbbVQ0tVNa3l7g/l6XOSo0PiXJfBNxqZi8C04Aq59yBALyuiEivcM5RVtvM/soG9h9u4EBVAweqGjlY1cjB6kYOVTdSUtNEs6+ty/+fnhhL/6RY+ifHkZkST15WKumJsaQlxpKWEPPJx9SEWFLiY0iOjyE1of1jUmw0UVFdTXgEVrflbmYLgLOBDDMrAv4diAVwzj0OLAZmATuBeuAbvRVWRMRfbW2O4qoGdpfVsaesjoKyOvaV17Ovop7Cw/U0tny6uBNjoxnSL4FBqQnkD+/PoLQEMlPjyUyNJyMlnoEpcQxMjqd/Uiwx0cF/ilC35e6cm93N8w74dsASiYgcA+ccJTVNbDlQzdaDNWw/WMOOklp2ltTS0NL6yXFJcdHkDkhiZEYyZ43JJLt/IsP6JzGsXyLD+iWSlhiDWe+PqPuKZxuHiYj0REl1I2sKK1lXWMnG4mo2F1dRVtv8yfND0hMYnZXClVNzGJ2VwgmZKYzMSCYrNT6syrs7KncRCVptbY6tB2tYtbeClXsOs3pPBcVVjQDERBl5g1L54tgsJg5NY/yQNMYNTiM9Kdbj1MFB5S4iQcM5x+6yOt7bUcayXeUs311OZX0LAIPTEsgf0Z8bc/tzck46E4emkxAb7XHi4KVyFxFPNba0smxXOX/fWsI/tpdQWNEAwLB+iZw3fhAzThjIaSMGkN0/MaKmVY6Xyl1E+lxtk4+3txzijY0HeXd7KfXNrSTFRXP6CRnMPfMEzszLYPjAZK9jhjSVu4j0icaWVv6+tYRFa4t5Z1sJTb42slLj+copwzhvwiBOP2Eg8TGaZgkUlbuI9BrnHGsLK3lpdRGvryumutFHZmo8s6fmctGJQ5iS279PTuiJRCp3EQm4msYWXltbzAsr9rHlQDWJsdHMnDSYr506jNNPyCBahd7rVO4iEjB7y+t46oM9vLSqiNomHxOHpvGfX53EJScNJTVBSxT7kspdRI7b2sJKfvPOTt7ccoiYKOPiE4dy7YzhnJzTTytcPKJyF5EeW7arnMfe2cn7O8tIT4zl22eP5poZwxmUluB1tIincheRY/bRvsP8Ysk2PthVTmZqPPfMGsdV04aTEq9KCRb6SoiI33YcquHnf93K21tLyEiJ476LJ3DVtFydKRqEVO4i0q3y2iZ+9dYOXvhwH0lx0dxxwViuP30EyRqpBy19ZUTkqFrbHM8t38sv/raN+uZWrp6Wy23n5DEwJd7raNINlbuIdGltYSU/em0DG/dX8y+jM/j3L08gb1Cq17HETyp3EfmU+mYfDy/ZxlMf7CEzJZ5fzz6Fi08coiWNIUblLiKfWLarnB++sp59FfXMmZ7LD2eO08lHIUrlLiI0+Vp5+I1tzH9/N7kDklhw83RmnDDQ61hyHFTuIhFux6EavrNgDVsP1nDN9OHcPWscSXGqhlCnr6BIhHLO8YeVhfz7ok2kxMfw5HX5nDN+kNexJEBU7iIRqKG5lR+9tpFXPiriX0Zn8MsrTiIrVVsGhBOVu0iE2V1WxzefXc32khpuOyeP756Tpy14w5DKXSSCvLejlG8//xHRUcZT35jKWWMyvY4kvUTlLhIBnHM89cEefvKXLYzOTGH+dfnkDEjyOpb0IpW7SJjztbbx4z9v4rnl+zhvwiAeueJk7d4YAfQVFglj9c0+vrtgDW9tKeGWs0bxwwvG6ZqlEULlLhKmymqbuPHpVWwoquSBSydyzYwRXkeSPqRyFwlDxZUNzJm/guKqBh6fM4XzJw72OpL0MZW7SJjZXVbHnPkrqG5o4dkbp3HaiAFeRxIPqNxFwsjWg9XMmf8hbc6xYO50Jg1L9zqSeCTKn4PMbKaZbTOznWZ2VxfP55rZO2a2xszWm9mswEcVkc+z9WA1Vz2xgugoWHiLij3SdVvuZhYNPAZcCEwAZpvZhCMO+xGw0Dl3CnAl8JtABxWRo/u42OOio/jD3BmMztJFNSKdPyP3qcBO51yBc64ZeBG49IhjHJDWcTsdKA5cRBH5PNsO1nDVEyuIjTYWzJ3OiIxkryNJEPBnzn0YUNjpfhEw7Yhjfgz8zcy+AyQD5wYknYh8rt1ldVw9v73YX5w7g5Eqdungz8i9qzMe3BH3ZwNPOeeygVnAs2b2mdc2s7lmtsrMVpWWlh57WhH5xMfLHduc4/mbpqvY5VP8KfciIKfT/Ww+O+1yI7AQwDm3DEgAMo58IefcPOdcvnMuPzNTGxaJ9FRZbRNznmxf7vjMDVMZnZXidSQJMv6U+0ogz8xGmlkc7X8wXXTEMfuAcwDMbDzt5a6huUgvqG3ycf3vP6S4soEnrz9Nq2KkS92Wu3POB9wKLAG20L4qZpOZ3W9ml3Qcdjtws5mtAxYA1zvnjpy6EZHj1NLaxree/4gtB2r4zdWnMnWkTlCSrvl1EpNzbjGw+IjH7ut0ezNwRmCjiUhnzjnu+eMGlm4v5cGvT+ZL43RJPDk6v05iEhHv/eqtHby0uojbzsnjitNyvY4jQU7lLhICXluzn0ff3sHl+dl879w8r+NICFC5iwS51XsPc+cr65k+agA/+cpkzLQfu3RP5S4SxIoO13PLs6sYmp7Ab6+eQlyMfmTFP9oVUiRI1TX5uOnpVTT52nhx7mn0T47zOpKEEA0DRIKQc447X17P9kM1PHbVqTpJSY6Zyl0kCP1uaQF/2XCAH84cx5ljdDa3HDuVu0iQWbq9lIfe2MrFJw5h7pmjvI4jIUrlLhJECivq+c6CNYwZlMpDl52olTHSYyp3kSDR5Gvl1hc+oq3N8ficKSTFab2D9Jy+e0SCxE//soV1RVU8PmeKLrghx00jd5Eg8Od1xTy9bC83/ctIZk4a7HUcCQMqdxGP7S6r465X1jNleH9+eOE4r+NImFC5i3ioydfKdxZ8RGxMFL+efQqx0fqRlMDQnLuIhx56Yxsb91cz75opDO2X6HUcCSMaJoh45O9bD/Hk+7u5bsZwzp+oeXYJLJW7iAdKqhv5wUvrGT8kjbtnjfc6joQhlbtIH3PO8YOX11Pf7OPXs08mITba60gShlTuIn3smWV7Wbq9lHsvmsDorFSv40iYUrmL9KEdh2r46eItfHFsJnOm6VJ50ntU7iJ9pNnXxm0vriUlPoaHLjtJ+8ZIr9JSSJE+8ujb29l8oJr51+aTmRrvdRwJcxq5i/SBNfsO89t/7OLy/GzOnTDI6zgSAVTuIr2sobmV2xeuY0h6Iv/v4glex5EIoWkZkV720JKtFJTV8cJN00hNiPU6jkQIjdxFetHygnJ+/889XH/6CE4fneF1HIkgKneRXlLf7OPOl9czfGASd84c63UciTCalhHpJQ8v2ca+inpenDtdV1WSPqeRu0gvWLWngqc+2MN1M4YzfdRAr+NIBFK5iwRYY0srd7y8nuz+idw5UxffEG/od0WRAHvkze3s7lgdkxyvHzHxhl8jdzObaWbbzGynmd11lGMuN7PNZrbJzF4IbEyR0LChqIon3ivgytNytDpGPNXtsMLMooHHgPOAImClmS1yzm3udEwecDdwhnPusJll9VZgkWDV0trGna+sJyMlXnu0i+f8GblPBXY65wqcc83Ai8ClRxxzM/CYc+4wgHOuJLAxRYLfvKUFbDlQzQNfmUR6ok5WEm/5U+7DgMJO94s6HutsDDDGzP5pZsvNbGagAoqEgl2ltTz69g5mTR7MBbpkngQBf/7a09W+pK6L18kDzgaygffMbJJzrvJTL2Q2F5gLkJurvawlPLS1Oe7+4wYSYqL48SUTvY4jAvg3ci8CcjrdzwaKuzjmT865FufcbmAb7WX/Kc65ec65fOdcfmZmZk8ziwSVl1YX8uHuCu6ZNZ6s1ASv44gA/pX7SiDPzEaaWRxwJbDoiGNeA74IYGYZtE/TFAQyqEgwKq1p4j//soWpIwdweX5O9/9BpI90W+7OOR9wK7AE2AIsdM5tMrP7zeySjsOWAOVmthl4B7jDOVfeW6FFgsX9r2+msaWNn351MlFRurKSBA+/zrBwzi0GFh/x2H2dbjvg+x3/RCLCP7aV8Od1xfzbuWMYnZXidRyRT9H2AyI90NDcyo9e28gJmcl88+xRXscR+QydGy3SA4++vYOiww38Ye504mOivY4j8hkauYsco60Hq5n/XgGX52czTTs+SpBSuYscg7Y2xz1/3EBaYix3X6gtBiR4qdxFjsGClfv4aF8l984aT//kOK/jiByVyl3ET6U1TTz4163MGDWQr5165A4cIsFF5S7ip58u3kJjSxs/+eokzLSmXYKbyl3EDx/sKuPVNfv55lmjOCFTa9ol+KncRbrR5Gtf0547IIlvfXG013FE/KJ17iLdmPduAQWldTz1jdNIiNWadgkNGrmLfI695XX8+p2dXDR5CGeP1QXGJHSo3EWOwjnHfX/aRFx0FPd9eYLXcUSOicpd5CgWbzjIu9tL+f55YxiUpn3aJbSo3EW6UNPYwv2vb2LCkDSunTHc6zgix0x/UBXpwiNv7qCkponH50whJlpjIAk9+q4VOcLG/VU89cFuZk/N5ZTc/l7HEekRlbtIJ21tjh+9tpH+SXH88IJxXscR6TGVu0gnL64sZG1hJfdeNJ70pFiv44j0mMpdpENZbRMPvrGV6aMG8NVTtDGYhDaVu0iHny3eSn2zj598RRuDSehTuYsAy3aV88pHRdz8hVGMzkr1Oo7IcVO5S8Rr9rXxo9c2kDMgke98Kc/rOCIBoXXuEvGeeK+AXaV1/P7600iM08ZgEh40cpeItq+8nv9+ewezJg/mi+O0MZiED5W7RCznHPct2khMlHHfxRO9jiMSUCp3iViLNxzkH9tK+f75Yxmcro3BJLyo3CUiVTe28OM/b2LSsDSu08ZgEob0B1WJSA+/sY3y2iaevC5fG4NJWNJ3tUScNfsO89yKvVw7YwQnZvfzOo5Ir1C5S0RpaW3jnlc3kpUaz+3nj/E6jkiv0bSMRJQn39/NlgPV/PbqU0lN0MZgEr78Grmb2Uwz22ZmO83srs857jIzc2aWH7iIIoGxr7yeX721nfMmDGLmpMFexxHpVd2Wu5lFA48BFwITgNlm9pmrBZtZKvBdYEWgQ4ocL+cc9762gWgz7r90ojYGk7Dnz8h9KrDTOVfgnGsGXgQu7eK4B4CHgMYA5hMJiD+tLea9HWXcOXMcQ9ITvY4j0uv8KfdhQGGn+0Udj33CzE4Bcpxzrwcwm0hAVNQ188Drmzk5px9zpmtNu0QGf/6g2tXvr+6TJ82igEeA67t9IbO5wFyA3Nxc/xKKHKcHXt9MVUMLz399MtFRmo6RyODPyL0IyOl0Pxso7nQ/FZgE/MPM9gDTgUVd/VHVOTfPOZfvnMvPzMzseWoRP/1jWwmvrtnPt84+gXGD07yOI9Jn/Cn3lUCemY00szjgSmDRx08656qccxnOuRHOuRHAcuAS59yqXkks4qfaJh/3vrqR0VkpfPtLo72OI9Knui1355wPuBVYAmwBFjrnNpnZ/WZ2SW8HFOmpXyzZRnFVAw9+fTLxMdqnXSKLXycxOecWA4uPeOy+oxx79vHHEjk+q/ZU8PSyPVw7fThThg/wOo5In9P2AxJ2GltaufPl9QxNT+TOmeO8jiPiCW0/IGHnl29up6CsjudvmkZyvL7FJTJp5C5h5aN9h5n/XgGzp+ZyxugMr+OIeEblLmHj4+mYwWkJ3DNL0zES2fQ7q4SNR97czs6SWp6+Yap2fJSIp5G7hIVVeyqY1zEdc9YYnSAnonKXkFff7OP2l9aR3T+Rey8a73UckaCgaRkJeT//61b2VdSz4ObppGh1jAigkbuEuKXbS3lm2V5uOGMk00cN9DqOSNBQuUvIOlzXzA9eWkdeVgp3XDDW6zgiQUW/w0pIcs5x9x83cLi+md9/4zQSYrV3jEhnGrlLSHppdRFvbDrID84fy8Sh6V7HEQk6KncJOXvL6/iPRZuYPmoAN31hlNdxRIKSyl1CSrOvje8uWEN0lPFfl5+sKyuJHIXm3CWk/NfftrGuqIrfXn0qw/rpQtciR6ORu4SMd7eX8rulBVw1LZcLJw/xOo5IUFO5S0goqWnk9oVrGTsolfsunuB1HJGgp2kZCXqtbY7bFqyltsnHCzdP17JHET+o3CXoPfLmdpYVlPPwZScyZlCq13FEQoKmZSSovbOthP95ZyeX52fzr/k5XscRCRkqdwla+ysb+Lc/rGXc4FTuv3SS13FEQorKXYJSY0sr33x2Nb5Wx2/nTNE8u8gx0py7BB3nHPe+upEN+6t44tp8RmYkex1JJORo5C5B55lle3nloyK+d24e500Y5HUckZCkcpegsqKgnAde38y54wfx3S/leR1HJGSp3CVo7C2v45vPrSZ3YBK/vOIkorRvjEiPqdwlKFQ3tnDj06toc/DkdaeRlhDrdSSRkKZyF8/5Wtu49YU17Cmr4/E5U/QHVJEA0GoZ8ZRzjvtf38zS7aX8/GuTmXGCroMqEggauYunHn+3gGeW7WXumaO4cmqu13FEwobKXTzz2pr9PPjGVr580lDumjnO6zgiYcWvcjezmWa2zcx2mtldXTz/fTPbbGbrzextMxse+KgSTv65s4w7Xl7H9FED+MW/nqiVMSIB1m25m1k08BhwITABmG1mR26ovQbId86dCLwMPBTooBI+1hZWMveZVYzKSOF31+QTH6OtBUQCzZ+R+1Rgp3OuwDnXDLwIXNr5AOfcO865+o67y4HswMaUcLHtYA3X//5DBqbE88yNU0lP1JJHkd7gT7kPAwo73S/qeOxobgT+ejyhJDztLa9jzpMriIuO4vmbpjEoLcHrSCJhy5+lkF1NhrouDzSbA+QDZx3l+bnAXIDcXK2MiCSFFfVc9cQKWlrbWHjLDHIGJHkdSSSs+TNyLwI6XyUhGyg+8iAzOxe4F7jEOdfU1Qs55+Y55/Kdc/mZmZk9ySshqOhwPbOfWE5NYwvP3jBNV1MS6QP+lPtKIM/MRppZHHAlsKjzAWZ2CvA72ou9JPAxJVQVHa7nynnLqW5o4fmbpjM5O93rSCIRodtyd875gFuBJcAWYKFzbpOZ3W9ml3Qc9jCQArxkZmvNbNFRXk4iyN7yuk+K/bmbpqnYRfqQX9sPOOcWA4uPeOy+TrfPDXAuCXHbDtZwzZPtc+wasYv0Pe0tIwG3rrCS637/IfExUSy8ZQZ5mmMX6XMqdwmod7eX8q3nVjMgJY7nb5xO7kCtihHxgvaWkYBZuLKQG55aSe7AZF7+5ukqdhEPaeQux805x6Nv7+BXb+3gC3kZ/ObqU0nVxTZEPKVyl+PS0NzKHS+v4/X1B7hsSjY/+9pkYqP1C6GI11Tu0mPFlQ3MfXYVm4qruevCcdxy5ijMtLujSDBQuUuPLC8o59YX1tDY0sr8a/M5Z/wgryOJSCcqdzkmbW2O3y0t4OElWxkxMJkXbtZ2AiLBSOUufjtc18wdL6/jrS0lXHTiEB78+omkxOtbSCQY6SdT/PL+jjJuf2ktFXXN/PjLE7ju9BGaXxcJYip3+VyNLa38Ysk25r+/m9FZKfzv9acxcai2EhAJdip3OarVew9z58vr2FVaxzXTh3PPrPEkxumSeCKhQOUun1Hf7OOXf9vOk//czdD0RJ65YSpnjtH++yKhROUun/K3TQf5jz9vZn9lA1dPy+WuC8fpbFOREKRyF6B97/UHXt/MW1tKGDsolYW3zGDqyAFexxKRHlK5R7iq+hZ+/fcdPL1sD7HRUdw7azzXnzFCWwiIhDiVe4RqbGnlueV7eeydnVQ2tHD5lBxuP38MWWkJXkcTkQBQuUeYZl8bC1cV8uu/7+BQdRNfyMvg7gvHM2FomtfRRCSAVO4RoqG5lRdX7mPe0gIOVDWSP7w/j155CtNHDfQ6moj0ApV7mCuvbeL5Fft4+oM9lNc1M3XEAH72tcmcNSZTZ5iKhDGVe5jaXFzN0x/s4dW1+2n2tXH22Ey+dfZorYARiRAq9zDS0NzKn9cX88KKfawtrCQhNorL87O5/vSRjM5K8TqeiPQhlXuIa2tzfLingldWF/HXjQepbfIxOiuF+y6ewNdOHUa/pDivI4qIB1TuIcg5x7qiKv6yvpjFGw6yv7KB5LhoZk0ewmVTspk6coDm00UinMo9RLS0tvHh7gre3HyINzcfYn9lA7HRxpl5mdxxwVgumDhYm3qJyCdU7kGsuLKBpdtLeXd7Ke/vLKOm0Ud8TBRfyMvgtnPzuGDCYNKTtO+LiHyWyj2IHKxqZOWeCpYVlLNsVzm7y+oAGJKewKxJQ/jS+Cy+kJdBUpy+bCLy+dQSHmn2tbH1YDVrCytZs6+SVXsrKKxoACA1PoapIwdw9bRczhyTSV5WiubQReSYqNz7QG2Tj20Ha9h6sJqN+6vZVFzF1gM1NLe2AZCREk/+8P5cN2MEp40YwMShacRo4y4ROQ4q9wBxzlFR18zusjoKSuvYWVrLzpJadpTUfDIiB0hPjGXi0DSuP2MEJ2X346ScdIb1S9TIXEQCSuV+DOqafBRXNlBU2cD+ww0UHW6gsKKefRX17C2vo7rR98mxcdFRjMpM5qTsflyRn8O4wWmMHZxKdn8VuYj0vogv97Y2R1VDC+V1zZTXNlFW20xpTSOltU0cqm7iUHUjh6obOVDVSE2n8gaIjTZy+ieRMyCJk3P6MSIjmVEZyYzISCanf6KmVkTEM36Vu5nNBB4FooH5zrmfH/F8PPAMMAUoB65wzu0JbNSuOedo8rVR2+SjrslHTaOP2iYftY0+qhtbqGn0Ud3QQlVDC5Uff6xv5nD9///Y2uY+87rRUUZWajxZqfEMH5jMjFEDGZyeyNB+CQzrl8iw/olkpSYQHaVRuIgEn27L3cyigceA84AiYKWZLXLObe502I3AYefcaDO7EngQuKI3Ai9cWcjjS3dR39RKXbOP+ubWLsv5SElx0aQnxpKeGEu/pFjyslLolxTHwOQ4BiTHMTAljoHJ8WSkxpGREs+ApDiiVNwiEqL8GblPBXY65woAzOxF4FKgc7lfCvy44/bLwP+YmTnnum/dY9Q/OY4JQ9JIiosmKS6GpLhokuNjSImPITk+htSEGFLjY0hJiCEtIZa0xFhS4mOIi9EUiYhEDn/KfRhQ2Ol+ETDtaMc453xmVgUMBMo6H2Rmc4G5ALm5uT0KfN6EQZw3YVCP/q+ISKTwZzjb1dzEkSNyf47BOTfPOZfvnMvPzMz0J5+IiPSAP+VeBOR0up8NFB/tGDOLAdKBikAEFBGRY+dPua8E8sxspJnFAVcCi444ZhFwXcfty4C/98Z8u4iI+KfbOfeOOfRbgSW0L4X8X+fcJjO7H1jlnFsEPAk8a2Y7aR+xX9mboUVE5PP5tc7dObcYWHzEY/d1ut0I/Gtgo4mISE9pfaCISBhSuYuIhCGVu4hIGDKvFrWYWSmw15NPfnwyOOLkrAgRie9b7zlyhNL7Hu6c6/ZEIc/KPVSZ2SrnXL7XOfpaJL5vvefIEY7vW9MyIiJhSOUuIhKGVO7Hbp7XATwSie9b7zlyhN371py7iEgY0shdRCQMqdyPg5n9wMycmWV4naW3mdnDZrbVzNab2atm1s/rTL3JzGaa2TYz22lmd3mdp7eZWY6ZvWNmW8xsk5nd5nWmvmJm0Wa2xsxe9zpLIKnce8jMcmi/9OA+r7P0kTeBSc65E4HtwN0e5+k1nS4teSEwAZhtZhO8TdXrfMDtzrnxwHTg2xHwnj92G7DF6xCBpnLvuUeAO+nioiThyDn3N+ecr+Puctr39Q9Xn1xa0jnXDHx8acmw5Zw74Jz7qON2De1lN8zbVL3PzLKBi4D5XmcJNJV7D5jZJcB+59w6r7N45Abgr16H6EVdXVoy7IvuY2Y2AjgFWOFtkj7xK9oHaW1eBwk0v7b8jURm9hYwuIun7gXuAc7v20S97/Pes3PuTx3H3Ev7r/DP92W2PubXZSPDkZmlAK8A33POVXudpzeZ2cVAiXNutZmd7XWeQFO5H4Vz7tyuHjezycBIYJ2ZQfv0xEdmNtU5d7APIwbc0d7zx8zsOuBi4Jwwv9KWP5eWDDtmFkt7sT/vnPuj13n6wBnAJWY2C0gA0szsOefcHI9zBYTWuR8nM9sD5DvnQmXToR4xs5nAL4GznHOlXufpTR3XAd4OnAPsp/1Sk1c55zZ5GqwXWftI5Wmgwjn3Pa/z9LWOkfsPnHMXe50lUDTnLv76HyAVeNPM1prZ414H6i0dfzj++NKSW4CF4VzsHc4ArgG+1PH1XdsxopUQpZG7iEgY0shdRCQMqdxFRMKQyl1EJAyp3EVEwpDKXUQkDKncRUTCkMpdRCQMqdxFRMJ8hsSHAAAAB0lEQVTQ/wFqiBP0W7Nw1QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x256afa3f9e8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "x = torch.arange(-5,5,0.1)\n",
    "y = torch.sigmoid(x)\n",
    "plt.plot(x.numpy(), y.numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x256b0fff668>]"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD8CAYAAABzTgP2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XmYXHWd7/H3t6u37Ft39u4sJJKEJQHasEQuWwJRZhJmREVHjAqT0SvjOCoK41ydi+M8eH1G5voMo+QqEmUTopgogUBYVAYI6UBI0oGQTsjS6U6ns3fSe9X3/lEnWN10p5NUdZ+uqs/reWrOOb/zO1XfGkN9+my/Y+6OiIjICTlhFyAiIn2LgkFERNpRMIiISDsKBhERaUfBICIi7SgYRESkHQWDiIi0o2AQEZF2FAwiItJObtgFnImioiKfOHFi2GWIiKSVdevW7Xf34u76pWUwTJw4kfLy8rDLEBFJK2a281T66VCSiIi0o2AQEZF2FAwiItKOgkFERNpRMIiISDspCQYzu9/M9pnZpi7Wm5n9yMwqzWyDmV2YsG6RmW0NXotSUY+IiJy5VO0xPADMP8n6DwNTg9di4McAZjYc+A5wMTAb+I6ZDUtRTSIicgZSch+Du//RzCaepMtC4Bcef47oq2Y21MzGAFcCz7r7QQAze5Z4wDySirpEJPO1RWM0t8VoaYtPW6MxWqLxaWub0xqL0RZ12qIx2mJONHi1xZyYx+dPTKMxxx1i7sSCqQfz702Jz7uDc2JKu+UTTvSDeB+Cfu+txxP6JrYn6PD45UWXTWTEwIIU/H+ua711g9s4YHfCclXQ1lX7+5jZYuJ7G5SWlvZMlSLSq6Ix58DxZurqmzl0vJWDDS0cOt7CkcZWjja2cqSxlWPNbe+9GpqjNLS20dgSpak1RlNrlLZY5j+33uzP8wtmjcuYYLBO2vwk7e9vdF8CLAEoKyvL/H8JIhkgGnN2H2xgx4Hj7DzQwM4DDVQfbqTmSCPVR5o4cKyZrn7X++dHGFyYx8DCXAYWxF/FAwvonx+hX34u/fIiFOblUJgXIT83h4LcHApyI+RFjPzcHPIjOeRFcsiNGLk5J6ZGpOPLjJxgGsmJz+cYGEZODuSYYQTToN1y4j9eFqx7rz34RTux/Of5E+1BW8L3TPzRN+vsJ7H39VYwVAElCcvjgeqg/coO7S/2Uk0ikkINLW1s2nOUDVWHqag+ypa99VTWHaOlLfZen355EcYN68eYIYVMGz2YUYMLKB5UQNHAAkYMLGD4gDyG9s9nSL888iK6aDIsvRUMK4DbzOxR4ieaj7h7jZmtAv4t4YTztcCdvVSTiCThWHMba7Yf4NXtB3h1+0Eqqo+899f/6MGFTBsziA9NLWLKyIFMLhpA6Yj+FA8s6DN/FUvXUhIMZvYI8b/8i8ysiviVRnkA7v4TYCXwEaASaAA+F6w7aGbfBdYGb3XXiRPRItL31B5tYlXFXp7dXMua7QdpicbIz83hwtKh3HbVFGaWDOX88UMpHtSzx8ClZ5l7+h2uLysrc42uKtI7GluirKrYy69fr+Klyv24w6SiAcydPpKrpo3kwtJhFOZFwi5TToGZrXP3su76peWw2yLS82qONLL05Z088toujjS2Mn5YP/7+qiksmDWWKSMHhV2e9CAFg4i0s/tgA/esfocV66uJuTP/3NF85tKJzJ44nJwcnR/IBgoGEQGgrr6Z/3x+Kw+/toscMz5z6UQ+N2ciJcP7h12a9DIFg0iWi8Wch1/bxfefepuG1iif+GAJX756KqOHFIZdmoREwSCSxSr31XPHrzdSvvMQc6aM4LsLz2Vy8cCwy5KQKRhEspC786u1u/n2igr650f4wY3nc+NF43WPgQAKBpGs09DSxj//dhO/eX0PH5pSxD2fmKX7DqQdBYNIFqk+3Mjnfr6Wd/bV85W5U/n7q6cS0ZVG0oGCQSRLvFNbz2d+9hrHm9tY+rnZ/I8PFIddkvRRCgaRLLB2x0FueWAthXkRHvvCpUwfMzjskqQPUzCIZLhXtx9g0f2vMW5YP37x+dmMH6b7EuTkFAwiGezN3Ye55YG1lA7vz6OLL+nxB7xIZtCA5yIZ6p3aehb9/DWGD8znl7dcrFCQU6ZgEMlA1Ycb+fRP15AfyeGhWy7RXcxyWnQoSSTDNLVG+cKD62hoibLsi5dSOkLnFOT0pGSPwczmm9kWM6s0szs6WX+Pma0PXu+Y2eGEddGEdStSUY9ItnJ3/vm3m9hQdYQffnwm00br6iM5fUnvMZhZBLgXmEf8Gc5rzWyFu28+0cfd/zGh/98DFyS8RaO7z0q2DhGBX7yyk2XrqvjyNVO59pzRYZcjaSoVewyzgUp33+7uLcCjwMKT9P8k8EgKPldEEryx6xDf/f1mrpk2kq9cMzXsciSNpSIYxgG7E5argrb3MbMJwCTg+YTmQjMrN7NXzeyGFNQjknUaW6J89bE3GTW4kB9+YpYeqCNJScXJ587+BXb1IOmbgGXuHk1oK3X3ajObDDxvZhvdfdv7PsRsMbAYoLS0NNmaRTLK3U+9xbv7j/Pw317MkH55YZcjaS4VewxVQEnC8niguou+N9HhMJK7VwfT7cCLtD//kNhvibuXuXtZcbHGeBE54aWt+1n6yk4+P2cSl51VFHY5kgFSEQxrgalmNsnM8on/+L/v6iIzOxsYBryS0DbMzAqC+SJgDrC547Yi0rkjja3cvuxNzioewDfmnx12OZIhkj6U5O5tZnYbsAqIAPe7e4WZ3QWUu/uJkPgk8Ki7Jx5mmg7cZ2Yx4iF1d+LVTCJycv/+zBZqjzbxxP+cQ2FeJOxyJEOk5AY3d18JrOzQ9u0Oy//SyXYvA+elogaRbFNRfYQHX93JzZdMYGbJ0LDLkQyiITFE0pC7853lFQzrn89Xr9UhJEktBYNIGnrijT2U7zzEN+dP01VIknIKBpE0U9/Uyr+tfJuZJUO58aLxYZcjGUiD6ImkmZ/8YRv7jzXzs0VlupFNeoT2GETSSF19M/e/tIO/nDlWJ5ylxygYRNLIvS9U0hKN8Y9zNRaS9BwFg0ia2HO4kYfX7OLGC8czuXhg2OVIBlMwiKSJH63eCsCXtbcgPUzBIJIG3t1/nGWvV/Gpi0sZN7Rf2OVIhlMwiKSBe1+oJC9ifOmqKWGXIllAwSDSx9UcaWT5+j18oqyE4kEFYZcjWUDBINLH3f/Su8Qcbr18ctilSJZQMIj0YUcaW3l4zS6uP28MJcP7h12OZAkFg0gf9tCanRxvifJ3V2hvQXqPgkGkj2pqjXL/Szu4fGoR54wdEnY5kkUUDCJ91BNv7GH/sWa+cMVZYZciWSYlwWBm881si5lVmtkdnaz/rJnVmdn64HVrwrpFZrY1eC1KRT0i6c7dWfryDmaMGcxlZ40IuxzJMkmPrmpmEeBeYB5QBaw1sxWdPKLzV+5+W4dthwPfAcoAB9YF2x5Kti6RdFa+8xBv763n7r8+DzONoCq9KxV7DLOBSnff7u4twKPAwlPc9jrgWXc/GITBs8D8FNQkktZ++cpOBhXmsmDW2LBLkSyUimAYB+xOWK4K2jr6qJltMLNlZlZymtuKZI26+mae2lTDjReNp3++HpkivS8VwdDZfq53WP4dMNHdzwdWA0tPY9t4R7PFZlZuZuV1dXVnXKxIX/dY+W5ao86nL5kQdimSpVIRDFVAScLyeKA6sYO7H3D35mDx/wEXneq2Ce+xxN3L3L2suLg4BWWL9D3RmPPQqzuZM2UEZ2lobQlJKoJhLTDVzCaZWT5wE7AisYOZjUlYXAC8FcyvAq41s2FmNgy4NmgTyUrPv72P6iNN3HzJxLBLkSyW9AFMd28zs9uI/6BHgPvdvcLM7gLK3X0F8GUzWwC0AQeBzwbbHjSz7xIPF4C73P1gsjWJpKuH1+xk1OAC5k4fGXYpksVScmbL3VcCKzu0fTth/k7gzi62vR+4PxV1iKSz2qNN/OGdOr545VnkRnTvqYRH//pE+ojfvL6HmMONF5V031mkBykYRPoAd+fxdbv54MRhTCoaEHY5kuUUDCJ9wOu7DrO97jg3XjQ+7FJEFAwifcGydbvplxfh+vN1p7OET8EgErLGlii/e7OGD583moEFutNZwqdgEAnZ0xU1HGtu42M66Sx9hIJBJGS/eX0PJcP7cfGk4WGXIgIoGERCta++if+u3M8Ns8aRk6PhtaVvUDCIhOjJDTXEHBZqeG3pQxQMIiFavr6aGWMGM2XkoLBLEXmPgkEkJDsPHGf97sPaW5A+R8EgEpIV6+MjzP/lTAWD9C0KBpEQuDvL36xm9sThjB3aL+xyRNpRMIiE4K2aeir3HdMznaVPUjCIhGD5m3vIzTE+ct6Y7juL9DIFg0gvc3d+/2YNl08tYviA/LDLEXmflASDmc03sy1mVmlmd3Sy/qtmttnMNpjZc2Y2IWFd1MzWB68VHbcVyTQbqo6w53CjBsyTPivpEbvMLALcC8wDqoC1ZrbC3TcndHsDKHP3BjP7IvB/gE8E6xrdfVaydYiki5Uba8iLGPOmjwq7FJFOpWKPYTZQ6e7b3b0FeBRYmNjB3V9w94Zg8VVAg85LVnJ3ntxYw5wpRQzpnxd2OSKdSkUwjAN2JyxXBW1duQV4KmG50MzKzexVM7uhq43MbHHQr7yuri65ikVCsnHPEaoONeqks/RpqRj8vbORv7zTjmafBsqAKxKaS9292swmA8+b2UZ33/a+N3RfAiwBKCsr6/T9Rfq6JzfWkJtjXDtDh5Gk70rFHkMVkDiQ/HigumMnM5sLfAtY4O7NJ9rdvTqYbgdeBC5IQU0ifY6789TGvVw2pYih/XU1kvRdqQiGtcBUM5tkZvnATUC7q4vM7ALgPuKhsC+hfZiZFQTzRcAcIPGktUjGqKg+yq6DDVx/3uiwSxE5qaQPJbl7m5ndBqwCIsD97l5hZncB5e6+AvgBMBB43MwAdrn7AmA6cJ+ZxYiH1N0drmYSyRhPbqwhkmNcO0PBIH1bSh4w6+4rgZUd2r6dMD+3i+1eBs5LRQ0ifVn8MFINl501gmG6qU36ON35LNILttTWs+NAA/PP1d6C9H0KBpFesGpTLWYwT1cjSRpQMIj0gqcr9nJR6TBGDioMuxSRbikYRHrYrgMNvFVzVIeRJG0oGER62KqKvQBcd46CQdKDgkGkh62q2MuMMYMpGd4/7FJETomCQaQH7atvYt2uQ9pbkLSiYBDpQc9ursUdnV+QtKJgEOlBT2/ay6SiAXxg1MCwSxE5ZQoGkR5ypLGVV7Yd4NoZowiGghFJCwoGkR7y4pZ9tMWca3V+QdKMgkGkhzyzuZaigQVcUDI07FJETouCQaQHNLdFefHtfcybMYqcHB1GkvSiYBDpAS9vO8DxlijXnqOxkST9KBhEesAzFbUMyI9w2Vkjwi5F5LSlJBjMbL6ZbTGzSjO7o5P1BWb2q2D9GjObmLDuzqB9i5ldl4p6RMIUizmr36rlymkjKciNhF2OyGlLOhjMLALcC3wYmAF80sxmdOh2C3DI3acA9wDfD7adQfxRoOcA84H/Ct5PJG2trzpMXX0z12qIbUlTqdhjmA1Uuvt2d28BHgUWduizEFgazC8DrrH4hd0LgUfdvdnd3wUqg/cTSVvPVNSSFzGumjYy7FJEzkgqgmEcsDthuSpo67SPu7cBR4ARp7itSFp5ZvNeLpk8gsGFeWGXInJGUhEMnV2L56fY51S2jb+B2WIzKzez8rq6utMsUaR3VO47xva643pSm6S1VARDFVCSsDweqO6qj5nlAkOAg6e4LQDuvsTdy9y9rLi4OAVli6TeM5vjz15QMEg6S0UwrAWmmtkkM8snfjJ5RYc+K4BFwfyNwPPu7kH7TcFVS5OAqcBrKahJJBTPVNQyc/wQxgzpF3YpImcs6WAIzhncBqwC3gIec/cKM7vLzBYE3X4GjDCzSuCrwB3BthXAY8Bm4GngS+4eTbYmkTDUHm1i/e7DGhtJ0l5uKt7E3VcCKzu0fTthvgn4WBfbfg/4XirqEAnTs5trAXSZqqQ93fkskiLPbq5lUtEApozUsxckvSkYRFKgvqmVl7ftZ56evSAZQMEgkgIvbqmjNeo6jCQZQcEgkgLxZy/kc0HpsLBLEUmagkEkSS1tMV58ex9zp48iomcvSAZQMIgk6eVt+6lvbtOzFyRjKBhEkrSqYi8DC3K57KyisEsRSQkFg0gSojHnmYparjy7mMI8jRgvmUHBIJKEdTsPceB4C/PP1d3OkjkUDCJJeHrTXvJzc7jybD17QTKHgkHkDLk7qyr28qEpRQwsSMnoMiJ9goJB5AxVVB9lz+FG5mvQPMkwCgaRM7SqYi85BtdM12EkySwKBpEz9PSmvcyeNJwRAwvCLkUkpRQMImdgW90xtu47xnU6jCQZSMEgcgZWbqgB0GWqkpGSCgYzG25mz5rZ1mD6vhHEzGyWmb1iZhVmtsHMPpGw7gEze9fM1gevWcnUI9JbntxYw4WlQ/UIT8lIye4x3AE85+5TgeeC5Y4agM+4+znAfOA/zGxowvrb3X1W8FqfZD0iPW573THe3lvPR84bE3YpIj0i2WBYCCwN5pcCN3Ts4O7vuPvWYL4a2AcUJ/m5IqF5atNeAAWDZKxkg2GUu9cABNOTXrdnZrOBfGBbQvP3gkNM95iZLu+QPu/JDTVcUDqUsUN1GEkyU7fBYGarzWxTJ6+Fp/NBZjYG+CXwOXePBc13AtOADwLDgW+eZPvFZlZuZuV1dXWn89EiKbNj/3E21xzleu0tSAbr9j5+d5/b1TozqzWzMe5eE/zw7+ui32DgSeCf3f3VhPeuCWabzeznwNdPUscSYAlAWVmZd1e3SE9YuUlXI0nmS/ZQ0gpgUTC/CFjesYOZ5QNPAL9w98c7rBsTTI34+YlNSdYj0qNWbqxhZslQxg/rH3YpIj0m2WC4G5hnZluBecEyZlZmZj8N+nwc+B/AZzu5LPUhM9sIbASKgH9Nsh6RHrPrQAOb9hzl+vO0tyCZLakhId39AHBNJ+3lwK3B/IPAg11sf3Uyny/Sm363oRrQ1UiS+XTns8gpcHd++8YePjhxmA4jScZTMIicgrf31rN13zEWzBoXdikiPU7BIHIKlq+vJjfHdJmqZAUFg0g3YjHnd29Wc/nUIoYPyA+7HJEep2AQ6cbruw6x53AjC3UYSbKEgkGkG8vXV1OYl8O8GaPCLkWkVygYRE6iNRrjyY01zJ0+igEFSV3dLZI2FAwiJ/HS1v0cPN7Cgpljwy5FpNcoGERO4vF1uxk+IJ8rzz7pwMEiGUXBINKFQ8dbWL15HzfMGkd+rv5Tkeyhf+0iXVi+fg8t0RgfKxsfdikivUrBINKFx9dVce64wUwfMzjsUkR6lYJBpBObq49SUX2Uj11UEnYpIr1OwSDSicfX7SY/kqOrkSQrKRhEOmhpi7F8fTVzZ4xkmIbAkCykYBDp4Lm3ajl4vEWHkSRrJRUMZjbczJ41s63BdFgX/aIJT29bkdA+yczWBNv/KngMqEioHlyzk7FDCrl8alHYpYiEItk9hjuA59x9KvBcsNyZRnefFbwWJLR/H7gn2P4QcEuS9YgkpXLfMf678gB/c8kEciPaoZbslOy//IXA0mB+KXDDqW5oZgZcDSw7k+1FesJDa3aSFzE+XqbDSJK9kg2GUe5eAxBMuxo3oNDMys3sVTM78eM/Ajjs7m3BchWgcY0lNA0tbSxbV8WHzx1D8aCCsMsRCU23w0Wa2WpgdCervnUan1Pq7tVmNhl43sw2Akc76ecnqWMxsBigtLT0ND5a5NQsX19NfVMbn7l0QtiliISq22Bw97ldrTOzWjMb4+41ZjYG2NfFe1QH0+1m9iJwAfBrYKiZ5QZ7DeOB6pPUsQRYAlBWVtZlgIicCXfnl6/sZNroQVw0odNrKESyRrKHklYAi4L5RcDyjh3MbJiZFQTzRcAcYLO7O/ACcOPJthfpDa/vOszmmqPcfOkE4qe/RLJXssFwNzDPzLYC84JlzKzMzH4a9JkOlJvZm8SD4G533xys+ybwVTOrJH7O4WdJ1iNyRn720nYGFeZygx7fKdL9oaSTcfcDwDWdtJcDtwbzLwPndbH9dmB2MjWIJGvH/uM8tWkvX7ziLD2lTQTd+SzCkj9tJy+Sw2fnTAy7FJE+QcEgWa2uvpll66r46IXjGTmoMOxyRPoEBYNktaUv76A1GuNvL58UdikifYaCQbLWseY2fvHKDq6bMZrJxQPDLkekz1AwSNZ6ZM0ujja18XdXTA67FJE+RcEgWelYcxs//sM2PjSliAtKdUObSCIFg2Sln7/0LgePt/D1684OuxSRPkfBIFnncEMLS/60nXkzRjGrZGjY5Yj0OQoGyTr3/XE7x5rb+Nq1Hwi7FJE+ScEgWWVffRMP/PcO/vL8sUwbPTjsckT6JAWDZJUfPbeVlmiMf5ynvQWRrigYJGtUVB/h4TW7+JuLS5lUNCDsckT6LAWDZAV359vLKxjWP5+vzdOVSCIno2CQrPDEG3tYt/MQ35w/jSH988IuR6RPUzBIxjva1Mq/rXybWSVDufGi8WGXI9LnafB5yXg/fOYdDhxv5v7PlpGTo6eziXQnqT0GMxtuZs+a2dZg+r6xBczsKjNbn/BqMrMbgnUPmNm7CetmJVOPSEcvb9vPAy/v4OZLJnD+eN3MJnIqkj2UdAfwnLtPBZ4Llttx9xfcfZa7zwKuBhqAZxK63H5ivbuvT7IekffUN7Vy++MbmFQ0gDs+PC3sckTSRrLBsBBYGswvBW7opv+NwFPu3pDk54p0667fbabmSCP//vGZ9M/XUVORU5VsMIxy9xqAYDqym/43AY90aPuemW0ws3vMrKCrDc1ssZmVm1l5XV1dclVLxnumYi+Pr6vii1eexYUaPVXktHQbDGa22sw2dfJaeDofZGZjgPOAVQnNdwLTgA8Cw4FvdrW9uy9x9zJ3LysuLj6dj5Yss+tAA7cv28CMMYP5h2t0h7PI6ep2/9rd53a1zsxqzWyMu9cEP/z7TvJWHweecPfWhPeuCWabzeznwNdPsW6RTjW0tLH4l+W4Oz/+9IXk5+qKbJHTlex/NSuARcH8ImD5Sfp+kg6HkYIwwcyM+PmJTUnWI1nM3fnGsg1sqa3nR5+8gAkjNOyFyJlINhjuBuaZ2VZgXrCMmZWZ2U9PdDKziUAJ8IcO2z9kZhuBjUAR8K9J1iNZ7L4/buf3G2q4/bqzufLs7k53iUhXkrpUw90PANd00l4O3JqwvAMY10m/q5P5fJETlq2r4u6n3ub688fwxSvOCrsckbSmA7CS9p7eVMM3lr3J5VOL+OHHZxI/MikiZ0rBIGntT1vr+PIj65lVMpT7br6IgtxI2CWJpD0Fg6StZyr2cuvSciYXD+Dnn52tm9hEUkTBIGnpkdd28YUH1zFtzGAe/ttLNJS2SArpTyxJK7GY86Pnt/Ifq7dy5dnF/NffXKg9BZEU039RkjYON7Twtcfe5Lm39/HRC8dz90fPIy+inV6RVFMwSFpYv/swX3rodfbVN/G/F5zDZy6doKuPRHqIgkH6tKbWKD96bitL/ridUYMLWfaFy5hZoucqiPQkBYP0WS9v288//WYjOw40cONF4/lf18/QSWaRXqBgkD5ny956frBqC6vfqmXCiP48dOvFzJlSFHZZIllDwSB9xpa99dz3h208sX4PA/Nzuf26s/n8nEn0y9dNayK9ScEgoYrGnD9ureP+l97lT1v3U5iXw+LLJ/OFK85i2ID8sMsTyUoKBgnF1tp6fvPGHn77xh5qjjQxclABt193Np+aXapAEAmZgkF6RWs0xvrdh1n9Vi2rN9eyre44kRzjig8U808fmc5154zWQ3VE+ggFg/SI481tbNpzhNd3HebV7Qco33GQ4y1RcnOMSyaP4OZLJnD9+WMpHtTlY75FJCRJBYOZfQz4F2A6MDt4DkNn/eYD/xeIAD919xMP9JkEPEr8ec+vAze7e0syNUnvao3GqDrUyLZ9x9hSW8/W2no21xylct8xYh7vM2XkQP7qwnFcOrmIyz9QxOBCXXIq0pclu8ewCfhr4L6uOphZBLiX+BPeqoC1ZrbC3TcD3wfucfdHzewnwC3Aj5OsSVLA3WlsjXLweAv7j7VQV99MXX0ze482UX24kZojjew+2Miew41ETyQAMHZIIWePHsT8c8cwc/wQZpYMpWig9gpE0kmyT3B7C+huaILZQKW7bw/6PgosNLO3gKuBTwX9lhLf+1AwdODuxDx+BU805kTdiUadtliMtpjTGo3RFiy3tMWXW6MxmttitLTFp81tUZpaozS1xmhoidLY0kZja5RjzVGONbdxvLmNo42tHG1q5WhjG4caWmhui72vFjMYNaiQMUMLOX/8EBbOGsuEEQOYVNSfqaMGaW9AJAP0xjmGccDuhOUq4GJgBHDY3dsS2t/3+M9U+tYTG1nz7sH3lt29037exYInbJPY58TbOI57wrJ7sE3CuhPL762L/+jHPL4+5h684iOJRoP2ntA/P8KAglwGFuQyoCDC4MI8JhcNZFBhLsMH5DNsQD7D+udRNLCA4kHxV9HAAg1cJ5Lhug0GM1sNjO5k1bfcffkpfEZnuxN+kvau6lgMLAYoLS09hY99v7FD+3H2qEHdV9ehOXGPyIj/1dxVHwv+j2GY/bn/e8tBhxz7c3skx96bzzGC5fbzETMiORDJySGSA7k5OeRG4tvmBfO5kRzyI0ZeJIe8SA75uTkU5ManhXmR+Cs3h/75uRTm5WgQOhHpVLfB4O5zk/yMKqAkYXk8UA3sB4aaWW6w13Civas6lgBLAMrKys7ob+gvXTXlTDYTEckqvXFMYC0w1cwmmVk+cBOwwuPHZF4Abgz6LQJOZQ9ERER6UFLBYGZ/ZWZVwKXAk2a2Kmgfa2YrAYK9gduAVcBbwGPuXhG8xTeBr5pZJfFzDj9Lph4REUmedXUCti8rKyvz8vJOb5kQEZEumNk6dy/rrp8uLxERkXYUDCIi0o6CQURE2lEwiIhIOwoGERFpJy2vSjKzOmBn2HWcgSLiN/Zlk2z8zpCd3zsbvzOk1/ee4O7F3XVKy2ByIPdjAAACpUlEQVRIV2ZWfiqXimWSbPzOkJ3fOxu/M2Tm99ahJBERaUfBICIi7SgYeteSsAsIQTZ+Z8jO752N3xky8HvrHIOIiLSjPQYREWlHwRASM/u6mbmZFYVdS08zsx+Y2dtmtsHMnjCzoWHX1FPMbL6ZbTGzSjO7I+x6eoOZlZjZC2b2lplVmNk/hF1TbzGziJm9YWa/D7uWVFIwhMDMSoB5wK6wa+klzwLnuvv5wDvAnSHX0yPMLALcC3wYmAF80sxmhFtVr2gDvubu04FLgC9lyfcG+AfijxPIKAqGcNwDfIOTPMo0k7j7MwnP9n6V+NP6MtFsoNLdt7t7C/AosDDkmnqcu9e4++vBfD3xH8oefX57X2Bm44HrgZ+GXUuqKRh6mZktAPa4+5th1xKSzwNPhV1EDxkH7E5YriILfiATmdlE4AJgTbiV9Ir/IP4HXizsQlKt22c+y+kzs9XA6E5WfQv4J+Da3q2o553sO7v78qDPt4gfdnioN2vrRdZJW1bsFQKY2UDg18BX3P1o2PX0JDP7C2Cfu68zsyvDrifVFAw9wN3ndtZuZucBk4A3zQzih1ReN7PZ7r63F0tMua6+8wlmtgj4C+Aaz9xrpKuAkoTl8UB1SLX0KjPLIx4KD7n7b8KupxfMARaY2UeAQmCwmT3o7p8Oua6U0H0MITKzHUCZu6fLAFxnxMzmAz8ErnD3urDr6Slmlkv85Po1wB5gLfCphGecZySL/5WzFDjo7l8Ju57eFuwxfN3d/yLsWlJF5xikN/wnMAh41szWm9lPwi6oJwQn2G8DVhE/AftYpodCYA5wM3B18L/v+uAvaUlT2mMQEZF2tMcgIiLtKBhERKQdBYOIiLSjYBARkXYUDCIi0o6CQURE2lEwiIhIOwoGERFp5/8DR+lbdw3lvnAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x256b112a1d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x = torch.arange(-5,5,0.1)\n",
    "y = torch.tanh(x)\n",
    "plt.plot(x.numpy(), y.numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x256b116fef0>]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAFodJREFUeJzt3Xd81fXZxvHrliGyRCUiMoyKAxxAiIDiqKOK+9FWZWjrxDIUrftRq+3T2tY6sApaWmutBHDgqnvhnhmAbAFBpgmyhZB1P38kx0YM5iRn/M7vnM/79eJlMIeT+wd4na/3OSeXubsAAOGxQ9ADAAAahuAGgJAhuAEgZAhuAAgZghsAQobgBoCQIbgBIGQIbgAIGYIbAEKmaSLutH379p6dnZ2IuwaAtFRQULDa3bOiuW1Cgjs7O1v5+fmJuGsASEtmtiTa27IqAYCQierEbWaLJW2UVCmpwt1zEzkUAGD7GrIqOdbdVydsEgBAVFiVAEDIRBvcLuk1Mysws2F13cDMhplZvpnll5SUxG9CAMD3RBvcA9w9R9LJkkaa2dHb3sDdx7t7rrvnZmVF9YoWAEAjRBXc7r6i5p/Fkp6R1DeRQwEAtq/e4DazVmbWJvKxpBMlzUz0YAAQJp9+uUYPv/+lklEHGc2rSjpIesbMIref6O6vJHQqAAiRko1bNWpioVrt2FSD+3ZRy+YJeW/jd+q9d3dfJKlnQqcAgJCqrHKNnlyk9VvK9ejFfRMe2lKC3vIOAJlizBvz9eHCb3Tnzw9V945tk/I1eR03ADTS2/OKdf9bC3ROn846N7dL0r4uwQ0AjbBi3RZd/fg0HbhHG/3uzIOT+rUJbgBooLKKKo2cWKjySte4oTnaqXmTpH59dtwA0EB/fmWuir5ap7FDcrRPVuukf31O3ADQAK/MXKmH3/9SFx6RrVMP7RjIDAQ3AERp8epvdd2TM9SzSzv97yndA5uD4AaAKJSWV2pEXqF22ME0dkhvNW8aXHyy4waAKNz+/CzNXrlBj1x4mDrv0jLQWThxA0A9phQs0+TPlmrksfvq2AN3D3ocghsAfsy8VRt187Ofq/8+u+rqE/YPehxJBDcAbNemrRUanlegNi2a6a+De6tpk9SIzNSYAgBSjLvrxikztHj1t/rroN7avU2LoEf6DsENAHV47OMlemHGSl1z4gE6fN/dgh7newhuANjG9KXr9H8vzNaxB2Rp+DH7Bj3ODxDcAFDLus1lGpFXqN3btNA95/bSDjtY0CP9AK/jBoAaVVWua56YruKNpXryV0dol1bNgx6pTpy4AaDG395dpDfnFuvmU7qrV5d2QY+zXQQ3AEj6ZNE3uuu1eTr1kI765RHZQY/zowhuABmveGOpRk0qUtddW+pPPztENeXoKYsdN4CMVlnlGj1pmjZsKde/L+6rNi2aBT1SvQhuABnt3tfn66NF3+gvSSz7jRWrEgAZa+q8Yj0wdYHOze2sc5JY9hsrghtARloeYNlvrAhuABmnrKJKI/MKVVHpevD8PmrRLLllv7Fixw0g4/zx5TmatnSdxg3N0d7tWwU9ToNx4gaQUV7+fKUe+WCxLjwiW6ccEkzZb6wIbgAZ48vV3+q6p2aoV8Blv7EiuAFkhNLySg2fUKCmTUxjh+YEWvYbK3bcADLCbc/N0txVG/XIRYepU7udgh4nJuF9yAGAKD1VsEyP59eU/R4QfNlvrKIObjNrYmZFZvZCIgcCgHiau2qDbkmxst9YNeTEPVrSnEQNAgDxtmlrhUbkFaZc2W+soroKM+ss6VRJ/0jsOAAQH7XLfu8fnFplv7GK9uFnjKTrJVUlcBYAiJtI2e91Jx2o/vukVtlvrOoNbjM7TVKxuxfUc7thZpZvZvklJSVxGxAAGmpaTdnv8QfursuP3ifoceIumhP3AElnmNliSZMlHWdmE7a9kbuPd/dcd8/NysqK85gAEJ11m8s0sqbs9+5ze6Zk2W+s6g1ud7/J3Tu7e7akQZLecvfzEz4ZADRQVZXr1zVlv+OG5qhdy9Qs+41VejzFCgCSHnp3od6aW6xbTu2hnilc9hurBr1z0t3flvR2QiYBgBh8vOgb3fXqPJ16aEf94vC9gh4noThxAwi94o2lumJSkbJ3a6U//+zQlC/7jRXfqwRAqEXKfjeWluuxS/qq9Y7pH2vpf4UA0lrtst8D9whH2W+sWJUACK2wlv3GiuAGEEqRst/uHduGruw3VgQ3gNCpXfY7bmhO6Mp+Y8WOG0Do3PFSddnvgyEt+40VJ24AofLijJX614eLddGAbJ0c0rLfWBHcAEJjUckm3TBlhnp3baebTg5v2W+sCG4AoVBaXqkReYVq1sQ0dki4y35jxY4bQCj85rmZmvf1Rj1y4WHaM+Rlv7HK3IcsAKHxZP5SPZG/TKOO7aafpEHZb6wIbgApbe6qDbr1uZk6Yt/ddFWalP3GiuAGkLI2lpZr+IRCtW3RTPcN6q0maViK0BjsuAGkJHfXjU9/rq/WbNbES/spq82OQY+UMjhxA0hJj364WC/OWKlrTzxA/dKs7DdWBDeAlFP01Vr94aU5aVv2GyuCG0BKWfttmUZNLErrst9YseMGkDKqy36nqWTjVj35q8PTtuw3Vpy4AaSMB99ZqKnzSnTLad3Tuuw3VgQ3gJTw0cJvdPdr83R6zz11Qf/0LvuNFcENIHDflf22b6U/nn1I2pf9xoodN4BAVVRW6cpJRdq0tVx5l/bLiLLfWPE7BCBQ974xXx8vWqO7z+mpA/ZoE/Q4ocCqBEBgps4t1tipCzXosC76WZ/OQY8TGgQ3gEAsX7dFVz8xTT06ttXtZxwU9DihQnADSLqyiiqNyCtUZYaW/caKHTeApLvjpTmavnSdHjo/R9kZWPYbK07cAJIqUvZ78YC9NfDgzCz7jRXBDSBpapf93njygUGPE1oEN4Ck2FJG2W+81Ps7Z2YtzOxTM5tuZrPM7LfJGAxAeomU/d57Xq+ML/uNVTRPTm6VdJy7bzKzZpLeN7OX3f3jBM8GIE08kb9UTxYs05XHUfYbD/UGt7u7pE01P21W88MTORSA9DFn5Qbd+uxMDei2m0ZT9hsXUS2ZzKyJmU2TVCzpdXf/JLFjAUgHG0vLNSKvUDvv1ExjzqPsN16iCm53r3T3XpI6S+prZgdvexszG2Zm+WaWX1JSEu85AYSMu+vGKdVlv/cP7k3Zbxw16Gldd18n6W1JA+v43Hh3z3X33KysrDiNByCsHv1wsV78fKWuO4my33iL5lUlWWbWrubjnSSdIGluogcDEF6Rst8Tuu+uYUdR9htv0byqpKOkR82siaqD/gl3fyGxYwEIq0jZb4e2LXT3Ob0o+02AaF5VMkNS7yTMAiDkqqpcV9eU/T41/HDt3LJZ0COlJd66BCBuHnxnod6eV6JbT++hQztT9psoBDeAuPhw4Wrd/do8ndFzT53fr2vQ46Q1ghtAzIo3lOrKSdO0N2W/ScH34wYQk4rKKl0xqUjfbq3QxMv6qRVlvwnH7zCAmNzz+nx98uUa3XNuT+3fgbLfZGBVAqDR3pr7tca9vVCD+3bR2TmU/SYLwQ2gUZat3ayrH5+uHh3b6rbTKftNJoIbQINtrajUyLxCVVVR9hsEdtwAGuyOF+do+rL1lP0GhBM3gAb5z/QVevSjJbrkSMp+g0JwA4jawpJNunHKDOVQ9hsoghtAVLaUVWrEhEI1b7qDHhiSo2ZNiI+gsOMGUC931y3PztT84o3610V9KfsNGA+ZAOr1RP5STSlcpiuO7aZj9qcoJWgEN4AfNXvFBv3muVmU/aYQghvAdm0oLdeIvAK1a9lM9w2i7DdVsOMGUCd31w1PzdDStVs0eVh/tW9N2W+q4MQNoE6PfLBYL89cpetPOkCHZe8a9DioheAG8AOFX63VHS/N0QndO2jY0ZT9phqCG8D3rP22TKPyCrXHzi109zk9KUVIQey4AXynqsp11ePTtHpTmaYMP4Ky3xTFiRvAd8ZOXaB35pfoN6f30CGddw56HGwHwQ1AkvThgtW69435OrPXnhpK2W9KI7gB6OsNpbpycpH2bt9Kd5xF2W+qY8cNZLj/lv1WauJl/Sn7DQH+hIAMd9dr8/UpZb+hwqoEyGBvzvlaD71D2W/YENxAhlq6ZrN+/QRlv2FEcAMZaGtFpUZOLFSVux48n7LfsGHHDWSgP7w4RzOWrddD5/fRXrtR9hs2nLiBDPOf6Sv074+W6NIj99bAg/cIehw0AsENZJBI2W+fvXbRDZT9hla9wW1mXcxsqpnNMbNZZjY6GYMBiK/NZRUaPqFAOzZrogeG9KbsN8Si2XFXSLrG3QvNrI2kAjN73d1nJ3g2AHESKfv9oniTHr2orzruTNlvmNX7kOvuK929sObjjZLmSOqU6MEAxM/jny3V04XLdcVx++loyn5Dr0H/r2Rm2ZJ6S/qkjs8NM7N8M8svKSmJz3QAYjZrxXr95vlZOrJbe40+fr+gx0EcRB3cZtZa0hRJV7n7hm0/7+7j3T3X3XOzsnhEB1JBddlvoXZp2UxjBvWi7DdNRPU6bjNrpurQznP3pxM7EoB4cHdd/+QMLaPsN+1E86oSk/SwpDnufk/iRwIQD//8YLFembVKNwyk7DfdRLMqGSDpAknHmdm0mh+nJHguADEoWLJWf3xpjn7ao4MuO4qy33RT76rE3d+XxGIMCIk135Zp1MRCdWzXQndR9puW+F4lQBqJlP1+Eyn73Ymy33TEW6eANDJ26gK9S9lv2iO4gTRB2W/mILiBNBAp+90nqzVlvxmAHTcQcrXLfiddlkPZbwbgTxgIuUjZ773n9dR+lP1mBFYlQIi9MTtS9ttVZ/Wm7DdTENxASFWX/U7TwZ3a6rbTewQ9DpKI4AZCKFL265LGDelD2W+GYccNhNDvX6gu+x1/QR913a1l0OMgyThxAyHz/PQVeuzjJRp29D468SDKfjMRwQ2EyILi6rLfw7J30XUnHRD0OAgIwQ2ExOayCo3IK9BOzZro/sE5lP1mMHbcQAjULvt97OJ+2mPnFkGPhADxkA2EQKTsd/Tx++nI/doHPQ4CRnADKS5S9nvUfu11xXGU/YLgBlJapOx315bNNeY8yn5RjR03kKJql/0+Pqy/dqPsFzU4cQMp6uH3v9Qrs1bpxoEHKpeyX9RCcAMpqGDJGv3p5bk6sUcHXXrU3kGPgxRDcAMpprrst0h7tttJf6HsF3Vgxw2kkO/Kfr8t09OU/WI7OHEDKeSBmrLf208/SAd3ouwXdSO4gRTxQU3Z71m9O2lw3y5Bj4MURnADKeDrDaUaPblI3bJa6w9nHcxeGz+KHTcQsPLKKo2aWKjNZZWaPCxHLZvznyV+HH9DgIDd9eo8fbZ4re4b1EvddqfsF/VjVQIE6PXZX+tv7y7S0H5ddWavTkGPg5AguIGALF2zWdfUlP3eehplv4gewQ0EoLS8UiPyKPtF47DjBgLw+xdn6/PllP2iceo9cZvZP82s2MxmJmMgIN09N225Jnz8FWW/aLRoViX/kjQwwXMAGWFB8Ubd9PTnlP0iJvUGt7u/K2lNEmYB0trmsgoNn1BI2S9ixo4bSAJ31y3PzNSCEsp+Ebu4PeSb2TAzyzez/JKSknjdLZAWJn+2VE8XUfaL+IhbcLv7eHfPdffcrKyseN0tEHozl6/XbTVlv1dS9os4YMkGJNCG0nKNnPjfst8dKPtFHETzcsBJkj6SdICZLTOzSxI/FhB+kbLf5Wu3aOzQ3pT9Im7qfXLS3QcnYxAg3UTKfm85tbv67EXZL+KHVQmQAJGy35MO6qBLjqTsF/FFcANx9s2mrRqZV132e+fPKftF/PE6biCOKmvKftdspuwXicOJG4ijB95aoPe+WE3ZLxKK4Abi5P0vVmvMm/N1NmW/SDCCG4iDVeury3732721fk/ZLxKM4AZiVF5ZpSsmFWpLeaXGDaXsF4nH3zAgRpT9Itk4cQMxiJT9nt+fsl8kD8ENNFKk7PeQTjtT9oukIriBRoiU/UrSuKE52rEpZb9IHnbcQCNEyn7//otcddmVsl8kFyduoIEiZb+XH72PftqjQ9DjIAMR3EAD1C77vZayXwSE4AaiFCn7bdm8iR4YQtkvgsOOG4iCu+vmmrLfCZf0U4e2lP0iOBwZgChM+nSpnilarqtP2F8DulH2i2AR3EA9Zi5fr9v/M0tH75+lUcd2C3ocgOAGfsz6LeUakVeo3VpR9ovUwY4b2A5313VPTteKdVv0+OX9tWur5kGPBEjixA1s18Pvf6nXZn+tG08+kLJfpBSCG6hDpOx34EF7UPaLlENwA9uIlP122mUn3XnOoZQiIOWw4wZq2bbst20Lyn6RejhxA7Xc/9YXeu+L1frtGZT9InUR3ECN974o0X1vfqGzczpp0GGU/SJ1EdyApJXrt+iqydOqy37/h7JfpDaCGxmvvLJKoyYW1ZT99qHsFymPv6HIeHe+MlcFS9bqr4N7q9vurYMeB6gXJ25ktFdnrdLf3/tSF/TfS2f03DPocYCoENzIWF99s1nXPjldh3beWbec1j3ocYCoRRXcZjbQzOaZ2QIzuzHRQwGJVlpeqeF5BTJJY4dQ9otwqTe4zayJpLGSTpbUQ9JgM+uR6MGARPrdC7M1a8UG3XNuL8p+ETrRnLj7Slrg7ovcvUzSZElnJnYsIHGeLVquiZ98pcuP2UcnUPaLEIrmVSWdJC2t9fNlkvolYpjT739fpeWVibhr4DtL1mxW3+xddd2JlP0inKIJ7rreieA/uJHZMEnDJKlr166NGmbfrFYqq6xq1K8FopXTdRddc+L+akrZL0IqmuBeJqn2+387S1qx7Y3cfbyk8ZKUm5v7g2CPxphBvRvzywAgo0Rz5PhM0n5mtreZNZc0SNLziR0LALA99Z643b3CzEZJelVSE0n/dPdZCZ8MAFCnqN7y7u4vSXopwbMAAKLAszMAEDIENwCEDMENACFDcANAyBDcABAy5t6o98r8+J2alUhaEvc7Trz2klYHPUSSZeI1S5l53VxzatvL3bOiuWFCgjuszCzf3XODniOZMvGapcy8bq45fbAqAYCQIbgBIGQI7u8bH/QAAcjEa5Yy87q55jTBjhsAQoYTNwCEDMFdBzO71szczNoHPUsymNlfzGyumc0ws2fMrF3QMyVKJhZfm1kXM5tqZnPMbJaZjQ56pmQxsyZmVmRmLwQ9SzwR3Nswsy6Sfirpq6BnSaLXJR3s7odKmi/ppoDnSYgMLr6ukHSNu3eX1F/SyAy5bkkaLWlO0EPEG8H9Q/dKul511LOlK3d/zd0ran76sapbjtJRRhZfu/tKdy+s+XijqoOsU7BTJZ6ZdZZ0qqR/BD1LvBHctZjZGZKWu/v0oGcJ0MWSXg56iASpq/g67QOsNjPLltRb0ifBTpIUY1R9CEu7ItuoihTSiZm9IWmPOj51s6T/lXRicidKjh+7bnd/ruY2N6v6f6vzkjlbEkVVfJ2uzKy1pCmSrnL3DUHPk0hmdpqkYncvMLOfBD1PvGVccLv7CXX9ezM7RNLekqabmVS9Lig0s77uviqJIybE9q47wsx+Kek0Scd7+r5GNKri63RkZs1UHdp57v500PMkwQBJZ5jZKZJaSGprZhPc/fyA54oLXse9HWa2WFKuu4flG9Q0mpkNlHSPpGPcvSToeRLFzJqq+snX4yUtV3UR9pB071C16pPIo5LWuPtVQc+TbDUn7mvd/bSgZ4kXdtyQpAcktZH0uplNM7OHgh4oEWqegI0UX8+R9ES6h3aNAZIukHRczZ/vtJqTKEKKEzcAhAwnbgAIGYIbAEKG4AaAkCG4ASBkCG4ACBmCGwBChuAGgJAhuAEgZP4f+6FA3C6YCEgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x256b07d6668>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x = torch.arange(-5,5,0.1)\n",
    "y = torch.relu(x)\n",
    "plt.plot(x.numpy(), y.numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x256b10bde10>]"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD8CAYAAABjAo9vAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAHl5JREFUeJzt3XtUVeeB9/Hvw00EucjFBEVEgdZLvEQRUHJpo53GJDXrHZMm9ZbEC3nTyftq1sykSefNmrRZnZVOu2qm03RNlJAULzVTM01bm3YmOtOaqKDghcZgIlCpeOUioiKC5zzvH1BX2poosGGfs8/vs5YrEM569m8L/HzOPs/Zj7HWIiIi3hHmdgAREXGWil1ExGNU7CIiHqNiFxHxGBW7iIjHqNhFRDxGxS4i4jEqdhERj1Gxi4h4TIQbB01JSbGZmZluHFpEJGhVVlY2WWtTr/c4V4o9MzOTiooKNw4tIhK0jDH1N/I4XYoREfEYR2bsxpijwHnAB1yx1uY6Ma6IiPSek5diPm+tbXJwPBER6QNdihER8Rinit0C/2WMqTTGFDk0poiI9IFTl2IKrbUnjDEjgHeMMYettTs+/oCewi8CyMjIcOiwIiLy5xyZsVtrT/T89wzwUyDvGo9Za63NtdbmpqZedxmmiIj0Ub+L3RgTa4yJ++PHwF8B7/d3XBERL2nvvMLzPz/EufauAT+WEzP2m4D3jDEHgT3AL621v3ZgXBERT+jo8vH4+kpKdx9l3x/ODvjx+n2N3VpbB0x1IIuIiOd0+fw8uWkf7x5p4rsPTuXz40cM+DG13FFEZID4/Jan3jjAtuozvHD/JB6YkT4ox1Wxi4gMAL/f8rU3q9hadZJn541nyazMQTu2il1ExGHWWv7x54fYUtnAqjk5PH5n1qAeX8UuIuIgay0v/uow68vqKbpjHKvn5gx6BhW7iIiD/mX7EV7ZUceSgjE8O288xphBz6BiFxFxyNodtby07QgLpqfzjfmTXCl1ULGLiDhifVk9//T2Ye6dksY/PzCFsDB3Sh1U7CIi/balsoHn3nqfuRNG8NJD0wh3sdRBxS4i0i9bq07w9JaD3Jadwg8WTicy3P1adT+BiEiQ2l59mtWbDzBjzHDWLp1BdGS425EAFbuISJ+8d6SJJzbsY+LIeEoenUlMlJMb0vWPil1EpJf2Hm1hZWkF41JjKV2WR1x0pNuR/oSKXUSkFw4ea+Wx1/aSlhDN+uX5JMZEuR3pL6jYRURu0OFTbTzy2h6Gx0aycWU+qXFD3I50TSp2EZEbUNt4gcXF5URHhLNpRQFpCUPdjvSJVOwiItdxrKWdRevKsRY2rMhndFKM25E+VeC8jCsiEoBOnetgYXEZl7p8bC4qIHvEMLcjXZdm7CIin6Dx/GUWFpdx9mIXpcvymJAW73akG6JiFxG5htb2Tpa8Ws6J1kuUPDqTqaMT3Y50w1TsIiJ/5nxHF4+U7KGu8SLrluaSNzbJ7Ui9omIXEfmY9s4rLH+9gkMn2vjhouncnpPqdqReU7GLiPTo6PLx+PpKKupbeOnhacydeJPbkfpEq2JERIAun58nN+3n3SNNfPfBqdw3ZaTbkfpMM3YRCXk+v+WpNw6wrfo0L9w/iQdmpLsdqV8cK3ZjTLgxZr8xZqtTY4qIDDS/3/K1N6vYWnWSr98zniWzMt2O1G9OzthXAdUOjiciMqCstfzjzw+xpbKBVXNyKLojy+1IjnCk2I0x6cC9QLET44mIDDRrLS/+6jDry+opumMcq+fmuB3JMU7N2F8Cngb8n/QAY0yRMabCGFPR2Njo0GFFRPrmX7Yf4ZUddSwuyODZeeMxxt19Sp3U72I3xtwHnLHWVn7a46y1a621udba3NTU4FsXKiLesXZHLS9tO8KC6el8c/4tnip1cGbGXgjMN8YcBTYDdxljNjgwroiI49bvPso/vX2Yeyen8e0FkwkL81apgwPFbq191lqbbq3NBB4G/ttau7jfyUREHPaTimM897NDzJ0wgjUPTSMi3Jsrvr15ViIif2Zr1Qm+9mYVt2Wn8IOF04mK8G79OfrOU2vtb4DfODmmiEh/ba8+zerNB5gxZjhrl84gOjLc7UgDyrv/ZImIAO8daeKJDfuYODKekkdnEhPl/TupqNhFxLP2Hm1hZWkF41JjKV2WR1x0pNuRBoWKXUQ8qaqhlcde20taYjTrl+eTGBPldqRBo2IXEc85fKqNpSV7GB4bycYV+aTGDXE70qBSsYuIp9Q2XmBxcTnREeFsWlFAWsJQtyMNOhW7iHjGsZZ2Fq0rB2DjynxGJ8W4nMgd3n95WERCwqlzHSwsLuNSl4/NRQVkpQ5zO5JrNGMXkaDXeP4yC4vLOHuxi9JleUxIi3c7kqtU7CIS1FrbO1nyajknWi9R8uhMpo5OdDuS61TsIhK0znd08UjJHuoaL7JuaS55Y5PcjhQQVOwiEpTaO6+w/PUKDp1o4+VF07k9R7cD/yMVu4gEnY4uH4+vr6SivoWXHp7GFybe5HakgKJVMSISVLp8fp7ctJ93jzTxnQemcN+UkW5HCjiasYtI0PD5LU+9cYBt1ad54f5JPJg72u1IAUnFLiJBwe+3fO3NKrZWneTZeeNZMivT7UgBS8UuIgHPWsvzvzjElsoGVs3J4fE7s9yOFNBU7CIS0Ky1vPjrw5TurufxO8axem6O25ECnopdRALa97fX8Mpv61hSMIZn5o3HGO9tPu00FbuIBKx1O+pYs+0jHpiRzjfmT1Kp3yAVu4gEpA1l9Xzr7WrunZLGtxdMISxMpX6jVOwiEnDerGzg/731PnMnjOClh6YRrlLvFRW7iASUX1ad5O+3HOT2nBR+sHA6keGqqd7S35iIBIzt1adZtXk/M8YM55UlM4iODHc7UlDqd7EbY6KNMXuMMQeNMYeMMd9wIpiIhJb3jjTxxMZ9TBwZT8mjM4mJ0h1P+sqJv7nLwF3W2gvGmEjgPWPMr6y1ZQ6MLSIhYO/RFlaWVjAuJZbSZXnERUe6HSmo9bvYrbUWuNDzaWTPH9vfcUUkNFQ1tPLYa3tJS4xm/fJ8EmOi3I4U9By5xm6MCTfGHADOAO9Ya8uv8ZgiY0yFMaaisbHRicOKSJA7fKqNpSV7SIyJZOOKfFLjhrgdyRMcKXZrrc9aOw1IB/KMMbdc4zFrrbW51trc1FTdEF8k1NU2XmBxcTnREeH8eGUBaQlD3Y7kGY6uirHWtgK/Ae52clwR8ZZjLe0sWleOtbBhRT6jk2LcjuQpTqyKSTXGJPZ8PBSYCxzu77gi4k2nznWwsLiMS10+NqzIJ3vEMLcjeY4Tq2LSgB8ZY8Lp/ofi3621Wx0YV0Q8punCZRYVl3H2YhcbV+QzIS3e7Uie5MSqmCrgVgeyiIiHtbZ3sri4nOOtlyhdls/U0YluR/IsvfNURAbc+Y4uHinZQ13jRdYtzSVvbJLbkTxNxS4iA6q98wrLX6/g0Ik2Xl40ndtztCpuoKnYRWTAdHT5eHx9JRX1Lax5aBpfmHiT25FCgm7GICIDosvn58lN+3n3SBPffXAqX5o60u1IIUMzdhFxnM9veeqNA2yrPs0L90/igRnpbkcKKSp2EXGU32/52ptVbK06ydfvGc+SWZluRwo5KnYRcYy1lud/cYgtlQ2smpND0R1ZbkcKSSp2EXGEtZYXf3WY0t31FN0xjtVzc9yOFLJU7CLiiO9vr+GVHXUsKRjDs/PGY4z2KXWLil1E+m3djjrWbPuIB2ak8435k1TqLlOxi0i/rC+r51tvV3PvlDS+vWAKYWEqdbep2EWkz7ZUNvDcW+8zZ/wI1nx5GuEq9YCgYheRPtladYKntxzktuwUXl40nagI1Umg0HdCRHpt2wenWb35ADPGDGft0hlER4a7HUk+RsUuIr3y3pEmvrpxHxNHxvPqozOJidKdSQKNil1Ebtjeoy2sLK1gXGospcvyiI+OdDuSXIOKXURuyMFjrTz22l7SEqJZvzyfxJgotyPJJ1Cxi8h1VZ9sY2nJHhJjItmwIp/UuCFuR5JPoWIXkU9V23iBJa+WMzQynE0rChiZONTtSHIdKnYR+UTHWtpZtK4ca2HDinwykmPcjiQ3QC9ni8g1nTx3iYXFZVzq8vHjlQVkjxjmdiS5QZqxi8hfaDx/mUXF5Zy92EXpsjwmjox3O5L0gopdRP5Ea3snS14t50TrJUoencnU0YluR5Je6nexG2NGG2P+xxhTbYw5ZIxZ5UQwERl85zu6eKRkD3WNF1m3NJe8sUluR5I+cOIa+xXgb621+4wxcUClMeYda+0HDowtIoOkvfMKy1+v4NCJNv5t8Qxuz0l1O5L0Ub9n7Nbak9bafT0fnweqgVH9HVdEBk9Hl4/H11dSUd/CmoemMXfiTW5Hkn5w9Bq7MSYTuBUov8bXiowxFcaYisbGRicPKyL90OXz8+Smfbx7pIkXF0zhS1NHuh1J+smxYjfGDAPeBFZba9v+/OvW2rXW2lxrbW5qqp7iiQQCn9/y1BsH2FZ9hhfun8SXc0e7HUkc4EixG2Mi6S71jdba/3BiTBEZWH6/5Zk3q9hadZJn541nyaxMtyOJQ5xYFWOAV4Fqa+33+h9JRAaatZbnf3GIn1Q2sGpODo/fmeV2JHGQEzP2QmAJcJcx5kDPn3scGFdEBoC1lhd/fZjS3fWsvH0sq+fmuB1JHNbv5Y7W2vcAbXQoEiS+v72GV35bx+KCDL5+zwS6n3SLl+idpyIhZN2OOtZs+4gF09P55vxbVOoepWIXCRHry+r51tvV3Ds5jW8vmExYmErdq1TsIiFgS2UDz731PnPGj2DNQ9OICNevvpfpuyvicb+sOsnTWw5SmJ3My4umExWhX3uv03dYxMO2fXCaVZv3Mz1jOOuW5hIdGe52JBkEKnYRj3rvSBNf3bSPiSPjKXlsJjFR2lcnVKjYRTyo4mgLK0srGJscy48eyyM+OtLtSDKIVOwiHlPV0Mqjr+0lLSGaDSvyGR4b5XYkGWQqdhEPOXyqjaUle0iMiWTjynxS44a4HUlcoGIX8YjaxgssLi4nOiKcTSsKSEsY6nYkcYmKXcQDjrW0s7i4exuEjSvzyUiOcTmRuEkvk4sEuVPnOlhYXEZ7p4/NRQVkpQ5zO5K4TDN2kSDWdOEyi4rLOHuxi9JleUxIi3c7kgQAFbtIkGpt72RxcTnHWy/x2mMzmTo60e1IEiBU7CJB6HxHF4+U7KGu8SLrluYyMzPJ7UgSQFTsIkHmUqeP5a9XcOhEGz9cNJ3bc7SHsPwpFbtIELl8xUfR+goq6lt46eFpzJ14k9uRJABpVYxIkOjy+fmbjft590gT33lgCvdNGel2JAlQmrGLBAGf3/LUGwfYVn2aF+6fxIO5o92OJAFMxS4S4Px+yzNvVrG16iTPzhvPklmZbkeSAKdiFwlg1lqe/8UhflLZwKo5OTx+Z5bbkSQIqNhFApS1lhd/fZjS3fUU3TGO1XNz3I4kQULFLhKgvr+9hld+W8eSgjE8O288xmjzabkxjhS7MabEGHPGGPO+E+OJhLp1O+pYs+0jFkxP5xvzJ6nUpVecmrG/Dtzt0FgiIW19WT3feruaeyen8e0FkwkLU6lL7zhS7NbaHUCLE2OJhLItlQ0899b7zBk/gjUPTSMiXFdLpff0UyMSIH5ZdZKntxzktuwUXl40nagI/XpK3wzaT44xpsgYU2GMqWhsbBysw4oEhe3Vp1m1eT8zxgxn7dIZREeGux1JgtigFbu1dq21Ntdam5uaqpsWifzRe0eaeGLjPiaOjKfk0ZnEROlOH9I/eq4n4qKKoy2sLK1gXEospcvyiIuOdDuSeIBTyx1/DOwGPmuMaTDGLHdiXBEvq2po5bHX9pKWEM365fkkxkS5HUk8wpHnfNbarzgxjkioOHyqjaUle0iMjWTjynxS44a4HUk8RJdiRAZZXeMFFhfvIToinE0rCkhLGOp2JPEYFbvIIDrW0s6i4nKstWxYkc/opBi3I4kH6eV3kUFy6lwHC4vLaO/0sbmogOwRw9yOJB6lGbvIIGi6cJlFxWWcvdhF6bI8JqTFux1JPEzFLjLAWts7WfLqHo63XqLk0ZlMHZ3odiTxOBW7yAA639HFIyV7qD1zgXVLc8kbm+R2JAkBKnaRAXKp08fy1ys4dKKNHy6azu05ese1DA4Vu8gA6OjyUbS+gor6FtY8NI25E29yO5KEEK2KEXFYl8/Pk5v28+6RJr7zwBS+NHWk25EkxGjGLuIgn9/y1BsH2FZ9mhfun8SDuaPdjiQhSMUu4hC/3/LMm1VsrTrJs/PGs2RWptuRJESp2EUcYK3l+V8c4ieVDfzfOTk8fmeW25EkhKnYRfrJWsuLvz5M6e56Vt4+lqfm5rgdSUKcil2kn/71v2t45bd1LC7I4Ov3TMAYbT4t7lKxi/TDuh11fO+dj/jr6aP45vxbVOoSEFTsIn20vqyeb71dzb2T0/jnBVMIC1OpS2BQsYv0wZbKBp57633mjB/BmoemERGuXyUJHPppFOmlrVUneHrLQW7LTuHlRdOJitCvkQQW/USK9ML26tOs3nyAGWOGs3bpDKIjw92OJPIXVOwiN2hnTRNPbNzHxJHxlDw6k5go3ZFDApOKXeQG7D3awoofVTAuJZbSZXnERUe6HUnkE6nYRa6jqqGVx17bS1pCNOuX55MYE+V2JJFPpWIX+RSHT7WxtGQPiTGRbFyZT2rcELcjiVyXI8VujLnbGPOhMabGGPOME2OKuK2u8QKLi/cQHRHOphUFpCUMdTuSyA3pd7EbY8KBl4F5wETgK8aYif0dV8RNx1raWVRcjrWWDSvyyUiOcTuSyA1zYsaeB9RYa+ustZ3AZuB+B8YVccWpcx0sLC6jvdPHhhX5ZI8Y5nYkkV5xYr3WKODYxz5vAPIdGFdkUFhrqW28yK7aJnbWNLGrthlrYeOKfCakxbsdT6TXnCj2a90gw/7Fg4wpAooAMjIyHDisSN+dPHeJnTXN7Oop8lNtHQCMShzK3ZNuZumsTCanJ7icUqRvnCj2BuDj+3+lAyf+/EHW2rXAWoDc3Ny/KH6RgdTa3snu2mZ21jaxq6aZuqaLACTFRjErK5nCrBRmZyUzJjlGd2iUoOdEse8FcowxY4HjwMPAQgfGFemz9s4r7Pl9C7tqm9lZ08QHJ9uwFmKjwskbm8TC/AxmZ6Uw/uY43ZVRPKffxW6tvWKMeRL4TyAcKLHWHup3MpFe6PL5OXCstfsaeU0z+4+dpctniQoPY1pGIqvnfIbC7GSmjk4kUndiFI9z5GYX1tq3gbedGEvkRvj9lg9OtrGrtvsa+Z7ft9De6cMYmDwqgWW3jaUwK4WZmUkMjdKNuiS06C5GEhSstRxtbu9ZtdLE7tpmzrZ3AZCVGsuC6ekUZidTMC5Zb/mXkKdil4B1pq2DnbVN7KxpZndtM8dbLwFwc3w0nx8/gsKsFAqzU7g5IdrlpCKBRcUuAePcpS7K6rqXIO6sbabmzAUAEmMimTUumf/9uSwKs5IZmxKrlSsin0LFLq7p6PKx92hL93ry2ibeP34Ov4WhkeHMHJvEgzPSKcxOYWJavFauiPSCil0GzRWfn4MN53pm5E3sq2+l0+cnIsxwa0Yi/+euHGZnJXNrxnBtNyfSDyp2GTB+v+XD0+fZWdP9Ymf571u4cPkKABPT4nlk9hhmZ6eQl5lE7BD9KIo4Rb9N4qhjLd0rV3bWNrO7tommC50AZCbHMH/aSAqzUpiVlUxSrFauiAwUFbv0S+P5y91ryWu6367fcLZ75cqIuCHclp3C7OzulSujEnUvc5HBomKXXjnf0UV5XcvVe658ePo8APHRERSMS2bl7eMozE4mK3WYVq6IuETFLp+qo8vHvj+cvTojr2o4h89vGRIRxszMJO6/tfvyyi2jEgjXyhWRgKBilz/h81t+d/zc1Rc89x5t4fIVP+FhhqnpCXz1c1nMzkrh1oxEoiP1Vn2RQKRiD3HWWmrOXLj6gmdZXTPnO7pXroy/OY5F+WMozE4mb2wScdGRLqcVkRuhYg9BDWfbr15a2VXbTOP5ywCMThrKvZPTmJ3dfW/ylGFDXE4qIn2hYg8BzRcus7uu+eo7POub2wFIGRbFrKwUCrOSKcxOYXSSNmwW8QIVuwddvNy9ycQfL69Un2wDIG5IBPnjknhkViaF2Sl85iatXBHxIhW7B1y+4mP/H1qv3jzr4LFWrvgtURFh5I4Zzt9/8bPMzkpm8qgEIrTJhIjnqdiDkM9v+eBEW88tbZvYe7SFji4/YQYmpydSdMc4CrNTmDFmuFauiIQgFXsQsNZS23iRXT1FXlbXwrlL3ZtM5IwYxsMzM5idlUz+uGQShmrlikioU7EHqJPnLl19sXNXTTOn2joAGJU4lC9OuonC7BRmjUtmRLw2mRCRP6ViDxBnL3ZSVtd89a36dU0XAUiKjWJWVnLPbkHJZCTF6AVPEflUKnaXtHd2r1zZVdvMzpomPjjZhrUQGxVO3tgkFuZnMDsrhfE3x2mTCRHpFRX7IOny+TlwrLV7M+aaZvYfO0uXzxIVHsa0jERWz/kMhdnJTB2dSKRWrohIP6jYB4jfb6k+1Xb1HZ57ft9Ce6cPY+CWkQksu20shVkpzMxMYmiUVq6IiHP6VezGmAeB54EJQJ61tsKJUMHIWsvR5varL3burmum5WL3JhPjUmNZMD2dwuxkCsYlkxijTSZEZOD0d8b+PvDXwCsOZAk6p9s6epYgNrOrpokT57pXrqQlRPP5z45gdlYys7OTSUvQJhMiMnj6VezW2mogZFZpnLvURVld89V3eNacuQBAwtBIZmcl88Tnu++7MjYlNmT+TkQk8Oga+6e41Omjor575cqumiZ+d/wcfgtDI8OZOTaJB2ekU5idwsS0eK1cEZGAcd1iN8ZsA26+xpf+wVr7sxs9kDGmCCgCyMjIuOGAg+mKz8/BhnM9M/Im9tW30unzExFmuDUjkSfvyqEwK5lbM4YTFaGVKyISmK5b7NbauU4cyFq7FlgLkJuba50Ys7+stXx4+vzVa+Tlv2/hwuXuTSYmpsXzyOwxzM5OIS8zidghenIjIsEh5NrqDz0rV3bWNrO7tommC90rVzKTY5g/rXv/zllZySTFauWKiASn/i53/F/AvwKpwC+NMQestV90JJlDGs9fvroEcWdtEw1nLwEwIm4It2WnMDs7hcLsFEYlauWKiHhDf1fF/BT4qUNZHHG+o4vyupar91z58PR5AOKiI5g1LpmVt4+jMDuZrFRtMiEi3hT0l2I6unzsqz/bc2/yZn53/Bw+v2VIRBgzM5O4/9buyyu3jEogXCtXRCQEBF2x+/yW3x0/d/Xyyt6jLVy+4ic8zDA1PYGvfi6LWVnJTM/QJhMiEpqCqti/v/0I696t43xH98qV8TfHsSh/DIXZyeSNTSIuWptMiIgEVbHfnBDNvZPTmJ2dwuysZFKGDXE7kohIwAmqYv9y7mi+nDva7RgiIgFNb58UEfEYFbuIiMeo2EVEPEbFLiLiMSp2ERGPUbGLiHiMil1ExGNU7CIiHmOsHfw9L4wxjUD9oB+4/1KAJrdDDLJQPGcIzfMOxXOG4DrvMdba1Os9yJViD1bGmAprba7bOQZTKJ4zhOZ5h+I5gzfPW5diREQ8RsUuIuIxKvbeWet2ABeE4jlDaJ53KJ4zePC8dY1dRMRjNGMXEfEYFXsfGWP+zhhjjTEpbmcZaMaY7xhjDhtjqowxPzXGJLqdaaAYY+42xnxojKkxxjzjdp7BYIwZbYz5H2NMtTHmkDFmlduZBosxJtwYs98Ys9XtLE5SsfeBMWY08AXgD25nGSTvALdYa6cAHwHPupxnQBhjwoGXgXnAROArxpiJ7qYaFFeAv7XWTgAKgL8JkfMGWAVUux3CaSr2vlkDPA2ExAsU1tr/stZe6fm0DEh3M88AygNqrLV11tpOYDNwv8uZBpy19qS1dl/Px+fpLrpR7qYaeMaYdOBeoNjtLE5TsfeSMWY+cNxae9DtLC5ZBvzK7RADZBRw7GOfNxACBfdxxphM4Fag3N0kg+IluidofreDOC2o9jwdLMaYbcDN1/jSPwBfB/5qcBMNvE87Z2vtz3oe8w90P23fOJjZBpG5xv8LiWdlAMaYYcCbwGprbZvbeQaSMeY+4Iy1ttIY8zm38zhNxX4N1tq51/r/xpjJwFjgoDEGui9J7DPG5FlrTw1iRMd90jn/kTHmEeA+YI717hrZBuDju6WnAydcyjKojDGRdJf6Rmvtf7idZxAUAvONMfcA0UC8MWaDtXaxy7kcoXXs/WCMOQrkWmuD5QZCfWKMuRv4HnCntbbR7TwDxRgTQfeLw3OA48BeYKG19pCrwQaY6Z6l/AhosdaudjvPYOuZsf+dtfY+t7M4RdfY5Ub8AIgD3jHGHDDG/JvbgQZCzwvETwL/SfcLiP/u9VLvUQgsAe7q+f4e6JnJSpDSjF1ExGM0YxcR8RgVu4iIx6jYRUQ8RsUuIuIxKnYREY9RsYuIeIyKXUTEY1TsIiIe8/8BvriDN/+hKaMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x256b102f630>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "prelu = nn.PReLU(num_parameters=1)\n",
    "x = torch.arange(-5,5,0.1)\n",
    "y = prelu(x).detach()\n",
    "plt.plot(x.numpy(), y.numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 0.6236, -0.0233, -0.8548]])\n",
      "tensor([[0.5709, 0.2990, 0.1302]])\n",
      "sum of probabilities = [1.]\n"
     ]
    }
   ],
   "source": [
    "softmax = nn.Softmax (dim=1)\n",
    "\n",
    "x = torch.randn([1, 3])\n",
    "y = softmax(x)\n",
    "print(x)\n",
    "print(y)\n",
    "print('sum of probabilities =', torch.sum(y, dim = 1).numpy())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loss Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.9876)\n"
     ]
    }
   ],
   "source": [
    "mse_loss = nn.MSELoss()\n",
    "\n",
    "output = torch.randn(3, 5)\n",
    "target = torch.randn(3, 5)\n",
    "\n",
    "print(mse_loss (output, target))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}