Multinomial Logistic Regression from scratch .ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.datasets import make_classification\n",
    "from sklearn.datasets import make_blobs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# set the dimensions\n",
    "N = 100\n",
    "D = 2\n",
    "K = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# set the initial weights\n",
    "W = np.random.randn(D,K)\n",
    "b = np.zeros(K)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2, 3)\n",
      "(3,)\n"
     ]
    }
   ],
   "source": [
    "print(W.shape)\n",
    "print(b.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1.30300696  0.67358626 -1.28849036]\n",
      " [-0.43770045  0.13552017  0.0382872 ]]\n",
      "\n",
      "[ 0.15947824  0.38399315 -1.74344171]\n"
     ]
    }
   ],
   "source": [
    "print(W)\n",
    "print()\n",
    "print(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X, t = make_blobs(n_features=2, centers=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(100, 2)\n",
      "(100,)\n"
     ]
    }
   ],
   "source": [
    "print(X.shape)\n",
    "print(t.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X1 = pd.DataFrame(X, columns=['f1', 'f2'])\n",
    "t1 = pd.DataFrame(t, columns=['target'])\n",
    "df = pd.concat([X1, t1], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD8CAYAAABzTgP2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X2UVPWd5/H3l6aRljC0QRKeB3BYsgEJkj7EqOTEUfGh\nnfgQn7Ln5GGzZxgnyQSzE2Zxk+MweZiQYROXbGIYnEnUbIwSo8YEIkYnu9ExaBrStqCSIKuxEQwh\nggZBmu7v/lG32qri3qpbD7duVffndU6frrr39q0ft5v7vb/f9/dg7o6IiEjWiLQLICIijUWBQURE\n8igwiIhIHgUGERHJo8AgIiJ5FBhERCSPAoOIiORRYBARkTwKDCIikmdk2gWoxMknn+wzZsxIuxgi\nIk1ly5Ytv3f3CaWOa8rAMGPGDLq6utIuhohIUzGz5+Mcp6YkERHJo8AgIiJ5FBhERCRPU+YYwvT1\n9dHb28uRI0fSLkpso0ePZurUqbS2tqZdFBGRQUMmMPT29jJ27FhmzJiBmaVdnJLcnf3799Pb28vM\nmTPTLo6IyKAhExiOHDnSNEEBwMwYP348+/btS7soItKg7v3VblZv2sGLBw4zub2N5efP4dLTpiT+\nuYnmGMxsjpl153y9YmbXFRzzXjM7mHPMDVV8XvWFrqNmK6+I1M+9v9rN9Xc/ye4Dh3Fg94HDXH/3\nk9z7q92Jf3aiNQZ33wEsADCzFmA3cE/IoQ+7+8VJlkVEpBHErQWs3rSDw339edsO9/WzetOOxGsN\n9eyVdA7wrLvHGmDRbA4cOMBNN900+P6CCy6gvb2diy9WvBORjHJqAS8eOBx6jqjttVTPwHAN8L2I\nfWeYWY+Z/cTM5oYdYGZLzazLzLoasV2+MDAsX76c73znOymWSEQaTbFaQKHJ7W2h54jaXkt1CQxm\nNgp4H/D9kN1bgenuPh/4X8C9Yedw93Xu3uHuHRMmlJzqo6R7f7WbM1f9GzNXbODMVf9WdbvdihUr\nePbZZ1mwYAHLly/nnHPOYezYsVWXU0SGjnJqAcvPn0Nba0vetrbWFpafPyeRsuWqV6+kC4Gt7v5S\n4Q53fyXn9UYzu8nMTnb33ydVmGx1Lhu5s9U5oOK2u1WrVrFt2za6u7trVk4RGVomt7exOyQIhNUC\nsveiNHol1SswfICIZiQzmwi85O5uZovI1GL2J1mYNJM6ItI46t0ddPn5c/IeSqF4LeDS06akck9K\nPDCY2RjgPOCvcrZdC+Dua4ErgL82s2PAYeAad/cky5RmUkdEGkMSLQelpFkLKEfigcHdDwHjC7at\nzXn9deDrSZcjVznVubjGjh3Lq6++Wk2xRKSOqmk5qKamkVYtoBzDchK9JJI648eP58wzz2TevHks\nX76cxYsXc+WVV/LQQw8xdepUNm3aVG2xRaSGKm05SHPgWb0MmSkxypFUde7222+vRfFEpA4qbTkY\nDjnKYRkYoDmqcyKSnOXnz2H5XU/Q1/9GSrO1xUq2HAyHHOWwbEoSEQGgsJtLjG4vaQ48qxcFBhEZ\nVrKDW6+7s5u+gfxI0DfgoaOQc6U58Kxehm1TkogMP4VdVMOUahJqli6n1VBgEJEhIbcLafuJrbjD\nwcN9eTfusMRxoThNQkM9R6nAICJNr7Am8PJrfYP7cgeulaoNJNkklNaiO5VQjqFGcmdX7e7u5t3v\nfjdz585l/vz53HnnnSmXTmRoK1UTyHYnLVYbmNLexpcuP7Wqm3XU5JzNNvZBgaFGcgPDiSeeyG23\n3cb27du5//77ue666zhw4EDKJRQZuuJ0FX3xwGHOftsECtdNbGtt4X9evYB/X/HnABXPulzs5l/O\ndNuNYPgGhp71cOM8WNme+d6zvqrT5U67ffPNNzN79mwAJk+ezFve8hat7SySoDh5gXFtrfxgy+7j\neqQunD6OS0+bUvVTfbGbf7ONfRiegaFnPfzok3DwBcAz33/0yaqCw6pVqzjllFPo7u5m9erVg9sf\nf/xxjh49yimnnFKDgotImLAupLnaWlswI7S56dFn/1CTp/piN/9mG/swPAPDQ5+DvoJfYt/hzPYa\n2rNnDx/84Af59re/zYgRw/NSi9TDpadN4UuXn8qU9jYMOOnEVtrbWjGgva2V0a0j8hLSuRxq8lRf\n7OZfzdiHWi8qFsfw7JV0sLe87RV45ZVX6Ozs5Itf/CKnn356zc4rIuHCupDGGbcAbzzVR82dFKdH\nUbG1Fiod+5DG1OAwXAPDuKlBM1LI9grlTrt99OhRLrvsMj70oQ9xxRVXVHxOEalOnHEL8MZTfeGN\n3YAZ49ti3ZxL3fwrGfuQ1oR9wzMwnHNDJqeQ25zU2pbZXqHcabcPHTpEb28v+/fv55ZbbgHglltu\nYcGCBVUWXETKEacZKPepvuv5P/Ddzb8dTFA7mRxEYcL6cF8///Cj7aFBoJY37LSS1sMzMMy/KvP9\noc9lmo/GTc0Ehez2CmnabZHGEtU81GLGgPtxT/U/e2Zf7Hn1Xn6tbzBvkVQTTxKLisUxPAMDZIJA\nlYFARBpbVLt/1EC2ap7Ek2jiKXeN6FoZvoFBRIa8cpO+UU/oRqwZuWMFlnKmxkhrwr7EA4OZPQe8\nCvQDx9y9o2C/AWuAi4DXgI+4+9akyyUiw0M57f5RT+gLp49j866X6XenxYxRI43DfQPH/XypJp5K\nehkVBofsuIokg0O9Otef7e4LCoNC4EJgdvC1FPhmncokIpKncDzElPY23v/OKWz97UH6PVNn6Hfn\n2IDTOiJ/co04TTyVDKJLY56lRhh1dQlwm2dsBtrNbFLahRIRAdjQs+e4m3lfv/Om0SPzAkicCfgq\n6WWUxjxL9cgxOPCgmfUD/+zu6wr2TwFyBxX0Btv21KFsIiKDwpp6ohx4rY9f3bCkrPNX0ssojS6r\n9agxnOXuC8g0GX3czN5TyUnMbKmZdZlZVyNOSJc7u+rzzz/PwoULWbBgAXPnzmXt2rUpl05E4og7\nIA4q6zJaydQYacyzlHhgcPfdwfffAfcAiwoO2Q1My3k/NdhWeJ517t7h7h0TJkxIqrgVyw0MkyZN\n4he/+AXd3d089thjrFq1ihdffDHlEoo0n3rPExT3KbzSLqNhOYxSTVBprDGdaFOSmY0BRrj7q8Hr\nJUDhTHX3AZ8wszuAdwEH3T3xZqQNuzawZusa9h7ay8QxE1m2cBmdszorPl/utNvnnXfe4Ayrr7/+\nOgMDx/deEJHi0pgnKKqpp72tlTEnjMxbNvRTd3azetOOsruPljs6Oo0uq0nnGN4K3JPpkcpI4HZ3\nv9/MrgVw97XARjJdVXeS6a76nxMuExt2bWDloys50n8EgD2H9rDy0ZUAFQeHVatWsW3bNrq7uwF4\n4YUX6OzsZOfOnaxevZrJkyfXpOwiw0Ua8wQtP38Oy+96gr7+N0YttLYYK983N2/NhnKDVbXLetZ7\njelEm5LcfZe7vyP4muvuXwy2rw2CAkFvpI+7+ynufqq7dyVZJoA1W9cMBoWsI/1HWLN1Tc0+Y9q0\nafT09LBz505uvfVWXnrppZqdW2Q4SG1xmyJzYkQFq3/40fbI0zXbsp7QGN1V627vob1lba/G5MmT\nmTdvHg8//HDNzy0ylCWVdC2Wt1i9aQd9A/mRoW/A+dv1TzBzxYbIXkovv9YXeaNvtmU9YZgGholj\nJpa1PY7cabd7e3s5fDjzB/Tyyy/zyCOPMGdOsnObiAw1SSRdSz29R9VG+t1LTokRdaNvtmU9YZjO\nlbRs4bK8HAPA6JbRLFu4rOJz5k673draSn9/P2aGu/PpT3+aU089tRZFFxk2kki6Rj29r7xvO5ee\nNiUy+RxH1I0+rRlSqzEsA0M2wVzLXkmgabdFaq1e6xscOJxpCgqbKymuqBt9WjOkVmNYBgbIBIdq\nA4GINJdiNYLVm3bw7yv+fPD1iwcOM8JscI6kXIWzrRa70ac1Q2o1hm1gEJGhr7Cb6Nlvm8D/3vzb\n0GOztYncWkrYmtFtrS28/51T+Nkz+5rmRl+uIRUY3J1gzERT8JAnERGpjbAxBz/YspsTW0fwWswp\ns2vxtJ/GQL1qDZnAMHr0aPbv38/48eObIji4O/v372f06NFpF0VkSIpKNJ90YiuOxW7zrzbPkcZA\nvWoNmcAwdepUent7acQJ9qKMHj2aqVOnpl0MkSEpMtH8Wh83Xr2gbm3+6q6aotbWVmbOnJl2MUSk\nQRTrJlrPKSaasbvqsBzgJiJDX9gAOQPOflt9Z2dOY3bUaikwiMiQdOlpU3j/O6eQm3F04Adbdtd1\nnqJKptpO25BpShIRKfSzZ/YdN5VFGonfes+OWi3VGERkyGrGxG8jUGAQkSErjWUxhwIFBhEZspox\n8dsIlGMQkSGrGecpagQKDCIypDVL4rfa5T9rSYFBRCRljTafknIMIiIpa7TlPxMNDGY2zcx+ZmZP\nmdl2MztuiTQze6+ZHTSz7uDrhiTLJCLSaBqtW23STUnHgL91961mNhbYYmY/dfenCo572N0vTrgs\nIiINqdHmU0q0xuDue9x9a/D6VeBpoPGzQCIiddRo3WrrlmMwsxnAacBjIbvPMLMeM/uJmc2N+Pml\nZtZlZl3NNLW2iEgpjTafktVjFTEzexPwf4EvuvvdBfv+BBhw9z+a2UXAGnefXex8HR0d3tXVlVyB\nRUSGIDPb4u4dpY5LvMZgZq3AD4DvFgYFAHd/xd3/GLzeCLSa2clJl0tERMIl3SvJgH8Fnnb3r0Yc\nMzE4DjNbFJRpf5LlEhGRaEn3SjoT+CDwpJl1B9v+OzAdwN3XAlcAf21mx4DDwDVej/YtEREJlWhg\ncPdHIG+djLBjvg58PclyiIhIfBr5LCIieRQYREQkjwKDiIjkUWAQEZE8CgwiIpJHgUFEGkPPerhx\nHqxsz3zvWZ92iYYtLdQjIunrWQ8/+iT0BTOMHnwh8x5g/lXplWuYUo1BRNL30OfeCApZfYcz26Xu\nFBhEJH0He8vbLolSYBCR9I2bWt52SZQCg4ik75wboLVgtbLWtsx2qTsFBhFJ3/yr4C++BuOmAZb5\n/hdfU+I5JeqVJCKNYf5VCgQNQjUGERHJo8AgIm/QIDNBTUkikqVBZhJQjUGkWST9NK9BZhJQjUEk\naT3rMzfXg72Zfvnn3FD+E3g9nuY1yEwCqjGIJCl7Qz/4AuBv3NDLfdqvx9O8BplJIPHAYGYXmNkO\nM9tpZitC9puZfS3Y32NmC5Muk0jd1OqGXu3TfJxmKA0yk0CiTUlm1gJ8AzgP6AV+aWb3uftTOYdd\nCMwOvt4FfDP4LtL8atU8M25qUOsI2R5lsAnrBcAADz47ohkq+7raZi9peknnGBYBO919F4CZ3QFc\nAuQGhkuA29zdgc1m1m5mk9x9T8JlE0leJTf0MOfckJ9jgOJP84U5iWxQyMrWWgpv+vUeZFaL/IvU\nXNJNSVOA3P8VvcG2co8RaU61ap4pd8qIsCasQmknlWuVf5Gaa5peSWa2FFgKMH369JRLIxJTLZtn\nynmaj3PTTzupXCz/olpDqpIODLuBaTnvpwbbyj0Gd18HrAPo6Ojwwv0iDSuNOYCimrCyRrTC0UOZ\nZHRaTTjqHtuwkm5K+iUw28xmmtko4BrgvoJj7gM+FPROOh04qPyCSJXCmrCwzLe2N4MZHP4DqTbh\nqHtsw0o0MLj7MeATwCbgaWC9u283s2vN7NrgsI3ALmAncDPwsSTLJDIshOUkLl8HKw/CqDHQfzT/\n+FqMiSh3ZLa6xzasxHMM7r6RzM0/d9vanNcOfDzpcogMO1FNWEk04VQyMnv+VfDbzbDlFvB+sBZ4\nx39SfqEBaOSzyHBTSRNOqdpAJQP5fvxfoetbmaAAme9P3K5eSQ1AgUFkuCm3CSdOt9JyayE96zNB\nIWp8haRKgUFkuKnFmIjCG3i5tZCHPsdxQSFLvZJS1zTjGESkQlGji8MCQdixcWoD5Y7MLnbzV6+k\n1KnGIDKUlTO6OOrYtpPCz517Ay+3FhJ58zf1SmoAqjGIDAVhT/oA91z7RnI3K2p0cVST0ci2zNN/\nWG2g0rmOwmoYGHR8VL2SGoACg0iaklrE596PZQaxFQaFrLCmnKjmncMvZ8ZAZMuZrUHc/ZfEmrU1\njGZybWgKDCJpqdWqbGFP+gN9xX8mrCmn7aRgNHTI9mxOotJZW8OkMVWIxKIcg0hakl7EJ0qlo4t7\n1sPdSxt/1lapmgKDSFpquYhPXNYSnRQ+/HL4zxx+OQgKf0VkF9NKyyMNSYFBJC21mkQudMK8EK1t\ncNna44NCdlRz1E1/3FT4yX8DBmIUxjJNYnHmSpKGpcAgkpZEFvEpIqymkNdFNURrG8xeEp57OE5I\nIlrBoSkpMIikpdy+/6XO9alt0cFh3LT882ZrCXf/ZXTOYNy0zKR2T9xe/LPb3hx8rqa3GCrUK0kk\nTbXumRNnBPJxPYvCWCbQ3DivdLL5wi9nktJhlIhuSqoxiAwlcWohcdaDzuY5St3YO/5L5txadGdI\nUWAQGWqyzUqXr8u8v3tpfjK41M0+t4YRdWO3Frj8Zrj4q5n3WnRnSFFgEBmKis2RVOwpvrCGEXXD\nL+zdVMt8iaTOMguoNZeOjg7v6upKuxgijevGeeE9jcZNy9zs7/3Y8aOjW0bBJd8I77mkqSuGBDPb\n4u4dpY5T8llkKCo2eG7+VZlxCYVdUPuPhk9noakrhp3EmpLMbLWZPWNmPWZ2j5m1Rxz3nJk9aWbd\nZqZqgEgtlEoGR41yVi8iIdkcw0+Bee4+H/g1cH2RY8929wVxqjgiEkOpZLB6EUkRiQUGd3/A3Y8F\nbzcD+osTqZdSyWD1IpIi6pVj+ChwZ8Q+Bx40s37gn919XZ3KJDK0FcsNaD0EKaKqwGBmDwITQ3Z9\nxt1/GBzzGeAY8N2I05zl7rvN7C3AT83sGXf/echnLQWWAkyfPr2aYosIKKkskaoKDO5+brH9ZvYR\n4GLgHI/oF+vuu4PvvzOze4BFwHGBIahJrINMd9Vqyi0iItGS7JV0AfB3wPvc/bWIY8aY2djsa2AJ\nsC2pMomISGlJ9kr6OjCWTPNQt5mtBTCzyWa2MTjmrcAjZvYE8Diwwd3vT7BMIiJSQmLJZ3f/s4jt\nLwIXBa93Ae9IqgwiUmPVjILWCOqmoZHPIhJP4XTd2fmXoPQNvpqflbrTJHoiEk/YdN1hi/FkFwFa\n2f7GrK5xf1YagmoMIhJPsfmXsqJqBlHrP2gKjoakGoOIxBNnGo2omoG1lHdOSZUCg4jEE2cajaga\ngPdrCo4mosAgIvHEWYwnslYxTQv5NBEt1CMitVOYY4BMzUBBoCHEXahHNQYRqR0t8TkkqFeSiNSW\nJudreqoxiIhIHgUGERHJo8AgIiJ5FBhERCSPAoOIiORRYBARkTwKDCIikkeBQURE8igwiIhIHgUG\nERHJo8AgIiJ5EgsMZrbSzHabWXfwdVHEcReY2Q4z22lmK5Iqj4iIxJP0JHo3uvv/iNppZi3AN4Dz\ngF7gl2Z2n7s/lXC5REQkQtpNSYuAne6+y92PAncAl6RcJhGRYS3pwPA3ZtZjZt8ys5NC9k8BXsh5\n3xtsO46ZLTWzLjPr2rdvXxJlFRERqgwMZvagmW0L+boE+CYwC1gA7AG+Us1nufs6d+9w944JEyZU\ncyoRESmiqhyDu58b5zgzuxn4cciu3cC0nPdTg20iIpKSJHslTcp5exmwLeSwXwKzzWymmY0CrgHu\nS6pMIiJSWpI5hn8ysyfNrAc4G/gUgJlNNrONAO5+DPgEsAl4Gljv7tsTLFNRG3ZtYMldS5h/63yW\n3LWEDbs2pFUUEZHUJNZd1d0/GLH9ReCinPcbgY1JlSOuDbs2sPLRlRzpPwLAnkN7WPnoSgA6Z3Wm\nWDIRkfpKu7tqw1izdc1gUMg60n+ENVvXpFQiEZF0KDAE9h7aW9Z2EZGhSoEhMHHMxLK2F1J+QkSG\niqSnxGgayxYuy8sxAIxuGc2yhcvyjtuwawNrtq5h76G9TBwzcXB/tfmJsPMqtyEiaTB3T7sMZevo\n6PCurq6an7fUzbkwQQ2Z4HFCywkcPHrwuPNNGjOJB654INbnhp135RkrFRxEpGbMbIu7d5Q8ToEh\nviV3LWHPoT2xjzeMng/3VHzeuIFFRCSOuIFBOYYylJuIjpufUOJbRBqJAkMZom707Se0M7pldN62\nsPxEueeNG1hERGpJgaEMyxYuCw0AKxatYOUZK5k0ZhKGMWnMpMH8QJzeSlHnjRtYRERqSTmGMpXT\neygsqQxw9Zyr+ezpny163vdMfQ8/7/25eimJSM0o+dwAiiWrVy1eNVijKNX9FdRLSUSqFzcwaBxD\ngoolj7NTbYSNfzih5YTI6TkUGEQkacoxJKhY8njvob2R8zOFjYnI/oyISNIUGGKodLqLYsnjiWMm\nJtb9VUSkGgoMJWQTyHsO7cFx9hzaw4qHV3DW984qGSA6Z3Vy9Zyrj9ue7XGUVPdXEZFqKMdQQlhz\nD8DBowcj50MqTChfPefqyB5GYUnmFYtWDH62eiWJSL2pV1IJ82+djxN9jQqnrSh33iNNnici9aIp\nMWqkVLt+YZ7gS499SQv+iEhTU2AoIWxUcq7cwLFh14ayehSF5S8++8hnWXzHYq3rICKpSSzHYGZ3\nAnOCt+3AAXdfEHLcc8CrQD9wLE41pxpf2PwFvv/r7zPgA4ywEVz5H648bhRyVraZ50j/EQw7rkkp\nmxDOHlds5tWwmkdY/uKYH+PA6wcArTstIulILDC4+2B3HDP7ChD+KJ1xtrv/PqmyZH1h8xe4c8ed\ng+8HfGDwfdgUFbm5AsdpsRb6vR+AETaCS/7sEuD4BHKYsB5FcbqramCbiNRb4k1JZmbAVcD3kv6s\nUtbvWB97e9jTfDYoQCao/HDnD1n1+KqSQaH9hPbQG3u103KLiCShHjmGxcBL7v6biP0OPGhmW8xs\naZIFiepdFLY97tN8ttknSm7300Kl8hdZZqZcg4jUTVWBwcweNLNtIV+X5Bz2AYrXFs4Kcg8XAh83\ns/dEfNZSM+sys659+/ZVU+xYajHKeISNKDrxXeeszrzpuseNGkfriNbjjhvwAVY8vILFdyxWgBCR\nxFUVGNz9XHefF/L1QwAzGwlcDtxZ5By7g++/A+4BFkUct87dO9y9Y8KECRWVt62lLXJfYS+guE/z\n40aNi9w34AMlf75zVicPXPEAPR/u4ZEPPMLnz/w8Iyz813Lg9QOsfHRlZHCodOoOEZFcSTclnQs8\n4+69YTvNbIyZjc2+BpYA25IqzN+f8feMiPgnZ7uLZm+8hU/z7Se0M9Lyc/WjW0Zz/buuLxpwit3I\nw3TO6qTYoMOoMRFhXV/L/WwREUg+MFxDQTOSmU02s43B27cCj5jZE8DjwAZ3vz+pwnTO6uQfF//j\n4M0+7Mk898ab+zT/8DUP84WzvpBXQxg9MlOjGNUyKvIzKxncVu6gOghPlmtgnYhUItG5ktz9IyHb\nXgQuCl7vAt6RZBkKdc7qHGzzn3/r/NBjiiWeX+9/ffB1tmmnVK+kcnsVLVu4rOh5wwJH1GeoR5OI\nlGtYj3yOejKP2h71VB6VEyh1vijZZqyw/EXULKvl/ltERKIM68AQlmAuNr111NP3gA9EJqornS67\nc1Ynj3zgEVYtXjXY9DVpzKTIXk7l/ltERKIM62m3szfYuLObThwzMXTai0ljJrFs4bLBaTFG2AgG\nfGBwezWjlnObvmr5bxERiaJpt8tQ7pTa5Z5bN3URSVLcabeHdY2hXOU+lce92RcGHE2eJyJpUo0h\nIeXULpbctSSyiSp3ESARkWpooZ6UlTOuQF1NRaSRKDAkpJybvbqaikgjUWBISDk3e3U1FZFGosCQ\nkLCb/UgbyeFjh4+b5K5wXqZi4xVERJKm5HOCcnsl/cmoP+G1Y6/RN9A3uL9WXV1FROJQ8rkB5E7C\nd2LriXlBAY5PRmvabBFpBBrHUCelktEayyAijUI1hjoplYzWtNki0igUGOqkVM8jjWUQkUahpqQ6\nKTWdRtQEfRrLICL1psBQR8VmSg1bnCdsLIMm2xORpCkwNIg4E/QpQS0i9aBxDE1Ek+2JSDXqMo7B\nzK40s+1mNmBmHQX7rjeznWa2w8zOj/j5N5vZT83sN8H3k6opz1CnBLWI1EO1vZK2AZcDP8/daGZv\nB64B5gIXADeZWUvIz68AHnL32cBDwXuJoMn2RKQeqgoM7v60u+8I2XUJcIe7v+7u/w/YCSyKOO7W\n4PWtwKXVlGeo02R7IlIPSSWfpwCbc973BtsKvdXds43me4G3JlSeIUHrOotIPZQMDGb2IBDWVvEZ\nd/9hrQri7m5mkZlwM1sKLAWYPn16rT626RTr8ioiUgslA4O7n1vBeXcD03LeTw22FXrJzCa5+x4z\nmwT8rkg51gHrINMrqYIyiYhIDElNiXEfcI2ZnWBmM4HZwOMRx304eP1hoGY1EBERqUy13VUvM7Ne\n4N3ABjPbBODu24H1wFPA/cDH3b0/+Jl/yenaugo4z8x+A5wbvBcRkRRpgJuIyDChhXpERKQiCgwi\nIpJHgUFERPI0ZY7BzPYBz9fxI08Gfl/Hz6tUM5SzGcoIzVFOlbF2mqGctSjjn7r7hFIHNWVgqDcz\n64qTsElbM5SzGcoIzVFOlbF2mqGc9SyjmpJERCSPAoOIiORRYIhnXdoFiKkZytkMZYTmKKfKWDvN\nUM66lVE5BhERyaMag4iI5FFgCDTbMqVmdqeZdQdfz5lZd8Rxz5nZk8FxdZ9HxMxWmtnunLJeFHHc\nBcH13WlmdV3Jz8xWm9kzZtZjZveYWXvEcalcy1LXxjK+FuzvMbOF9Spb8PnTzOxnZvZU8H/ouJWj\nzOy9ZnYw5+/ghnqWMaccRX+HDXAt5+Rco24ze8XMris4Jvlr6e76yjSn/UdgDvB/gI6c7W8HngBO\nAGYCzwItIT//T8CK4PUK4Mt1LPtXgBsi9j0HnJzidV0JfLrEMS3BdZ0FjAqu99vrWMYlwMjg9Zej\nfndpXMs41wa4CPgJYMDpwGN1LuMkYGHweizw65Ayvhf4cb3//sr9HaZ9LUN+93vJjD2o67VUjSHg\nTbpMqZnQSeqPAAAC8ElEQVQZcBXwvXp8XkIWATvdfZe7HwXuIHM968LdH3D3Y8HbzWTWD2kUca7N\nJcBtnrEZaA/WN6kLd9/j7luD168CTxO+YmMzSPVaFjgHeNbd6zmYF1BTUhxTgBdy3jfaMqWLgZfc\n/TcR+x140My2BKvgpeFvgmr5tyKa2OJe43r4KJknxjBpXMs416Zhrp+ZzQBOAx4L2X1G8HfwEzOb\nW9eCvaHU77BhriVwDdEPfIley6TWfG5IjbJMaVwxy/sBitcWznL33Wb2FuCnZvaMu/+82rLFLSfw\nTeDzZP5Dfp5Ms9dHa/n5ccS5lmb2GeAY8N2I0yR+LZuZmb0J+AFwnbu/UrB7KzDd3f8Y5JnuJbOA\nV701xe/QzEYB7wOuD9md+LUcVoHBG2SZ0rhKldfMRgKXA+8sco7dwfffmdk9ZJomavofIe51NbOb\ngR+H7Ip7jSsW41p+BLgYOMeDhtyQcyR+LUPEuTaJX79SzKyVTFD4rrvfXbg/N1C4+0Yzu8nMTnb3\nus5PFON3mPq1DFwIbHX3lwp31ONaqimptEZepvRc4Bl37w3baWZjzGxs9jWZJOu2OpQrtwy57bOX\nRXz+L4HZZjYzeFK6hsz1rAszuwD4O+B97v5axDFpXcs41+Y+4ENBj5rTgYM5zZqJC/Jc/wo87e5f\njThmYnAcZraIzL1nf73KGHxunN9hqtcyR2RLQF2uZVoZ90b7InPT6gVeB14CNuXs+wyZniE7gAtz\ntv8LQQ8mYDzwEPAb4EHgzXUo8y3AtQXbJgMbg9ezyPRieQLYTqbZpN7X9TvAk0APmf90kwrLGby/\niExvlmfrXU4yHQpeALqDr7WNdC3Drg1wbfZ3T6YHzTeC/U+S06uuTuU7i0xTYU/ONbyooIyfCK7b\nE2QS/Gek8LcY+jtspGsZlGEMmRv9uJxtdb2WGvksIiJ51JQkIiJ5FBhERCSPAoOIiORRYBARkTwK\nDCIikkeBQURE8igwiIhIHgUGERHJ8/8Bg7oW3y6B+QgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x112e196a0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(df.loc[df['target']==0, 'f1'], df.loc[df['target']==0, 'f2'],\n",
    "           label = 't1')\n",
    "plt.scatter(df.loc[df['target']==1, 'f1'], df.loc[df['target']==1, 'f2'],\n",
    "            label = 't2')\n",
    "plt.scatter(df.loc[df['target']==2, 'f1'], df.loc[df['target']==2, 'f2'],\n",
    "            label = 't3')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Log reg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Z(X, W, b):\n",
    "#     return X.dot(W) + b\n",
    "    return np.dot(X, W) + b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100, 3)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Z1 = Z(X, W, b)\n",
    "Z1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def predict(Z_single):\n",
    "    exp_Z = np.exp(Z_single)\n",
    "    return np.argmax(exp_Z/(np.exp(Z_single).sum()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def soft(Z):\n",
    "    exp_Z = np.exp(Z)\n",
    "    return exp_Z/(np.exp(Z).sum(axis=1).reshape((N,-1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100, 3)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = soft(Z1)\n",
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def ohe(t):\n",
    "    result = np.zeros((N,K))\n",
    "    for i in range(len(t)):\n",
    "        result[i, t[i]] = 1\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cost(t, y):\n",
    "    return -t*np.log(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100, 3)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cost1 = cost(ohe(t), y)\n",
    "cost1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def error(cost):\n",
    "    return (cost.sum())/N"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.77717031546423687"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "error1 = error(cost1)\n",
    "error1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 100)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(X.T).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100, 3)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(y - ohe(t)).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 3)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dw.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "initial error: 0.7771703154642369\n",
      "error:0.7771703154642369\n",
      "error:0.07835081839095223\n",
      "error:0.06914737104518072\n",
      "error:0.06626235140474115\n",
      "error:0.06492898143944979\n",
      "error:0.06409845364136679\n",
      "error:0.06348152636312779\n",
      "error:0.06298198678859679\n",
      "error:0.06255596883779911\n",
      "error:0.06217956464934552\n",
      "error:0.06183851504708294\n",
      "error:0.06152379400606911\n",
      "error:0.061229412543875775\n",
      "error:0.060951233842984696\n",
      "error:0.060686297317653876\n",
      "error:0.06043241580501953\n",
      "error:0.06018792664998334\n",
      "error:0.059951533036390094\n",
      "error:0.05972220006084346\n",
      "error:0.05949908499414123\n",
      "error:0.0592814894416358\n",
      "error:0.05906882584704937\n",
      "error:0.05886059357864932\n",
      "error:0.05865636153171243\n",
      "error:0.058455755234485283\n",
      "error:0.05825844711362362\n",
      "error:0.058064149007997085\n",
      "error:0.057872606304853316\n",
      "error:0.057683593263009134\n",
      "error:0.05749690921700999\n",
      "error:0.05731237544490125\n",
      "error:0.05712983254376202\n",
      "error:0.056949138200192866\n",
      "error:0.05677016527330581\n",
      "error:0.056592800129323335\n",
      "error:0.05641694118229518\n",
      "error:0.05624249760650751\n",
      "error:0.05606938819415115\n",
      "error:0.055897540337619456\n",
      "error:0.055726889120050596\n",
      "error:0.05555737650085271\n",
      "error:0.05538895058527489\n",
      "error:0.05522156496883524\n",
      "error:0.05505517814875325\n",
      "error:0.05488975299557131\n",
      "error:0.05472525627897496\n",
      "error:0.054561658242489576\n",
      "error:0.05439893222228611\n",
      "error:0.05423705430580005\n",
      "error:0.05407600302627549\n",
      "error:0.053915759089706895\n",
      "error:0.053756305130971634\n",
      "error:0.05359762549623889\n",
      "error:0.05343970604900385\n",
      "error:0.05328253399733915\n",
      "error:0.05312609774017884\n",
      "error:0.05297038673065286\n",
      "error:0.05281539135468038\n",
      "error:0.05266110282320076\n",
      "error:0.052507513076581275\n",
      "error:0.05235461469988421\n",
      "error:0.052202400847809416\n",
      "error:0.05205086517824875\n",
      "error:0.05190000179349872\n",
      "error:0.05174980518827765\n",
      "error:0.05160027020378331\n",
      "error:0.05145139198710958\n",
      "error:0.051303165955413184\n",
      "error:0.05115558776428873\n",
      "error:0.05100865327986952\n",
      "error:0.05086235855422439\n",
      "error:0.05071669980367016\n",
      "error:0.050571673389660336\n",
      "error:0.050427275801950396\n",
      "error:0.05028350364377318\n",
      "error:0.050140353618788616\n",
      "error:0.04999782251959951\n",
      "error:0.0498559072176479\n",
      "error:0.04971460465432956\n",
      "error:0.04957391183318182\n",
      "error:0.04943382581301743\n",
      "error:0.0492943437018916\n",
      "error:0.04915546265180343\n",
      "error:0.04901717985404343\n",
      "error:0.04887949253510998\n",
      "error:0.048742397953126966\n",
      "error:0.0486058933947018\n",
      "error:0.0484699761721711\n",
      "error:0.048334643621187896\n",
      "error:0.04819989309860809\n",
      "error:0.04806572198064135\n",
      "error:0.04793212766123311\n",
      "error:0.04779910755065106\n",
      "error:0.04766665907425012\n",
      "error:0.047534779671395\n",
      "error:0.04740346679452116\n",
      "error:0.04727271790831664\n",
      "error:0.04714253048901087\n",
      "error:0.04701290202375681\n",
      "error:0.04688383001009532\n"
     ]
    }
   ],
   "source": [
    "lr = 0.001\n",
    "print('initial error: {}'.format(error1))\n",
    "\n",
    "for e in range(10000):\n",
    "    \n",
    "    Z1 = Z(X, W, b)\n",
    "    y = soft(Z1)\n",
    "    cost1 = cost(ohe(t), y)\n",
    "    error1 = error(cost1)\n",
    "    \n",
    "    dw = np.dot(X.T,(y - ohe(t)))\n",
    "    db = ((y - ohe(t)).sum(axis=0))/N\n",
    "    \n",
    "    W -= lr*dw\n",
    "    b -= lr*db\n",
    "    \n",
    "    if e % 100 == 0:\n",
    "        print('error:{}'.format(error1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD8CAYAAACfF6SlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztnXd4VGX2xz8vIZBQJFTpAkov0kURGwooKkXsBdfC2lbQ\nFQULBhVFsbH+VlewgAUbIohxKWJBVgUBAakCsSWGTkAgQEje3x9nRiaTmZDJtDsz5/M888zMnXLP\nTCbnvvd8TzHWWhRFUZTEoly0DVAURVEijzp/RVGUBESdv6IoSgKizl9RFCUBUeevKIqSgKjzVxRF\nSUCCdv7GmEbGmC+MMWuNMWuMMcNd22sYY+YbYza6rqsHb66iKIoSCkywef7GmHpAPWvtcmNMVWAZ\nMBC4HthlrR1vjBkFVLfW3heswYqiKErwBL3yt9bmWGuXu27/CawDGgADgKmup01FDgiKoiiKAwh6\n5V/kzYxpAiwE2gG/WWvTXNsNsNt93+s1w4BhAJUrV+7SqlWrwHZ68CBs3Aj5+dCkCdSoEcxHUBRF\niTmWLVu2w1pbO5DXlA/Vzo0xVYAPgRHW2r3i7wVrrTXG+DzKWGsnAZMAunbtapcuXRr4znfuhEsu\nga++giuvhEcfhXKqZSuKkhgYY34N9DUh8ZDGmGTE8b9trZ3h2rzVpQe4dYFtodiXT2rWhHnz4Oab\n4fHHYcgQ2LcvbLtTFEWJdUKR7WOAV4F11tpnPR76GBjquj0UmBXsvkqkQgV4+WV4/nmYNQt69YLf\nfw/rLhVFUWKVUKz8ewLXAucYY1a4LhcA44HzjDEbgXNd98OLMTB8OGRkQGYmdOsG330X9t0qiqLE\nGkHH/K21iwDj5+Hewb5/mejXD779Fi66CM46C157Da66KiqmKIqiOJH4VUXbtIHFi6FHD7j6anjg\nASgsjLZViqIooSU3t0wvi1/nD1CrlgjBN92kQrCiKPFFQQG88go0b16ml8e38wcRgidNgueeUyFY\nUZT44Ntv4ZRTJMMx0NooF/Hv/EGE4BEj4JNPVAhWFCV2ycmBoUPhtNNgyxaYNg0WLizTWyWG83dz\n/vlyxKxcWYTgt9+OtkWKoijH5vBhePppaNkS3n0X7r8f1q+XolbjL9+mZBLL+YMIwUuWwKmnwjXX\nqBCsKIqzmTMHOnSAkSPhzDNhzRoYNw6qVAnqbRPP+YNUBM+dW7QieP/+aFulKIpylM2bYcAAiVgU\nFEj90uzZcNJJIXn7xHT+cLQi2C0En366CsGKokSf/fvhwQehbVv4/HMYPx5Wr4YLLgjpbhLX+UNR\nIXjzZhGCFy+OtlWKoiQi1sJ770n2zrhxcOmlsGED3HcfVKwY8t0ltvN34xaCK1WSmNq0adG2SFGU\nRGLVKjj7bLjiCqhdGxYtgjffhPr1w7ZLdf5u2rYVIfiUU6Qi+MEHVQhWFCW87NoFd9wBnTpJaOfl\nl+H776Fnz7DvWp2/J7Vqwfz5cOONctqlQrCiKOGgoEAcfYsW8NJLcNttMpRq2DBISoqICer8valQ\nASZPViFYUZTwsGgRdO0Kt9wC7dvDihXwwgtQvXpEzVDn7wtfQrBWBCuKEgzZ2VJb1KsX7Ngh4u7n\nn8sBIAqo8y+J888Xp++uCFYhWFGUQDl0CJ58Uqpzp08XPXH9erjssjJX54YCdf7Hwrs1tArBiqKU\nlk8/lZX9qFFw7rmwdq3MGK9cOdqWqfMvFZ6tod35tyoEK4rij02b4MILoX9/KFdOOgrMnAnNmkXb\nsr9Q519aPFtDz5ypraEVRSnOvn0werSkji9cKM3YVq2CPn2ibVkxQuL8jTGvGWO2GWNWe2xLN8Zk\ne831jW28heDu3bUiWFEUqc6dNk3i+uPHS7fNn36Cf/5TFo4OJFQr/ylAPx/bn7PWdnRdPg3RvqKP\nd0XwO+9E2yJFUaLFihVwxhmiCdavL75hyhSoWzfalpVISJy/tXYhsCsU7xUzeArBV10FDz2kQrCi\nJBI7d0pxVpcu0oPnlVeO+oQYINwx/38YY1a5wkKRrWCIBG4h+MYb4bHHVAhWlESgoABefFFm506a\nJO0ZfvpJ/EC52JFRw2npS0AzoCOQAzzj60nGmGHGmKXGmKXbt28Pozlhwl0R/OyzKgQrSryzcKGs\n9G+/HTp2lJDPxImQlhZtywImbM7fWrvVWltgrS0EJgPd/TxvkrW2q7W2a+3atcNlTngxBu66SwYt\nbNqkQrCixBtZWSLinnkm7N4NH3wACxZAu3bRtqzMhM35G2PqedwdBKz299y44YILROxJTdXW0IoS\nDxw8KNP+WraUM/sxY2DdOmn6GMXq3FBQPhRvYox5BzgLqGWMyQIeBs4yxnQELPAL8PdQ7MvxuFtD\nX3KJqP9r18Ijj8RULFBREh5r5Uz+rrsgMxMGD4ZnnoEmTaJtWcgIifO31l7pY/OroXjvmMTdGvq2\n26QieN06eOMNR5R0K4pyDDZskHqeOXOgdWtJ6jjvvGhbFXJ0ORouVAhWlNhi7164917pxfPNN/K/\nu3JlXDp+UOcfXlQIVhTnU1goIxNbtoQJE+DaayV18667IDk52taFDXX+kcBbCNaKYEVxBsuWyVn5\ndddB48ayOHv1VTj++GhbFnbU+UcKtxDcvbtWBCtKtNm+XUYmdusmZ+WvvSYLtO4+M9LjEnX+kaRW\nLfjsM7jhBqkIvuwyrQhWlEhy5IiMTGzRAl5/XUI7P/0Ef/tbwmXkJdandQIVKkgPkGeegY8+klPO\nrKxoW6Uo8c8XX0CnTnDnnTJDd9Uq+T+sVi3alkUFdf7RwBi4++6jQnC3bioEK0q4+O03Ocs+5xzp\ntz9jhqRvtm4dbcuiijr/aKJCsKKEj4MHJbzaqpUstMaOlaLLQYNivjo3FKjzjzYqBCtKaLEWZs2S\ntusPPSSjFNevl9YMqanRts4xqPN3AioEK0poWL8e+vWDgQNl2NKCBdKE7YQTom2Z41Dn7xRUCFaU\nsrN3L9xzj1TnLl4Mzz8v7ZbPOSfaljkWdf5OwpcQvGRJtK1SFOdSWAhTp0rq5rPPwvXXS+rm8OFQ\nPiSty+IWdf5OxFsIfvfdaFukKM7j++/htNPE4TdpIiv+yZOhTp1oWxYTqPN3Km4huFs3GSIxZowK\nwYoCsG0b3HQTnHIK/PKLDEv/5hv5X1FKjTp/J+MpBD/6qArBSmKTny8jE1u0kFDPP/8pIZ6hQxOu\nOjcUaFDM6biF4LZtRdDKzISPP4aGDaNtmaJEjgULJI6/Zg306SMHgVatom1VTKOHy1hAhWAlUfn1\nVxmZeO65kJcn+ftz5qjjDwHq/GOJ/v1VCFYSg7w8qcht1Qr++1+pf1mzBi6+WKtzQ4Q6/1ijbVvJ\nalAhWIlHrJXeO61bQ3q6OPv16+GBByAlJdrWxRUhcf7GmNeMMduMMas9ttUwxsw3xmx0XVcPxb4U\noHZtEYL/9jcVgpX4Yc0aGZl4ySVw3HHw5Zfw3nvQqFG0LYtLQrXynwL089o2ClhgrW0OLHDdV0JF\nhQoycejpp2WldMYZWhGsxCZ79khf/ZNPlslaL7wAy5dLaFMJGyFx/tbahcAur80DgKmu21OBgaHY\nl+KBMZLuNns2bNwozeFUCFZihcJCmaDVooVk79x4o/yO77hDq3MjQDhj/sdba3Nct7cAPodiGmOG\nGWOWGmOWbt++PYzmxDH9+0uRS8WKKgQrscHixdCjhzj8k06CpUvh5ZeltkWJCBERfK21FrB+Hptk\nre1qre1au3btSJgTn7RrpxXBivPZulWKFnv0kDDlW2/BokXQuXO0LUs4wun8txpj6gG4rreFcV8K\nFBeCL79chWDFGeTnS+O1Fi3E4d97L2zYAFdframbUSKczv9jYKjr9lBgVhj3pbjxFII//FCFYCX6\nzJ8PHTqIPtWzJ6xeDU8+CVWrRtuyhCZUqZ7vAN8CLY0xWcaYG4HxwHnGmI3Aua77SiRwC8Effyy9\nT1QIVqJBZqaMTOzTB44cgU8+gU8/ldW/EnVCle1zpbW2nrU22Vrb0Fr7qrV2p7W2t7W2ubX2XGut\ndzaQEm4uvFAqglUIViLJgQOiObVpI6v+xx+X1X7//tG2TPFAK3zjHbcQ3LWrCMEPP6xCsBIerJWR\nia1aieZ0ySUS1x89WhYgiqNQ558IeArBjzwiQvCBA9G2SoknfvxRRiZedhnUqAELF8Lbb0ODBtG2\nTPGDOv9EoWJFEYInTBAhWGcEK6Fg9264807o1AlWrYIXX5Qq3V69om2ZcgzU+ScSxshMABWClWAp\nKJCRiS1awL//DX//u/ymbr0VkpKibZ1SCtT5JyIqBCsezPwhm57jP6fpqAx6jv+cmT9kl7idb76R\nEYrDhkn3zWXL5ABQs2YUP4USKEaKb51B165d7dKlS6NtRuKwfbuIcl9/LdkZDz+s4/ASjJk/ZDN6\nxo/k5Rf8tS01OYlLujTgw2XZRbY3PriHtzbNoHHGhxLLnzABrrhCi7QcgDFmmbW2ayCv0e5JiUzt\n2pKKd+utIgSvXSuzUStVirZlSoSYMHdDEQcPkJdfwDuLf6fAtTBMLsjnb0s/5s5v3qVCwRG4/37J\n4KlSJRomKyFCnX+i4xaC27aFkSPh559lVJ5macQcM3/IZsLcDfyRm0f9tFRG9m3JwE4l/x3/yM3z\nud3t+M/MXMaYBZM4cVc280/qzrhzbuLLcTeH3HYl8qjzV45WBLdsKbUA3brJAaBbt2hbppQS7/BN\ndm4eo2f8CMDATg38Hhjqp6WS7eMA0Cw3h9ELJnPepiVkVq/P9UMe5ssTu9EgLTWin0sJH+r8laO4\nheCLLpKeQFOmSE2A4nj8hW8mzN0A4PfAMLJvyyKPVTqcx51LpnPT4hkcKleeJ866nte6DiA/KZnU\n5CRG9m0ZwU+lhBN1/kpR3BXBgweLmLdunYjBKgQ7lpk/ZPtcvYM4+pIODP8bdQ4AE+asp8u3c3nw\nq9eps3cHXHMNn107gk+W7+FIbh4NShlGUmIHdf5KcdwVwbfeCmPHihA8ZYoKwQ5g5g/ZjJ29ht0H\n8gFITS7HkUL/GXtJxviN67u3Dyy3g4FzHpGq3E6d4IWZ0LMn/YB+fUL+ERSHoMs5xTeeFcHTp0sY\nKDs72lYlNDN/yGbk9JV/OX6AvPxC8gv8O/8Ca6nvJ05/XN6fvHvKxRR06syu71fw5KC7mDlJHL8S\n/+jKX/GPuyK4VSsVgh3AhLkbSnT0vkgyhrNb1S6Ss1+usIArVs3jnoVvUu3gPt7sdAHPnX41e1Kr\nkjprLSQlaXgnAdCVv3JsLrzw6IzgM86A996LtkUJib/wTUkUWMuHy7K5pEsDGqSl0iVrLR+/cTeP\nz/03P9VqTP/rJ5J+3i3sSZXBKp4isd8KXyUu0JW/Ujrat1chOMr4S8s8Fnn5BSz4bAXvZ35Eo/9+\nxB9Va3H7xfeR0ep0n9W5f+TmHTN1VIl99D9XKT2eraHHjpWDgLaGjhgj+7YkOSmwVgoVjuRzy3fT\n+WzSMOrM+4R/nXo5vW/6Dxmte/lty1A/LfWYqaNK7KMrfyUw3EJwmzYyhDszUyuCI4R7xe2Z7VMS\nZ23+njELJtNs9x/Ma96Dx86+kd+q1yvxNQY4u1Vt3v7uN5+Pl+XMQ3EmYW/sZoz5BfgTKACOlNR8\nSBu7xRiffCJCcNWq0ia6a0B9pZQg6Dn+c7+O+ITdfzBmwWR6b/6ezTUa8kjvm/mqWRefz62UXI4D\n+UUnu6UmJ1GxfDly84ofYAzw3OUdNfTjMMrS2C1SYZ+zrbUdAzVOcTieraF79VIhOIKM7NuS1OSi\nffMrHc7j3q+mMO/V2+j++2oeO/sG+t3wgl/Hb4DqlYuPV8zLL8AYedwbCxr6iRM07KMER7t2sHix\nCsEh5lhN2ty3J8zdwB+7DzD012+57/NXSd2+lentevPkmUPZXqVGifuon5bqN4Mo90A+/mICZck6\nUpxHJJy/BT4zxhQAL1trJ3k+aIwZBgwDaNy4cQTMUUJOnTqwYAHccotWBIeA0mbaDOzUgIFmO/zj\nYVi0CLp0YXDfe1jeoPUx9+Hu05P+8Rqf4R13YZiv0JK/ojEltojE8ux0a21H4HzgdmPMGZ4PWmsn\nWWu7Wmu71q5dOwLmKGGhYkV47TV46imtCA6SUmXa7Nwp7Te6dOHQmnU8MfifNOv9MCsaHtvxA3Ru\nXA2A/YePFHssuZxhZN+WPkNL2twtfgj7yt9am+263maM+QjoDiwM936VKGCMzARo1QquukoqgVUI\nDpgSe/EcOQKTJsGDD8LevWy+/HouO74POyu6BquUMn/jm827WJvzp8+K4Sop5YucYQQ6I0CJDcLq\n/I0xlYFy1to/Xbf7AI+Ec5+KA7joIqkIvvhiEYK1NXRA+CvmOn/3RuhyP6xaBWefzYJbH+Cm5Qcp\nS8KeBb/porke2wd2aqDOPk4J98r/eOAjI8Uk5YFp1to5fp+9ciX06AHNmkHTpkWvGzaE8qpPxwzt\n2xcVgteu1RnBpcS7x37dvTt46KvX6b/2K2jcGD74gJnNejD6o9VlcvzHIq1ScujfVHEcYfWm1tpM\n4ORSvyAtTeaCfvcdvP8+FHjEPcuXlx++90HBfbtmTR0k7TQ8heBHHpFMIBWCj4l7pT3xkx+54LN3\nuOPb96lQzsrB8957oVIlJoz/vJguECgG31GiMJf+KA7BWUvpE06Q9gEgsc3ff5eZspmZcnHfnjkT\ntm8v+tqqVYseGDxvN2miDidauIXgNm3gvvu0Irg0WMvArOUMnDJCvq/Bg+GZZ+R37OJYlbb+HLub\n1OQkvwePPT6yf5T4w1nO35Py5Y868XPOKf74vn1yMHAfENy3N26EefOK95ypW7f4WYP7ukEDSEoq\nvg8lNKgQXHo2bIARI2DOHGjdGubPh3PPLfa0JGP+GrLuTXKSYcIQOeF2i7VplZKxVhy7W7idMHeD\npnImMM51/seiShWJK7dvX/wxa2HbtqIHBvf1okXwzjtQ6FHSnpwsZx3+Dg7Vq2tIKRS4heCLLhIh\neOpUuOyyaFvlDPbuhcceg+efh9RUeO45uP12+W36wJ/jB5gw5OS/QkfHEms9tQXQVM5EInadf0kY\nA8cfL5cePYo/np8Pv/1W9IzBfXv6dMmh9uS443wfFNwhpZSUiHysuMCzNfTllx+tCE7Ug2thIbz9\ntsTyt2yBG26Axx+X324JNPCTEdQgLbXU2TlFqoRdqZxnt6rNhLkbuOu9FZraGeeEvbFbIDimsdve\nvUUPCp4HiZ9/hoMHiz6/fv3iOoNbjK5fXzNcfHHoEPz970dX/6+/nni6zLJl8I9/SH+k7t3hhRfk\nuhR4VwGDrNqfGNy+zM46HO+pRIayNHZT5x8o1soKzfuMwX2QyMoqmi5RoYKcHfgLKaWlRe2jRB1r\n4emnRQju0kWE/EQQgrdvhwcegFdekRkJ48fD0KEBLxKO1f8nUPx1Cm2Qlsr/RvnQ3RTHoM7fCRw+\nDL/+6vusITMTdu8u+vy0NP8hpRNOkGyZeOfjj+HqqyW8NmtW/ArBR47ASy9JmGvfPrjzTrldrVq0\nLQOg6agMnxlCBvh5fP9Im6MEQFmcf3zG/KNJhQrQvLlcfJGb6/us4ccfYfZsOXi4MUZWwv5SWOvV\ni4+Q0sUXw//+d7QiOB6F4C++EGe/ejWcdx5MnCjZPA7CX2WxZv/EJ+r8I01aGnTqJBdvCgshJ6d4\n+mpmphRLZWcXDSlVrCghJX9nDg5ZUZaKDh2KCsHuiuBYF4J/+w3uuQc++ED+VjNmwMCBjvxc3pXF\noNk/8YyGfWKJQ4fgl1/86w179hR9fo0a/rWGxo3lLMVpxIsQnJcnesYTT8j90aPlIJDq7FV0qHUE\nJTJozD/R2b3bv9bwyy+S4uqmXDnpl+Tr4NC0qRTFRWt1GstCsLVi7913y3d+6aUwYYLoN4oSJjTm\nn+hUry6Xzp2LP1ZQAH/8UTxtNTNTqklzcoo+PzW15JBS1arh+xzuiuCWLUUI7t49NoTgdetg+HCp\nym3XDj7/HM4+O7D3WPU+LHgE9mRBtYbQewx0iDP9Q3EEuvJXhLw8yVLyroh2n0X8+WfR59eq5T+k\n1KiR38rUgFm1SoTgbdukKZwTheA9e6Rx3b/+JZXnjzwig1YC7UK76n2YfSfke4iuyalw0b/0AKCU\niIZ9lPBgLeza5bsiOjNTDhpHPCZCJSXJAcDfwaF27cBCStu2waBB0hoiPd05FcGFhfDGGzBqlNh4\n000wbpx8vrLwXDvY83vx7dUawV2rg7NViWs07KOEB2OkZXbNmtKUzZuCAilu86U3ZGRIUZwnlSsX\nT1v1vF25ctHn16kjIZRhw8T5r1snQnA0xdPvv5fq3MWLpYXIJ58EH5bakxXYdkUJAnX+SvAkJYmg\necIJcNZZxR8/cOBolpJ3OOmLL6TgyZM6dXyfNaSnS278/ffD5s2iA9SvH4EP6MG2bZK589pr0n9n\n6lS45prQ1FtUa+hn5d8w+PdWFC/U+Svhp1Il6effpk3xx6yFHTt8nzX4G+pTpw788IMU0l1/vQyL\ndx8kQj3Uxy3A7vodfqwEC/bAwXxJ23zoIalK9veaQEXb3mN8x/x7jwnd51EUFxrzV5yN91Af9/Xq\n1VII5tmaG0RwLakDq2fNwLGctFuA3fAnzDkI2wvhpArw9HgYcJdve4MVbTXbRykDjhR8jTH9gIlA\nEvCKtXa8v+eq81cCYts2yQRavBiuvFL0CO/wkr+hPtXLwYEVcFyB3E4rBzUrwYAXjjrbh1rAhz/D\nuiOQZqBvCrQsD2mNxSn/9z7I2yXPTa0B5z/pctwq2iqRxXHO3xiTBPwEnAdkAd8DV1pr1/p6vjp/\nJWAOHRIh+I03pC2EpxDsOdTH+8xh5SLYnV901mE5oHoFOPl0Sd/8YZl0NeuYDGdWhCrmaEgpqQIU\nHC5qS7lkKPQ3AtFAeq7/z6ErfiUInJjt0x3Y5BrkjjHmXWAA4NP5K0rAVKwo+f9t20rKpacQXNJQ\nn/Q0KCiEvRZ2F8plVwH8XiDT3jwb7C3Ll0tF5AyhehJUP+i6bY6eOZQvYfZtSaKtd6hoz+9yH/QA\noISNcDv/BoDnOXAWcIrnE4wxw4BhAI0bNw6zOUpcYoxMwvKeEdyli//XuDNr3M57ewGsyRfn376N\nDFap8Au8fTfsyDt6gNhjYMcR2FQIR7zes6pxHRDKSZiouut2rUow4EF5jq8V/oJHimoEIPcXPKLO\nXwkb4Q77DAH6WWtvct2/FjjFWnuHr+dr2EcJmlWrZEbw9u2ShnnppX6e51pt/3kAvjwESw5DRQMj\nrodHJx2tzvXnrHN/g30Wcgtht+vsIbfw6PUer/+rJAPHV4PKB+A4e/TAUDsVquRDqq8MpWOEihTF\nhRPDPtlAI4/7DV3bFCU8dOggBViDBkkriLFjJSXTO/2z3RD46Av492TYVwCnVocnnoIzb/J6v8t8\nr75n3Q5VD0PVckV/4W4KLOyrUPSsIXc/7C6ALAt57oODa8WfgsdZg/uMIRV++ilxhvooESXczv97\noLkxpini9K8ArgrzPpVEx7Mi+OGHJSXUUwhevFiqc7//Hk47TUI8vprh+cN9MPhkBBze7/s5SQaq\n5UM1P/9iB+3RM4W/Dg4WthXCT0egAOAgvNEycYb6KBElrM7fWnvEGHMHMBdJ9XzNWrsmnPtUFKC4\nEJyZCZMnw/PPy/Z69eCtt0QjMKZ02Taez0muBPl+HH9pSDFQN0ku3lgLf7oODme9FPhQH/eBIRaH\n+igRQ4u8lPhnxgypA8jPl1YU//ynDFB3t6UuTWGWr+eEguRUOHIIbGHxx0wSPLyr+PZDh47Oid68\nuejBIV6G+igB4cSYv6JEl/nzxdEfPgwpKbKtS5ei8whKk23j6zmh4GRXFHTpq8Uf63K979dUrAgt\nWsjFF55DfTwPCitWSBqsZxqrk4f6KGFFnb8Sn/z8s0zTmjkTTjwRZs+WoTC+hODSdNMMV2fNldPk\nDANg2RSwBbLi73I9XPhs2d6zNEN9fM2JnjtXHvPEPdTH35lDOIf6KGFFnb8SXxw4AOPHw1NPSYjn\n8cflIODOlvElBJemm6a/5wSL+wzjrtVld/aB4J610KiRNMTzxj3Ux9eZw6JFsHdv0efXrOm/l1Lj\nxqEb6qOEHHX+SnxgLUyfLvH8338XIffJJyWk4YmviuAn7oDF6SV30/TVcTNUOKlff2qqFMu1alX8\nMWslpORr2tvy5aKteA71KVdODjK+Dg5lGeqjhBR1/krss3o13HmnzAY4+WR4+23o1cv/870rgoc+\nCs+MgJy3/Gf7uG97NnMLFbHSr98YEY9r1PA9uMZzqI/3weHTT4sP9alUybfO4G+ojxJSNNtHiV12\n75bwzYsvSjrjuHFw880S2igtpa0I9iQ9hKmTiTSj199QH/e191Cf2rX9h5QaNQp8RnIc47iunoGi\nzl8pFQUFMknr/vtltvAtt8Cjj8qKtCxs3QqDB8uMYH8VwZ74m7Xri2qNip5NgHbv9IXnUB9fB4df\nfy061CcpSTQFfweHWrUSKqSkzl+Jf775RkI8y5ZJaOeFFyTUEywHD4oQ/OabcMUVcnDxNyO4tDn/\n2sM/dBw5IiEl72lv7tvbthV9fpUq/jOUvIf6xAHq/JX4JScH7rtPnHODBvD009K/P5SrO2slS2j0\naIlpz5zpf0awZ7VvanU49GfRXv7ucA4ce1qYngkEz759ElLyzlByHyT8DfXx1hmaNZMkgUBChw5A\nnb8Sfxw+DBMnwiOPyO177hHnXKVK+PY5axZcfTWkpcntklpDu/HlxMF35fDJV8HGea7QkaHIRJlE\n0gAihbWi6fg6Y8jMhN9+KzoOtHx5aabn3SbDfV2jhuNCSur8lfjiv/+FESOks+WFF8Jzz8FJJ0Vm\n3ytXyojI7dtlStiQIYG/h19twMvhe6PhosiSn190TrT3WcOOHUWff9xxJYeU3JXkEUTbOyjxwaZN\nUpg1e7a0MPj0Uzj//MjacPLJsGSJCMGXXipnHg8+GNiKz2/+/jEWXE7K+08EkpOPrvB79y7++J9/\n+hah169QY6v3AAAajUlEQVSXBcrBg0WfX79+8XCS+3b9+o4JKanzV5zD/v1Skfv009Js7KmnYPjw\n6DUeO/546aI5bBiMGSMVwSUJwd6UtSo4VvL+E4WqVWVORIcOxR+zVuoXfIWTFi6EadOKhpSSk0tu\nl1G9esQ+ljp/JfpYC++9J/H87Gy45hqpzvUntkaSlBTJ/2/bVrSGzZtLFoI98VkVfIyQj3dlseJs\njJH24PXqyWwIbw4fFk3B15nD0qWSquxJtWr+01ebNAnpUB91/kp0WblSBqt8/bU0InvvPejZM9pW\nFcUYyTRq1UqE4O7dZUbwsQbAeHYEdQvBzftIMzdfB4RqjTTbJ96oUEF0Kn9a1Z49vs8a1q6FjAxp\n3+3Gc6iP98GhDKjgq0SHnTsllPKf/8ip7uOPw403OiYe6pdQCMGa3qmUhsJCCSn5q23wGOpjQLN9\nFIdTUACTJol4mpsLt90mYmoEY51Bs3WrtIb+9tuyCcGKEgo8hvqYfv0Cdv46+FOJHF9/LTnzt90G\n7dvLcJEXXogtxw8iBH/+OVx7rZy9XHWVtEJWlEjiHurTt2+ZXh4252+MSTfGZBtjVrguF4RrX4rD\nyc6WWPkZZ0i45733pANn+/bRtqzsuIXg8ePl85x5plQhK0qMEO6V/3PW2o6uy6dh3pfiNA4dEufY\nsiV8+KGMU1y/XiZpxUOYxC0Ef/SRCHTduklfe0WJATTso4SHjAxo107SI889V5zjY4/FZ4/2AQPg\nf/8Tsfr00+VApygOJ9zO/x/GmFXGmNeMMT4Du8aYYcaYpcaYpdu3bw+zOfFNRmYGfab3ocPUDvSZ\n3oeMzIzIG7Fpk7RiuPBCcYZz5khefBnT0WIGd0Vwx46SAfTYY39lYiiKEwkq28cY8xlQ18dDDwDf\nATuQipZHgXrW2htKej/N9ik7GZkZpH+TzsGCo6XmKUkppJ+WTv9m/cNvwL59Mkzl2WdFiHr4Ycnf\nj1Z1brTwbA195ZXw6qulrwhWlDIS8d4+1tpzS/M8Y8xk4JNg9qWUzMTlE4s4foCDBQeZuHwi/Zv1\nJyMzg4nLJ7Jl/xbqVq7L8M7DQ3NQsBbeeQdGjoQ//oChQ+GJJ6TiMRFxC8Ft2siwGXdFcKJ+H4pj\nCVuFrzGmnrXWnf4wCNA2hWFky/4tPrfn7M+h17u9yD2UW2Rb+jfpAEUOAAEfIFaskNX9okWSwjl9\nOpx6akg+T0xjjAyHb9VKWlV061a6imBFiSDhjPk/ZYz50RizCjgbuCuM+0p46lb2FX0TPB2/G/dZ\ngRt32Chnfw4W+9cBwqdusHMn3HqrOPz162HyZIl3q+MvysCBIgSXK6dCsOI4tMI3TvAV8y8NBkPd\nynXJO5Ln8yBRr3I95g2ZJ3eOHDlanbt3L9xxB6Sny9ATxT9aEayEmbLE/DXVM07o36w/6aelU69y\nPQyGepVLF2N2r/J9OX6QEFFGZoa0p+3aFW6/XTJaVqyA559Xx18atCJYcSDa1TOO6N+sf5EYfZ/p\nfcjZH1zV6fG78il/9XXw3S5o3Bg++AAuuURXroHiLQRnZqoQrEQVdf5xSkZmBnsO7Snz6yscLuS6\nuTu4efZ2yll489JmXDvlR6hUKYRWJhjeQrC7NXSnTtG2TElANOwTh2RkZvDQ/x7iwJEDxR7rUbdH\nyS+2ljN/2MvMBzYy/MNt/K99VQY83pwJ/Sur4w8VAwdKhpQxIgTPmBFti5QERJ1/HDJ+yXjyC/N9\nPvbdlu/8vq5JziFeevZX/m/ibxxOLsewe5pw9z8a80ftCiVmEylloGNHyZBq317CaOPGaUWwElE0\n7BPDZGRmMPabseQViHhoMJxS9xS/4q0/KucVMOzj7Vw7bycHKxievLIu7/auyZHyEtcvb8ozvPPw\nkNuf8NStC19+CTfdJBlAa9ZoRbASMdT5xygZmRnc//X9FHJ0OLTFlriy98YUWvp/m8vd72+l9p4j\nzOiVxr+G1GVntaI/iwJbEDK7FS9SUqQVRNu2WhGsRBTN849Rgs3kafNLHqPf+oOOm/JY1SyVJ66p\nx+pm/mP6yeWSGdx8MAuzFoa+RYQifPSRCME1aqgQrAREWfL81fnHKB2mdsAS+N+u+t4j3PnhVgYv\n3M3uqkk8f2ld/turDiYpKeACsYg2jksUVqyQGcE7d8qM4EsuibZFSgygRV4JRKACbFKB5ar5O/lk\n1E8MWLSbN/vU5MLxLZjZqzopFSqRflo61SpUC+g9vVtEKCHALQR36CCtoVUIVsKEOv8YZXjn4ZQr\n5Z+v27p9fPDwJka/ncOapqlc8uhJPH1lPfZVSgJg7+G99G/Wn9GnjMYQWPGWv4ZyShDUrStjLq+5\nRoTgq6/WimAl5KjgG6O4Qy2jvh7l9zl1dx7mnne30Pf7vWTVSmb4Pxrzeeeqxapzj6tw3F+9gQIN\nJWkKaJhISZGwj7aGVsKEOv8Yxn0A8G7oVq2gAtfP2cHVs34H4P8G1WHK+bU4VMH3mcKBIwd4YvET\nfmP+aRXT6NukL7M2zSo2LEZTQMOIMTIGUyuClTCgzj/GcR8AJi6fyJZ9OQxem8S972RTKWsrn3Wr\nzlOX1yanVsnTtPIL89lz2HcrCIPh6yu+BqBTnU7hGQijlMygQdIa+uKLpSL4zTdh8OBoW6XEOOr8\n44D+zfrT/1AzGDEC5s2TnPGp0zjUJA+WT8R4OOvRX48OKLTjGdbxbhynRBC3EDxwoGQAPfaYhIO0\nwZ5SRtT5xzp79kiP+H/9C6pUketbb4Xy5ekPxZz1xOUTA6oPOKPhGSE2WCkz3hXBa9fCK69oRbBS\nJjTbJ1YpLIQpU6BlS3juObj+evjpJxmrWN7/MX145+GkJKWUejcLsxYGb6sSOtwVwePGwbRpcNZZ\nkBNc224lMVHnH4t8/z2cdhr87W/QrJncnzwZatc+5kvdQ1/KmdL96XP259Bneh86TO1An+l9fI91\nVCKLMRLymTEDVq8WIfiHH6JtlRJjBOX8jTGXGmPWGGMKjTFdvR4bbYzZZIzZYIzpG5yZCgDbtsGN\nN8o/+6+/Siqge3h6APRv1p/HT3+81GcApZrrq0QetxCsraGVMhDsyn81MBgoEhswxrQBrgDaAv2A\nF40xSUHuK3HJz5eRic2byyn/PffAhg0yFrBc2f6E3mMf0yqmUd4cWwLSql6Hoa2hlTISlOBrrV0H\nYIpnHAwA3rXWHgJ+NsZsAroD3wazv4Tks8/gzjth3Tro2xcmTpQ4fwjwzt7JyMwoksrpTxjWql6H\noUKwUgbCle3TAPDsLZzl2lYMY8wwYBhA48aNw2RODPLLL/DPf8qpfLNmMGsWXHRRWFP7SjsDWKt6\nHYh3a+hNm7QiWCmRY8YMjDGfGWNW+7gMCIUB1tpJ1tqu1tqutUshWMY9Bw5Aejq0bg1z5kg+95o1\nUuAT4ZxuX5lBWtXrYNwVwSoEK6XgmCt/a+25ZXjfbKCRx/2Grm2KP6yVf9q774bffoPLL4cJE6BR\no2O/NkwUqR7Wqt7YQSuCS0V+fj5ZWVkcPBhYK/NokpKSQsOGDUlOTg76vcIV9vkYmGaMeRaoDzQH\nloRpX7HPmjUwfDgsWCDC3ZdfwplnRtsqQKt6YxatCD4mWVlZVK1alSZNmvjSLR2HtZadO3eSlZVF\n06ZNg36/YFM9BxljsoBTgQxjzFyXkWuA94G1wBzgdmt1FmAxcnOlJcPJJ8Py5fDCC3LtEMevxDhu\nIfjqq0UIvvZaiKFVbrg5ePAgNWvWjAnHD5JYU7NmzZCdqQSb7fMR8JGfx8YB44J5/7ilsBBef13i\nszt2wLBhsjKrVSvalinxhlsIbtMGHnjgqBBcV0V78Jmp6GhCaa9W+EaaxYuhRw9Jy2veHJYuhf/8\nRx2/Ej48K4J//FGFYAVQ5x85tmyRdgw9ekBWlqzGFi2Czp2jbZmSKAwaJL85a7Ui2AHk5uby4osv\n/nV/6tSpNG/enObNmzN16tSw71+df7g5fBieeQZatIC334b77pPq3GuuUfFNiTydOkkvKK0IDpiZ\nP2TTc/znNB2VQc/xnzPzh+ASGD2d/65duxg7diyLFy9myZIljB07lt27d4fCbL+o8w8n8+eLmHvP\nPbLSWr0axo+HqlWjbZmSyLiF4KuuUiG4lMz8IZvRM34kOzcPC2Tn5jF6xo9BHQBGjRrF5s2b6dix\nI6eccgrnnXceNWrUoHr16px33nnMmTMndB/AB+r8w0Fmppxi9+kjfXlmz4ZPP5XVv6I4gZQUeOst\nWfm//ba0ht6ibTv8MWHuBvLyiyYs5uUXMGHuhjK/5/jx4znxxBNZsWIFf//732nkUdPTsGFDsrPD\nWxqlzj+UHDgAY8ZIZsX8+fDEE5LDf+GF0bZMUYrjSwhesSLaVjmSP3LzAtoeC6jzDwXWwgcfyKDt\nRx+VWOqGDTBqFFSsGG3rFKVkPIXgnj3hI5/Z2wlN/TTfTfL8bQ+UBg0a8Pvvv/91PysriwYNfLZD\nCxnq/IPlxx/hnHPgssugRg1YuFBOo8P8h1OUkOIpBA8eDI8/rkKwByP7tiQ1uWhX+tTkJEb2LXuH\n3apVq/Lnn38C0LdvX+bNm8fu3bvZvXs38+bNo2/f8I5B0Rm+ZWX3bnj4YXjxRahWDV56CW6+GZJ0\nbIESo7iF4BtvlIIwd2volNKP/YxXBnaSxdyEuRv4IzeP+mmpjOzb8q/tZaFmzZr07NmTdu3acf75\n5/PQQw/RrVs3AMaMGUONGjVCYrs/1PkHSkEBvPaaxEp37YJbbpEB6jVrRtsyRQketxDctq1WBHsx\nsFODoJy9L6ZNm1bk/g033BDS9y8JDfsEwjffwCmnSDuG1q1h2TL497/V8SvxhVsI/vBDFYLjGHX+\npSEnB667TsSwLVvgnXfgq6+kc6KixCuDB6sQHMeo8y+Jw4elp36LFvDee7IaWr8errhCq3OVxECF\n4LhFnb8/5syRH/y990oBzJo1UhBTpUq0LVOUyFK3LnzxBVx5pegAWhEcF6jz92bTJpmAdP75cv/T\nT6VC96STomuXokST1FRJYX7sMbk++2ytCI5x1Pm72b9fVjVt28oq56mnROxyHwQUJdExRv5Hpk+H\nVatUCI5x1PlbC+++Cy1bSjzzssukOnfkSKhQIdrWKYrzuOSSokLwzJnRtigm8W7p3K9fP9LS0rgw\nQu1gEtv5r1wpIxOvvBLq1JEf9JtvQv360bZMUZxNp04yI7h9e2kP8cQT8S8Er3ofnmsH6Wlyver9\noN7O2/mPHDmSN998M1grS02wM3wvNcasMcYUGmO6emxvYozJM8ascF3+E7ypIWTnTrj9dhmksnYt\nvPyyZDT07BltyxQldqhX76gQfP/9kg4dr0Lwqvdh9p2w53fAyvXsO4M6AHi2dB45ciS9e/emagTb\nvQdb4bsaGAy87OOxzdZaZyXCFxTA5MkSt8zNhdtug7FjpSePoiiB4xaC27aV2QCbNkk9QLxVBC94\nBPK9Onjm58n2DpeV6S3Hjx/P6tWrWREl3SSolb+1dp21tuwNrSPJokXQtSvceit06CAzTF94QR2/\nogSLWwj+8MP4FYL3ZAW2PQYIZ8y/qSvk85UxplcY91My2dlw9dXQq5eEe95/Hz7/XA4AiqKEDu+K\n4HgSgqs1DGx7DHBM52+M+cwYs9rHZUAJL8sBGrvCPncD04wxx/l5/2HGmKXGmKXbt28v26fwxaFD\nMjKxZUtZkTz4IKxbB5deqtW5ihIu3EJwu3bxJQT3HgPJXr37k1NlexnxbOkcDY4Z87fWnhvom1pr\nDwGHXLeXGWM2Ay2ApT6eOwmYBNC1a9fQ/EoyMmDECIk/DhwoA9SbNQvJWyuKcgzq1ZPW0DfcIELw\n2rWitcVya2h3XH/BIxLqqdZQHH8Z4/1QvKXzd999x/r169m3bx8NGzbk1VdfDWtP/7C0dDbG1AZ2\nWWsLjDHNgOZAZjj2VYSNG+Guu8T5t2wJc+fKHF1FUSJLaipMmyZC8EMPHW0Nffzx0bas7HS4LChn\n7wvvls6RJNhUz0HGmCzgVCDDGDPX9dAZwCpjzApgOnCLtXZXcKaWwL59MjKxbVuZpDVhgghP6vgV\nJXoYI+HW6dOlpqZbt/gTgmOYYLN9PrLWNrTWVrTWHm+t7eva/qG1tq21tqO1trO1dnZozC1mgKSZ\ntWwJTz4JV10l1bn33KPVuYriFNwVwYWF8ScExzCxW+H7ww9wxhlwzTVSkfvttzBlisQbFUVxFp07\nSyFlvAnBMUzsOf+dOyVXv0sXWeVPngyLF0OPHtG2TFGUknALwVdcEf8VwTFA7Dj/I0dkWHrz5uLw\n77wTfvoJbroJysXOx1CUhMYtBD/2mMwKPvts2Lo12lYlJLHhNRculOrc22+XPOKVK+H55yEtLdqW\nKYoSKJ6tod1C8MqV0bYq4XC288/KkqZRZ54Ju3fDBx/AZ59JVo+iKLFNggvBnl09V6xYwamnnkrb\ntm3p0KED7733Xtj370znf/Cg9NZv2VJ+EA8/LNW5Q4Zoda6ixBNuIbhNG2kPMX68Y4XgjMwM+kzv\nQ4epHegzvQ8ZmRlBvZ+n869UqRJvvPEGa9asYc6cOYwYMYLc3NxQmO0X5zn/2bMlI+CBB6BfP3H6\n6elQqVK0LVMUJRzUqwdffQWXXw6jR8PQoY4TgjMyM0j/Jp2c/TlYLDn7c0j/Jj2oA4BnS+fJkyfT\nvHlzAOrXr0+dOnUIabsbH4SlwrfMbNwo83Nbt4Z58+C886JtkaIokcAtBLdpA2PGHG0N7ZCK4InL\nJ3KwoOgB6WDBQSYun0j/Zv3L9J7+WjovWbKEw4cPc+KJJ5bZ3tLgrJX/vn3w3HMi/qjjV5TEwhhp\nBfHBB1IJ7CAheMt+38Pq/W0vKzk5OVx77bW8/vrrlAtzFqOznH+7dtKQLTk52pYoihIthgxxnBBc\nt7Lv4TT+tpeFvXv30r9/f8aNG0ePCNQtOcv5q9NXFAWKC8FRrgge3nk4KUlFu5KmJKUwvPPwMr+n\nZ0vnw4cPM2jQIK677jqGDBkSlK2lxVkxf0VRFDduIdgBraHdcf2JyyeyZf8W6lauy/DOw8sc74ei\nLZ33799PVlYWO3fuZMqUKQBMmTKFjh3DNwlXnb+iKM7FQa2h+zfrH5Sz90XMtnRWFEUJO75aQztE\nCI5l1PkrihIbJHhFcKhR568oSuzgFoLbtg1JRbB1aDWxP0Jprzp/RVFiC3dr6CArglNSUti5c2fM\nHACstezcuZOUEAneKvgqihJ7hKAiuGHDhmRlZYW9jUIoSUlJoWHDhiF5L3X+iqLEJu6K4NatZTBM\n9+7w8cdw8smlenlycjJNmzYNs5HOJdgB7hOMMeuNMauMMR8ZY9I8HhttjNlkjNlgjOkbvKmKoig+\nGDIEvv4aCgpECJ41K9oWxQTBxvznA+2stR2An4DRAMaYNsAVQFugH/CiMSYpyH0piqL4pksXWLJE\nwkCDBsGTTzq2NbRTCMr5W2vnWWuPuO5+B7iDUQOAd621h6y1PwObgO7B7EtRFKVE6tc/2hp61Ci4\n/nrHtYZ2EqGM+d8AuMfPNEAOBm6yXNuKYYwZBgxz3T1kjFkdQpvCRS1gR7SNKAVqZ2hRO0NH+G18\n4w25BEcsfJcALQN9wTGdvzHmM8BX67oHrLWzXM95ADgCvB2oAdbaScAk1/sstdZ2DfQ9Io3aGVrU\nztASC3bGgo0QW3YG+ppjOn9r7bnH2On1wIVAb3s0YTYbaOTxtIaubYqiKIoDCDbbpx9wL3CxtfaA\nx0MfA1cYYyoaY5oCzYElwexLURRFCR3Bxvz/D6gIzDcyWP07a+0t1to1xpj3gbVIOOh2a21BKd5v\nUpD2RAq1M7SonaElFuyMBRshju00sVLarCiKooQO7e2jKIqSgKjzVxRFSUAc4fxjpU2EMeZSY8wa\nY0yhMaarx/Ymxpg8Y8wK1+U/TrTT9Zhjvk9PjDHpxphsj+/wgmjb5MYY08/1fW0yxoyKtj3+MMb8\nYoz50fX9BZz6Fy6MMa8ZY7Z51vAYY2oYY+YbYza6rqtH00aXTb7sdNzv0hjTyBjzhTFmrev/fLhr\ne2DfqbU26hegD1DedftJ4EnX7TbASkRUbgpsBpKiaGdrpJjiS6Crx/YmwOpof4+lsNNR36eXzenA\nPdG2w4ddSa7vqRlQwfX9tYm2XX5s/QWoFW07fNh1BtDZ838EeAoY5bo9yv0/70A7Hfe7BOoBnV23\nqyKtddoE+p06YuVvY6RNhLV2nbV2Q7T2X1pKsNNR32eM0B3YZK3NtNYeBt5FvkellFhrFwK7vDYP\nAKa6bk8FBkbUKB/4sdNxWGtzrLXLXbf/BNYhHRQC+k4d4fy9uAH4r+t2A+B3j8f8tolwAE1dp4Vf\nGWN6RdsYPzj9+/yHK/T3mhPCAC6c/p15YoHPjDHLXG1TnMzx1toc1+0tQOQnspceJ/4uAQk5A52A\nxQT4nUasn3+420SEitLY6YMcoLG1dqcxpgsw0xjT1lq712F2RpWSbAZeAh5FHNijwDPIQkApPadb\na7ONMXWQ2pv1rtWso7HWWmOMU3POHfu7NMZUAT4ERlhr97pqrYDSfacRc/42RtpEHMtOP685BBxy\n3V5mjNkMtADCJrqVxU6i3HajtDYbYyYDn4TZnNISM61KrLXZruttxpiPkJCVU53/VmNMPWttjjGm\nHrAt2gb5wlq71X3bSb9LY0wy4vjfttbOcG0O6Dt1RNgn1ttEGGNqu+cVGGOaIXZmRtcqnzj2+3T9\nWN0MApzS3fV7oLkxpqkxpgIyp+LjKNtUDGNMZWNMVfdtJInCKd+hLz4GhrpuDwWcerbquN+lkSX+\nq8A6a+2zHg8F9p1GW7l2LfI3IXHVFa7LfzweewDJttgAnB9lOwchMd9DwFZgrmv7JcAal+3LgYuc\naKfTvk8vm98EfgRWuX7E9aJtk4dtFyAZFZuRsFrUbfJhYzMkE2ml67foGDuBd5DQaL7rd3kjUBNY\nAGwEPgNqONROx/0ugdORMNQqD595QaDfqbZ3UBRFSUAcEfZRFEVRIos6f0VRlAREnb+iKEoCos5f\nURQlAVHnryiKkoCo81cURUlA1PkriqIkIP8P/iFOvRc2hNkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11607fc50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ex = np.linspace(-30, 30, 100)\n",
    "ey1 = -(W[0,0]*ex + b[0])/W[1,0]\n",
    "ey2 = -(W[0,1]*ex + b[1])/W[1,1]\n",
    "ey3 = -(W[0,2]*ex + b[2])/W[1,2]\n",
    "\n",
    "plt.axes(xlim=(-20,20), ylim=(-20,20))\n",
    "plt.scatter(df.loc[df['target']==0, 'f1'], df.loc[df['target']==0, 'f2'],\n",
    "           label = 't0')\n",
    "plt.scatter(df.loc[df['target']==1, 'f1'], df.loc[df['target']==1, 'f2'],\n",
    "            label = 't1')\n",
    "plt.scatter(df.loc[df['target']==2, 'f1'], df.loc[df['target']==2, 'f2'],\n",
    "            label = 't2')\n",
    "\n",
    "plt.plot(ex, ey1, color='r')\n",
    "plt.plot(ex, ey2, color='r')\n",
    "plt.plot(ex, ey3, color='r')\n",
    "\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  4.53644806e-09,   7.85011939e-06,   9.99992145e-01],\n",
       "       [  9.99965035e-01,   3.49639384e-05,   1.55996807e-09],\n",
       "       [  6.56698446e-03,   9.93154107e-01,   2.78908615e-04],\n",
       "       [  5.38947826e-09,   6.06850925e-04,   9.99393144e-01],\n",
       "       [  3.76199431e-11,   2.73408503e-05,   9.99972659e-01]])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.,  0.,  1.],\n",
       "       [ 1.,  0.,  0.],\n",
       "       [ 0.,  1.,  0.],\n",
       "       [ 0.,  0.,  1.],\n",
       "       [ 0.,  0.,  1.]])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ohe(t)[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test = Z([-10,5],W,b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2, 0, 1, 2, 2, 0, 0, 0, 0, 1, 1, 1, 2, 0, 2, 0, 2, 1, 2, 2, 0, 1, 1,\n",
       "       0, 1, 0, 2, 0, 0, 2, 2, 1, 1, 0, 2, 2, 0, 0, 2, 2, 2, 0, 0, 2, 2, 2,\n",
       "       0, 2, 2, 1, 1, 0, 1, 2, 1, 1, 0, 0, 1, 0, 0, 2, 1, 1, 1, 2, 0, 0, 0,\n",
       "       1, 2, 1, 1, 0, 2, 2, 2, 0, 0, 2, 2, 1, 1, 1, 1, 1, 1, 2, 0, 2, 0, 0,\n",
       "       0, 0, 0, 1, 1, 0, 2, 0])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argmax(y, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2, 0, 1, 2, 2, 0, 0, 0, 0, 1, 1, 1, 2, 0, 2, 0, 2, 1, 2, 2, 0, 1, 1,\n",
       "       0, 1, 0, 2, 0, 0, 2, 2, 1, 1, 0, 2, 2, 0, 0, 2, 2, 2, 0, 0, 2, 2, 2,\n",
       "       0, 2, 2, 1, 1, 0, 1, 2, 1, 1, 0, 0, 1, 0, 0, 2, 1, 1, 1, 2, 0, 0, 0,\n",
       "       1, 2, 1, 1, 0, 2, 2, 2, 0, 0, 2, 2, 1, 1, 1, 1, 1, 1, 2, 0, 2, 0, 0,\n",
       "       1, 1, 0, 1, 1, 1, 2, 0])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argmax(ohe(t), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "97"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(np.argmax(y, axis=1)==np.argmax(ohe(t), axis=1)).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "        0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "        0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "        0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "        0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "        0,  0,  0,  0,  0,  0,  0, -1, -1,  0,  0,  0, -1,  0,  0])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argmax(y, axis=1)-np.argmax(ohe(t), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}