logistic_regression_from__scratch.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "logistic_regression_from _scratch.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AURzfkIhxkxP",
        "outputId": "3b03977e-5c53-43a6-e7b4-e559e9be7365"
      },
      "source": [
        "!pip install plotly\n"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: plotly in /usr/local/lib/python3.7/dist-packages (4.4.1)\n",
            "Requirement already satisfied: retrying>=1.3.3 in /usr/local/lib/python3.7/dist-packages (from plotly) (1.3.3)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from plotly) (1.15.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E9d7b386HkDV"
      },
      "source": [
        "# LOGISTIC REGRESSION IMPLEMNTATION USING PYTHON\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W_FudGFFG50Y"
      },
      "source": [
        "import necessary **libraries**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y8RtICK6yKQ_"
      },
      "source": [
        "import pandas as pd \n",
        "import numpy as np\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "from sklearn.model_selection import train_test_split"
      ],
      "execution_count": 106,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KmeC53xo5tnl"
      },
      "source": [
        "data=pd.read_csv(\"/content/Breast_cancer_data.csv\")\n",
        "X=data.iloc[:,:-1]\n",
        "y=data.iloc[:,-1]\n",
        "X_train,X_test,y_train,y_test=train_test_split(X,y,random_state=0)"
      ],
      "execution_count": 114,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0gN8YOSGIe0s"
      },
      "source": [
        "Now cut the main part that is implementation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TfaLIyXlIsQ8"
      },
      "source": [
        "Sigmoid function : logistic regression uses a function that gives outputs between 0 and 1 for all values of X.\n",
        "<br>Here our X refers to the dependent variable which is used to predict outputs."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OuD7OZg8KO-P"
      },
      "source": [
        "sig(z)=$\\frac{1}{1+e^z}$\n",
        "<br>$z \\rightarrow +\\infty, \\space\\space g(z)\\rightarrow 1 $\n",
        "<br>$z \\rightarrow -\\infty, \\space\\space g(z)\\rightarrow 10 $\n",
        "<br>\n",
        "here z=$\\theta_0 +\\theta_1x_1+\\theta_2x2 $"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5C3qikiRK899"
      },
      "source": [
        "<img  src = \"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAUsAAACYCAMAAABatDuZAAAA3lBMVEX////x8fH8/Py0tLT19fX/AAD09PT5+fm9vb23t7e6urr/9/f/4+Pl5eXt7e3S0tLGxsbf39+wsLDLy8ulpaXZ2dn/goL/b2//ysqhoaH/2Nj/+Pj/ZmbOzs7/4eH/YWH/jY3/8PCRkZGFhYV6enr/qKj/l5f/dnb/RUX/e3v/tbX/wMD/ERFzc3NkZGT/oKD/Jyf/NTX/UFD/MjL/QUH/0tL/S0v/WlpWVlaLi4v/urq2e3v/nJz/k5NNTU30///njY21xMTkAABBQUHklJTTlZXJl5e7nJy0p6cAAADo3JCeAAAMBklEQVR4nO1dC3uiuhYNkYhaeYjiq9aqrbY++rK2nfZMZ6Zn7txz7v//QzcBtaDARuQRqev7SiVud8Ii2TvJJgGhI5IB5kQCRiS5xFlULIIiQiISPyLQkQd5kKK4XC8E4DKfhIT6o7Z/LiLIJZZAJXA2vAMXnOfy8u+IECha/y6tfzmEDCSr6RUnAAoRYeDQGoUVYFyumijOl0xWWzvqgO1VgDYOK1miAIuE0BOF76FcVtSCcoty6oWqyUi7kZHThEbie2AuA/ueeLiEbyUsQbkUbltoQOvjDapJ6KxKW7qymw6YqAD1EpZYIjSXku6451Hdk09QLlWkVgdIq0qopKABrqBaFN3S2ODBwepenNE/w/3+1xzJsXBpaFWxVtFbg6pQMOtlC/5ZivDgQLe6HzpCmoDcO3o1R93f315izcynpFat8+LqixZSdFYS+iNtx1wSt5en3fInun3ajTMEQ0KGXKUsyrcVVHTV9lrZ0LNT9i4S+IIe5Bq6sE7XXKIKtZKY3VwD1LEBmMsAI0Sgu4DzVEBiB9rXaBM7hkhqIbVo6PKFIhfMnkiFVgcpVyqVig4dznqZp6d564CRaOmnVkBcJeHlYZku2NKXRREu6HVpOrrEoqgoAqv3pqgk0k+ieXCokBR7lngzS3YQWLrIyvoToZ8/2UGSTk/65W67/dScTieTyXA4vPs+en/v9Xp1E+Px1dVVo9HoUJxTzMvoM8u85LgkkwKMlwdWn07soGn6bUUzNNplpCTKtHExSrHMQIWFYoGyyv4sFIpmwytgu9bPg0vS8rCVJLEKWc1RLukpZr1zT1HrgKAsLZz2283JXa/emS9mxInZ7P7+8WFx/Tyfn59T7hoMlMqr8XhcHy+57XuX2llDXeylUER66UwtnCFRR8XKGTqjbQtXGNjXMq3K6LNvIg5c9IRqWrSNYzXfWo10itsSsA47Ttpv38dzi77Zw0ujProbTptP7XK5fxJLf8CFS1xRUUXW5corNfuYDts2fA8jj7V6VZE1TV76gwjsZUGXMXPXSzsMc+mdS3dSf2YULhqjSZO5AE9EOU/k2ZepqoaGzJqoVJ3fDJBaGBSEC7PJI1F10RPRuCeMDtzsLSiL56Npl1pHUEeC4x521zb8mHybN+tlTbZP20TQxjcQpo13Rw+EzEfN08C5BEBgJSH72JXNhFj66ruh2yPkcdSOvCCBoeWKG9hK2AKVKG2mOeeJohjkwlzadeDhjCyGm5YxQPuARQKPx8P8NJkoxi72sj8mZOTiYpId92yBm3hPcC7L52TRdJVImUtuENRe9hvkJUUjeRAIyOU7WXDLZCSTo4n5niYh071yiXIueFv54djLPLoiPX+JlH1PFAHjqGJn/miTWddfIpLY2eHHx2Euh+QqiXJkARCXDV9LyQUOpI2fPpAyPHORchs/DN/TJw+Yu3jPFg6iT1QmLxHlEmufiBv4cFkmjeTKkQV4c1k+FAeenr1UNWbdKtXlFLMnl/0Vlfzby7S4VDX0Sq+tsAoveXF5QjqBc0l7niiKydEwMYqBjG7pv2/68iFLLy7JfJ9cQogAl4sFSrbEDqIgspg8lRdYYN6ZLgRMzzMVy3S8mS7Z0pcqNtIt1QMDf6Nf5gY3NF1RFPa8kkuWD4/u6V5FkUIUJTArUK1Kax2FbqAbhM5k1LK+c6+XDXK6/pxQZOwA11HgmqZKr6hWXT6A5crlHSnvkstX9T2bcOPyidhjEdyvSQmAZKyAC5d98r6bjkjA9fOzweDC5eNz8sXYC1H0ieIZj9eJs2yRjMfjjI/zay+bZCNIdvQ9QSU2uTwlo511pD3u4QWbXM4fUilGJrDB5YT4PUbJJyJZ1BbRWikb+mQSQkckc8GB1/BtK+fTXi7m2xLc28uE6uWOsbMhOQmjI9m1pfzCzqVbCz8iOOxcvlynVox9wMseEXYup+4+nPs9Inj0PeR7SB38+56kx+PjWXy5xPu8Oi9Yc9ndHIcfsSvWXD4e7IMF6dlLwVrSbCxt/YrLiS3Cs2su/O/1FA+Xii6wBfctfGOdL7nEZBg+l7TnidKKQ7YqLD6uvK4W3C+5HN/HWo79lfAbH9dukD0+nu+SJ2mvonzJ+LhVLyst9M06t+rl9Xm85chmfFzRFR2/oldsX4vvMeIJmkvaffX0/HiF1QG82rPN5HIrLrFbLvzHx5MB4/I7SbsU2UCRPR74lnYp9gJP4/HGY+y5fJX4eBkYiB99T1CJIlp0/CWi4PJrrOErNQ8wisspPma9tIuwL7iJj/8C+0PH+HhACQyHHvn3PZzUyzrcTee+XnKCPvkr7SJkBi8vH2kXYX/wYS+fSH+ffQZXiKR/eeh99fvxXns2rsC/74lfgsXLdtvPLaxE9uPj5C7EPoNHuKHH+kMZ4JIDe9k3V5clZC8zGR/HtaoZnlA/OuaSqAz4npTX6SJU+8taP7r3PtaBJGKPj+eBdJegtOgIPvvGxyX34LMVH5c04+/6z/x6/fjeRfGNj3sUJTArvMbHB2Z83Pi4I5aVOZR66ffLVPfbKCBh9fxQBuxlen78TMNYRY3HD+vyMuDHU0Z7HS/LQP8yZdyvt3HKAJfpjseHn0vEMzAeT3XcY18T9TV8T3zre14Wn5+D7BcM4Qu/A9bxhGAG7GWKwI41URngMsU9bhuOR9MjaeOgREbj403i2OE7C74ntfi4cyOnAFzy9g5YftDZWHySAXuZFqb2Xe8YMsBlSvayv7W8LAv2Mh0uH7a2assAl+ns5/a+vYA0A+PxVPBEnrbSMmAvAyDyGMXGvpYWEopRwDisGMW9y0ZOWZhXT4HLjvXYqqaxQaGaWy56/hq+J2KJkTU7pJ2xt7jLLfTbSj6YNm57WffnwSXJKx1QsUv6G3vihX6yYrpYQT/Yl7b3jydXlHDporg82F46L2KWRCVd01nSMp1JuaWjdbq4TpIcKj5VC6v0J9ZJZ+kDg60cF1FLoemKItQkp+rtogj2LF2LImBblq5FoadeRVxlibdUr+TzgIFI2F521y68pLIdDZAqQe/22CGXL/VccJmM10m6JkstfHtzYZ1mwI8nCr+XGn2Nvnpk8H0/VAa4TDA+3vZ9P1RC4/E454KTs5dT/xe4ZqCvnlDsTETfvTfEMsFN7Iz/+HiHuL9+fY0M2MtkUJ7NyoBIBrhMpI0PCbD9A8pEG0/A9/SvyRv8Ltyv4Xv2lBiReX/Pd8BGUg4TBx0fn5KA77rOgL2MF8173x3a7MgAl3Hay7d7Uj8JqiML9jIuLvvvhPRO/CSc4J1L1T3ZgVji46eTa3I/8ZPYQhS+J0AUI/TlXoI/jAPl4TMh9S4s6ADn9lJ/VWChaCNj5UmDkFlva5M7WAfna/j0IFRGYS9NHd233pwQ0pi4bct28PbyAsNk7s9lvz29qzMWycv71Gt7O264DHu5WhDfEwIn/XK7OR3e9a7mj4xD8tB5n7Rd3gT1iWTaeNpoOjCdTt/e3iYUQ4a7u++j9169Pr5qdF7mi4fHGVnj8blTHw2b7Yh2WixFoyZG1MxWLug5LxSJK2az+/uHxWJx/Tx/eek0rsb1eq/369d//vz5899//vn339yHiVKpWCzmcjVP9SuAEsXL4t46Akjo3kxASoqX/4NMJjyDIyciAddLWEcA9wAbXc9sqtAvJTj/ZLiE7WWAXGDfE8XleiHAjYKzhyWi8ONR5ILhmd4AHUnewfm4Jwh42Us0obngSKYfPHWLSNX8K76ABM0AJBD2t8wCkgv+vV2QS6FYgEQkUMLQoCaslkIHfLCkyqjmKyKiC9QCJJAO6bhEt77mDOTyEg2gsYf8AxCo5BSAbpANX0i/c4DrOhuokHOrqFAJDHTre8NBLg2YSxWaGLvQNKCRS7+LoRy5UioUSnJeu/HkIc8kjEJt9cKibWAmUUWG4s1lgUKjF+pfIwL4cf/GQe85OMl4I0uAiODDBghMSzjwlZAqNeTfeCRZ0y99bbaIFMO/twFyKZVQxV9C034AEi1ZuvWXqEFs+ABLSu7Mv+0IqKb51ylqCHMDXy4FdHPjWbdN5Hy/pRjcfgMtzSXApdSqATqUHOBnfcDFHrcMoa/AhkhmkviNjx9xxBFHHBD+D5/8rqw0pzbnAAAAAElFTkSuQmCC\" width=\"400\" height=\"200\">"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MnlTZaOW82hq"
      },
      "source": [
        "def sigmoid(X,theta):\n",
        "  z=np.dot(theta,X.T) #transposing the X matrix because every column will have its own value of theta and transposing will serve this purpose\n",
        "  return 1/(1+np.exp(-(z)))-0.0000001 #to avoid exception in cost function we add a factor -0.0000001\n"
      ],
      "execution_count": 117,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vwoTgNJOqQRk"
      },
      "source": [
        "\n",
        "\n",
        "\n",
        "\n",
        "cost function is defined as<br>\n",
        "Cost($h_ {\\Theta}$(x),y) = \\{\\begin{matrix} 0 &if  &h_{\\Theta}(x)=y\\\\  \\infty  & if & y=0 &and &h_{\\Theta}(x)\\rightarrow 1 \\\\   \\infty & if &y=1  &and  &h_{\\Theta}(x)\\rightarrow 0  \\end{matrix}\n",
        "<br><br>simplified cost function\n",
        "<br>  $Cost(h_{\\Theta}(x),y)$ = $-y log(h_{\\Theta}(x)) - (1-y) log(1-h_{\\Theta}$(x))  \n",
        "  $J({\\Theta}) $=$ \\frac{-1}{m}\\sum_{i=1}^{m} Cost(h_{\\Theta}(x),y)  $\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xifbli1jaOF9"
      },
      "source": [
        "def cost(X, y, theta):\n",
        "    y1 = sigmoid(X, theta)\n",
        "    return -(1/len(X)) * np.sum(y*np.log(y1) + (1-y)*np.log(1-y1))"
      ],
      "execution_count": 118,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 168
        },
        "id": "f04xmF9MaOQu",
        "outputId": "5d324d3f-088b-44b4-e103-27a8fc9acc85"
      },
      "source": [
        ""
      ],
      "execution_count": 80,
      "outputs": [
        {
          "output_type": "error",
          "ename": "TypeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-80-ed8b3113cdf6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mcost\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m: cost() missing 2 required positional arguments: 'y' and 'theta'"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IgB0zipYSJ0H"
      },
      "source": [
        "gradient decent\n",
        "<br>\n",
        " $\\Theta_{j} := \\Theta_{j} - \\alpha \\sum_{i = 1}^{m}(h_\\Theta(x^{(i)})- y^{(i)})x_j^{(i)} $\n",
        " <br>\n",
        " where\n",
        " <br>\n",
        "alpha= learning rate\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TI-Ig4XVaOfX"
      },
      "source": [
        "def gradient_descent(X, y, theta, lr, epochs):\n",
        "    m =len(X) #no\n",
        "    J = [cost(X, y, theta)] \n",
        "    for i in range(0, epochs): #this loop will run for certain (mentioned=epochs)amount of iteration  until cost is minimized\n",
        "        h = sigmoid(X, theta)\n",
        "        for i in range(0, len(X.columns)): #now this loop finds theta for each column and this loop repeats itself oaccording to the outer loop\n",
        "            theta[i] -= (lr/m) * np.sum((h-y)*X.iloc[:, i])\n",
        "            #applying the gradient descent function and finding theta \n",
        "            #as you can see for the next iteration of outer loop the value of theta in the array gets updatedn with new value of theta for that particular  column\n",
        "        J.append(cost(X, y, theta))\n",
        "    return J, theta"
      ],
      "execution_count": 119,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mGcYZ3XFayAg"
      },
      "source": [
        "theta = [0.5]*len(X.columns)\n"
      ],
      "execution_count": 120,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rRZJt0egZKYL",
        "outputId": "642484c4-49da-4bd5-e5e8-db4bee6405e9"
      },
      "source": [
        "X.columns"
      ],
      "execution_count": 121,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['mean_radius', 'mean_texture', 'mean_perimeter', 'mean_area',\n",
              "       'mean_smoothness'],\n",
              "      dtype='object')"
            ]
          },
          "metadata": {},
          "execution_count": 121
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bIJRve7vaOkJ",
        "outputId": "16bbf2d3-a3c0-4669-cf25-61f2f7fa7b9c"
      },
      "source": [
        "\n",
        "L, theta = gradient_descent(X_train, y_train, theta, 0.03, 2)"
      ],
      "execution_count": 122,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:3: RuntimeWarning:\n",
            "\n",
            "overflow encountered in exp\n",
            "\n",
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:3: RuntimeWarning:\n",
            "\n",
            "invalid value encountered in log\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hZpETkpnD_Mt",
        "outputId": "988f8710-fc6c-4f3b-df9d-4d161f6dbe9c"
      },
      "source": [
        "len(theta)"
      ],
      "execution_count": 123,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "5"
            ]
          },
          "metadata": {},
          "execution_count": 123
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o11_jpq9Dw10"
      },
      "source": [
        "def predict(X, y, theta, alpha, epochs):\n",
        "    J, th = gradient_descent(X, y, theta, alpha, epochs) \n",
        "    h = sigmoid(X, theta)\n",
        "    for i in range(len(h)):\n",
        "        h[i]=1 if h[i]>=0.5 else 0\n",
        "    y = list(y)\n",
        "    acc = np.sum([y[i] == h[i] for i in range(len(y))])/len(y)\n",
        "    return J, acc"
      ],
      "execution_count": 124,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ll92XvKkFR10"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rzviGzeHFTa6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "79763df6-2687-4852-de23-77d37c7217ae"
      },
      "source": [
        "J, acc = predict(X, y, theta, 0.000002, 50000)"
      ],
      "execution_count": 125,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:3: RuntimeWarning:\n",
            "\n",
            "overflow encountered in exp\n",
            "\n",
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:3: RuntimeWarning:\n",
            "\n",
            "invalid value encountered in log\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "H9SCFalhHVHP"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IJq3XR6kFXWL",
        "outputId": "e74c49ee-6764-4926-8ebb-a4ea24b490ed"
      },
      "source": [
        "acc"
      ],
      "execution_count": 126,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8471001757469244"
            ]
          },
          "metadata": {},
          "execution_count": 126
        }
      ]
    }
  ]
}