TF_IDF_and_Count_Vectorizer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "TF IDF and Count Vectorizer.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyOn0Ys/826N+yamkhts4yoO",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/srivatsan88/Natural-Language-Processing/blob/master/TF_IDF_and_Count_Vectorizer.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3Tib-VoNf3SU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
        "import numpy as np"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WShP9UhDdqqz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sentences=['I have a credit card account','My account card, debit card is lost','My credit card stopped working']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bWwAqyZubjXg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "vectorizer=CountVectorizer()\n",
        "countvec=vectorizer.fit_transform(sentences)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gidCFkK2bjUm",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        },
        "outputId": "b722a8a3-9b0e-40f5-bf89-5eb5a4f9e1e0"
      },
      "source": [
        "countvec.A"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[1, 1, 1, 0, 1, 0, 0, 0, 0, 0],\n",
              "       [1, 2, 0, 1, 0, 1, 1, 1, 0, 0],\n",
              "       [0, 1, 1, 0, 0, 0, 0, 1, 1, 1]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IfQXzKR8bjRY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 196
        },
        "outputId": "2615baa2-635f-4b5b-947d-98e9be758267"
      },
      "source": [
        "vectorizer.get_feature_names()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['account',\n",
              " 'card',\n",
              " 'credit',\n",
              " 'debit',\n",
              " 'have',\n",
              " 'is',\n",
              " 'lost',\n",
              " 'my',\n",
              " 'stopped',\n",
              " 'working']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0HX9sTs-jGv-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "vectorizer=CountVectorizer(max_features=4)\n",
        "countvec=vectorizer.fit_transform(sentences)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n0dnuMHijISd",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 88
        },
        "outputId": "f67abf47-7193-4f20-b57b-e0883635e80c"
      },
      "source": [
        "print(countvec.A)\n",
        "print(vectorizer.get_feature_names())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[1 1 1 0]\n",
            " [1 2 0 1]\n",
            " [0 1 1 1]]\n",
            "['account', 'card', 'credit', 'my']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qeGcIJgHc-rG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "vectorizer=CountVectorizer(max_features=4, stop_words='english')\n",
        "countvec=vectorizer.fit_transform(sentences)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oMprZNb6c-oo",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 88
        },
        "outputId": "a9c7b5ce-302f-4c62-fcd1-62b32062cff5"
      },
      "source": [
        "print(countvec.A)\n",
        "print(vectorizer.get_feature_names())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[1 1 1 0]\n",
            " [1 2 0 1]\n",
            " [0 1 1 0]]\n",
            "['account', 'card', 'credit', 'debit']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vJOj7ucXbjIg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "vectorizer=CountVectorizer(max_features=6, ngram_range=(1,2))\n",
        "countvec=vectorizer.fit_transform(sentences)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9VFcQsvKc-uy",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 88
        },
        "outputId": "5998c84a-304f-44d8-c4de-29681cbe0a07"
      },
      "source": [
        "print(countvec.A)\n",
        "print(vectorizer.get_feature_names())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[1 1 1 1 0 0]\n",
            " [1 2 0 0 1 0]\n",
            " [0 1 1 1 1 1]]\n",
            "['account', 'card', 'credit', 'credit card', 'my', 'stopped']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wKy15maGfXIJ",
        "colab_type": "text"
      },
      "source": [
        "TF = (# occurrences of term t in document) / (# of words in document)\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tqjK7ppXc-kp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "vectorizer=TfidfVectorizer(use_idf=False, norm='l1')\n",
        "tfvec=vectorizer.fit_transform(sentences)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hpgLWhqodzOq",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 142
        },
        "outputId": "eff13d46-7ca5-4ee5-ec5a-d2b856ce0bdd"
      },
      "source": [
        "print(tfvec.A)\n",
        "print(vectorizer.get_feature_names())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[0.25       0.25       0.25       0.         0.25       0.\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.14285714 0.28571429 0.         0.14285714 0.         0.14285714\n",
            "  0.14285714 0.14285714 0.         0.        ]\n",
            " [0.         0.2        0.2        0.         0.         0.\n",
            "  0.         0.2        0.2        0.2       ]]\n",
            "['account', 'card', 'credit', 'debit', 'have', 'is', 'lost', 'my', 'stopped', 'working']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6DGVPbtbh097",
        "colab_type": "text"
      },
      "source": [
        "sentences=['I have a credit card account','My card, debit card account','My credit card is not working']"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wERaF8RodzLj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "vectorizer=TfidfVectorizer(use_idf=False, norm='l2')\n",
        "tfvec=vectorizer.fit_transform(sentences)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vfUA6DledzIr",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 142
        },
        "outputId": "6dc691d7-ba7d-4891-e41d-a7ab3649ce3c"
      },
      "source": [
        "print(tfvec.A)\n",
        "print(vectorizer.get_feature_names())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[0.5        0.5        0.5        0.         0.5        0.\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.33333333 0.66666667 0.         0.33333333 0.         0.33333333\n",
            "  0.33333333 0.33333333 0.         0.        ]\n",
            " [0.         0.4472136  0.4472136  0.         0.         0.\n",
            "  0.         0.4472136  0.4472136  0.4472136 ]]\n",
            "['account', 'card', 'credit', 'debit', 'have', 'is', 'lost', 'my', 'stopped', 'working']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F0R2lhAVdzGK",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "c267aa06-d313-4a35-d989-60ca623fa99c"
      },
      "source": [
        "print(2/np.sqrt(9))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.6666666666666666\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BKO1_ap3dzDP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "vectorizer=TfidfVectorizer(use_idf=False, norm=None)\n",
        "tfvec=vectorizer.fit_transform(sentences)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aucBmRAOdzAo",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 88
        },
        "outputId": "705b9204-bc16-4e3b-da84-83a763c08687"
      },
      "source": [
        "print(tfvec.A)\n",
        "print(vectorizer.get_feature_names())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[1. 1. 1. 0. 1. 0. 0. 0. 0. 0.]\n",
            " [1. 2. 0. 1. 0. 1. 1. 1. 0. 0.]\n",
            " [0. 1. 1. 0. 0. 0. 0. 1. 1. 1.]]\n",
            "['account', 'card', 'credit', 'debit', 'have', 'is', 'lost', 'my', 'stopped', 'working']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cuvVmX32AUup",
        "colab_type": "text"
      },
      "source": [
        "IDF = log(# of documents / # documents with term t in it)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5UHJcaQMdy9h",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "vectorizer_idf=TfidfVectorizer(smooth_idf=False)\n",
        "tfidfvec=vectorizer_idf.fit_transform(sentences)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TKFsfbK3AMLR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        },
        "outputId": "f0587a66-6d9f-45ba-e62b-b2d65cb89a55"
      },
      "source": [
        "print(vectorizer_idf.idf_)\n",
        "print(vectorizer_idf.get_feature_names())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[1.40546511 1.         1.40546511 2.09861229 2.09861229 2.09861229\n",
            " 2.09861229 1.40546511 2.09861229 2.09861229]\n",
            "['account', 'card', 'credit', 'debit', 'have', 'is', 'lost', 'my', 'stopped', 'working']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MHhC4HZ_A53x",
        "colab_type": "text"
      },
      "source": [
        "sentences=['I have a credit card account','My card, debit card account','My credit card is not working']"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x9Tsup0ZAMHe",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "4e7e1f7b-d771-4195-90c5-ca8df7a7e282"
      },
      "source": [
        "print(np.log(3/2)+1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1.4054651081081644\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "olxtjaNE2FYW",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 124
        },
        "outputId": "82502547-aa20-46c8-93f8-ba300bd12aba"
      },
      "source": [
        "tfidfvec.A\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.45951737, 0.3269504 , 0.45951737, 0.        , 0.68614212,\n",
              "        0.        , 0.        , 0.        , 0.        , 0.        ],\n",
              "       [0.3055129 , 0.43474989, 0.        , 0.45618573, 0.        ,\n",
              "        0.45618573, 0.45618573, 0.3055129 , 0.        , 0.        ],\n",
              "       [0.        , 0.26959162, 0.37890161, 0.        , 0.        ,\n",
              "        0.        , 0.        , 0.37890161, 0.56576828, 0.56576828]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eQrN5c3a9y2F",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}