industry_classification_selftrained_vectors.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Libs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-08T02:42:17.402320Z",
     "start_time": "2019-10-08T02:42:13.387716Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from nltk.tokenize import sent_tokenize, word_tokenize \n",
    "import warnings \n",
    "from scipy import spatial\n",
    "import gensim\n",
    "from gensim.models import Word2Vec \n",
    "# import nltk\n",
    "# nltk.download('punkt')\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "import re\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "porter_stemmer = PorterStemmer()\n",
    "stop_words = set(stopwords.words('english'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pip install stop-words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-08T02:42:20.050915Z",
     "start_time": "2019-10-08T02:42:18.182399Z"
    }
   },
   "outputs": [],
   "source": [
    "desc = pd.ExcelFile(r\"C:\\Users\\Aishwarya\\Desktop\\ISB task\\OneDrive_1_01-10-2019\\company descriptions.xlsx\")\n",
    "keyw = pd.ExcelFile(r\"C:\\Users\\Aishwarya\\Desktop\\ISB task\\OneDrive_1_01-10-2019\\Industry Segments - Top 10 Keywords.xlsx\")\n",
    "\n",
    "desc = desc.parse('Sheet1')\n",
    "keyw = keyw.parse('Sheet1')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Combine descriptions and remove special chars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-08T02:42:20.544591Z",
     "start_time": "2019-10-08T02:42:20.052905Z"
    }
   },
   "outputs": [],
   "source": [
    "desc.replace(u'\\xa0',u' ', regex = True, inplace = True)\n",
    "desc.replace(u'\\n',u' ', regex = True, inplace = True)\n",
    "desc = desc.replace(np.nan, '', regex = True)\n",
    "\n",
    "desc['desc_tot'] = desc['company_short_description'] + \" \" + desc['company_description'] + desc['company_name'].str.split().str[0] + \".\"\n",
    "\n",
    "desc['desc_tot'] = desc['desc_tot'].str.replace(r\"[\\\"\\',]\", '')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collapse desc_tot into single string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-08T02:42:21.233982Z",
     "start_time": "2019-10-08T02:42:21.182120Z"
    }
   },
   "outputs": [],
   "source": [
    "corp = ' '.join(desc['desc_tot'].tolist())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convert to list and remove stop words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-08T02:42:41.368067Z",
     "start_time": "2019-10-08T02:42:22.691852Z"
    }
   },
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "stop_words = set(stopwords.words('english'))\n",
    "\n",
    "data = []\n",
    "  \n",
    "# iterate through each sentence in the file \n",
    "for i in sent_tokenize(corp):\n",
    "    temp = []\n",
    "      \n",
    "    # tokenize the sentence into words \n",
    "    for j in word_tokenize(i):\n",
    "        if j not in stop_words:\n",
    "            temp.append(j.lower()) \n",
    "    data.append(temp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-06T13:13:13.216767Z",
     "start_time": "2019-10-06T13:13:13.049216Z"
    }
   },
   "source": [
    "## word2vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-08T02:42:41.371029Z",
     "start_time": "2019-10-08T02:42:23.917Z"
    }
   },
   "outputs": [],
   "source": [
    "# Create CBOW model \n",
    "model1 = gensim.models.Word2Vec(data, min_count = 1,  \n",
    "                              size = 100, window = 5) \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sample results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-08T02:41:51.095201Z",
     "start_time": "2019-10-08T02:41:50.998458Z"
    }
   },
   "outputs": [],
   "source": [
    "# Print results \n",
    "print(\"Cosine similarity between 'food' \" + \n",
    "               \"and 'drink' - CBOW : \", \n",
    "    model1.similarity('food', 'drink'))\n",
    "\n",
    "# Print results \n",
    "print(\"Cosine similarity between 'food' \" + \n",
    "               \"and 'machine' - CBOW : \", \n",
    "    model1.similarity('food', 'machine')) \n",
    "\n",
    "# Print results \n",
    "print(\"Cosine similarity between 'machine' \" + \n",
    "               \"and 'work' - CBOW : \", \n",
    "    model1.similarity('machine', 'work')) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## vector for each company"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-08T01:33:43.792947Z",
     "start_time": "2019-10-08T01:33:43.779979Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_mean_vector(word2vec_model, words):\n",
    "    # remove out-of-vocabulary words\n",
    "    words = [word for word in words if word in word2vec_model.wv.vocab]\n",
    "    if len(words) >= 1:\n",
    "        return np.mean(word2vec_model[words], axis=0)\n",
    "    else:\n",
    "        return []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-08T01:34:12.366864Z",
     "start_time": "2019-10-08T01:33:43.796932Z"
    }
   },
   "outputs": [],
   "source": [
    "vecs = np.zeros((desc.shape[0],100))\n",
    "err = []\n",
    "\n",
    "for i in range(0,desc.shape[0]):\n",
    "    comp_desc = desc.iloc[i,3]\n",
    "\n",
    "    data = []\n",
    "    # tokenize the sentence into words \n",
    "    for j in word_tokenize(comp_desc):\n",
    "        if j not in stop_words:\n",
    "            data.append(j.lower())\n",
    "    try:\n",
    "        vecs[i] = get_mean_vector(model1, data) \n",
    "        err.append(1)\n",
    "    except:\n",
    "        err.append(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-08T01:34:13.049034Z",
     "start_time": "2019-10-08T01:34:12.368763Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "vecs_all = []\n",
    "\n",
    "for i in range(0,desc.shape[0]):\n",
    "    vecs_all.append(vecs[i].tolist())\n",
    "\n",
    "comps = pd.DataFrame(vecs_all)\n",
    "comps['list_vec'] = comps.values.tolist()\n",
    "comps = comps['list_vec'].to_frame()\n",
    "comps['company'] = desc[\"company_name\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Industry keyword data cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-08T01:34:13.072970Z",
     "start_time": "2019-10-08T01:34:13.051948Z"
    }
   },
   "outputs": [],
   "source": [
    "keyw.replace(u'\\xa0',u' ', regex=True, inplace=True)\n",
    "keyw.replace(u'\\n',u' ', regex=True, inplace=True)\n",
    "keyw = keyw.replace(np.nan, '', regex=True)\n",
    "keyw = keyw.apply(lambda x: x.str.replace(',',''))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-08T01:34:13.103825Z",
     "start_time": "2019-10-08T01:34:13.077863Z"
    }
   },
   "outputs": [],
   "source": [
    "vecs = np.zeros((keyw.shape[0],100))\n",
    "err = []\n",
    "\n",
    "for i in range(0,keyw.shape[0]):\n",
    "    comp_desc = keyw.iloc[i,1]\n",
    "\n",
    "    data = []\n",
    "    # tokenize the sentence into words \n",
    "    for j in word_tokenize(comp_desc):\n",
    "        if j not in stop_words:\n",
    "            data.append(j.lower())\n",
    "    try:\n",
    "        vecs[i] = get_mean_vector(model1, data)\n",
    "        err.append(1)\n",
    "    except:\n",
    "        err.append(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-08T01:34:13.155676Z",
     "start_time": "2019-10-08T01:34:13.105787Z"
    }
   },
   "outputs": [],
   "source": [
    "vecs_all = []\n",
    "\n",
    "for i in range(0,keyw.shape[0]):\n",
    "    vecs_all.append(vecs[i].tolist())\n",
    "\n",
    "segs = pd.DataFrame(vecs_all)\n",
    "segs['list_vec_segs'] = segs.values.tolist()\n",
    "segs = segs['list_vec_segs'].to_frame()\n",
    "segs['Industry segment'] = keyw[\"Industry segment\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Matching"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-08T01:42:45.268766Z",
     "start_time": "2019-10-08T01:42:45.179007Z"
    }
   },
   "outputs": [],
   "source": [
    "comps['key'] = 1\n",
    "segs['key'] = 1\n",
    "\n",
    "cross = pd.merge(comps, segs, on = 'key')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-08T01:42:45.268766Z",
     "start_time": "2019-10-08T01:42:45.179007Z"
    }
   },
   "outputs": [],
   "source": [
    "for i in range(0, cross.shape[0]):\n",
    "    vec1 = cross.iloc[i,3]\n",
    "    vec2 = cross.iloc[i,5]\n",
    "    result = 1 - spatial.distance.cosine(vec1, vec2)\n",
    "    cross.iloc[i,2] = result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_fin = pd.merge(cross, cross.groupby(['company'])[\"cosine_sim\"].max(), on = [\"company\", \"cosine_sim\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_fin[['company', 'Industry segment', 'cosine_sim']].to_csv(\"classify_selftrained_vectors.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "303.6px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}