w2v_tsne.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import sent_tokenize, word_tokenize\n",
    "\n",
    "data_path = \"/Users/yun/Dev/humanet3/human-centered-eu/concat_results/all-eu-treaties-20240418-162444.txt\"\n",
    "with open(data_path, 'r') as file:\n",
    "    data = file.read()\n",
    "    \n",
    "    \n",
    "sentences = sent_tokenize(data)\n",
    "sentences = [word_tokenize(sent) for sent in sentences]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "cannot import name 'triu' from 'scipy.linalg' (/Users/yun/Dev/humanet3/human-centered-eu/w2v_tsne/lib/python3.12/site-packages/scipy/linalg/__init__.py)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[19], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mgensim\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmodels\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Word2Vec\n\u001b[1;32m      3\u001b[0m model \u001b[38;5;241m=\u001b[39m Word2Vec(sentences, vector_size\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m100\u001b[39m, window\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m5\u001b[39m, min_count\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m, workers\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m4\u001b[39m)\n\u001b[1;32m      4\u001b[0m model\u001b[38;5;241m.\u001b[39mtrain(sentences, total_examples\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mlen\u001b[39m(sentences), epochs\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m10\u001b[39m)\n",
      "File \u001b[0;32m~/Dev/humanet3/human-centered-eu/w2v_tsne/lib/python3.12/site-packages/gensim/__init__.py:11\u001b[0m\n\u001b[1;32m      7\u001b[0m __version__ \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m4.3.2\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m      9\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mlogging\u001b[39;00m\n\u001b[0;32m---> 11\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mgensim\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m parsing, corpora, matutils, interfaces, models, similarities, utils  \u001b[38;5;66;03m# noqa:F401\u001b[39;00m\n\u001b[1;32m     14\u001b[0m logger \u001b[38;5;241m=\u001b[39m logging\u001b[38;5;241m.\u001b[39mgetLogger(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mgensim\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m     15\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m logger\u001b[38;5;241m.\u001b[39mhandlers:  \u001b[38;5;66;03m# To ensure reload() doesn't add another one\u001b[39;00m\n",
      "File \u001b[0;32m~/Dev/humanet3/human-centered-eu/w2v_tsne/lib/python3.12/site-packages/gensim/corpora/__init__.py:6\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;124;03mThis package contains implementations of various streaming corpus I/O format.\u001b[39;00m\n\u001b[1;32m      3\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m      5\u001b[0m \u001b[38;5;66;03m# bring corpus classes directly into package namespace, to save some typing\u001b[39;00m\n\u001b[0;32m----> 6\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mindexedcorpus\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m IndexedCorpus  \u001b[38;5;66;03m# noqa:F401 must appear before the other classes\u001b[39;00m\n\u001b[1;32m      8\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmmcorpus\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m MmCorpus  \u001b[38;5;66;03m# noqa:F401\u001b[39;00m\n\u001b[1;32m      9\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mbleicorpus\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m BleiCorpus  \u001b[38;5;66;03m# noqa:F401\u001b[39;00m\n",
      "File \u001b[0;32m~/Dev/humanet3/human-centered-eu/w2v_tsne/lib/python3.12/site-packages/gensim/corpora/indexedcorpus.py:14\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mlogging\u001b[39;00m\n\u001b[1;32m     12\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mnumpy\u001b[39;00m\n\u001b[0;32m---> 14\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mgensim\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m interfaces, utils\n\u001b[1;32m     16\u001b[0m logger \u001b[38;5;241m=\u001b[39m logging\u001b[38;5;241m.\u001b[39mgetLogger(\u001b[38;5;18m__name__\u001b[39m)\n\u001b[1;32m     19\u001b[0m \u001b[38;5;28;01mclass\u001b[39;00m \u001b[38;5;21;01mIndexedCorpus\u001b[39;00m(interfaces\u001b[38;5;241m.\u001b[39mCorpusABC):\n",
      "File \u001b[0;32m~/Dev/humanet3/human-centered-eu/w2v_tsne/lib/python3.12/site-packages/gensim/interfaces.py:19\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[38;5;124;03m\"\"\"Basic interfaces used across the whole Gensim package.\u001b[39;00m\n\u001b[1;32m      8\u001b[0m \n\u001b[1;32m      9\u001b[0m \u001b[38;5;124;03mThese interfaces are used for building corpora, model transformation and similarity queries.\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     14\u001b[0m \n\u001b[1;32m     15\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m     17\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mlogging\u001b[39;00m\n\u001b[0;32m---> 19\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mgensim\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m utils, matutils\n\u001b[1;32m     22\u001b[0m logger \u001b[38;5;241m=\u001b[39m logging\u001b[38;5;241m.\u001b[39mgetLogger(\u001b[38;5;18m__name__\u001b[39m)\n\u001b[1;32m     25\u001b[0m \u001b[38;5;28;01mclass\u001b[39;00m \u001b[38;5;21;01mCorpusABC\u001b[39;00m(utils\u001b[38;5;241m.\u001b[39mSaveLoad):\n",
      "File \u001b[0;32m~/Dev/humanet3/human-centered-eu/w2v_tsne/lib/python3.12/site-packages/gensim/matutils.py:20\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mscipy\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01msparse\u001b[39;00m\n\u001b[1;32m     19\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mscipy\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mstats\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m entropy\n\u001b[0;32m---> 20\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mscipy\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mlinalg\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m get_blas_funcs, triu\n\u001b[1;32m     21\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mscipy\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mlinalg\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mlapack\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m get_lapack_funcs\n\u001b[1;32m     22\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mscipy\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mspecial\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m psi  \u001b[38;5;66;03m# gamma function utils\u001b[39;00m\n",
      "\u001b[0;31mImportError\u001b[0m: cannot import name 'triu' from 'scipy.linalg' (/Users/yun/Dev/humanet3/human-centered-eu/w2v_tsne/lib/python3.12/site-packages/scipy/linalg/__init__.py)"
     ]
    }
   ],
   "source": [
    "from gensim.models import Word2Vec\n",
    "\n",
    "model = Word2Vec(sentences, vector_size=100, window=5, min_count=1, workers=4)\n",
    "model.train(sentences, total_examples=len(sentences), epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.13.0\n"
     ]
    }
   ],
   "source": [
    "import scipy\n",
    "print(scipy.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_vectors = model.wv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.manifold import TSNE\n",
    "\n",
    "words = list(word_vectors.key_to_index)\n",
    "vectors = [word_vectors[word] for word in words]\n",
    "\n",
    "tsne = TSNE(n_components=2)\n",
    "X_tsne = tsne.fit_transform(vectors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.scatter(X_tsne[:, 0], X_tsne[:, 1])\n",
    "for i, word in enumerate(words):\n",
    "    plt.annotate(word, xy=(X_tsne[i, 0], X_tsne[i, 1]))\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}