L3Z3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Zad 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from tqdm import  tqdm_notebook, tnrange"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "initialize = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_lemmas = []\n",
    "for line in open('Dane/polimorfologik-2.1.txt', 'r', encoding='utf8'):\n",
    "    raw_lemmas.append(line.split(';')[:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "lemmas = {}\n",
    "inverted_lemmas = {}\n",
    "for [lemma, word] in raw_lemmas:\n",
    "    if word in lemmas:\n",
    "        lemmas[word].append(lemma)\n",
    "    else:\n",
    "        lemmas[word] = [lemma]\n",
    "    if initialize: \n",
    "        if lemma in inverted_lemmas:\n",
    "            inverted_lemmas[lemma].append(word)\n",
    "        else:\n",
    "            inverted_lemmas[lemma] = [word]\n",
    "\n",
    "del raw_lemmas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "super_lemmas = {}\n",
    "if initialize:\n",
    "    for word in tqdm_notebook(lemmas):\n",
    "        super_lemmas[word] = lemmas[word]\n",
    "        for word_lemma in lemmas[word]:\n",
    "            for inv_lemma in inv_lemmas[word_lemma]:\n",
    "                super_lemmas[word] = np.unique(np.concatenate((super_lemmas[word], np.array(lemmas[inv_lemma]))))\n",
    "\n",
    "    del inv_lemmas\n",
    "    with open('Dane/super_lemmas.txt', 'w', encoding='utf8') as file:\n",
    "        for line in super_lemmas:\n",
    "            file.write(line + ' ' + ' '.join(super_lemmas[line]))\n",
    "            file.write('\\n')\n",
    "else:\n",
    "    for line in open('Dane/super_lemmas.txt', 'r', encoding='utf8'):\n",
    "        line = line.split()\n",
    "        super_lemmas[line[0]] = line[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_super_lemma(query):\n",
    "    query = query.split()\n",
    "    super_lemma = ''\n",
    "    for word in query:\n",
    "        if word not in super_lemmas:\n",
    "            return False\n",
    "        super_lemma += '|'.join(super_lemmas[word]) + ' '\n",
    "    return super_lemma[:-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'niepijąca|niepijący|picie|pila|pilić|pita|pić|piła wina|wino|winąć na barek|bark|barka|barki'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_super_lemma('piłem wino na barce')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2-grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "sl_2gram = {}\n",
    "for line in open('Dane/2grams', 'r', encoding='utf8'):\n",
    "    gram = line.split(' ')\n",
    "    if int(gram[-3]) > 100:\n",
    "        gram = ' '.join(gram[-2:])[:-1]\n",
    "        sup_lemma = get_super_lemma(gram)\n",
    "        if not sup_lemma:\n",
    "            continue\n",
    "        if sup_lemma not in sl_2gram:\n",
    "            sl_2gram[sup_lemma] = [gram]\n",
    "        else:\n",
    "            sl_2gram[sup_lemma].append(gram)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "for key, array in list(sl_2gram.items()):\n",
    "    if len(array) < 2:\n",
    "        del sl_2gram[key]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('Dane/output_2grams.txt', 'w', encoding='utf8') as file:\n",
    "    for key, value in sl_2gram.items():\n",
    "        n = len(value)\n",
    "        for i in range(n):\n",
    "            ref_lemmas = [lemmas[tok] for tok in value[i].split()]\n",
    "            for j in range(i + 1, n):\n",
    "                comp_lemmas = [lemmas[tok] for tok in value[j].split()]\n",
    "                fun_words = []\n",
    "                for k in range(2):\n",
    "                    if len([l for l in comp_lemmas[k] if l in ref_lemmas[k]]) == 0:\n",
    "                        fun_words.append(k)\n",
    "                \n",
    "                if len(fun_words) != 0:\n",
    "                    file.write('Superlemma: ' + key + '\\n')\n",
    "                    file.write('First:\\n')\n",
    "                    for k in range(2):\n",
    "                        if k in fun_words:\n",
    "                            file.write('- [' + value[i].split()[k] + ']' + ' (' + ', '.join(ref_lemmas[k]) + ') ')\n",
    "                        else:\n",
    "                            file.write('- ' + value[i].split()[k] + ' ')\n",
    "                    file.write('\\nSecond:\\n')\n",
    "                    for k in range(2):\n",
    "                        if k in fun_words:\n",
    "                            file.write('- [' + value[j].split()[k] + ']' + ' (' + ', '.join(comp_lemmas[k]) + ') ')\n",
    "                        else:\n",
    "                            file.write('- ' + value[j].split()[k] + ' ')\n",
    "                    file.write('\\n' + '-' * 30 + '\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3-grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "sl_3gram = {}\n",
    "for line in open('Dane/3grams', 'r', encoding='utf8'):\n",
    "    gram = line.split(' ')\n",
    "    if int(gram[-4]) > 10:\n",
    "        gram = ' '.join(gram[-3:])[:-1]\n",
    "        sup_lemma = get_super_lemma(gram)\n",
    "        if not sup_lemma: # if invalid token in gram\n",
    "            continue\n",
    "        if sup_lemma not in sl_3gram:\n",
    "            sl_3gram[sup_lemma] = [gram]\n",
    "        else:\n",
    "            sl_3gram[sup_lemma].append(gram)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "for key, array in list(sl_3gram.items()):\n",
    "    if len(array) < 2:\n",
    "        del sl_3gram[key]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('Dane/output_3grams.txt', 'w', encoding='utf8') as file:\n",
    "    for key, key_lemmas in sl_3gram.items():\n",
    "        n = len(key_lemmas)\n",
    "        for i in range(n):\n",
    "            ref_lemmas = [lemmas[tok] for tok in key_lemmas[i].split()]\n",
    "            for j in range(i + 1, n):\n",
    "                comp_lemmas = [lemmas[tok] for tok in key_lemmas[j].split()]\n",
    "                fun_words = []\n",
    "                for k in range(3):\n",
    "                    if len([l for l in comp_lemmas[k] if l in ref_lemmas[k]]) == 0:\n",
    "                        fun_words.append(k)\n",
    "                \n",
    "                if len(fun_words) != 0:\n",
    "                    file.write('Superlemma: ' + key + '\\n')\n",
    "                    file.write('First:\\n')\n",
    "                    for k in range(3):\n",
    "                        if k in fun_words:\n",
    "                            file.write('- [' + key_lemmas[i].split()[k] + ']' + ' (' + ', '.join(ref_lemmas[k]) + ') ')\n",
    "                        else:\n",
    "                            file.write('- ' + key_lemmas[i].split()[k] + ' ')\n",
    "                    file.write('\\nSecond:\\n')\n",
    "                    for k in range(3):\n",
    "                        if k in fun_words:\n",
    "                            file.write('- [' + key_lemmas[j].split()[k] + ']' + ' (' + ', '.join(comp_lemmas[k]) + ') ')\n",
    "                        else:\n",
    "                            file.write('- ' + key_lemmas[j].split()[k] + ' ')\n",
    "                    file.write('\\n' + '-' * 30 + '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}