L3Z1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from collections import defaultdict as dd\n",
    "from typing import List, Dict, Set\n",
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-2-12f404b335d7>, line 14)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-2-12f404b335d7>\"\u001b[0;36m, line \u001b[0;32m14\u001b[0m\n\u001b[0;31m    for sentence in line.split(\".\")\u001b[0m\n\u001b[0m                                   ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "class ConversationBot:\n",
    "    def _to_lemma_mapping(self) -> Dict[str, str]:\n",
    "        all_lemmas = {}\n",
    "        for line in open('Dane/polimorfologik-2.1.txt', encoding='utf-8'):\n",
    "            L = line.split(';')[:2]\n",
    "            if L[1].lower() not in all_lemmas or L[0].lower() == L[1].lower():\n",
    "                all_lemmas[L[1].lower()] = L[0].lower()\n",
    "        return all_lemmas\n",
    "\n",
    "    def _load_quotes(self) -> List[List[str]]:\n",
    "        result = []\n",
    "        with open(\"Dane/tokenized_quotes.txt\", encoding='utf-8') as f:\n",
    "            for line in f:\n",
    "                for sentence in line.split(\".\")\n",
    "                    result.append(nltk.word_tokenize(sentence))\n",
    "                # result.append(line.split(\" \"))\n",
    "        return result\n",
    "    \n",
    "    def _to_lemmas(self, words):\n",
    "        result = []\n",
    "        for w in words:\n",
    "            if w in self.lemma_mapping:\n",
    "                result.append(self.lemma_mapping[w])\n",
    "            else:\n",
    "                result.append(w)\n",
    "        return result\n",
    "    \n",
    "\n",
    "    def _get_reverse_index(self, lemmas_dict, quotes):\n",
    "        result = dd(lambda:set())\n",
    "        for i, quote in enumerate(quotes):\n",
    "            for word in quote:\n",
    "                word = word.lower()\n",
    "                if word in lemmas_dict:\n",
    "                    lemma = lemmas_dict[word]\n",
    "                    result[lemma].add(i)\n",
    "                else:\n",
    "                    result[word].add(i)\n",
    "        return result\n",
    "\n",
    "    def __init__(self) -> None:\n",
    "        print(\"Loading quotes\")\n",
    "        self.quotes = self._load_quotes()\n",
    "        print(\"Quotes loaded\")\n",
    "        print(\"Creating lemmas mapping\")\n",
    "        self.counts = dd(lambda:0)\n",
    "        self.lemma_mapping = self._to_lemma_mapping()\n",
    "        print(\"Creating reverse index\")\n",
    "        self.reverse_index = self._get_reverse_index(self.lemma_mapping, self.quotes)\n",
    "        # print(self.reverse_index)\n",
    "        print(\"Reverse index created\")\n",
    "        # print(self.reverse_index)\n",
    "        \n",
    "\n",
    "    def query(self, words: str) -> Set[int]:\n",
    "        if words[-1] == \"\\n\":\n",
    "            words = words[:-1]\n",
    "        splitted_words = self._to_lemmas(nltk.word_tokenize(words))\n",
    "        \n",
    "        # print(splitted_words)\n",
    "        result = self.reverse_index[splitted_words[0]].copy()\n",
    "        # print(result)\n",
    "        for word in splitted_words:\n",
    "            result &= self.reverse_index[word]\n",
    "            # print(result)\n",
    "        return result\n",
    "\n",
    "    def highlight(self, words):\n",
    "        indices = self.query(words)\n",
    "        words = self._to_lemmas(nltk.word_tokenize(words))\n",
    "        # print(indices)\n",
    "        for i in indices:\n",
    "            quote = self.quotes[i]\n",
    "            for word in quote:\n",
    "                if word in self.lemma_mapping and self.lemma_mapping[word] in words:\n",
    "                    print(f\"[{word}]\", end=\" \")\n",
    "                else:\n",
    "                    print(word, end=\" \")\n",
    "            print(\"\\n\")\n",
    "    \n",
    "    def group(self, lines):\n",
    "        result = dd(lambda:set())\n",
    "        for i, line in enumerate(lines):\n",
    "            query_result = frozenset(self.query(line))\n",
    "            # print(query_result)\n",
    "            result[query_result].add(i)\n",
    "            # for j in query_result:\n",
    "            #     result[j].add(i)\n",
    "        for _, v in result.items():\n",
    "            for i in v:\n",
    "                print(lines[i][:-1])\n",
    "            print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}