L4Z5.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from collections import defaultdict as dd\n",
    "from typing import List, Dict, Set\n",
    "import nltk\n",
    "from math import log, ceil"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_lemma_mapping():\n",
    "    all_lemmas = dd(list)\n",
    "    for line in open('Dane/polimorfologik-2.1.txt', encoding='utf-8'):\n",
    "        L = line.split(';')[:2]\n",
    "        all_lemmas[L[1].lower()].append(L[0].lower())\n",
    "    return all_lemmas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "wiki_list = []\n",
    "key = \"\"\n",
    "with open(\"Dane/fp_wiki.txt\") as f:\n",
    "    cntr = -1\n",
    "    for line in f:\n",
    "        tokenized_line = line.split(\": \")\n",
    "        if tokenized_line[0] == \"TITLE\":\n",
    "            if tokenized_line[-1][-1] == \"\\n\":\n",
    "                tokenized_line[-1] = tokenized_line[-1][:-1]\n",
    "            cntr+=1\n",
    "            wiki_list.append((tokenized_line[-1], []))\n",
    "            continue\n",
    "        wiki_list[-1][1].append(nltk.word_tokenize(line))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "lemmas = to_lemma_mapping()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "positional_index = dd(list)\n",
    "for i, article in enumerate(wiki_list):\n",
    "    lines = article[1]\n",
    "    for words in lines:\n",
    "        for w in words:\n",
    "            if w not in lemmas:\n",
    "                continue\n",
    "            lemmas_ = lemmas[w]\n",
    "            for l in lemmas_:\n",
    "                if len(positional_index[l]) == 0 or positional_index[l][-1] != i+1:\n",
    "                    positional_index[l].append(i+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_vb_size(n, base):\n",
    "    bit_num = int(log(n, 2)) + 1\n",
    "    parts = ceil(bit_num / (base-1))\n",
    "    return parts*base"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_gamma_size(n):\n",
    "    m = int(log(n, 2))\n",
    "    return 2*m + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "int32size = 0\n",
    "vbsize = 0\n",
    "vb4size = 0\n",
    "gammasize = 0\n",
    "for k, v in positional_index.items():\n",
    "    prev_index = 0\n",
    "    for index in v:\n",
    "        int32size += 32\n",
    "        diff = index - prev_index\n",
    "#         print(diff)\n",
    "        prev_index = index\n",
    "        vbsize += compute_vb_size(diff, 8)\n",
    "        vb4size += compute_vb_size(diff, 4)      \n",
    "        gammasize += compute_gamma_size(diff)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1647220960\n"
     ]
    }
   ],
   "source": [
    "print(int32size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "521425384\n"
     ]
    }
   ],
   "source": [
    "print(vbsize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "431271520\n"
     ]
    }
   ],
   "source": [
    "print(vb4size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "474647051\n"
     ]
    }
   ],
   "source": [
    "print(gammasize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}