parser.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import re\n",
    "import json\n",
    "from lxml import html\n",
    "import urllib.request"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "URL_BASE = 'http://www.mathnet.ru'\n",
    "\n",
    "URL = '''http://www.mathnet.ru/php/archive.phtml?jrnid=uzku&wshow=issue&bshow=contents&series=0&year=2017&volume=158&issue=1&option_lang=rus&bookID=1621'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ANNOTATION_XPATH_EXPR = '''\n",
    "    //b[contains(text(), 'Аннотация')]/following-sibling::text()[not(preceding-sibling::b[contains(text(), 'Ключевые')])]\n",
    "'''\n",
    "\n",
    "TITLE_XPATH_EXPR = '''\n",
    "    //td//td//td//a[@class='SLink']\n",
    "'''\n",
    "\n",
    "KEYWORDS_XPATH_EXPR = '''\n",
    "    //b[contains(text(), 'Ключевые')]/following-sibling::i\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Stemmer:\n",
    "    # Helper regex strings.\n",
    "    _vowel = \"[аеиоуыэюя]\"\n",
    "    _non_vowel = \"[^аеиоуыэюя]\"\n",
    "\n",
    "    # Word regions.\n",
    "    _re_rv = re.compile(_vowel)\n",
    "    _re_r1 = re.compile(_vowel + _non_vowel)\n",
    "\n",
    "    # Endings.\n",
    "    _re_perfective_gerund = re.compile(\n",
    "        r\"(((?P<ignore>[ая])(в|вши|вшись))|(ив|ивши|ившись|ыв|ывши|ывшись))$\"\n",
    "    )\n",
    "    _re_adjective = re.compile(\n",
    "        r\"(ее|ие|ые|ое|ими|ыми|ей|ий|ый|ой|ем|им|ым|ом|его|ого|ему|ому|их|ых|\"\n",
    "        r\"ую|юю|ая|яя|ою|ею)$\"\n",
    "    )\n",
    "    _re_participle = re.compile(\n",
    "        r\"(((?P<ignore>[ая])(ем|нн|вш|ющ|щ))|(ивш|ывш|ующ))$\"\n",
    "    )\n",
    "    _re_reflexive = re.compile(\n",
    "        r\"(ся|сь)$\"\n",
    "    )\n",
    "    _re_verb = re.compile(\n",
    "        r\"(((?P<ignore>[ая])(ла|на|ете|йте|ли|й|л|ем|н|ло|но|ет|ют|ны|ть|ешь|\"\n",
    "        r\"нно))|(ила|ыла|ена|ейте|уйте|ите|или|ыли|ей|уй|ил|ыл|им|ым|ен|ило|\"\n",
    "        r\"ыло|ено|ят|ует|уют|ит|ыт|ены|ить|ыть|ишь|ую|ю))$\"\n",
    "    )\n",
    "    _re_noun = re.compile(\n",
    "        r\"(а|ев|ов|ие|ье|е|иями|ями|ами|еи|ии|и|ией|ей|ой|ий|й|иям|ям|ием|ем|\"\n",
    "        r\"ам|ом|о|у|ах|иях|ях|ы|ь|ию|ью|ю|ия|ья|я)$\"\n",
    "    )\n",
    "    _re_superlative = re.compile(\n",
    "        r\"(ейш|ейше)$\"\n",
    "    )\n",
    "    _re_derivational = re.compile(\n",
    "        r\"(ост|ость)$\"\n",
    "    )\n",
    "    _re_i = re.compile(\n",
    "        r\"и$\"\n",
    "    )\n",
    "    _re_nn = re.compile(\n",
    "        r\"((?<=н)н)$\"\n",
    "    )\n",
    "    _re_ = re.compile(\n",
    "        r\"ь$\"\n",
    "    )\n",
    "\n",
    "    def stem(self, word):\n",
    "        \"\"\"\n",
    "        Gets the stem.\n",
    "        \"\"\"\n",
    "\n",
    "        rv_pos, r2_pos = self._find_rv(word), self._find_r2(word)\n",
    "        word = self._step_1(word, rv_pos)\n",
    "        word = self._step_2(word, rv_pos)\n",
    "        word = self._step_3(word, r2_pos)\n",
    "        word = self._step_4(word, rv_pos)\n",
    "        return word\n",
    "\n",
    "    def _find_rv(self, word):\n",
    "        \"\"\"\n",
    "        Searches for the RV region.\n",
    "        \"\"\"\n",
    "\n",
    "        rv_match = self._re_rv.search(word)\n",
    "        if not rv_match:\n",
    "            return len(word)\n",
    "        return rv_match.end()\n",
    "\n",
    "    def _find_r2(self, word):\n",
    "        \"\"\"\n",
    "        Searches for the R2 region.\n",
    "        \"\"\"\n",
    "\n",
    "        r1_match = self._re_r1.search(word)\n",
    "        if not r1_match:\n",
    "            return len(word)\n",
    "        r2_match = self._re_r1.search(word, r1_match.end())\n",
    "        if not r2_match:\n",
    "            return len(word)\n",
    "        return r2_match.end()\n",
    "\n",
    "    def _cut(self, word, ending, pos):\n",
    "        \"\"\"\n",
    "        Tries to cut the specified ending after the specified position.\n",
    "        \"\"\"\n",
    "\n",
    "        match = ending.search(word, pos)\n",
    "        if match:\n",
    "            try:\n",
    "                ignore = match.group(\"ignore\") or \"\"\n",
    "            except IndexError:\n",
    "                # No ignored characters in pattern.\n",
    "                return True, word[:match.start()]\n",
    "            else:\n",
    "                # Do not cut ignored part.\n",
    "                return True, word[:match.start() + len(ignore)]\n",
    "        else:\n",
    "            return False, word\n",
    "\n",
    "    def _step_1(self, word, rv_pos):\n",
    "        match, word = self._cut(word, self._re_perfective_gerund, rv_pos)\n",
    "        if match:\n",
    "            return word\n",
    "        _, word = self._cut(word, self._re_reflexive, rv_pos)\n",
    "        match, word = self._cut(word, self._re_adjective, rv_pos)\n",
    "        if match:\n",
    "            _, word = self._cut(word, self._re_participle, rv_pos)\n",
    "            return word\n",
    "        match, word = self._cut(word, self._re_verb, rv_pos)\n",
    "        if match:\n",
    "            return word\n",
    "        _, word = self._cut(word, self._re_noun, rv_pos)\n",
    "        return word\n",
    "\n",
    "    def _step_2(self, word, rv_pos):\n",
    "        _, word = self._cut(word, self._re_i, rv_pos)\n",
    "        return word\n",
    "\n",
    "    def _step_3(self, word, r2_pos):\n",
    "        _, word = self._cut(word, self._re_derivational, r2_pos)\n",
    "        return word\n",
    "\n",
    "    def _step_4(self, word, rv_pos):\n",
    "        _, word = self._cut(word, self._re_superlative, rv_pos)\n",
    "        match, word = self._cut(word, self._re_nn, rv_pos)\n",
    "        if not match:\n",
    "            _, word = self._cut(word, self._re_, rv_pos)\n",
    "        return word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stemmer = Stemmer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def stem(s):\n",
    "    if re.match('^[А-Яа-я0-9]+[?:;.,!]$', s):\n",
    "        return stemmer.stem(s[:len(s) - 2]).lower() + s[-1]\n",
    "    else:\n",
    "        return stemmer.stem(s).lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pymystem3 import Mystem\n",
    "\n",
    "mystem = Mystem()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def main():\n",
    "    main_page = html.fromstring(urllib.request.urlopen(URL).read())\n",
    "\n",
    "    articles = []\n",
    "    \n",
    "    links = list(filter(lambda x: x.text is not None, main_page.xpath(TITLE_XPATH_EXPR)))\n",
    "    for link in links:\n",
    "        article_page = html.fromstring(urllib.request.urlopen(URL_BASE + link.get(\"href\")).read())\n",
    "\n",
    "        title = link.text;\n",
    "        title_porter = re.sub('\\t|\\n|\\.|\\,|!|\\?|:|;|–|\\(|\\)', '', ' '.join(map(stem, title.split(' '))))\n",
    "        title_mystem = re.sub('\\t|\\n|\\.|\\,|!|\\?|:|;|–|\\(|\\)', '', ''.join(mystem.lemmatize(title)))\n",
    "        \n",
    "        annotation = str.join('', article_page.xpath(ANNOTATION_XPATH_EXPR))\n",
    "        annotation = re.sub('\\t|\\n', '', annotation)\n",
    "        annotation_porter = re.sub('\\n|\\t|\\.|\\,|!|\\?|:|;|–|\\(|\\)', '', ' '.join(map(stem, annotation.split(' '))))\n",
    "        annotation_mystem = re.sub('\\n|\\t|\\.|\\,|!|\\?|:|;|–|\\(|\\)', '', ''.join(mystem.lemmatize(annotation)))\n",
    "        \n",
    "        keywords = list(filter(None, re.split(', ', article_page.xpath(KEYWORDS_XPATH_EXPR)[0].text)))\n",
    "\n",
    "        articles.append({\n",
    "            'title': {\n",
    "                'origin': title,\n",
    "                'porter': title_porter,\n",
    "                'mystem': title_mystem\n",
    "            },\n",
    "            'link': URL_BASE + link.get(\"href\"),\n",
    "            'annotation': {\n",
    "                'origin': annotation,\n",
    "                'porter': annotation_porter,\n",
    "                'mystem': annotation_mystem\n",
    "            },\n",
    "            'keywords': keywords\n",
    "        })\n",
    "\n",
    "    result = {\n",
    "        'issue': {\n",
    "            'URL': URL,\n",
    "            'articles': articles\n",
    "        }\n",
    "    }\n",
    "        \n",
    "    with open('articles_porter_mystem.json', 'w') as f:\n",
    "        json.dump(result, f, ensure_ascii=False, indent=4, sort_keys=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "main()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}