dasha_ensemble_news.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bf67f9a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import pymorphy2\n",
    "import pandas as pd\n",
    "from rake_nltk import Rake\n",
    "from sklearn import metrics\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9e9aa103",
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_corpus(dir_path: str, eval: str):\n",
    "    results = []\n",
    "    morph = pymorphy2.MorphAnalyzer()\n",
    "    r_model = Rake(language='russian')\n",
    "    for item in os.listdir(dir_path):\n",
    "        path = dir_path + '/' + item\n",
    "        if os.path.isfile(path):\n",
    "            with open(path, encoding='utf-8') as f:\n",
    "                text = f.read()\n",
    "                text = re.split('<\\d+>', text)\n",
    "                for instance in text:\n",
    "                    words = instance.split(' ')\n",
    "                    lemmatized_text = [morph.parse(word)[0].normal_form for word in words]\n",
    "                    lemmatized_text = ' '.join(lemmatized_text)\n",
    "                    r_model.extract_keywords_from_text(instance)\n",
    "                    raked_text = ' '.join(r_model.get_ranked_phrases())\n",
    "                    results.append((instance, lemmatized_text, raked_text, eval))\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9904ad45",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_data_news(rel_path: str):\n",
    "    path = os.path.abspath('') + rel_path\n",
    "    for item in os.listdir(path):\n",
    "        item_path = path + '/' + item\n",
    "        if os.path.isdir(item_path):\n",
    "            results = process_corpus(item_path, item)\n",
    "            df = pd.DataFrame(results)\n",
    "            df.to_csv('news_processed.csv', mode='a', index=False, header=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2fd3f6dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "get_data_news('/corpus_news')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9a2db973",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('news_processed.csv', names=['Raw', 'Lemmatized', 'Rake', 'Evaluation'])\n",
    "\n",
    "#df.head(-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0db15df8",
   "metadata": {},
   "outputs": [],
   "source": [
    "evals = df['Evaluation']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f3e51ec9",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer\n",
    "\n",
    "raw_text = df['Raw']\n",
    "lemmatized_text = df['Lemmatized']\n",
    "raked_text = df['Rake']\n",
    "evals = df['Evaluation']\n",
    "\n",
    "raw_train, raw_test, raw_target_train, raw_target_test = train_test_split(raw_text, evals, train_size=0.5, random_state=45)\n",
    "lemmatized_train, lemmatized_test, lemmatized_target_train, lemmatized_target_test = train_test_split(lemmatized_text, evals, train_size=0.5, random_state=45)\n",
    "raked_train, raked_test, raked_target_train, raked_target_test = train_test_split(raked_text, evals, train_size=0.5, random_state=45)\n",
    "\n",
    "#print(raw_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "02838966",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "     culture       0.99      1.00      1.00       831\n",
      "     hi-tech       1.00      1.00      1.00      1233\n",
      "    politics       1.00      1.00      1.00      2154\n",
      "     science       1.00      0.99      1.00      1052\n",
      "\n",
      "    accuracy                           1.00      5270\n",
      "   macro avg       1.00      1.00      1.00      5270\n",
      "weighted avg       1.00      1.00      1.00      5270\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "\n",
    "clf = Pipeline([('vect', CountVectorizer()),\n",
    "                ('tfdif', TfidfTransformer()),\n",
    "                ('clf', GradientBoostingClassifier())])\n",
    "\n",
    "clf.fit(raw_train, raw_target_train)\n",
    "\n",
    "gbc_predicted = clf.predict(raw_test)\n",
    "\n",
    "print(metrics.classification_report(raw_target_test, gbc_predicted))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e909256b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "     culture       0.99      1.00      1.00       831\n",
      "     hi-tech       1.00      1.00      1.00      1233\n",
      "    politics       1.00      1.00      1.00      2154\n",
      "     science       1.00      0.99      1.00      1052\n",
      "\n",
      "    accuracy                           1.00      5270\n",
      "   macro avg       1.00      1.00      1.00      5270\n",
      "weighted avg       1.00      1.00      1.00      5270\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "clf = Pipeline([('vect', CountVectorizer()),\n",
    "                ('tfdif', TfidfTransformer()),\n",
    "                ('clf', RandomForestClassifier())])\n",
    "\n",
    "clf.fit(raw_train, raw_target_train)\n",
    "\n",
    "rfc_predicted = clf.predict(raw_test)\n",
    "\n",
    "print(metrics.classification_report(raw_target_test, rfc_predicted))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7408c1b3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "     culture       0.99      1.00      1.00       831\n",
      "     hi-tech       1.00      1.00      1.00      1233\n",
      "    politics       1.00      1.00      1.00      2154\n",
      "     science       1.00      0.99      1.00      1052\n",
      "\n",
      "    accuracy                           1.00      5270\n",
      "   macro avg       1.00      1.00      1.00      5270\n",
      "weighted avg       1.00      1.00      1.00      5270\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "\n",
    "clf = Pipeline([('vect', CountVectorizer()),\n",
    "                ('tfdif', TfidfTransformer()),\n",
    "                ('clf', ExtraTreesClassifier())])\n",
    "\n",
    "clf.fit(raw_train, raw_target_train)\n",
    "\n",
    "efc_predicted = clf.predict(raw_test)\n",
    "\n",
    "print(metrics.classification_report(raw_target_test, efc_predicted))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b0ad93d0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "     culture       0.99      1.00      1.00       831\n",
      "     hi-tech       1.00      1.00      1.00      1233\n",
      "    politics       1.00      1.00      1.00      2154\n",
      "     science       1.00      0.99      1.00      1052\n",
      "\n",
      "    accuracy                           1.00      5270\n",
      "   macro avg       1.00      1.00      1.00      5270\n",
      "weighted avg       1.00      1.00      1.00      5270\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.svm import SVC\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "\n",
    "clf = Pipeline([('vect', CountVectorizer()),\n",
    "                ('tfdif', TfidfTransformer()),\n",
    "                ('clf', BaggingClassifier(estimator=SVC()))])\n",
    "\n",
    "clf.fit(raw_train, raw_target_train)\n",
    "\n",
    "bc_predicted = clf.predict(raw_test)\n",
    "\n",
    "print(metrics.classification_report(raw_target_test, bc_predicted))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "83630d9c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "     culture       0.62      0.34      0.44       831\n",
      "     hi-tech       1.00      1.00      1.00      1233\n",
      "    politics       0.75      0.97      0.85      2154\n",
      "     science       0.97      0.76      0.85      1052\n",
      "\n",
      "    accuracy                           0.83      5270\n",
      "   macro avg       0.84      0.77      0.78      5270\n",
      "weighted avg       0.83      0.83      0.82      5270\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "\n",
    "clf = Pipeline([('vect', CountVectorizer()),\n",
    "                ('tfdif', TfidfTransformer()),\n",
    "                ('clf', AdaBoostClassifier())])\n",
    "\n",
    "clf.fit(raw_train, raw_target_train)\n",
    "\n",
    "abc_predicted = clf.predict(raw_test)\n",
    "\n",
    "print(metrics.classification_report(raw_target_test, abc_predicted))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6f38396f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "     culture       0.99      1.00      1.00       831\n",
      "     hi-tech       1.00      1.00      1.00      1233\n",
      "    politics       1.00      1.00      1.00      2154\n",
      "     science       1.00      0.99      1.00      1052\n",
      "\n",
      "    accuracy                           1.00      5270\n",
      "   macro avg       1.00      1.00      1.00      5270\n",
      "weighted avg       1.00      1.00      1.00      5270\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import StackingClassifier\n",
    "\n",
    "estimators = [('rf', RandomForestClassifier(n_estimators=10, random_state=42))]\n",
    "\n",
    "clf = Pipeline([('vect', CountVectorizer()),\n",
    "                ('tfdif', TfidfTransformer()),\n",
    "                ('clf', StackingClassifier(estimators=estimators, final_estimator=LogisticRegression()))])\n",
    "\n",
    "clf.fit(raw_train, raw_target_train)\n",
    "\n",
    "stackclass_predicted = clf.predict(raw_test)\n",
    "\n",
    "print(metrics.classification_report(raw_target_test, stackclass_predicted))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "26d7e875",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "     culture       0.99      1.00      1.00       831\n",
      "     hi-tech       1.00      1.00      1.00      1233\n",
      "    politics       1.00      1.00      1.00      2154\n",
      "     science       1.00      0.99      1.00      1052\n",
      "\n",
      "    accuracy                           1.00      5270\n",
      "   macro avg       1.00      1.00      1.00      5270\n",
      "weighted avg       1.00      1.00      1.00      5270\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.ensemble import RandomForestClassifier, VotingClassifier\n",
    "\n",
    "clf1 = LogisticRegression(multi_class='multinomial', random_state=1)\n",
    "clf2 = RandomForestClassifier(n_estimators=50, random_state=1)\n",
    "clf3 = MultinomialNB()\n",
    "\n",
    "\n",
    "eclf1 = Pipeline([('vect', CountVectorizer()),\n",
    "                ('tfdif', TfidfTransformer()),\n",
    "                ('clf', VotingClassifier(estimators=[('lr', clf1), ('rf', clf2), ('mnb', clf3)], voting='hard'))])\n",
    "\n",
    "eclf1.fit(raw_train, raw_target_train)\n",
    "\n",
    "vc1_predicted = eclf1.predict(raw_test)\n",
    "\n",
    "print(metrics.classification_report(raw_target_test, vc1_predicted))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d5f69af3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "     culture       1.00      1.00      1.00       831\n",
      "     hi-tech       1.00      1.00      1.00      1233\n",
      "    politics       1.00      1.00      1.00      2154\n",
      "     science       1.00      1.00      1.00      1052\n",
      "\n",
      "    accuracy                           1.00      5270\n",
      "   macro avg       1.00      1.00      1.00      5270\n",
      "weighted avg       1.00      1.00      1.00      5270\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.ensemble import RandomForestClassifier, VotingClassifier\n",
    "\n",
    "clf1 = LogisticRegression(multi_class='multinomial', random_state=1)\n",
    "clf2 = RandomForestClassifier(n_estimators=50, random_state=1)\n",
    "clf3 = MultinomialNB()\n",
    "\n",
    "\n",
    "eclf2 = Pipeline([('vect', CountVectorizer()),\n",
    "                ('tfdif', TfidfTransformer()),\n",
    "                ('clf', VotingClassifier(estimators=[('lr', clf1), ('rf', clf2), ('mnb', clf3)], voting='soft'))])\n",
    "\n",
    "eclf2.fit(raw_train, raw_target_train)\n",
    "\n",
    "vc2_predicted = eclf2.predict(raw_test)\n",
    "\n",
    "print(metrics.classification_report(raw_target_test, vc2_predicted))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c7f0a21",
   "metadata": {},
   "source": [
    "------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "fa03da60",
   "metadata": {},
   "outputs": [],
   "source": [
    "classifiers = {\n",
    "    'AdaBoostClassifier': abc_predicted,\n",
    "    'BaggingClassifier': bc_predicted,\n",
    "    'ExtraTreesClassifier': efc_predicted\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "62b8054a",
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_target_test_array = raw_target_test.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "bf6aea84",
   "metadata": {},
   "outputs": [],
   "source": [
    "reports = {}\n",
    "for name, predicted in classifiers.items():\n",
    "    report = metrics.classification_report(raw_target_test_array, predicted, output_dict=True)\n",
    "    reports[name] = report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "e86551d7",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'AdaBoostClassifier': {'culture': {'precision': 0.6160520607375272,\n",
       "   'recall': 0.3417569193742479,\n",
       "   'f1-score': 0.43962848297213625,\n",
       "   'support': 831},\n",
       "  'hi-tech': {'precision': 1.0,\n",
       "   'recall': 1.0,\n",
       "   'f1-score': 1.0,\n",
       "   'support': 1233},\n",
       "  'politics': {'precision': 0.7532608695652174,\n",
       "   'recall': 0.9651810584958217,\n",
       "   'f1-score': 0.8461538461538463,\n",
       "   'support': 2154},\n",
       "  'science': {'precision': 0.9742647058823529,\n",
       "   'recall': 0.7557034220532319,\n",
       "   'f1-score': 0.8511777301927195,\n",
       "   'support': 1052},\n",
       "  'accuracy': 0.8332068311195446,\n",
       "  'macro avg': {'precision': 0.8358944090462744,\n",
       "   'recall': 0.7656603499808254,\n",
       "   'f1-score': 0.7842400148296755,\n",
       "   'support': 5270},\n",
       "  'weighted avg': {'precision': 0.8334705210824666,\n",
       "   'recall': 0.8332068311195446,\n",
       "   'f1-score': 0.8190485059066358,\n",
       "   'support': 5270}},\n",
       " 'BaggingClassifier': {'culture': {'precision': 0.992831541218638,\n",
       "   'recall': 1.0,\n",
       "   'f1-score': 0.9964028776978416,\n",
       "   'support': 831},\n",
       "  'hi-tech': {'precision': 1.0,\n",
       "   'recall': 1.0,\n",
       "   'f1-score': 1.0,\n",
       "   'support': 1233},\n",
       "  'politics': {'precision': 1.0,\n",
       "   'recall': 1.0,\n",
       "   'f1-score': 1.0,\n",
       "   'support': 2154},\n",
       "  'science': {'precision': 1.0,\n",
       "   'recall': 0.9942965779467681,\n",
       "   'f1-score': 0.9971401334604385,\n",
       "   'support': 1052},\n",
       "  'accuracy': 0.9988614800759014,\n",
       "  'macro avg': {'precision': 0.9982078853046595,\n",
       "   'recall': 0.998574144486692,\n",
       "   'f1-score': 0.9983857527895701,\n",
       "   'support': 5270},\n",
       "  'weighted avg': {'precision': 0.998869641509049,\n",
       "   'recall': 0.9988614800759014,\n",
       "   'f1-score': 0.9988618997660886,\n",
       "   'support': 5270}},\n",
       " 'ExtraTreesClassifier': {'culture': {'precision': 0.992831541218638,\n",
       "   'recall': 1.0,\n",
       "   'f1-score': 0.9964028776978416,\n",
       "   'support': 831},\n",
       "  'hi-tech': {'precision': 1.0,\n",
       "   'recall': 1.0,\n",
       "   'f1-score': 1.0,\n",
       "   'support': 1233},\n",
       "  'politics': {'precision': 1.0,\n",
       "   'recall': 1.0,\n",
       "   'f1-score': 1.0,\n",
       "   'support': 2154},\n",
       "  'science': {'precision': 1.0,\n",
       "   'recall': 0.9942965779467681,\n",
       "   'f1-score': 0.9971401334604385,\n",
       "   'support': 1052},\n",
       "  'accuracy': 0.9988614800759014,\n",
       "  'macro avg': {'precision': 0.9982078853046595,\n",
       "   'recall': 0.998574144486692,\n",
       "   'f1-score': 0.9983857527895701,\n",
       "   'support': 5270},\n",
       "  'weighted avg': {'precision': 0.998869641509049,\n",
       "   'recall': 0.9988614800759014,\n",
       "   'f1-score': 0.9988618997660886,\n",
       "   'support': 5270}}}"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3114da9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}