Make_train_test_splits.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This competition is a \"Kernels only competition\" and each Kaggle kernel runs for nine hours on a GPU. In nine hours, BERT can run approximately half of the training examples. So the dataset was split into:\n",
    "\n",
    "1. Training part 1\n",
    "2. Training part 2\n",
    "3. Validation\n",
    "\n",
    "\n",
    "- Two forms of the data were saved, one which keeps the target as float, and one which binarizes the target. \n",
    "- BERT-base uses binary targets out of the box. A modified BERT was also used to enable using targets as float.\n",
    "- The evaluation metric provided on Kaggle converts floats into binary labels prior to evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import pathlib\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv(pathlib.Path.cwd().joinpath('data_raw', 'train.csv'))\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#alpha is dummy column for bert (shows that there are no sentence pairs 'a' and 'b')\n",
    "train['alpha'] = ['a']*train.shape[0]\n",
    "train = train[['id', 'target', 'alpha', 'comment_text']]\n",
    "train['comment_text'] = train['comment_text'].replace({r'\\s+$': '', r'^\\s+': ''}, regex=True).replace(r'\\n',  ' ', regex=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#split train and validation\n",
    "train_sub,dev = train_test_split(train, test_size=0.05, random_state=42)\n",
    "\n",
    "#shuffle train\n",
    "train_sub = train_sub.sample(frac=1, random_state=42).reset_index(drop=True)\n",
    "\n",
    "#split train into two parts\n",
    "train_sub_p1 = train_sub.iloc[0:train_sub.shape[0]//2,:]\n",
    "train_sub_p2 = train_sub.iloc[train_sub.shape[0]//2:,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#save as tsv files for bert, keep targets as float - this will be used with modified bert that accepts floats for targets\n",
    "train_sub.to_csv(pathlib.Path.cwd().joinpath('data_float','complete','train.tsv'), sep='\\t', header=False, index=False)\n",
    "dev.to_csv(pathlib.Path.cwd().joinpath('data_float','complete','dev.tsv'), sep='\\t', header=False, index=False)\n",
    "train_sub_p1.to_csv(pathlib.Path.cwd().joinpath('data_float','p1','train.tsv'), sep='\\t', header=False, index=False)\n",
    "train_sub_p2.to_csv(pathlib.Path.cwd().joinpath('data_float','p2','train.tsv'), sep='\\t', header=False, index=False)\n",
    "dev.to_csv(pathlib.Path.cwd().joinpath('data_float','p1','dev.tsv'), sep='\\t', header=False, index=False)\n",
    "dev.to_csv(pathlib.Path.cwd().joinpath('data_float','p2','dev.tsv'), sep='\\t', header=False, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#also save files with binarized targets\n",
    "train_bin = train_sub.copy(deep=True)\n",
    "dev_bin = dev.copy(deep=True)\n",
    "train_bin_p1 = train_sub_p1.copy(deep=True)\n",
    "train_bin_p2 = train_sub_p2.copy(deep=True)\n",
    "\n",
    "for df in [train_bin, dev_bin, train_bin_p1, train_bin_p2]:\n",
    "    df['target'] = (df['target']>=0.5).astype(int)\n",
    "\n",
    "train_bin.to_csv(pathlib.Path.cwd().joinpath('data_bin','complete','train.tsv'), sep='\\t', header=False, index=False)\n",
    "dev_bin.to_csv(pathlib.Path.cwd().joinpath('data_bin','complete','dev.tsv'), sep='\\t', header=False, index=False)\n",
    "train_bin_p1.to_csv(pathlib.Path.cwd().joinpath('data_bin','p1','train.tsv'), sep='\\t', header=False, index=False)\n",
    "train_bin_p2.to_csv(pathlib.Path.cwd().joinpath('data_bin','p2','train.tsv'), sep='\\t', header=False, index=False)\n",
    "dev_bin.to_csv(pathlib.Path.cwd().joinpath('data_bin','p1','dev.tsv'), sep='\\t', header=False, index=False)\n",
    "dev_bin.to_csv(pathlib.Path.cwd().joinpath('data_bin','p2','dev.tsv'), sep='\\t', header=False, index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}