1.split_data/split_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Split feature data\n",
    "## Create tsv file with indexes for held out data, training data, and testing data\n",
    "### Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pathlib\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import sys\n",
    "sys.path.append(\"../utils\")\n",
    "from split_utils import get_features_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Specify datasets, test ratio, save path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# datasets to split\n",
    "datasets = [\"ic\", \"no_ic\"]\n",
    "\n",
    "# ratio of data to be used for testing (ex 0.15 = 15%)\n",
    "test_ratio = 0.15\n",
    "\n",
    "# make results dir for saving\n",
    "results_dir = pathlib.Path(\"indexes/\")\n",
    "results_dir.mkdir(parents=True, exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load, split, save data indexes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Splitting data for ic dataset\n",
      "Dataset shape: (2862, 1450)\n",
      "Training data has shape: (2432, 1450)\n",
      "Testing data has shape: (430, 1450)\n",
      "Saved index data\n",
      "\n",
      "Splitting data for no_ic dataset\n",
      "Dataset shape: (2841, 1450)\n",
      "Training data has shape: (2414, 1450)\n",
      "Testing data has shape: (427, 1450)\n",
      "Saved index data\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for dataset in datasets:\n",
    "    print(f\"Splitting data for {dataset} dataset\")\n",
    "    \n",
    "    # load x (features) and y (labels) dataframes\n",
    "    labeled_data_path = pathlib.Path(f\"../0.download_data/data/labeled_data__{dataset}.csv.gz\")\n",
    "    labeled_data = get_features_data(labeled_data_path)\n",
    "    print(f\"Dataset shape: {labeled_data.shape}\")\n",
    "\n",
    "    # get indexes of training and testing data\n",
    "    training_data, testing_data = train_test_split(\n",
    "        labeled_data,\n",
    "        test_size=test_ratio,\n",
    "        stratify=labeled_data[[\"Mitocheck_Phenotypic_Class\"]],\n",
    "        random_state=1,\n",
    "    )\n",
    "    train_indexes = training_data.index.to_numpy()\n",
    "    test_indexes = testing_data.index.to_numpy()\n",
    "\n",
    "    print(f\"Training data has shape: {training_data.shape}\")\n",
    "    print(f\"Testing data has shape: {testing_data.shape}\")\n",
    "    \n",
    "    # create pandas dataframe with all indexes and their respective labels, stratified by phenotypic class\n",
    "    index_data = []\n",
    "    for index in train_indexes:\n",
    "        index_data.append({\"labeled_data_index\": index, \"label\": \"train\"})\n",
    "    for index in test_indexes:\n",
    "        index_data.append({\"labeled_data_index\": index, \"label\": \"test\"})\n",
    "\n",
    "    # make index data a dataframe and sort it by labeled data index\n",
    "    index_data = (\n",
    "        pd.DataFrame(index_data)\n",
    "        .sort_values([\"labeled_data_index\"])\n",
    "    )\n",
    "    \n",
    "    # save indexes as tsv file\n",
    "    index_data.to_csv(f\"{results_dir}/data_split_indexes__{dataset}.tsv\", sep=\"\\t\")\n",
    "    print(f\"Saved index data\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preview index data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>labeled_data_index</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1359</th>\n",
       "      <td>0</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1554</th>\n",
       "      <td>1</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>2</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>957</th>\n",
       "      <td>3</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1390</th>\n",
       "      <td>4</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2609</th>\n",
       "      <td>2889</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114</th>\n",
       "      <td>2890</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1275</th>\n",
       "      <td>2891</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1882</th>\n",
       "      <td>2892</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1674</th>\n",
       "      <td>2893</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2841 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      labeled_data_index  label\n",
       "1359                   0  train\n",
       "1554                   1  train\n",
       "149                    2  train\n",
       "957                    3  train\n",
       "1390                   4  train\n",
       "...                  ...    ...\n",
       "2609                2889   test\n",
       "114                 2890  train\n",
       "1275                2891  train\n",
       "1882                2892  train\n",
       "1674                2893  train\n",
       "\n",
       "[2841 rows x 2 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "index_data"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.13 ('phenotypic_profiling')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "f9df586d1764dbc68785000a153dad1832127ac564b5e2e4c94e83fc43160b30"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}