generate_challenging_test_cases.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate hard test cases\n",
    "\n",
    "#### Motivation\n",
    "Since we have achieved a high baseline (97.8% F1 score) on the seizure dataset, the potential for improvement on the overall dataset is limited. Therefore, we will create challenging test cases where seizure classification is more difficult to evaluate the model's robustness.\n",
    "\n",
    "#### Methodologies\n",
    "Use SVM and ANN(Artificial Neural Network) to generate hard test cases\n",
    "\n",
    "1. **Misclassified Data**:\n",
    "   - Identify data misclassified by SVM and ANN.\n",
    "\n",
    "2. **Data Near the Decision Boundary**:\n",
    "   - **SVM**: Data whose distance to the hyperplanes is within a certain threshold.\n",
    "   - **ANN**: Data whose predicted probability is around 0.5.\n",
    "\n",
    "3. **Support Vectors (SVM)**:\n",
    "   - Identify data points classified as support vectors by the SVM."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing necessary libraries\n",
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_auc_score, cohen_kappa_score, accuracy_score, classification_report\n",
    "\n",
    "import keras.backend as K\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-20T02:37:01.180011Z",
     "start_time": "2024-10-20T02:37:01.092757Z"
    }
   },
   "outputs": [],
   "source": [
    "# Read data\n",
    "df = pd.read_csv(\"data/data_cleaned.csv\")\n",
    "X = df.loc[:, 'X1':'X178']  # All feature columns\n",
    "y = df['y']  # Labels (non-seizure/seizure)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Necassary functions\n",
    "\n",
    "def evaluate_classification_metrics(y_ground_truth, y_pred, y_pred_prob):\n",
    "    # Calculate accuracy\n",
    "    accuracy = accuracy_score(y_ground_truth, y_pred)\n",
    "    \n",
    "    # Calculate precision\n",
    "    precision = precision_score(y_ground_truth, y_pred)\n",
    "    \n",
    "    # Calculate recall\n",
    "    recall = recall_score(y_ground_truth, y_pred)\n",
    "    \n",
    "    # Calculate F1 score\n",
    "    f1 = f1_score(y_ground_truth, y_pred, average='weighted')\n",
    "    \n",
    "    if len(set(y_ground_truth)) > 1:\n",
    "        # Check if ROC-AUC can be calculated (i.e., both classes are present)\n",
    "        roc_auc = roc_auc_score(y_ground_truth, y_pred_prob)\n",
    "        # Calculate Cohen's Kappa\n",
    "        kappa = cohen_kappa_score(y_ground_truth, y_pred)\n",
    "    else:\n",
    "        roc_auc = None  # Not computable, only one class in y_true\n",
    "        kappa = None\n",
    "    \n",
    "    # Calculate metrics for seizure class (y_label=1)\n",
    "    precision_seizure = precision_score(y_ground_truth, y_pred, pos_label=1)\n",
    "    recall_seizure = recall_score(y_ground_truth, y_pred, pos_label=1)\n",
    "    f1_seizure = f1_score(y_ground_truth, y_pred, pos_label=1)\n",
    "    \n",
    "    # Calculate metrics for non-seizure class (y_label=0)\n",
    "    precision_non_seizure = precision_score(y_ground_truth, y_pred, pos_label=0)\n",
    "    recall_non_seizure = recall_score(y_ground_truth, y_pred, pos_label=0)\n",
    "    f1_non_seizure = f1_score(y_ground_truth, y_pred, pos_label=0)\n",
    "    \n",
    "    print(f'\\nSeizure (y=1):')\n",
    "    print(f'  Precision: {precision_seizure * 100:.2f} %')\n",
    "    print(f'  Recall: {recall_seizure * 100:.2f} %')\n",
    "    print(f'  F1 Score: {f1_seizure * 100:.2f} %')\n",
    "    \n",
    "    print(f'\\nNon-Seizure (y=0):')\n",
    "    print(f'  Precision: {precision_non_seizure * 100:.2f} %')\n",
    "    print(f'  Recall: {recall_non_seizure * 100:.2f} %')\n",
    "    print(f'  F1 Score: {f1_non_seizure * 100:.2f} %')\n",
    "    \n",
    "    # Print metrics\n",
    "    print(f'\\nOverall:')\n",
    "    print(f'  Accuracy: {accuracy * 100:.2f} %')\n",
    "    print(f'  Precision: {precision * 100:.2f} %')\n",
    "    print(f'  Recall: {recall * 100:.2f} %')\n",
    "    print(f'  F1 Score: {f1 * 100:.2f} %')\n",
    "    if roc_auc is not None:\n",
    "        print(f'  ROC-AUC: {roc_auc * 100:.2f} %')\n",
    "    if kappa is not None:\n",
    "        print(f'  Cohen\\'s Kappa: {kappa * 100:.2f} %')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-20T02:43:19.392215Z",
     "start_time": "2024-10-20T02:43:15.946393Z"
    }
   },
   "outputs": [],
   "source": [
    "# Define ANN\n",
    "\n",
    "def ann(X_train, y_train):\n",
    "    # Initializing the ANN\n",
    "    classifier = Sequential()\n",
    "    \n",
    "    # Adding the input layer and the first hidden layer\n",
    "    classifier.add(Dense(units=80, kernel_initializer='uniform', activation='relu', input_dim=178))\n",
    "    \n",
    "    # Adding the second hidden layer\n",
    "    classifier.add(Dense(units=80, kernel_initializer='uniform', activation='relu'))\n",
    "    \n",
    "    # Adding the output layer\n",
    "    classifier.add(Dense(units=1, kernel_initializer='uniform', activation='sigmoid'))\n",
    "    \n",
    "    classifier.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n",
    "    \n",
    "    # Fitting the ANN to the training set without printing the epoch output\n",
    "    classifier.fit(X_train, y_train, batch_size=10, epochs=100, verbose=0)\n",
    "    \n",
    "    return classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Print the metrics on the hard test cases\n",
    "verbose = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Misclassified Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-20T02:39:43.944643Z",
     "start_time": "2024-10-20T02:39:43.941451Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_error_points(X_test, y_test, y_pred, y_pred_prob):\n",
    "    # Identify error cases (misclassified points)\n",
    "    error_mask = y_test != y_pred\n",
    "    X_error = X_test[error_mask]\n",
    "    y_error_true = y_test[error_mask]\n",
    "    y_error_pred = y_pred[error_mask]\n",
    "    y_error_prob = y_pred_prob[error_mask]\n",
    "    \n",
    "    if verbose:\n",
    "        evaluate_classification_metrics(y_error_true, y_error_pred, y_error_prob)\n",
    "        \n",
    "    return X_error, y_error_true"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-20T02:39:51.802718Z",
     "start_time": "2024-10-20T02:39:45.948718Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "--------------------misclassified_data_svm--------------------\n",
      "\n",
      "Seizure (y=1):\n",
      "  Precision: 0.00 %\n",
      "  Recall: 0.00 %\n",
      "  F1 Score: 0.00 %\n",
      "\n",
      "Non-Seizure (y=0):\n",
      "  Precision: 0.00 %\n",
      "  Recall: 0.00 %\n",
      "  F1 Score: 0.00 %\n",
      "\n",
      "Overall:\n",
      "  Accuracy: 0.00 %\n",
      "  Precision: 0.00 %\n",
      "  Recall: 0.00 %\n",
      "  F1 Score: 0.00 %\n",
      "  ROC-AUC: 0.00 %\n",
      "  Cohen's Kappa: -73.86 %\n",
      "\n",
      "Number of error cases: 49\n",
      "Label distribution in error cases (actual labels):\n",
      " 0    34\n",
      "1    15\n",
      "Name: y, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "def misclassified_data_svm():\n",
    "    if verbose:\n",
    "        print (\"\\n--------------------misclassified_data_svm--------------------\")\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "    clf = SVC(class_weight='balanced', probability=True, random_state=42)\n",
    "    clf.fit(X_train, y_train)\n",
    "    y_pred = clf.predict(X_test)\n",
    "    y_pred_prob = clf.predict_proba(X_test)[:, 1]\n",
    "    X_error, y_error = get_error_points(X_test, y_test, y_pred, y_pred_prob)\n",
    "    return X_error, y_error\n",
    "\n",
    "X_error, y_error = misclassified_data_svm()\n",
    "print(f\"\\nNumber of error cases: {len(y_error)}\")\n",
    "print(f\"Label distribution in error cases (actual labels):\\n {pd.Series(y_error).value_counts()}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ANN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-20T02:44:07.581662Z",
     "start_time": "2024-10-20T02:43:38.425531Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "--------------------misclassified_data_ann--------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/mikky/miniconda3/envs/epileptic/lib/python3.9/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n",
      "  super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m72/72\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 445us/step\n",
      "\n",
      "Seizure (y=1):\n",
      "  Precision: 0.00 %\n",
      "  Recall: 0.00 %\n",
      "  F1 Score: 0.00 %\n",
      "\n",
      "Non-Seizure (y=0):\n",
      "  Precision: 0.00 %\n",
      "  Recall: 0.00 %\n",
      "  F1 Score: 0.00 %\n",
      "\n",
      "Overall:\n",
      "  Accuracy: 0.00 %\n",
      "  Precision: 0.00 %\n",
      "  Recall: 0.00 %\n",
      "  F1 Score: 0.00 %\n",
      "  ROC-AUC: 0.00 %\n",
      "  Cohen's Kappa: -50.97 %\n",
      "\n",
      "Number of error cases: 93\n",
      "Label distribution in error cases (actual labels):\n",
      "1    73\n",
      "0    20\n",
      "Name: y, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "def misclassified_data_ann():\n",
    "    if verbose:\n",
    "        print (\"\\n--------------------misclassified_data_ann--------------------\")\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "    ann_clf = ann(X_train, y_train)\n",
    "    y_pred_prob = ann_clf.predict(X_test)\n",
    "    y_pred = (y_pred_prob > 0.5).astype(int).ravel()\n",
    "    X_error, y_error = get_error_points(X_test, y_test, y_pred, y_pred_prob)\n",
    "    return X_error, y_error\n",
    "\n",
    "X_error, y_error = misclassified_data_ann()\n",
    "print(f\"\\nNumber of error cases: {len(y_error)}\")\n",
    "print(f\"Label distribution in error cases (actual labels):\\n{pd.Series(y_error).value_counts()}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Data Near the Decision Boundary\n",
    "### SVM\n",
    "- Points within a distance to the hyperplanes.\n",
    "- Points are from the test data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_near_decision_boundary_svm():\n",
    "    # Define distance threshold from hyperplanes\n",
    "    distance_threshold = 0.3\n",
    "\n",
    "    # Train a SVM\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "    clf = SVC(class_weight='balanced', probability=True, random_state=42)\n",
    "    clf.fit(X_train, y_train)\n",
    "    y_pred = clf.predict(X_test)\n",
    "\n",
    "    # Get the decision function (distance from the decision boundary)\n",
    "    decision_distances = clf.decision_function(X_test)\n",
    "    # Find the cases where the absolute value of the distance is close to 0 (i.e., near the boundary)\n",
    "    boundary_cases = np.where(np.abs(decision_distances) < distance_threshold)[0]\n",
    "    \n",
    "    # Get the vectors and labels near the decision boundary\n",
    "    X_near_boundary = X_test.iloc[boundary_cases]\n",
    "    y_near_boundary = y_test.iloc[boundary_cases]\n",
    "    y_pred_near_boundary = y_pred[boundary_cases]\n",
    "    decision_distances_near_boundary = decision_distances[boundary_cases]  # Filter decision distances\n",
    "    \n",
    "    if verbose:\n",
    "        print (\"\\n--------------------data_near_decision_boundary_svm--------------------\")\n",
    "        evaluate_classification_metrics(y_near_boundary, y_pred_near_boundary, decision_distances_near_boundary)\n",
    "\n",
    "    return X_near_boundary, y_near_boundary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-20T20:32:24.363405Z",
     "start_time": "2024-10-20T20:32:18.233413Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "--------------------data_near_decision_boundary_svm--------------------\n",
      "\n",
      "Seizure (y=1):\n",
      "  Precision: 50.00 %\n",
      "  Recall: 70.59 %\n",
      "  F1 Score: 58.54 %\n",
      "\n",
      "Non-Seizure (y=0):\n",
      "  Precision: 73.68 %\n",
      "  Recall: 53.85 %\n",
      "  F1 Score: 62.22 %\n",
      "\n",
      "Overall:\n",
      "  Accuracy: 60.47 %\n",
      "  Precision: 50.00 %\n",
      "  Recall: 70.59 %\n",
      "  F1 Score: 60.77 %\n",
      "  ROC-AUC: 76.02 %\n",
      "  Cohen's Kappa: 22.81 %\n",
      "\n",
      "Number of points near the decision boundary: 43\n",
      "Label distribution in points near the decision boundary (actual labels):\n",
      "0    26\n",
      "1    17\n",
      "Name: y, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "X_near_boundary, y_near_boundary = data_near_decision_boundary_svm()\n",
    "print(f\"\\nNumber of points near the decision boundary: {y_near_boundary.shape[0]}\")\n",
    "print(f\"Label distribution in points near the decision boundary (actual labels):\\n{pd.Series(y_near_boundary).value_counts()}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ANN\n",
    "- Get points where the predicted probability is near the decision threshold (e.g., near 0.5).\n",
    "- prob=0 means non-seizure, prob=1 means seizure, prob=0.5 means not sure.\n",
    "- Assuming distance threshold is 0.2, any data points whose probabilities is within 0.3 to 0.7 will be considered near decision boundary."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-20T20:35:32.930512Z",
     "start_time": "2024-10-20T20:35:32.924822Z"
    }
   },
   "outputs": [],
   "source": [
    "def data_near_decision_boundary_ann():\n",
    "\n",
    "    # Define the distance threshold\n",
    "    distance_threshold = 0.2\n",
    "\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "    ann_clf = ann(X_train, y_train)\n",
    "    y_pred_prob = ann_clf.predict(X_test)\n",
    "    y_pred = (y_pred_prob > 0.5).astype(int).ravel()\n",
    "    boundary_cases = np.where(np.abs(y_pred_prob - 0.5) < distance_threshold)[0]\n",
    "    \n",
    "    X_near_boundary = X_test.iloc[boundary_cases]\n",
    "    y_near_boundary = y_test.iloc[boundary_cases]\n",
    "    y_pred_near_boundary = y_pred[boundary_cases]\n",
    "    decision_prob_near_boundary = y_pred_prob[boundary_cases]\n",
    "    \n",
    "    if verbose:\n",
    "        print (\"\\n--------------------data_near_decision_boundary_ann--------------------\")\n",
    "        evaluate_classification_metrics(y_near_boundary, y_pred_near_boundary, decision_prob_near_boundary)\n",
    "    return X_near_boundary, y_near_boundary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-20T20:36:07.689359Z",
     "start_time": "2024-10-20T20:35:35.857539Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/mikky/miniconda3/envs/epileptic/lib/python3.9/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n",
      "  super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m72/72\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 429us/step\n",
      "\n",
      "--------------------data_near_decision_boundary_ann--------------------\n",
      "\n",
      "Seizure (y=1):\n",
      "  Precision: 18.18 %\n",
      "  Recall: 25.00 %\n",
      "  F1 Score: 21.05 %\n",
      "\n",
      "Non-Seizure (y=0):\n",
      "  Precision: 45.45 %\n",
      "  Recall: 35.71 %\n",
      "  F1 Score: 40.00 %\n",
      "\n",
      "Overall:\n",
      "  Accuracy: 31.82 %\n",
      "  Precision: 18.18 %\n",
      "  Recall: 25.00 %\n",
      "  F1 Score: 33.11 %\n",
      "  ROC-AUC: 24.11 %\n",
      "  Cohen's Kappa: -36.36 %\n",
      "\n",
      "Number of points near the decision boundary: 22\n",
      "Label distribution in points near the decision boundary (actual labels):\n",
      "0    14\n",
      "1     8\n",
      "Name: y, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "X_near_boundary, y_near_boundary = data_near_decision_boundary_ann()\n",
    "print(f\"\\nNumber of points near the decision boundary: {y_near_boundary.shape[0]}\")\n",
    "print(f\"Label distribution in points near the decision boundary (actual labels):\\n{pd.Series(y_near_boundary).value_counts()}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Support Vectors (SVM)\n",
    "- Support vectors are the data points that lie closest to the decision boundary. These points have the highest influence on determining the position of the boundary.\n",
    "- These points are from training data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-20T20:52:21.672054Z",
     "start_time": "2024-10-20T20:52:21.668462Z"
    }
   },
   "outputs": [],
   "source": [
    "def data_support_vectors():\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "    clf = SVC(class_weight='balanced', probability=True, random_state=42)\n",
    "    clf.fit(X_train, y_train)\n",
    "\n",
    "    support_vectors_indices = clf.support_\n",
    "    support_vectors = clf.support_vectors_\n",
    "    X_support_vectors = X_train.iloc[support_vectors_indices]\n",
    "    y_support_vectors = y_train.iloc[support_vectors_indices]\n",
    "\n",
    "    if verbose:\n",
    "        print (\"\\n--------------------support_vectors_svm--------------------\")\n",
    "        y_pred_within_support_vectors = clf.predict(support_vectors)\n",
    "        y_pred_prob_within_support_vectors = clf.decision_function(support_vectors)\n",
    "        evaluate_classification_metrics(y_support_vectors, y_pred_within_support_vectors, y_pred_prob_within_support_vectors)\n",
    "        \n",
    "    return X_support_vectors, y_support_vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-20T20:52:27.771605Z",
     "start_time": "2024-10-20T20:52:22.256799Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "--------------------support_vectors_svm--------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/mikky/miniconda3/envs/epileptic/lib/python3.9/site-packages/sklearn/base.py:493: UserWarning: X does not have valid feature names, but SVC was fitted with feature names\n",
      "  warnings.warn(\n",
      "/Users/mikky/miniconda3/envs/epileptic/lib/python3.9/site-packages/sklearn/base.py:493: UserWarning: X does not have valid feature names, but SVC was fitted with feature names\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Seizure (y=1):\n",
      "  Precision: 62.12 %\n",
      "  Recall: 90.11 %\n",
      "  F1 Score: 73.54 %\n",
      "\n",
      "Non-Seizure (y=0):\n",
      "  Precision: 96.24 %\n",
      "  Recall: 82.16 %\n",
      "  F1 Score: 88.65 %\n",
      "\n",
      "Overall:\n",
      "  Accuracy: 84.11 %\n",
      "  Precision: 62.12 %\n",
      "  Recall: 90.11 %\n",
      "  F1 Score: 84.95 %\n",
      "  ROC-AUC: 87.44 %\n",
      "  Cohen's Kappa: 62.73 %\n",
      "\n",
      "Number of support vectors: 1114\n",
      "Label distribution of support vectors (actual labels):\n",
      " 0    841\n",
      "1    273\n",
      "Name: y, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "X_support_vectors, y_support_vectors = data_support_vectors()\n",
    "print(f\"\\nNumber of support vectors: {y_support_vectors.shape[0]}\")\n",
    "# Convert y_support_vectors to Pandas Series for label distribution\n",
    "y_support_vectors_series = pd.Series(y_support_vectors)\n",
    "print(\"Label distribution of support vectors (actual labels):\\n\", y_support_vectors_series.value_counts())  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Function to get hard test cases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-20T21:39:39.763964Z",
     "start_time": "2024-10-20T21:39:39.759807Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_hard_test_cases(method_list):\n",
    "    total_test_data = []\n",
    "    for method in method_list:\n",
    "        result = eval(method + '()')\n",
    "\n",
    "        # Get hard test data\n",
    "        if isinstance(result, tuple):\n",
    "            X_test, y_test = result\n",
    "            y_test = pd.Series(y_test, name='y')\n",
    "            combined_df = pd.concat([X_test, y_test], axis=1)\n",
    "        total_test_data.append(combined_df)\n",
    "        total_test_data_df = pd.concat(total_test_data)\n",
    "\n",
    "    # Drop duplicate rows\n",
    "    hard_test_df_cleaned = total_test_data_df.drop_duplicates()\n",
    "    print (\"\\nNumber of hard test cases:\", hard_test_df_cleaned.shape[0])\n",
    "    print (\"Distribution of label: \", hard_test_df_cleaned['y'].value_counts())\n",
    "\n",
    "    # Turn rest of the dataset into training data\n",
    "    train_data = df.loc[~df.index.isin(hard_test_df_cleaned.index), df.columns[df.columns.get_loc('X1'):df.columns.get_loc('X178') + 1].tolist() + ['y']]\n",
    "    hard_test_df_cleaned.sort_index(inplace=True)\n",
    "\n",
    "    return train_data, hard_test_df_cleaned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-20T21:39:45.709501Z",
     "start_time": "2024-10-20T21:39:40.288490Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "--------------------misclassified_data_svm--------------------\n",
      "\n",
      "Seizure (y=1):\n",
      "  Precision: 0.00 %\n",
      "  Recall: 0.00 %\n",
      "  F1 Score: 0.00 %\n",
      "\n",
      "Non-Seizure (y=0):\n",
      "  Precision: 0.00 %\n",
      "  Recall: 0.00 %\n",
      "  F1 Score: 0.00 %\n",
      "\n",
      "Overall:\n",
      "  Accuracy: 0.00 %\n",
      "  Precision: 0.00 %\n",
      "  Recall: 0.00 %\n",
      "  F1 Score: 0.00 %\n",
      "  ROC-AUC: 0.00 %\n",
      "  Cohen's Kappa: -73.86 %\n",
      "\n",
      "Number of hard test cases: 49\n",
      "Distribution of label:  0    34\n",
      "1    15\n",
      "Name: y, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "method_list = [\n",
    "                \"misclassified_data_svm\",\n",
    "                \"misclassified_data_ann\",\n",
    "                \"data_near_decision_boundary_svm\",\n",
    "                \"data_near_decision_boundary_ann\",\n",
    "                \"data_support_vectors\",\n",
    "                ]\n",
    "train_data, test_data = get_hard_test_cases(method_list)\n",
    "\n",
    "os.makedirs(\"data/hard_test_cases\", exist_ok = True)\n",
    "train_data.to_csv('data/hard_test_cases/train.csv')\n",
    "test_data.to_csv('datahard_test_cases/test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-20T21:41:44.673484Z",
     "start_time": "2024-10-20T21:41:43.407580Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Seizure (y=1):\n",
      "  Precision: 0.00 %\n",
      "  Recall: 0.00 %\n",
      "  F1 Score: 0.00 %\n",
      "\n",
      "Non-Seizure (y=0):\n",
      "  Precision: 31.82 %\n",
      "  Recall: 20.59 %\n",
      "  F1 Score: 25.00 %\n",
      "\n",
      "Overall:\n",
      "  Accuracy: 14.29 %\n",
      "  Precision: 0.00 %\n",
      "  Recall: 0.00 %\n",
      "  F1 Score: 17.35 %\n",
      "  ROC-AUC: 3.92 %\n",
      "  Cohen's Kappa: -64.90 %\n"
     ]
    }
   ],
   "source": [
    "# Evaluate performance on the hard test cases using SVM\n",
    "\n",
    "train_data = pd.read_csv('hard_test_cases/train.csv')\n",
    "test_data = pd.read_csv('hard_test_cases/test.csv')\n",
    "\n",
    "X_train = train_data.drop(columns=['y']).values\n",
    "y_train = train_data['y'].values\n",
    "X_test = test_data.drop(columns=['y']).values\n",
    "y_test = test_data['y'].values\n",
    "\n",
    "# Train the SVM model\n",
    "clf = SVC(class_weight='balanced', probability=True, random_state=42)\n",
    "clf.fit(X_train, y_train)\n",
    "y_pred = clf.predict(X_test)\n",
    "decision_distances = clf.decision_function(X_test)\n",
    "\n",
    "evaluate_classification_metrics(y_test, y_pred, decision_distances)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "epileptic",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.20"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}