analyzing_overall_results.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:57.831731Z",
     "start_time": "2024-12-07T21:26:51.077401Z"
    }
   },
   "source": [
    "import json\n",
    "import os.path\n",
    "from collections.abc import Callable\n",
    "from statistics import mean\n",
    "from typing import Literal, NamedTuple\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from evaluation_utils import MetricsForDatasetProbes, ConfusionMetrics, TraditionalMetrics\n",
    "from phi_3_5_constants import dsets_index_path, four_way_topics_index_path, misc_datasets_index_path, \\\n",
    "    directions_reconstruction_losses_path, separation_by_layer_analysis_path, train_split_classification_metrics_path, \\\n",
    "    validation_split_classification_metrics_path, test_classification_metrics_path"
   ],
   "outputs": [],
   "execution_count": 1
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "",
   "id": "6ea586ec4f8ec0c8"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:57.839161Z",
     "start_time": "2024-12-07T21:26:57.832737Z"
    }
   },
   "cell_type": "code",
   "source": [
    "dsets_index_df = pd.read_csv(dsets_index_path, index_col=\"Idx\")\n",
    "num_dsets = dsets_index_df.shape[0]"
   ],
   "id": "c9e2862acb47faeb",
   "outputs": [],
   "execution_count": 2
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:57.844553Z",
     "start_time": "2024-12-07T21:26:57.840167Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# top-level key is the name of a topic that has pos/neg/conj/disj variants, second level key is one of those variant names\n",
    "with four_way_topics_index_path.open(\"r\") as f:\n",
    "    dset_idxs_for_4way_topics: dict[str, dict[str, int]] = json.load(f)\n",
    "with misc_datasets_index_path.open(\"r\") as f:\n",
    "    idxs_for_other_dsets: dict[str, int] = json.load(f)"
   ],
   "id": "e117f3a660c753be",
   "outputs": [],
   "execution_count": 3
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:57.851018Z",
     "start_time": "2024-12-07T21:26:57.845558Z"
    }
   },
   "cell_type": "code",
   "source": [
    "for dset_idx, dset_dtls in dsets_index_df.iterrows():\n",
    "    categ_nm = dset_dtls[\"Categ_Folder\"]    \n",
    "    if dset_dtls['in_german']:\n",
    "        assert categ_nm in dset_idxs_for_4way_topics, f\"{categ_nm} topic has german variant but wasn't captured in earlier tracking of standard ('4-way') topics\"\n",
    "        if dset_dtls['is_negated']:\n",
    "            dset_idxs_for_4way_topics[categ_nm][\"de_neg\"] = dset_idx\n",
    "        else:\n",
    "            dset_idxs_for_4way_topics[categ_nm][\"de_affirm\"] = dset_idx"
   ],
   "id": "da3099d3ad1e4698",
   "outputs": [],
   "execution_count": 4
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:57.856159Z",
     "start_time": "2024-12-07T21:26:57.852023Z"
    }
   },
   "cell_type": "code",
   "source": [
    "affirm_german_dset_idxs = dsets_index_df.loc[dsets_index_df.in_german & (~dsets_index_df.is_negated)].index.to_list()\n",
    "negated_german_dset_idxs = dsets_index_df.loc[dsets_index_df.in_german & dsets_index_df.is_negated].index.to_list()"
   ],
   "id": "8d430ff5d0402b7b",
   "outputs": [],
   "execution_count": 5
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:57.863224Z",
     "start_time": "2024-12-07T21:26:57.857164Z"
    }
   },
   "cell_type": "code",
   "source": [
    "directions_reconstruction_losses = pd.read_csv(directions_reconstruction_losses_path)\n",
    "\n",
    "dir_probe_scenarios_strs = directions_reconstruction_losses['scenario_identifier'].to_list()\n",
    "dir_probe_scenarios_specs = [tuple(map(int, scenario_str.split(' '))) for scenario_str in dir_probe_scenarios_strs]"
   ],
   "id": "fb38b28c40aa367b",
   "outputs": [],
   "execution_count": 6
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-08T18:25:54.423765Z",
     "start_time": "2024-12-08T18:25:54.419384Z"
    }
   },
   "cell_type": "code",
   "source": [
    "num_standard_topics = dsets_index_df[dsets_index_df[\"is_other\"] == False][\"Categ_Folder\"].nunique()\n",
    "assert num_standard_topics == len(dset_idxs_for_4way_topics)"
   ],
   "id": "5060e7b84ef6e579",
   "outputs": [],
   "execution_count": 87
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:57.874193Z",
     "start_time": "2024-12-07T21:26:57.864228Z"
    }
   },
   "cell_type": "code",
   "source": [
    "#'scenarios' describe the different combinations of data that directions/probes were trained on \n",
    "other_categs: set[str] = set(dsets_index_df.loc[dsets_index_df.is_other].Categ_Folder.to_list())\n",
    "\n",
    "scenario_standard_categs: dict[tuple[int,...], list[str]] = {}\n",
    "scenario_other_dset_names: dict[tuple[int,...], list[str]] = {}\n",
    "for scenario_id in dir_probe_scenarios_specs:\n",
    "    scenario_standard_categs[scenario_id] = []\n",
    "    scenario_other_dset_names[scenario_id] = []\n",
    "    for dset_idx_in_scenario in scenario_id:\n",
    "        if not dsets_index_df.loc[dset_idx_in_scenario, \"is_other\"]:\n",
    "            curr_standard_categ = dsets_index_df.loc[dset_idx_in_scenario, \"Categ_Folder\"]\n",
    "            if curr_standard_categ not in scenario_standard_categs[scenario_id]:\n",
    "                scenario_standard_categs[scenario_id].append(curr_standard_categ)\n",
    "        else:\n",
    "            curr_other_dset_name = os.path.splitext(dsets_index_df.loc[dset_idx_in_scenario, \"Dataset_File\"])[0]\n",
    "            assert curr_other_dset_name not in scenario_other_dset_names[scenario_id]\n",
    "            scenario_other_dset_names[scenario_id].append(curr_other_dset_name)\n",
    "\n",
    "scenario_data_variants: dict[tuple[int,...], set[Literal[\"affirm\", \"neg\", \"conj\", \"disj\", \"other\"]]] = {}\n",
    "for scenario_id in dir_probe_scenarios_specs:\n",
    "    scenario_data_variants[scenario_id] = set()\n",
    "    for dset_idx_in_scenario in scenario_id:\n",
    "        assert not dsets_index_df.at[dset_idx_in_scenario, \"in_german\"]\n",
    "        scenario_data_variants[scenario_id].add(\n",
    "            \"other\" if dsets_index_df.at[dset_idx_in_scenario, \"is_other\"] else \"neg\" if dsets_index_df.at[dset_idx_in_scenario, \"is_negated\"] else \"conj\" if dsets_index_df.at[dset_idx_in_scenario, \"is_conj\"] else \"disj\" if dsets_index_df.at[dset_idx_in_scenario, \"is_disj\"] else \"affirm\")\n",
    "\n",
    "scenario_labels: dict[tuple[int,...], str] = {}\n",
    "for scenario_id in dir_probe_scenarios_specs:\n",
    "    label = \"+\".join(scenario_standard_categs[scenario_id])\n",
    "    was_earlier_variant_included=False\n",
    "    if \"affirm\" in scenario_data_variants[scenario_id]:\n",
    "        label += \"__affirmative\"\n",
    "        was_earlier_variant_included = True\n",
    "    if \"neg\" in scenario_data_variants[scenario_id]:\n",
    "        label += (\"+\" if was_earlier_variant_included else \"__\") + \"negated\"\n",
    "        was_earlier_variant_included = True\n",
    "    if \"conj\" in scenario_data_variants[scenario_id]:\n",
    "        label += (\"+\" if was_earlier_variant_included else \"__\") + \"conjunction\"\n",
    "        was_earlier_variant_included = True\n",
    "    if \"disj\" in scenario_data_variants[scenario_id]:\n",
    "        label += (\"+\" if was_earlier_variant_included else \"__\") + \"disjunction\"\n",
    "    # assuming no training scenarios include german\n",
    "    if scenario_other_dset_names[scenario_id]:\n",
    "        label += (\"_and_\" if label else \"\") + \"+\".join(scenario_other_dset_names[scenario_id])\n",
    "    scenario_labels[scenario_id] = label"
   ],
   "id": "e75dc4b3ecccfbdc",
   "outputs": [],
   "execution_count": 7
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:57.880551Z",
     "start_time": "2024-12-07T21:26:57.875217Z"
    }
   },
   "cell_type": "code",
   "source": [
    "t_f_sep_by_layer_df = pd.read_csv(separation_by_layer_analysis_path, index_col=\"Idx\")\n",
    "t_f_sep_by_layer_df.at[idxs_for_other_dsets[\"ambiguous_truthful_reply\"], \"Separation after 18\"] = t_f_sep_by_layer_df.at[idxs_for_other_dsets[\"ambiguous_lie\"], \"Separation after 18\"]\n",
    "t_f_sep_by_layer_df.at[idxs_for_other_dsets[\"ambiguous_truthful_reply\"], \"Separation after 25\"] = t_f_sep_by_layer_df.at[idxs_for_other_dsets[\"ambiguous_lie\"], \"Separation after 25\"]\n",
    "t_f_sep_by_layer_df.at[idxs_for_other_dsets[\"unambiguous_truthful_reply\"], \"Separation after 18\"] = t_f_sep_by_layer_df.at[idxs_for_other_dsets[\"unambiguous_lie\"], \"Separation after 18\"]\n",
    "t_f_sep_by_layer_df.at[idxs_for_other_dsets[\"unambiguous_truthful_reply\"], \"Separation after 25\"] = t_f_sep_by_layer_df.at[idxs_for_other_dsets[\"unambiguous_lie\"], \"Separation after 25\"]"
   ],
   "id": "fc5dff01d50b386e",
   "outputs": [],
   "execution_count": 8
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:57.889539Z",
     "start_time": "2024-12-07T21:26:57.881550Z"
    }
   },
   "cell_type": "code",
   "source": [
    "with train_split_classification_metrics_path.open(\"r\") as f:\n",
    "    serialized_train_metrics = json.load(f)\n",
    "with validation_split_classification_metrics_path.open(\"r\") as f:\n",
    "    serialized_validation_metrics = json.load(f)\n",
    "train_metrics: dict[tuple[int,...], MetricsForDatasetProbes] = {\n",
    "    tuple(map(int, scenario_str.split(' '))): MetricsForDatasetProbes.from_dict(metrics_dict) for scenario_str, metrics_dict in serialized_train_metrics.items()\n",
    "}\n",
    "validation_metrics: dict[tuple[int,...], MetricsForDatasetProbes] = {\n",
    "    tuple(map(int, scenario_str.split(' '))): MetricsForDatasetProbes.from_dict(metrics_dict) for scenario_str, metrics_dict in serialized_validation_metrics.items()\n",
    "}"
   ],
   "id": "73743a80f4d4594f",
   "outputs": [],
   "execution_count": 9
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:57.969503Z",
     "start_time": "2024-12-07T21:26:57.890540Z"
    }
   },
   "cell_type": "code",
   "source": [
    "with test_classification_metrics_path.open(\"r\") as f:\n",
    "    serialized_probes_metrics_on_test_dsets = json.load(f)\n",
    "# first level key identifies the scenario (of one or more datasets) which the probe was trained on;\n",
    "# second level key identifies the previously-unseen dataset which the probe is tested on\n",
    "probes_metrics_on_test_dsets: dict[tuple[int,...], dict[int, MetricsForDatasetProbes]] = {\n",
    "    tuple(map(int, scenario_str.split(' '))): {\n",
    "        int(test_dset_idx): MetricsForDatasetProbes.from_dict(serialized_test_dset_metrics) for test_dset_idx, serialized_test_dset_metrics in serialized_test_dsets_metrics.items()\n",
    "    } for scenario_str, serialized_test_dsets_metrics in serialized_probes_metrics_on_test_dsets.items()\n",
    "}"
   ],
   "id": "5606b30bd90afa97",
   "outputs": [],
   "execution_count": 10
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:57.979590Z",
     "start_time": "2024-12-07T21:26:57.970508Z"
    }
   },
   "cell_type": "code",
   "source": [
    "class TruthPolarityDirsEval(NamedTuple):\n",
    "    rel_loss_reduct_on_train: float\n",
    "    rel_loss_reduct_on_validation: float\n",
    "    rel_loss_reduct_on_validation_using_validation_mean: float\n",
    "\n",
    "lyr18_dirs_evals: dict[tuple[int,...], TruthPolarityDirsEval] = {}\n",
    "lyr25_dirs_evals: dict[tuple[int,...], TruthPolarityDirsEval] = {}\n",
    "lyrs18_and_25_dirs_evals: dict[tuple[int,...], TruthPolarityDirsEval] = {}\n",
    "\n",
    "for _, recon_losses_row in directions_reconstruction_losses.iterrows():\n",
    "    scenario_id = tuple(map(int, recon_losses_row[\"scenario_identifier\"].split(' ')))\n",
    "    lyr18_dirs_evals[scenario_id] = TruthPolarityDirsEval(\n",
    "        rel_loss_reduct_on_train=1-recon_losses_row[\"lyr18_train_mean_activ_and_t_p_dirs_loss_on_train\"]/recon_losses_row[\"lyr18_train_mean_activ_loss_on_train\"],\n",
    "        rel_loss_reduct_on_validation=1-recon_losses_row[\"lyr18_train_mean_activ_and_t_p_dirs_loss_on_validation\"]/recon_losses_row[\"lyr18_train_mean_activ_loss_on_validation\"],\n",
    "        rel_loss_reduct_on_validation_using_validation_mean=1-recon_losses_row[\"lyr18_validation_mean_activ_and_t_p_dirs_loss_on_validation\"]/recon_losses_row[\"lyr18_validation_mean_activ_loss_on_validation\"]\n",
    "    )\n",
    "    lyr25_dirs_evals[scenario_id] = TruthPolarityDirsEval(\n",
    "        rel_loss_reduct_on_train=1-recon_losses_row[\"lyr25_train_mean_activ_and_t_p_dirs_loss_on_train\"]/recon_losses_row[\"lyr25_train_mean_activ_loss_on_train\"],\n",
    "        rel_loss_reduct_on_validation=1-recon_losses_row[\"lyr25_train_mean_activ_and_t_p_dirs_loss_on_validation\"]/recon_losses_row[\"lyr25_train_mean_activ_loss_on_validation\"],\n",
    "        rel_loss_reduct_on_validation_using_validation_mean=1-recon_losses_row[\"lyr25_validation_mean_activ_and_t_p_dirs_loss_on_validation\"]/recon_losses_row[\"lyr25_validation_mean_activ_loss_on_validation\"]\n",
    "    )\n",
    "    lyrs18_and_25_dirs_evals[scenario_id] = TruthPolarityDirsEval(\n",
    "        rel_loss_reduct_on_train=1-recon_losses_row[\"lyrs18_and_25_train_mean_activ_and_t_p_dirs_loss_on_train\"]/recon_losses_row[\"lyrs18_and_25_train_mean_activ_loss_on_train\"],\n",
    "        rel_loss_reduct_on_validation=1-recon_losses_row[\"lyrs18_and_25_train_mean_activ_and_t_p_dirs_loss_on_validation\"]/recon_losses_row[\"lyrs18_and_25_train_mean_activ_loss_on_validation\"],\n",
    "        rel_loss_reduct_on_validation_using_validation_mean=1-recon_losses_row[\"lyrs18_and_25_validation_mean_activ_and_t_p_dirs_loss_on_validation\"]/recon_losses_row[\"lyrs18_and_25_validation_mean_activ_loss_on_validation\"]\n",
    "    )\n"
   ],
   "id": "146c3d6246ad2bf7",
   "outputs": [],
   "execution_count": 11
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:57.990852Z",
     "start_time": "2024-12-07T21:26:57.980592Z"
    }
   },
   "cell_type": "code",
   "source": [
    "class ProbeEvalOnData(NamedTuple):\n",
    "    acc: float\n",
    "    precision: float\n",
    "    recall: float\n",
    "    f1: float\n",
    "    brier: float\n",
    "    soft_precision: float\n",
    "    soft_recall: float\n",
    "    soft_f1: float\n",
    "\n",
    "lyr18_probes_train_set_evals: dict[tuple[int,...], ProbeEvalOnData] = {}\n",
    "lyr25_probes_train_set_evals: dict[tuple[int,...], ProbeEvalOnData] = {}\n",
    "lyrs18_and_25_probes_train_set_evals: dict[tuple[int,...], ProbeEvalOnData] = {}\n",
    "\n",
    "lyr18_baseline_linear_probes_train_set_evals: dict[tuple[int,...], ProbeEvalOnData] = {}\n",
    "\n",
    "for scenario_id, probes_metrics_on_train in train_metrics.items():\n",
    "    lyr18_train_metrics = probes_metrics_on_train.lyr18_probe_metrics\n",
    "    lyr18_train_trad_metrics = lyr18_train_metrics.get_traditional_metrics()\n",
    "    lyr18_train_soft_metrics = lyr18_train_metrics.get_soft_metrics()\n",
    "    lyr25_train_metrics = probes_metrics_on_train.lyr25_probe_metrics\n",
    "    lyr25_train_trad_metrics = lyr25_train_metrics.get_traditional_metrics()\n",
    "    lyr25_train_soft_metrics = lyr25_train_metrics.get_soft_metrics()\n",
    "    lyrs18_and_25_train_metrics = probes_metrics_on_train.lyrs18_and_25_probe_metrics\n",
    "    lyrs18_and_25_train_trad_metrics = lyrs18_and_25_train_metrics.get_traditional_metrics()\n",
    "    lyrs18_and_25_train_soft_metrics = lyrs18_and_25_train_metrics.get_soft_metrics()\n",
    "    \n",
    "    lyr18_baseline_linear_probes_train_metrics = probes_metrics_on_train.lyr18_baseline_linear_probe_metrics\n",
    "    lyr18_baseline_linear_probes_train_trad_metrics = lyr18_baseline_linear_probes_train_metrics.get_traditional_metrics()\n",
    "    lyr18_baseline_linear_probes_train_soft_metrics = lyr18_baseline_linear_probes_train_metrics.get_soft_metrics()\n",
    "    \n",
    "    lyr18_probes_train_set_evals[scenario_id] = ProbeEvalOnData(\n",
    "        acc=lyr18_train_trad_metrics.accuracy, precision=lyr18_train_trad_metrics.precision, recall=lyr18_train_trad_metrics.recall,\n",
    "        f1=lyr18_train_trad_metrics.f1, brier=lyr18_train_metrics.get_brier_score(), soft_precision=lyr18_train_soft_metrics.soft_precision,\n",
    "        soft_recall=lyr18_train_soft_metrics.soft_recall, soft_f1=lyr18_train_soft_metrics.soft_f1)\n",
    "    lyr25_probes_train_set_evals[scenario_id] = ProbeEvalOnData(\n",
    "        acc=lyr25_train_trad_metrics.accuracy, precision=lyr25_train_trad_metrics.precision, recall=lyr25_train_trad_metrics.recall,\n",
    "        f1=lyr25_train_trad_metrics.f1, brier=lyr25_train_metrics.get_brier_score(), soft_precision=lyr25_train_soft_metrics.soft_precision,\n",
    "        soft_recall=lyr25_train_soft_metrics.soft_recall, soft_f1=lyr25_train_soft_metrics.soft_f1)\n",
    "    lyrs18_and_25_probes_train_set_evals[scenario_id] = ProbeEvalOnData(\n",
    "        acc=lyrs18_and_25_train_trad_metrics.accuracy, precision=lyrs18_and_25_train_trad_metrics.precision, recall=lyrs18_and_25_train_trad_metrics.recall,\n",
    "        f1=lyrs18_and_25_train_trad_metrics.f1, brier=lyrs18_and_25_train_metrics.get_brier_score(), soft_precision=lyrs18_and_25_train_soft_metrics.soft_precision,\n",
    "        soft_recall=lyrs18_and_25_train_soft_metrics.soft_recall, soft_f1=lyrs18_and_25_train_soft_metrics.soft_f1)\n",
    "    \n",
    "    lyr18_baseline_linear_probes_train_set_evals[scenario_id] = ProbeEvalOnData(\n",
    "        acc=lyr18_baseline_linear_probes_train_trad_metrics.accuracy, precision=lyr18_baseline_linear_probes_train_trad_metrics.precision,\n",
    "        recall=lyr18_baseline_linear_probes_train_trad_metrics.recall, f1=lyr18_baseline_linear_probes_train_trad_metrics.f1,\n",
    "        brier=lyr18_baseline_linear_probes_train_metrics.get_brier_score(), soft_precision=lyr18_baseline_linear_probes_train_soft_metrics.soft_precision,\n",
    "        soft_recall=lyr18_baseline_linear_probes_train_soft_metrics.soft_recall, soft_f1=lyr18_baseline_linear_probes_train_soft_metrics.soft_f1\n",
    "    )\n",
    "\n",
    "lyr18_probes_validation_set_evals: dict[tuple[int,...], ProbeEvalOnData] = {}\n",
    "lyr25_probes_validation_set_evals: dict[tuple[int,...], ProbeEvalOnData] = {}\n",
    "lyrs18_and_25_probes_validation_set_evals: dict[tuple[int,...], ProbeEvalOnData] = {}\n",
    "\n",
    "lyr18_baseline_linear_probes_validation_set_evals: dict[tuple[int,...], ProbeEvalOnData] = {}\n",
    "\n",
    "for scenario_id, probes_metrics_on_validation in validation_metrics.items():\n",
    "    lyr18_validation_metrics = probes_metrics_on_validation.lyr18_probe_metrics\n",
    "    lyr18_validation_trad_metrics = lyr18_validation_metrics.get_traditional_metrics()\n",
    "    lyr18_validation_soft_metrics = lyr18_validation_metrics.get_soft_metrics()\n",
    "    lyr25_validation_metrics = probes_metrics_on_validation.lyr25_probe_metrics\n",
    "    lyr25_validation_trad_metrics = lyr25_validation_metrics.get_traditional_metrics()\n",
    "    lyr25_validation_soft_metrics = lyr25_validation_metrics.get_soft_metrics()\n",
    "    lyrs18_and_25_validation_metrics = probes_metrics_on_validation.lyrs18_and_25_probe_metrics\n",
    "    lyrs18_and_25_validation_trad_metrics = lyrs18_and_25_validation_metrics.get_traditional_metrics()\n",
    "    lyrs18_and_25_validation_soft_metrics = lyrs18_and_25_validation_metrics.get_soft_metrics()\n",
    "    \n",
    "    lyr18_baseline_linear_probes_validation_metrics = probes_metrics_on_validation.lyr18_baseline_linear_probe_metrics\n",
    "    lyr18_baseline_linear_probes_validation_trad_metrics = lyr18_baseline_linear_probes_validation_metrics.get_traditional_metrics()\n",
    "    lyr18_baseline_linear_probes_validation_soft_metrics = lyr18_baseline_linear_probes_validation_metrics.get_soft_metrics()\n",
    "    \n",
    "    lyr18_probes_validation_set_evals[scenario_id] = ProbeEvalOnData(\n",
    "        acc=lyr18_validation_trad_metrics.accuracy, precision=lyr18_validation_trad_metrics.precision, recall=lyr18_validation_trad_metrics.recall,\n",
    "        f1=lyr18_validation_trad_metrics.f1, brier=lyr18_validation_metrics.get_brier_score(), soft_precision=lyr18_validation_soft_metrics.soft_precision,\n",
    "        soft_recall=lyr18_validation_soft_metrics.soft_recall, soft_f1=lyr18_validation_soft_metrics.soft_f1)\n",
    "    lyr25_probes_validation_set_evals[scenario_id] = ProbeEvalOnData(\n",
    "        acc=lyr25_validation_trad_metrics.accuracy, precision=lyr25_validation_trad_metrics.precision, recall=lyr25_validation_trad_metrics.recall,\n",
    "        f1=lyr25_validation_trad_metrics.f1, brier=lyr25_validation_metrics.get_brier_score(), soft_precision=lyr25_validation_soft_metrics.soft_precision,\n",
    "        soft_recall=lyr25_validation_soft_metrics.soft_recall, soft_f1=lyr25_validation_soft_metrics.soft_f1)\n",
    "    lyrs18_and_25_probes_validation_set_evals[scenario_id] = ProbeEvalOnData(\n",
    "        acc=lyrs18_and_25_validation_trad_metrics.accuracy, precision=lyrs18_and_25_validation_trad_metrics.precision, recall=lyrs18_and_25_validation_trad_metrics.recall,\n",
    "        f1=lyrs18_and_25_validation_trad_metrics.f1, brier=lyrs18_and_25_validation_metrics.get_brier_score(), soft_precision=lyrs18_and_25_validation_soft_metrics.soft_precision,\n",
    "        soft_recall=lyrs18_and_25_validation_soft_metrics.soft_recall, soft_f1=lyrs18_and_25_validation_soft_metrics.soft_f1)\n",
    "    \n",
    "    lyr18_baseline_linear_probes_validation_set_evals[scenario_id] = ProbeEvalOnData(\n",
    "        acc=lyr18_baseline_linear_probes_validation_trad_metrics.accuracy, precision=lyr18_baseline_linear_probes_validation_trad_metrics.precision,\n",
    "        recall=lyr18_baseline_linear_probes_validation_trad_metrics.recall, f1=lyr18_baseline_linear_probes_validation_trad_metrics.f1,\n",
    "        brier=lyr18_baseline_linear_probes_validation_metrics.get_brier_score(), soft_precision=lyr18_baseline_linear_probes_validation_soft_metrics.soft_precision,\n",
    "        soft_recall=lyr18_baseline_linear_probes_validation_soft_metrics.soft_recall, soft_f1=lyr18_baseline_linear_probes_validation_soft_metrics.soft_f1\n",
    "    )"
   ],
   "id": "a36a8fa093405711",
   "outputs": [],
   "execution_count": 12
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:58.006634Z",
     "start_time": "2024-12-07T21:26:57.991855Z"
    }
   },
   "cell_type": "code",
   "source": [
    "lyr18_probe_row_dicts: list[dict[str, str|float]] = []\n",
    "lyr25_probe_row_dicts: list[dict[str, str|float]] = []\n",
    "lyrs18_and_25_probe_row_dicts: list[dict[str, str|float]] = []\n",
    "\n",
    "lyr18_baseline_linear_probe_row_dicts: list[dict[str, str|float]] = []\n",
    "\n",
    "lyr25_rel_to_18_probe_row_dicts: list[dict[str, str|float]] = []\n",
    "lyrs18_and_25_rel_to_just_18_probe_row_dicts: list[dict[str, str|float]] = []\n",
    "\n",
    "lyr18_ttpd_probe_to_baseline_linear_probe_row_dicts: list[dict[str, str | float]] = []\n",
    "\n",
    "for scenario_id in dir_probe_scenarios_specs:\n",
    "    lyr18_t_f_sep = mean([t_f_sep_by_layer_df.at[dset_idx_in_scenario, \"Separation after 18\"] for dset_idx_in_scenario in scenario_id])\n",
    "    lyr25_t_f_sep = mean([t_f_sep_by_layer_df.at[dset_idx_in_scenario, \"Separation after 25\"] for dset_idx_in_scenario in scenario_id])\n",
    "    lyr18_probe_row_dict = {\n",
    "        \"scenario\": scenario_labels[scenario_id],\n",
    "        \"tf_sep\": lyr18_t_f_sep, \"train_recon_improv\": lyr18_dirs_evals[scenario_id].rel_loss_reduct_on_train,\n",
    "        \"val_recon_improv\": lyr18_dirs_evals[scenario_id].rel_loss_reduct_on_validation,\n",
    "        \"val_recon_improv_w_val_mean\": lyr18_dirs_evals[\n",
    "            scenario_id].rel_loss_reduct_on_validation_using_validation_mean,\n",
    "        \"val_acc\": lyr18_probes_validation_set_evals[scenario_id].acc,\n",
    "        \"val_f1\": lyr18_probes_validation_set_evals[scenario_id].f1,\n",
    "        \"val_brier\": lyr18_probes_validation_set_evals[scenario_id].brier,\n",
    "        \"val_soft_f1\": lyr18_probes_validation_set_evals[scenario_id].soft_f1,\n",
    "        \"train_acc\": lyr18_probes_train_set_evals[scenario_id].acc,\n",
    "        \"train_f1\": lyr18_probes_train_set_evals[scenario_id].f1,\n",
    "        \"train_brier\": lyr18_probes_train_set_evals[scenario_id].brier,\n",
    "        \"train_soft_f1\": lyr18_probes_train_set_evals[scenario_id].soft_f1,\n",
    "    }\n",
    "    lyr18_probe_row_dicts.append(lyr18_probe_row_dict)\n",
    "    lyr25_probe_row_dict = {\n",
    "        \"scenario\": scenario_labels[scenario_id],\n",
    "        \"tf_sep\": lyr25_t_f_sep, \"train_recon_improv\": lyr25_dirs_evals[scenario_id].rel_loss_reduct_on_train,\n",
    "        \"val_recon_improv\": lyr25_dirs_evals[scenario_id].rel_loss_reduct_on_validation,\n",
    "        \"val_recon_improv_w_val_mean\": lyr25_dirs_evals[\n",
    "            scenario_id].rel_loss_reduct_on_validation_using_validation_mean,\n",
    "        \"val_acc\": lyr25_probes_validation_set_evals[scenario_id].acc,\n",
    "        \"val_f1\": lyr25_probes_validation_set_evals[scenario_id].f1,\n",
    "        \"val_brier\": lyr25_probes_validation_set_evals[scenario_id].brier,\n",
    "        \"val_soft_f1\": lyr25_probes_validation_set_evals[scenario_id].soft_f1,\n",
    "        \"train_acc\": lyr25_probes_train_set_evals[scenario_id].acc,\n",
    "        \"train_f1\": lyr25_probes_train_set_evals[scenario_id].f1,\n",
    "        \"train_brier\": lyr25_probes_train_set_evals[scenario_id].brier,\n",
    "        \"train_soft_f1\": lyr25_probes_train_set_evals[scenario_id].soft_f1,\n",
    "    }\n",
    "    lyr25_probe_row_dicts.append(lyr25_probe_row_dict)\n",
    "    lyrs18_and_25_probe_row_dict = {\n",
    "        \"scenario\": scenario_labels[scenario_id],\n",
    "        \"tf_sep\": (lyr18_t_f_sep + lyr25_t_f_sep) / 2,\n",
    "        \"train_recon_improv\": lyrs18_and_25_dirs_evals[scenario_id].rel_loss_reduct_on_train,\n",
    "        \"val_recon_improv\": lyrs18_and_25_dirs_evals[scenario_id].rel_loss_reduct_on_validation,\n",
    "        \"val_recon_improv_w_val_mean\": lyrs18_and_25_dirs_evals[\n",
    "            scenario_id].rel_loss_reduct_on_validation_using_validation_mean,\n",
    "        \"val_acc\": lyrs18_and_25_probes_validation_set_evals[scenario_id].acc,\n",
    "        \"val_f1\": lyrs18_and_25_probes_validation_set_evals[scenario_id].f1,\n",
    "        \"val_brier\": lyrs18_and_25_probes_validation_set_evals[scenario_id].brier,\n",
    "        \"val_soft_f1\": lyrs18_and_25_probes_validation_set_evals[scenario_id].soft_f1,\n",
    "        \"train_acc\": lyrs18_and_25_probes_train_set_evals[scenario_id].acc,\n",
    "        \"train_f1\": lyrs18_and_25_probes_train_set_evals[scenario_id].f1,\n",
    "        \"train_brier\": lyrs18_and_25_probes_train_set_evals[scenario_id].brier,\n",
    "        \"train_soft_f1\": lyrs18_and_25_probes_train_set_evals[scenario_id].soft_f1,\n",
    "    }\n",
    "    lyrs18_and_25_probe_row_dicts.append(lyrs18_and_25_probe_row_dict)\n",
    "    \n",
    "    lyr18_baseline_linear_probe_row_dict = {\n",
    "        \"scenario\": scenario_labels[scenario_id],\n",
    "        \"val_acc\": lyr18_baseline_linear_probes_validation_set_evals[scenario_id].acc,\n",
    "        \"val_f1\": lyr18_baseline_linear_probes_validation_set_evals[scenario_id].f1,\n",
    "        \"val_brier\": lyr18_baseline_linear_probes_validation_set_evals[scenario_id].brier,\n",
    "        \"val_soft_f1\": lyr18_baseline_linear_probes_validation_set_evals[scenario_id].soft_f1,\n",
    "        \"train_acc\": lyr18_baseline_linear_probes_train_set_evals[scenario_id].acc,\n",
    "        \"train_f1\": lyr18_baseline_linear_probes_train_set_evals[scenario_id].f1,\n",
    "        \"train_brier\": lyr18_baseline_linear_probes_train_set_evals[scenario_id].brier,\n",
    "        \"train_soft_f1\": lyr18_baseline_linear_probes_train_set_evals[scenario_id].soft_f1,\n",
    "    }\n",
    "    lyr18_baseline_linear_probe_row_dicts.append(lyr18_baseline_linear_probe_row_dict)\n",
    "    \n",
    "    lyr25_rel_to_18_probe_row_dicts.append({\n",
    "        col_nm: (lyr18_probe_row_dict[col_nm] if col_nm == \"scenario\" else lyr25_probe_row_dict[col_nm]-lyr18_probe_row_dict[col_nm]) for col_nm in lyr18_probe_row_dict.keys()\n",
    "    })\n",
    "    lyrs18_and_25_rel_to_just_18_probe_row_dicts.append({\n",
    "        col_nm: (lyr18_probe_row_dict[col_nm] if col_nm == \"scenario\" else lyrs18_and_25_probe_row_dict[col_nm]-lyr18_probe_row_dict[col_nm]) for col_nm in lyr18_probe_row_dict.keys()\n",
    "    })\n",
    "    \n",
    "    lyr18_ttpd_probe_to_baseline_linear_probe_row_dicts.append({\n",
    "        col_nm: (lyr18_probe_row_dict[col_nm] if col_nm == \"scenario\" else lyr18_probe_row_dict[col_nm]-lyr18_baseline_linear_probe_row_dict[col_nm]) for col_nm in lyr18_baseline_linear_probe_row_dict.keys()\n",
    "    })\n",
    "    \n",
    "\n",
    "lyr18_eval_except_test_df = pd.DataFrame(lyr18_probe_row_dicts)\n",
    "lyr25_eval_except_test_df = pd.DataFrame(lyr25_probe_row_dicts)\n",
    "lyrs18_and_25_eval_except_test_df = pd.DataFrame(lyrs18_and_25_probe_row_dicts)\n",
    "\n",
    "lyr18_baseline_linear_probes_eval_except_test_df = pd.DataFrame(lyr18_baseline_linear_probe_row_dicts)\n",
    "\n",
    "lyr25_rel_to_18_eval_except_test_df = pd.DataFrame(lyr25_rel_to_18_probe_row_dicts)\n",
    "lyrs18_and_25_rel_to_18_eval_except_test_df = pd.DataFrame(lyrs18_and_25_rel_to_just_18_probe_row_dicts)\n",
    "\n",
    "lyr18_ttpd_probe_to_baseline_linear_probe_eval_except_test_df = pd.DataFrame(lyr18_ttpd_probe_to_baseline_linear_probe_row_dicts)\n"
   ],
   "id": "bca6e0d7cfc21ad9",
   "outputs": [],
   "execution_count": 13
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "For the \"tf_sep\" aka \"true-false separation\" column, and the ambiguous/unambiguous truth/lie (plus \"honest reply despite incentive to lie\") datasets,  \n",
    "the value in the \"ambiguous lie\" and \"ambiguous truthful reply\" rows represent the separation between ambiguous lies and truths,  \n",
    "the value in the \"unambiguous lie\" and \"unambiguous truthful reply\" rows represent the separation between unambiguous lies and truths,\n",
    "and the 0.0 values for the \"honest reply despite incentive to lie\" row should be ignored.\n",
    "\n",
    "When evaluating the below numbers, please note that brier scores are better if they are lower (unlike all other metrics here, which are better if they are higher)"
   ],
   "id": "289bc5227313ec8c"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:58.032390Z",
     "start_time": "2024-12-07T21:26:58.007652Z"
    }
   },
   "cell_type": "code",
   "source": "lyr18_eval_except_test_df",
   "id": "b6d7eeede8a6ce9b",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "                                             scenario    tf_sep  \\\n",
       "0                           animal_class__affirmative  0.873458   \n",
       "1                           animal_class__conjunction  0.314961   \n",
       "2                           animal_class__disjunction  0.029296   \n",
       "3                               animal_class__negated  0.579465   \n",
       "4                                 cities__affirmative  1.690080   \n",
       "5                                 cities__conjunction  0.388754   \n",
       "6                                 cities__disjunction  0.058523   \n",
       "7                                     cities__negated  0.868476   \n",
       "8                           element_symb__affirmative  0.577514   \n",
       "9                           element_symb__conjunction  0.425471   \n",
       "10                          element_symb__disjunction  0.066850   \n",
       "11                              element_symb__negated  0.365157   \n",
       "12                                 facts__affirmative  0.093213   \n",
       "13                                 facts__conjunction  0.035627   \n",
       "14                                 facts__disjunction  0.017876   \n",
       "15                                     facts__negated  0.028115   \n",
       "16                             inventors__affirmative  0.059885   \n",
       "17                             inventors__conjunction  0.024496   \n",
       "18                             inventors__disjunction  0.011326   \n",
       "19                                 inventors__negated  0.085275   \n",
       "20                                      ambiguous_lie  0.051308   \n",
       "21                           ambiguous_truthful_reply  0.051308   \n",
       "22              honest_reply_despite_incentive_to_lie  0.000000   \n",
       "23                                    unambiguous_lie  0.107328   \n",
       "24                         unambiguous_truthful_reply  0.107328   \n",
       "25                                        larger_than  0.456852   \n",
       "26                                       smaller_than  0.347493   \n",
       "27                               sp_en_trans__negated  0.679175   \n",
       "28                           sp_en_trans__affirmative  0.763526   \n",
       "29                           sp_en_trans__conjunction  0.360656   \n",
       "30                           sp_en_trans__disjunction  0.055794   \n",
       "31                            common_claim_true_false  0.016943   \n",
       "32                             counterfact_true_false  0.012648   \n",
       "33                  animal_class__affirmative+negated  0.726462   \n",
       "34              animal_class__affirmative+disjunction  0.451377   \n",
       "35                  animal_class__negated+conjunction  0.447213   \n",
       "36  animal_class__affirmative+negated+conjunction+...  0.449295   \n",
       "37                        cities__affirmative+negated  1.279278   \n",
       "38                    cities__affirmative+disjunction  0.874302   \n",
       "39                        cities__negated+conjunction  0.628615   \n",
       "40  cities__affirmative+negated+conjunction+disjun...  0.751458   \n",
       "41                  element_symb__affirmative+negated  0.471336   \n",
       "42              element_symb__affirmative+disjunction  0.322182   \n",
       "43                  element_symb__negated+conjunction  0.395314   \n",
       "44  element_symb__affirmative+negated+conjunction+...  0.358748   \n",
       "45                         facts__affirmative+negated  0.060664   \n",
       "46                     facts__affirmative+disjunction  0.055544   \n",
       "47                         facts__negated+conjunction  0.031871   \n",
       "48  facts__affirmative+negated+conjunction+disjunc...  0.043708   \n",
       "49                     inventors__affirmative+negated  0.072580   \n",
       "50                 inventors__affirmative+disjunction  0.035606   \n",
       "51                     inventors__negated+conjunction  0.054885   \n",
       "52  inventors__affirmative+negated+conjunction+dis...  0.045245   \n",
       "53                   sp_en_trans__affirmative+negated  0.721350   \n",
       "54               sp_en_trans__affirmative+disjunction  0.409660   \n",
       "55                   sp_en_trans__negated+conjunction  0.519915   \n",
       "56  sp_en_trans__affirmative+negated+conjunction+d...  0.464788   \n",
       "57  ambiguous_truthful_reply+unambiguous_lie+unamb...  0.088655   \n",
       "58  animal_class+facts+inventors__affirmative+nega...  0.232722   \n",
       "59  animal_class+facts+inventors__affirmative+nega...  0.213619   \n",
       "\n",
       "    train_recon_improv  val_recon_improv  val_recon_improv_w_val_mean  \\\n",
       "0         4.543415e-01      5.163863e-01                 3.967561e-01   \n",
       "1         2.398567e-01      2.314483e-01                 2.357424e-01   \n",
       "2         2.734168e-02      3.243016e-02                 3.037365e-02   \n",
       "3         3.598452e-01      4.085457e-01                 3.995034e-01   \n",
       "4         6.311205e-01      6.175942e-01                 6.166543e-01   \n",
       "5         2.749372e-01      2.993844e-01                 2.982197e-01   \n",
       "6         5.700467e-02      4.585128e-02                 4.644103e-02   \n",
       "7         4.610974e-01      4.793974e-01                 4.679124e-01   \n",
       "8         3.498783e-01      4.370834e-01                 4.490662e-01   \n",
       "9         3.073251e-01      2.748850e-01                 2.742458e-01   \n",
       "10        6.149318e-02      6.555922e-02                 6.573485e-02   \n",
       "11        2.844515e-01      2.087251e-01                 2.134846e-01   \n",
       "12        8.781164e-02      7.354002e-02                 6.904972e-02   \n",
       "13        3.673653e-02      2.145067e-02                 2.171824e-02   \n",
       "14        1.650338e-02      1.990205e-02                 2.031295e-02   \n",
       "15        2.862351e-02      2.065965e-02                 2.092022e-02   \n",
       "16        5.634922e-02      5.107413e-02                 5.130731e-02   \n",
       "17        2.481788e-02      1.691028e-02                 1.747544e-02   \n",
       "18        1.231043e-02      5.683728e-03                 5.092700e-03   \n",
       "19        8.420518e-02      4.965519e-02                 4.958424e-02   \n",
       "20        5.361147e-08      3.629734e-07                 6.957434e-08   \n",
       "21       -6.544491e-08      9.936519e-08                 0.000000e+00   \n",
       "22       -1.319963e-07      2.438925e-07                 0.000000e+00   \n",
       "23        1.258997e-07     -2.674763e-07                 0.000000e+00   \n",
       "24        6.411020e-08     -2.047305e-07                -7.574867e-08   \n",
       "25        3.142809e-01      3.110852e-01                 3.111883e-01   \n",
       "26        2.578911e-01      2.581460e-01                 2.549792e-01   \n",
       "27        4.083760e-01      3.927502e-01                 3.957529e-01   \n",
       "28        4.254077e-01      4.657559e-01                 4.151790e-01   \n",
       "29        2.654025e-01      2.726139e-01                 2.707033e-01   \n",
       "30        5.717739e-02      3.746276e-02                 3.632303e-02   \n",
       "31        1.601768e-02      1.895602e-02                 1.898451e-02   \n",
       "32        1.257900e-02      1.211249e-02                 1.211160e-02   \n",
       "33        3.255649e-01      3.730479e-01                 3.483443e-01   \n",
       "34        3.461717e-02      3.647969e-02                 3.132026e-02   \n",
       "35        1.274588e-01      1.192823e-01                 1.191230e-01   \n",
       "36        5.486206e-02      5.498721e-02                 5.459481e-02   \n",
       "37        3.803487e-01      3.786022e-01                 3.774681e-01   \n",
       "38        2.039240e-01      2.019980e-01                 2.012607e-01   \n",
       "39        1.256688e-01      1.276570e-01                 1.258128e-01   \n",
       "40        1.131659e-01      1.183838e-01                 1.180848e-01   \n",
       "41        1.768124e-01      1.959989e-01                 1.974226e-01   \n",
       "42        2.541350e-02      3.067232e-02                 3.052880e-02   \n",
       "43        1.584398e-01      1.283977e-01                 1.277602e-01   \n",
       "44        4.414859e-02      3.996578e-02                 4.002296e-02   \n",
       "45        5.364384e-02      4.356713e-02                 4.226291e-02   \n",
       "46        2.839450e-02      2.716368e-02                 2.710478e-02   \n",
       "47        2.251311e-02      1.294852e-02                 1.317613e-02   \n",
       "48        2.225074e-02      1.678023e-02                 1.706678e-02   \n",
       "49        4.958832e-02      4.507503e-02                 4.572443e-02   \n",
       "50        1.016311e-02      8.969028e-03                 8.869333e-03   \n",
       "51        4.034714e-02      4.136835e-02                 4.150045e-02   \n",
       "52        1.717407e-02      1.567208e-02                 1.575162e-02   \n",
       "53        2.160448e-01      2.094281e-01                 2.005184e-01   \n",
       "54        5.649454e-02      5.059661e-02                 4.968711e-02   \n",
       "55        1.346869e-01      1.178831e-01                 1.173058e-01   \n",
       "56        7.256485e-02      5.993876e-02                 5.999693e-02   \n",
       "57        8.594724e-02      4.916947e-02                 6.062967e-02   \n",
       "58        2.317787e-02      2.236908e-02                 2.232274e-02   \n",
       "59        1.702797e-02      1.806127e-02                 1.802021e-02   \n",
       "\n",
       "     val_acc    val_f1     val_brier  val_soft_f1  train_acc  train_f1  \\\n",
       "0   1.000000  1.000000  2.445273e-08     0.999943   0.992366  0.992908   \n",
       "1   0.890000  0.899083  6.859379e-02     0.876145   0.917500  0.920097   \n",
       "2   0.740000  0.697674  1.855859e-01     0.586721   0.717500  0.729017   \n",
       "3   1.000000  1.000000  6.577489e-04     0.995938   0.984733  0.984848   \n",
       "4   0.983333  0.982578  1.536337e-02     0.974531   0.983278  0.983526   \n",
       "5   0.960000  0.964286  3.396462e-02     0.934809   0.930718  0.933759   \n",
       "6   0.800000  0.795918  1.366856e-01     0.716584   0.855000  0.852041   \n",
       "7   0.990000  0.991150  8.395773e-03     0.984477   0.985786  0.985482   \n",
       "8   0.973684  0.975610  2.134477e-02     0.965086   0.905405  0.912500   \n",
       "9   0.880000  0.884615  7.882613e-02     0.853128   0.932500  0.940659   \n",
       "10  0.800000  0.818182  1.413988e-01     0.752431   0.790000  0.787879   \n",
       "11  1.000000  1.000000  9.240658e-19     1.000000   1.000000  1.000000   \n",
       "12  0.858407  0.885714  1.054875e-01     0.820781   0.850446  0.863544   \n",
       "13  0.720000  0.730769  1.866593e-01     0.650655   0.790000  0.814978   \n",
       "14  0.760000  0.727273  1.681977e-01     0.633569   0.712500  0.679666   \n",
       "15  0.823009  0.818182  1.239086e-01     0.773496   0.904018  0.903803   \n",
       "16  0.841463  0.860215  1.071005e-01     0.804620   0.833333  0.844828   \n",
       "17  0.720000  0.754386  1.596683e-01     0.733773   0.810000  0.829596   \n",
       "18  0.700000  0.732143  2.037729e-01     0.636309   0.740000  0.754717   \n",
       "19  0.658537  0.631579  1.951255e-01     0.606508   0.706790  0.686469   \n",
       "20  0.500000       NaN  2.412132e-01          NaN   0.428571       NaN   \n",
       "21  0.600000  0.750000  2.086981e-01     0.710639   0.555556  0.714286   \n",
       "22  1.000000  1.000000  7.010247e-02     0.847460   0.250000  0.400000   \n",
       "23  0.600000       NaN  2.176609e-01          NaN   0.538462       NaN   \n",
       "24  0.727273  0.842105  2.202386e-01     0.697597   0.500000  0.666667   \n",
       "25  0.972222  0.971867  2.691245e-02     0.946148   0.970960  0.972019   \n",
       "26  0.949495  0.955357  3.484066e-02     0.932294   0.945076  0.946919   \n",
       "27  1.000000  1.000000  2.294588e-04     0.997626   0.996466  0.996466   \n",
       "28  0.971831  0.977778  1.804558e-02     0.974164   0.968198  0.967273   \n",
       "29  0.930000  0.930693  5.284401e-02     0.918356   0.967500  0.970787   \n",
       "30  0.840000  0.818182  1.090786e-01     0.762275   0.832500  0.813370   \n",
       "31  0.782022  0.786813  1.528305e-01     0.684293   0.747472  0.749093   \n",
       "32  0.707962  0.729851  1.875083e-01     0.634437   0.710219  0.735300   \n",
       "33  1.000000  1.000000  3.724002e-03     0.988765   0.984733  0.985294   \n",
       "34  0.796992  0.761062  1.403436e-01     0.657099   0.779661  0.789189   \n",
       "35  0.917293  0.925170  5.825581e-02     0.896909   0.930320  0.932849   \n",
       "36  0.748120  0.712446  1.358077e-01     0.718068   0.740113  0.714286   \n",
       "37  0.983333  0.984076  1.378936e-02     0.976110   0.981605  0.981621   \n",
       "38  0.920000  0.913043  6.200888e-02     0.864782   0.906015  0.901575   \n",
       "39  0.930000  0.939828  4.939174e-02     0.909276   0.926065  0.929172   \n",
       "40  0.891000  0.899724  7.653691e-02     0.863456   0.865163  0.870983   \n",
       "41  0.986842  0.987342  1.476023e-02     0.972173   0.959459  0.960784   \n",
       "42  0.840580  0.851351  1.039167e-01     0.806696   0.843066  0.848057   \n",
       "43  0.891304  0.896552  8.803703e-02     0.847493   0.906934  0.918138   \n",
       "44  0.804348  0.812500  1.324841e-01     0.739445   0.814781  0.819556   \n",
       "45  0.787611  0.811024  1.386779e-01     0.765212   0.844866  0.852285   \n",
       "46  0.788732  0.798206  1.572402e-01     0.704486   0.768868  0.753769   \n",
       "47  0.661972  0.669725  2.013792e-01     0.581925   0.742925  0.759382   \n",
       "48  0.708920  0.718182  1.832747e-01     0.649696   0.747052  0.751016   \n",
       "49  0.829268  0.829268  1.245958e-01     0.744966   0.830247  0.826498   \n",
       "50  0.736264  0.762376  1.608515e-01     0.695569   0.766575  0.771930   \n",
       "51  0.675824  0.725581  2.141323e-01     0.583113   0.647790  0.698225   \n",
       "52  0.692308  0.738318  2.098124e-01     0.603974   0.652624  0.690080   \n",
       "53  0.971831  0.975309  2.340497e-02     0.960256   0.971731  0.971326   \n",
       "54  0.801170  0.817204  1.244469e-01     0.787459   0.828697  0.822995   \n",
       "55  0.953216  0.953488  3.660722e-02     0.930899   0.956076  0.959350   \n",
       "56  0.827485  0.822823  1.349240e-01     0.732068   0.841874  0.837349   \n",
       "57  0.807692  0.814815  1.229308e-01     0.829563   0.950495  0.958678   \n",
       "58  0.750000  0.769231  1.661825e-01     0.687835   0.764471  0.779577   \n",
       "59  0.805430  0.819517  1.450034e-01     0.711798   0.787832  0.797455   \n",
       "\n",
       "     train_brier  train_soft_f1  \n",
       "0   8.771814e-03       0.989470  \n",
       "1   6.192252e-02       0.879679  \n",
       "2   1.874590e-01       0.632118  \n",
       "3   1.469421e-02       0.975943  \n",
       "4   1.203893e-02       0.978910  \n",
       "5   4.737542e-02       0.912977  \n",
       "6   1.151929e-01       0.742089  \n",
       "7   1.017570e-02       0.979065  \n",
       "8   6.313708e-02       0.913252  \n",
       "9   5.868069e-02       0.892681  \n",
       "10  1.353580e-01       0.737389  \n",
       "11  6.743369e-25       1.000000  \n",
       "12  1.072506e-01       0.803707  \n",
       "13  1.412838e-01       0.727692  \n",
       "14  1.881253e-01       0.617878  \n",
       "15  7.790436e-02       0.832414  \n",
       "16  1.331661e-01       0.777379  \n",
       "17  1.303819e-01       0.757822  \n",
       "18  1.790653e-01       0.644249  \n",
       "19  1.714797e-01       0.652882  \n",
       "20  2.524060e-01            NaN  \n",
       "21  2.632712e-01       0.667486  \n",
       "22  2.680697e-01       0.664283  \n",
       "23  2.550336e-01            NaN  \n",
       "24  2.593466e-01       0.666987  \n",
       "25  2.098595e-02       0.955345  \n",
       "26  3.824428e-02       0.920902  \n",
       "27  2.715710e-03       0.995245  \n",
       "28  1.792615e-02       0.966510  \n",
       "29  2.128471e-02       0.955200  \n",
       "30  1.089132e-01       0.770467  \n",
       "31  1.691688e-01       0.665531  \n",
       "32  1.858964e-01       0.638215  \n",
       "33  9.318734e-03       0.984407  \n",
       "34  1.506604e-01       0.710305  \n",
       "35  5.293857e-02       0.895870  \n",
       "36  1.394676e-01       0.725212  \n",
       "37  1.456238e-02       0.971601  \n",
       "38  6.242136e-02       0.870892  \n",
       "39  5.756781e-02       0.886600  \n",
       "40  8.988929e-02       0.840710  \n",
       "41  3.085962e-02       0.954207  \n",
       "42  1.078788e-01       0.784090  \n",
       "43  6.595096e-02       0.888005  \n",
       "44  1.243532e-01       0.755632  \n",
       "45  1.077000e-01       0.782647  \n",
       "46  1.613407e-01       0.679229  \n",
       "47  1.715787e-01       0.629935  \n",
       "48  1.631878e-01       0.668476  \n",
       "49  1.359260e-01       0.731713  \n",
       "50  1.627316e-01       0.681231  \n",
       "51  2.191090e-01       0.569212  \n",
       "52  2.149256e-01       0.583715  \n",
       "53  1.558442e-02       0.958855  \n",
       "54  1.080418e-01       0.785671  \n",
       "55  3.132679e-02       0.941975  \n",
       "56  1.220694e-01       0.752624  \n",
       "57  4.795936e-02       0.911356  \n",
       "58  1.603947e-01       0.688521  \n",
       "59  1.535762e-01       0.695825  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>scenario</th>\n",
       "      <th>tf_sep</th>\n",
       "      <th>train_recon_improv</th>\n",
       "      <th>val_recon_improv</th>\n",
       "      <th>val_recon_improv_w_val_mean</th>\n",
       "      <th>val_acc</th>\n",
       "      <th>val_f1</th>\n",
       "      <th>val_brier</th>\n",
       "      <th>val_soft_f1</th>\n",
       "      <th>train_acc</th>\n",
       "      <th>train_f1</th>\n",
       "      <th>train_brier</th>\n",
       "      <th>train_soft_f1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>animal_class__affirmative</td>\n",
       "      <td>0.873458</td>\n",
       "      <td>4.543415e-01</td>\n",
       "      <td>5.163863e-01</td>\n",
       "      <td>3.967561e-01</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.445273e-08</td>\n",
       "      <td>0.999943</td>\n",
       "      <td>0.992366</td>\n",
       "      <td>0.992908</td>\n",
       "      <td>8.771814e-03</td>\n",
       "      <td>0.989470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>animal_class__conjunction</td>\n",
       "      <td>0.314961</td>\n",
       "      <td>2.398567e-01</td>\n",
       "      <td>2.314483e-01</td>\n",
       "      <td>2.357424e-01</td>\n",
       "      <td>0.890000</td>\n",
       "      <td>0.899083</td>\n",
       "      <td>6.859379e-02</td>\n",
       "      <td>0.876145</td>\n",
       "      <td>0.917500</td>\n",
       "      <td>0.920097</td>\n",
       "      <td>6.192252e-02</td>\n",
       "      <td>0.879679</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>animal_class__disjunction</td>\n",
       "      <td>0.029296</td>\n",
       "      <td>2.734168e-02</td>\n",
       "      <td>3.243016e-02</td>\n",
       "      <td>3.037365e-02</td>\n",
       "      <td>0.740000</td>\n",
       "      <td>0.697674</td>\n",
       "      <td>1.855859e-01</td>\n",
       "      <td>0.586721</td>\n",
       "      <td>0.717500</td>\n",
       "      <td>0.729017</td>\n",
       "      <td>1.874590e-01</td>\n",
       "      <td>0.632118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>animal_class__negated</td>\n",
       "      <td>0.579465</td>\n",
       "      <td>3.598452e-01</td>\n",
       "      <td>4.085457e-01</td>\n",
       "      <td>3.995034e-01</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.577489e-04</td>\n",
       "      <td>0.995938</td>\n",
       "      <td>0.984733</td>\n",
       "      <td>0.984848</td>\n",
       "      <td>1.469421e-02</td>\n",
       "      <td>0.975943</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cities__affirmative</td>\n",
       "      <td>1.690080</td>\n",
       "      <td>6.311205e-01</td>\n",
       "      <td>6.175942e-01</td>\n",
       "      <td>6.166543e-01</td>\n",
       "      <td>0.983333</td>\n",
       "      <td>0.982578</td>\n",
       "      <td>1.536337e-02</td>\n",
       "      <td>0.974531</td>\n",
       "      <td>0.983278</td>\n",
       "      <td>0.983526</td>\n",
       "      <td>1.203893e-02</td>\n",
       "      <td>0.978910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>cities__conjunction</td>\n",
       "      <td>0.388754</td>\n",
       "      <td>2.749372e-01</td>\n",
       "      <td>2.993844e-01</td>\n",
       "      <td>2.982197e-01</td>\n",
       "      <td>0.960000</td>\n",
       "      <td>0.964286</td>\n",
       "      <td>3.396462e-02</td>\n",
       "      <td>0.934809</td>\n",
       "      <td>0.930718</td>\n",
       "      <td>0.933759</td>\n",
       "      <td>4.737542e-02</td>\n",
       "      <td>0.912977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>cities__disjunction</td>\n",
       "      <td>0.058523</td>\n",
       "      <td>5.700467e-02</td>\n",
       "      <td>4.585128e-02</td>\n",
       "      <td>4.644103e-02</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.795918</td>\n",
       "      <td>1.366856e-01</td>\n",
       "      <td>0.716584</td>\n",
       "      <td>0.855000</td>\n",
       "      <td>0.852041</td>\n",
       "      <td>1.151929e-01</td>\n",
       "      <td>0.742089</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>cities__negated</td>\n",
       "      <td>0.868476</td>\n",
       "      <td>4.610974e-01</td>\n",
       "      <td>4.793974e-01</td>\n",
       "      <td>4.679124e-01</td>\n",
       "      <td>0.990000</td>\n",
       "      <td>0.991150</td>\n",
       "      <td>8.395773e-03</td>\n",
       "      <td>0.984477</td>\n",
       "      <td>0.985786</td>\n",
       "      <td>0.985482</td>\n",
       "      <td>1.017570e-02</td>\n",
       "      <td>0.979065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>element_symb__affirmative</td>\n",
       "      <td>0.577514</td>\n",
       "      <td>3.498783e-01</td>\n",
       "      <td>4.370834e-01</td>\n",
       "      <td>4.490662e-01</td>\n",
       "      <td>0.973684</td>\n",
       "      <td>0.975610</td>\n",
       "      <td>2.134477e-02</td>\n",
       "      <td>0.965086</td>\n",
       "      <td>0.905405</td>\n",
       "      <td>0.912500</td>\n",
       "      <td>6.313708e-02</td>\n",
       "      <td>0.913252</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>element_symb__conjunction</td>\n",
       "      <td>0.425471</td>\n",
       "      <td>3.073251e-01</td>\n",
       "      <td>2.748850e-01</td>\n",
       "      <td>2.742458e-01</td>\n",
       "      <td>0.880000</td>\n",
       "      <td>0.884615</td>\n",
       "      <td>7.882613e-02</td>\n",
       "      <td>0.853128</td>\n",
       "      <td>0.932500</td>\n",
       "      <td>0.940659</td>\n",
       "      <td>5.868069e-02</td>\n",
       "      <td>0.892681</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>element_symb__disjunction</td>\n",
       "      <td>0.066850</td>\n",
       "      <td>6.149318e-02</td>\n",
       "      <td>6.555922e-02</td>\n",
       "      <td>6.573485e-02</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.818182</td>\n",
       "      <td>1.413988e-01</td>\n",
       "      <td>0.752431</td>\n",
       "      <td>0.790000</td>\n",
       "      <td>0.787879</td>\n",
       "      <td>1.353580e-01</td>\n",
       "      <td>0.737389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>element_symb__negated</td>\n",
       "      <td>0.365157</td>\n",
       "      <td>2.844515e-01</td>\n",
       "      <td>2.087251e-01</td>\n",
       "      <td>2.134846e-01</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>9.240658e-19</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.743369e-25</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>facts__affirmative</td>\n",
       "      <td>0.093213</td>\n",
       "      <td>8.781164e-02</td>\n",
       "      <td>7.354002e-02</td>\n",
       "      <td>6.904972e-02</td>\n",
       "      <td>0.858407</td>\n",
       "      <td>0.885714</td>\n",
       "      <td>1.054875e-01</td>\n",
       "      <td>0.820781</td>\n",
       "      <td>0.850446</td>\n",
       "      <td>0.863544</td>\n",
       "      <td>1.072506e-01</td>\n",
       "      <td>0.803707</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>facts__conjunction</td>\n",
       "      <td>0.035627</td>\n",
       "      <td>3.673653e-02</td>\n",
       "      <td>2.145067e-02</td>\n",
       "      <td>2.171824e-02</td>\n",
       "      <td>0.720000</td>\n",
       "      <td>0.730769</td>\n",
       "      <td>1.866593e-01</td>\n",
       "      <td>0.650655</td>\n",
       "      <td>0.790000</td>\n",
       "      <td>0.814978</td>\n",
       "      <td>1.412838e-01</td>\n",
       "      <td>0.727692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>facts__disjunction</td>\n",
       "      <td>0.017876</td>\n",
       "      <td>1.650338e-02</td>\n",
       "      <td>1.990205e-02</td>\n",
       "      <td>2.031295e-02</td>\n",
       "      <td>0.760000</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>1.681977e-01</td>\n",
       "      <td>0.633569</td>\n",
       "      <td>0.712500</td>\n",
       "      <td>0.679666</td>\n",
       "      <td>1.881253e-01</td>\n",
       "      <td>0.617878</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>facts__negated</td>\n",
       "      <td>0.028115</td>\n",
       "      <td>2.862351e-02</td>\n",
       "      <td>2.065965e-02</td>\n",
       "      <td>2.092022e-02</td>\n",
       "      <td>0.823009</td>\n",
       "      <td>0.818182</td>\n",
       "      <td>1.239086e-01</td>\n",
       "      <td>0.773496</td>\n",
       "      <td>0.904018</td>\n",
       "      <td>0.903803</td>\n",
       "      <td>7.790436e-02</td>\n",
       "      <td>0.832414</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>inventors__affirmative</td>\n",
       "      <td>0.059885</td>\n",
       "      <td>5.634922e-02</td>\n",
       "      <td>5.107413e-02</td>\n",
       "      <td>5.130731e-02</td>\n",
       "      <td>0.841463</td>\n",
       "      <td>0.860215</td>\n",
       "      <td>1.071005e-01</td>\n",
       "      <td>0.804620</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>0.844828</td>\n",
       "      <td>1.331661e-01</td>\n",
       "      <td>0.777379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>inventors__conjunction</td>\n",
       "      <td>0.024496</td>\n",
       "      <td>2.481788e-02</td>\n",
       "      <td>1.691028e-02</td>\n",
       "      <td>1.747544e-02</td>\n",
       "      <td>0.720000</td>\n",
       "      <td>0.754386</td>\n",
       "      <td>1.596683e-01</td>\n",
       "      <td>0.733773</td>\n",
       "      <td>0.810000</td>\n",
       "      <td>0.829596</td>\n",
       "      <td>1.303819e-01</td>\n",
       "      <td>0.757822</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>inventors__disjunction</td>\n",
       "      <td>0.011326</td>\n",
       "      <td>1.231043e-02</td>\n",
       "      <td>5.683728e-03</td>\n",
       "      <td>5.092700e-03</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>0.732143</td>\n",
       "      <td>2.037729e-01</td>\n",
       "      <td>0.636309</td>\n",
       "      <td>0.740000</td>\n",
       "      <td>0.754717</td>\n",
       "      <td>1.790653e-01</td>\n",
       "      <td>0.644249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>inventors__negated</td>\n",
       "      <td>0.085275</td>\n",
       "      <td>8.420518e-02</td>\n",
       "      <td>4.965519e-02</td>\n",
       "      <td>4.958424e-02</td>\n",
       "      <td>0.658537</td>\n",
       "      <td>0.631579</td>\n",
       "      <td>1.951255e-01</td>\n",
       "      <td>0.606508</td>\n",
       "      <td>0.706790</td>\n",
       "      <td>0.686469</td>\n",
       "      <td>1.714797e-01</td>\n",
       "      <td>0.652882</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>ambiguous_lie</td>\n",
       "      <td>0.051308</td>\n",
       "      <td>5.361147e-08</td>\n",
       "      <td>3.629734e-07</td>\n",
       "      <td>6.957434e-08</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.412132e-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.428571</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.524060e-01</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>ambiguous_truthful_reply</td>\n",
       "      <td>0.051308</td>\n",
       "      <td>-6.544491e-08</td>\n",
       "      <td>9.936519e-08</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>2.086981e-01</td>\n",
       "      <td>0.710639</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>2.632712e-01</td>\n",
       "      <td>0.667486</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>honest_reply_despite_incentive_to_lie</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-1.319963e-07</td>\n",
       "      <td>2.438925e-07</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>7.010247e-02</td>\n",
       "      <td>0.847460</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>2.680697e-01</td>\n",
       "      <td>0.664283</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>unambiguous_lie</td>\n",
       "      <td>0.107328</td>\n",
       "      <td>1.258997e-07</td>\n",
       "      <td>-2.674763e-07</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.176609e-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.538462</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.550336e-01</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>unambiguous_truthful_reply</td>\n",
       "      <td>0.107328</td>\n",
       "      <td>6.411020e-08</td>\n",
       "      <td>-2.047305e-07</td>\n",
       "      <td>-7.574867e-08</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>0.842105</td>\n",
       "      <td>2.202386e-01</td>\n",
       "      <td>0.697597</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>2.593466e-01</td>\n",
       "      <td>0.666987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>larger_than</td>\n",
       "      <td>0.456852</td>\n",
       "      <td>3.142809e-01</td>\n",
       "      <td>3.110852e-01</td>\n",
       "      <td>3.111883e-01</td>\n",
       "      <td>0.972222</td>\n",
       "      <td>0.971867</td>\n",
       "      <td>2.691245e-02</td>\n",
       "      <td>0.946148</td>\n",
       "      <td>0.970960</td>\n",
       "      <td>0.972019</td>\n",
       "      <td>2.098595e-02</td>\n",
       "      <td>0.955345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>smaller_than</td>\n",
       "      <td>0.347493</td>\n",
       "      <td>2.578911e-01</td>\n",
       "      <td>2.581460e-01</td>\n",
       "      <td>2.549792e-01</td>\n",
       "      <td>0.949495</td>\n",
       "      <td>0.955357</td>\n",
       "      <td>3.484066e-02</td>\n",
       "      <td>0.932294</td>\n",
       "      <td>0.945076</td>\n",
       "      <td>0.946919</td>\n",
       "      <td>3.824428e-02</td>\n",
       "      <td>0.920902</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>sp_en_trans__negated</td>\n",
       "      <td>0.679175</td>\n",
       "      <td>4.083760e-01</td>\n",
       "      <td>3.927502e-01</td>\n",
       "      <td>3.957529e-01</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.294588e-04</td>\n",
       "      <td>0.997626</td>\n",
       "      <td>0.996466</td>\n",
       "      <td>0.996466</td>\n",
       "      <td>2.715710e-03</td>\n",
       "      <td>0.995245</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>sp_en_trans__affirmative</td>\n",
       "      <td>0.763526</td>\n",
       "      <td>4.254077e-01</td>\n",
       "      <td>4.657559e-01</td>\n",
       "      <td>4.151790e-01</td>\n",
       "      <td>0.971831</td>\n",
       "      <td>0.977778</td>\n",
       "      <td>1.804558e-02</td>\n",
       "      <td>0.974164</td>\n",
       "      <td>0.968198</td>\n",
       "      <td>0.967273</td>\n",
       "      <td>1.792615e-02</td>\n",
       "      <td>0.966510</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>sp_en_trans__conjunction</td>\n",
       "      <td>0.360656</td>\n",
       "      <td>2.654025e-01</td>\n",
       "      <td>2.726139e-01</td>\n",
       "      <td>2.707033e-01</td>\n",
       "      <td>0.930000</td>\n",
       "      <td>0.930693</td>\n",
       "      <td>5.284401e-02</td>\n",
       "      <td>0.918356</td>\n",
       "      <td>0.967500</td>\n",
       "      <td>0.970787</td>\n",
       "      <td>2.128471e-02</td>\n",
       "      <td>0.955200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>sp_en_trans__disjunction</td>\n",
       "      <td>0.055794</td>\n",
       "      <td>5.717739e-02</td>\n",
       "      <td>3.746276e-02</td>\n",
       "      <td>3.632303e-02</td>\n",
       "      <td>0.840000</td>\n",
       "      <td>0.818182</td>\n",
       "      <td>1.090786e-01</td>\n",
       "      <td>0.762275</td>\n",
       "      <td>0.832500</td>\n",
       "      <td>0.813370</td>\n",
       "      <td>1.089132e-01</td>\n",
       "      <td>0.770467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>common_claim_true_false</td>\n",
       "      <td>0.016943</td>\n",
       "      <td>1.601768e-02</td>\n",
       "      <td>1.895602e-02</td>\n",
       "      <td>1.898451e-02</td>\n",
       "      <td>0.782022</td>\n",
       "      <td>0.786813</td>\n",
       "      <td>1.528305e-01</td>\n",
       "      <td>0.684293</td>\n",
       "      <td>0.747472</td>\n",
       "      <td>0.749093</td>\n",
       "      <td>1.691688e-01</td>\n",
       "      <td>0.665531</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>counterfact_true_false</td>\n",
       "      <td>0.012648</td>\n",
       "      <td>1.257900e-02</td>\n",
       "      <td>1.211249e-02</td>\n",
       "      <td>1.211160e-02</td>\n",
       "      <td>0.707962</td>\n",
       "      <td>0.729851</td>\n",
       "      <td>1.875083e-01</td>\n",
       "      <td>0.634437</td>\n",
       "      <td>0.710219</td>\n",
       "      <td>0.735300</td>\n",
       "      <td>1.858964e-01</td>\n",
       "      <td>0.638215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>animal_class__affirmative+negated</td>\n",
       "      <td>0.726462</td>\n",
       "      <td>3.255649e-01</td>\n",
       "      <td>3.730479e-01</td>\n",
       "      <td>3.483443e-01</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.724002e-03</td>\n",
       "      <td>0.988765</td>\n",
       "      <td>0.984733</td>\n",
       "      <td>0.985294</td>\n",
       "      <td>9.318734e-03</td>\n",
       "      <td>0.984407</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>animal_class__affirmative+disjunction</td>\n",
       "      <td>0.451377</td>\n",
       "      <td>3.461717e-02</td>\n",
       "      <td>3.647969e-02</td>\n",
       "      <td>3.132026e-02</td>\n",
       "      <td>0.796992</td>\n",
       "      <td>0.761062</td>\n",
       "      <td>1.403436e-01</td>\n",
       "      <td>0.657099</td>\n",
       "      <td>0.779661</td>\n",
       "      <td>0.789189</td>\n",
       "      <td>1.506604e-01</td>\n",
       "      <td>0.710305</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>animal_class__negated+conjunction</td>\n",
       "      <td>0.447213</td>\n",
       "      <td>1.274588e-01</td>\n",
       "      <td>1.192823e-01</td>\n",
       "      <td>1.191230e-01</td>\n",
       "      <td>0.917293</td>\n",
       "      <td>0.925170</td>\n",
       "      <td>5.825581e-02</td>\n",
       "      <td>0.896909</td>\n",
       "      <td>0.930320</td>\n",
       "      <td>0.932849</td>\n",
       "      <td>5.293857e-02</td>\n",
       "      <td>0.895870</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>animal_class__affirmative+negated+conjunction+...</td>\n",
       "      <td>0.449295</td>\n",
       "      <td>5.486206e-02</td>\n",
       "      <td>5.498721e-02</td>\n",
       "      <td>5.459481e-02</td>\n",
       "      <td>0.748120</td>\n",
       "      <td>0.712446</td>\n",
       "      <td>1.358077e-01</td>\n",
       "      <td>0.718068</td>\n",
       "      <td>0.740113</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>1.394676e-01</td>\n",
       "      <td>0.725212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>cities__affirmative+negated</td>\n",
       "      <td>1.279278</td>\n",
       "      <td>3.803487e-01</td>\n",
       "      <td>3.786022e-01</td>\n",
       "      <td>3.774681e-01</td>\n",
       "      <td>0.983333</td>\n",
       "      <td>0.984076</td>\n",
       "      <td>1.378936e-02</td>\n",
       "      <td>0.976110</td>\n",
       "      <td>0.981605</td>\n",
       "      <td>0.981621</td>\n",
       "      <td>1.456238e-02</td>\n",
       "      <td>0.971601</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>cities__affirmative+disjunction</td>\n",
       "      <td>0.874302</td>\n",
       "      <td>2.039240e-01</td>\n",
       "      <td>2.019980e-01</td>\n",
       "      <td>2.012607e-01</td>\n",
       "      <td>0.920000</td>\n",
       "      <td>0.913043</td>\n",
       "      <td>6.200888e-02</td>\n",
       "      <td>0.864782</td>\n",
       "      <td>0.906015</td>\n",
       "      <td>0.901575</td>\n",
       "      <td>6.242136e-02</td>\n",
       "      <td>0.870892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>cities__negated+conjunction</td>\n",
       "      <td>0.628615</td>\n",
       "      <td>1.256688e-01</td>\n",
       "      <td>1.276570e-01</td>\n",
       "      <td>1.258128e-01</td>\n",
       "      <td>0.930000</td>\n",
       "      <td>0.939828</td>\n",
       "      <td>4.939174e-02</td>\n",
       "      <td>0.909276</td>\n",
       "      <td>0.926065</td>\n",
       "      <td>0.929172</td>\n",
       "      <td>5.756781e-02</td>\n",
       "      <td>0.886600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>cities__affirmative+negated+conjunction+disjun...</td>\n",
       "      <td>0.751458</td>\n",
       "      <td>1.131659e-01</td>\n",
       "      <td>1.183838e-01</td>\n",
       "      <td>1.180848e-01</td>\n",
       "      <td>0.891000</td>\n",
       "      <td>0.899724</td>\n",
       "      <td>7.653691e-02</td>\n",
       "      <td>0.863456</td>\n",
       "      <td>0.865163</td>\n",
       "      <td>0.870983</td>\n",
       "      <td>8.988929e-02</td>\n",
       "      <td>0.840710</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>element_symb__affirmative+negated</td>\n",
       "      <td>0.471336</td>\n",
       "      <td>1.768124e-01</td>\n",
       "      <td>1.959989e-01</td>\n",
       "      <td>1.974226e-01</td>\n",
       "      <td>0.986842</td>\n",
       "      <td>0.987342</td>\n",
       "      <td>1.476023e-02</td>\n",
       "      <td>0.972173</td>\n",
       "      <td>0.959459</td>\n",
       "      <td>0.960784</td>\n",
       "      <td>3.085962e-02</td>\n",
       "      <td>0.954207</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>element_symb__affirmative+disjunction</td>\n",
       "      <td>0.322182</td>\n",
       "      <td>2.541350e-02</td>\n",
       "      <td>3.067232e-02</td>\n",
       "      <td>3.052880e-02</td>\n",
       "      <td>0.840580</td>\n",
       "      <td>0.851351</td>\n",
       "      <td>1.039167e-01</td>\n",
       "      <td>0.806696</td>\n",
       "      <td>0.843066</td>\n",
       "      <td>0.848057</td>\n",
       "      <td>1.078788e-01</td>\n",
       "      <td>0.784090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>element_symb__negated+conjunction</td>\n",
       "      <td>0.395314</td>\n",
       "      <td>1.584398e-01</td>\n",
       "      <td>1.283977e-01</td>\n",
       "      <td>1.277602e-01</td>\n",
       "      <td>0.891304</td>\n",
       "      <td>0.896552</td>\n",
       "      <td>8.803703e-02</td>\n",
       "      <td>0.847493</td>\n",
       "      <td>0.906934</td>\n",
       "      <td>0.918138</td>\n",
       "      <td>6.595096e-02</td>\n",
       "      <td>0.888005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>element_symb__affirmative+negated+conjunction+...</td>\n",
       "      <td>0.358748</td>\n",
       "      <td>4.414859e-02</td>\n",
       "      <td>3.996578e-02</td>\n",
       "      <td>4.002296e-02</td>\n",
       "      <td>0.804348</td>\n",
       "      <td>0.812500</td>\n",
       "      <td>1.324841e-01</td>\n",
       "      <td>0.739445</td>\n",
       "      <td>0.814781</td>\n",
       "      <td>0.819556</td>\n",
       "      <td>1.243532e-01</td>\n",
       "      <td>0.755632</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>facts__affirmative+negated</td>\n",
       "      <td>0.060664</td>\n",
       "      <td>5.364384e-02</td>\n",
       "      <td>4.356713e-02</td>\n",
       "      <td>4.226291e-02</td>\n",
       "      <td>0.787611</td>\n",
       "      <td>0.811024</td>\n",
       "      <td>1.386779e-01</td>\n",
       "      <td>0.765212</td>\n",
       "      <td>0.844866</td>\n",
       "      <td>0.852285</td>\n",
       "      <td>1.077000e-01</td>\n",
       "      <td>0.782647</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>facts__affirmative+disjunction</td>\n",
       "      <td>0.055544</td>\n",
       "      <td>2.839450e-02</td>\n",
       "      <td>2.716368e-02</td>\n",
       "      <td>2.710478e-02</td>\n",
       "      <td>0.788732</td>\n",
       "      <td>0.798206</td>\n",
       "      <td>1.572402e-01</td>\n",
       "      <td>0.704486</td>\n",
       "      <td>0.768868</td>\n",
       "      <td>0.753769</td>\n",
       "      <td>1.613407e-01</td>\n",
       "      <td>0.679229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>facts__negated+conjunction</td>\n",
       "      <td>0.031871</td>\n",
       "      <td>2.251311e-02</td>\n",
       "      <td>1.294852e-02</td>\n",
       "      <td>1.317613e-02</td>\n",
       "      <td>0.661972</td>\n",
       "      <td>0.669725</td>\n",
       "      <td>2.013792e-01</td>\n",
       "      <td>0.581925</td>\n",
       "      <td>0.742925</td>\n",
       "      <td>0.759382</td>\n",
       "      <td>1.715787e-01</td>\n",
       "      <td>0.629935</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>facts__affirmative+negated+conjunction+disjunc...</td>\n",
       "      <td>0.043708</td>\n",
       "      <td>2.225074e-02</td>\n",
       "      <td>1.678023e-02</td>\n",
       "      <td>1.706678e-02</td>\n",
       "      <td>0.708920</td>\n",
       "      <td>0.718182</td>\n",
       "      <td>1.832747e-01</td>\n",
       "      <td>0.649696</td>\n",
       "      <td>0.747052</td>\n",
       "      <td>0.751016</td>\n",
       "      <td>1.631878e-01</td>\n",
       "      <td>0.668476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>inventors__affirmative+negated</td>\n",
       "      <td>0.072580</td>\n",
       "      <td>4.958832e-02</td>\n",
       "      <td>4.507503e-02</td>\n",
       "      <td>4.572443e-02</td>\n",
       "      <td>0.829268</td>\n",
       "      <td>0.829268</td>\n",
       "      <td>1.245958e-01</td>\n",
       "      <td>0.744966</td>\n",
       "      <td>0.830247</td>\n",
       "      <td>0.826498</td>\n",
       "      <td>1.359260e-01</td>\n",
       "      <td>0.731713</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>inventors__affirmative+disjunction</td>\n",
       "      <td>0.035606</td>\n",
       "      <td>1.016311e-02</td>\n",
       "      <td>8.969028e-03</td>\n",
       "      <td>8.869333e-03</td>\n",
       "      <td>0.736264</td>\n",
       "      <td>0.762376</td>\n",
       "      <td>1.608515e-01</td>\n",
       "      <td>0.695569</td>\n",
       "      <td>0.766575</td>\n",
       "      <td>0.771930</td>\n",
       "      <td>1.627316e-01</td>\n",
       "      <td>0.681231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>inventors__negated+conjunction</td>\n",
       "      <td>0.054885</td>\n",
       "      <td>4.034714e-02</td>\n",
       "      <td>4.136835e-02</td>\n",
       "      <td>4.150045e-02</td>\n",
       "      <td>0.675824</td>\n",
       "      <td>0.725581</td>\n",
       "      <td>2.141323e-01</td>\n",
       "      <td>0.583113</td>\n",
       "      <td>0.647790</td>\n",
       "      <td>0.698225</td>\n",
       "      <td>2.191090e-01</td>\n",
       "      <td>0.569212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>inventors__affirmative+negated+conjunction+dis...</td>\n",
       "      <td>0.045245</td>\n",
       "      <td>1.717407e-02</td>\n",
       "      <td>1.567208e-02</td>\n",
       "      <td>1.575162e-02</td>\n",
       "      <td>0.692308</td>\n",
       "      <td>0.738318</td>\n",
       "      <td>2.098124e-01</td>\n",
       "      <td>0.603974</td>\n",
       "      <td>0.652624</td>\n",
       "      <td>0.690080</td>\n",
       "      <td>2.149256e-01</td>\n",
       "      <td>0.583715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>sp_en_trans__affirmative+negated</td>\n",
       "      <td>0.721350</td>\n",
       "      <td>2.160448e-01</td>\n",
       "      <td>2.094281e-01</td>\n",
       "      <td>2.005184e-01</td>\n",
       "      <td>0.971831</td>\n",
       "      <td>0.975309</td>\n",
       "      <td>2.340497e-02</td>\n",
       "      <td>0.960256</td>\n",
       "      <td>0.971731</td>\n",
       "      <td>0.971326</td>\n",
       "      <td>1.558442e-02</td>\n",
       "      <td>0.958855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>sp_en_trans__affirmative+disjunction</td>\n",
       "      <td>0.409660</td>\n",
       "      <td>5.649454e-02</td>\n",
       "      <td>5.059661e-02</td>\n",
       "      <td>4.968711e-02</td>\n",
       "      <td>0.801170</td>\n",
       "      <td>0.817204</td>\n",
       "      <td>1.244469e-01</td>\n",
       "      <td>0.787459</td>\n",
       "      <td>0.828697</td>\n",
       "      <td>0.822995</td>\n",
       "      <td>1.080418e-01</td>\n",
       "      <td>0.785671</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>sp_en_trans__negated+conjunction</td>\n",
       "      <td>0.519915</td>\n",
       "      <td>1.346869e-01</td>\n",
       "      <td>1.178831e-01</td>\n",
       "      <td>1.173058e-01</td>\n",
       "      <td>0.953216</td>\n",
       "      <td>0.953488</td>\n",
       "      <td>3.660722e-02</td>\n",
       "      <td>0.930899</td>\n",
       "      <td>0.956076</td>\n",
       "      <td>0.959350</td>\n",
       "      <td>3.132679e-02</td>\n",
       "      <td>0.941975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>sp_en_trans__affirmative+negated+conjunction+d...</td>\n",
       "      <td>0.464788</td>\n",
       "      <td>7.256485e-02</td>\n",
       "      <td>5.993876e-02</td>\n",
       "      <td>5.999693e-02</td>\n",
       "      <td>0.827485</td>\n",
       "      <td>0.822823</td>\n",
       "      <td>1.349240e-01</td>\n",
       "      <td>0.732068</td>\n",
       "      <td>0.841874</td>\n",
       "      <td>0.837349</td>\n",
       "      <td>1.220694e-01</td>\n",
       "      <td>0.752624</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>ambiguous_truthful_reply+unambiguous_lie+unamb...</td>\n",
       "      <td>0.088655</td>\n",
       "      <td>8.594724e-02</td>\n",
       "      <td>4.916947e-02</td>\n",
       "      <td>6.062967e-02</td>\n",
       "      <td>0.807692</td>\n",
       "      <td>0.814815</td>\n",
       "      <td>1.229308e-01</td>\n",
       "      <td>0.829563</td>\n",
       "      <td>0.950495</td>\n",
       "      <td>0.958678</td>\n",
       "      <td>4.795936e-02</td>\n",
       "      <td>0.911356</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>animal_class+facts+inventors__affirmative+nega...</td>\n",
       "      <td>0.232722</td>\n",
       "      <td>2.317787e-02</td>\n",
       "      <td>2.236908e-02</td>\n",
       "      <td>2.232274e-02</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.769231</td>\n",
       "      <td>1.661825e-01</td>\n",
       "      <td>0.687835</td>\n",
       "      <td>0.764471</td>\n",
       "      <td>0.779577</td>\n",
       "      <td>1.603947e-01</td>\n",
       "      <td>0.688521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>animal_class+facts+inventors__affirmative+nega...</td>\n",
       "      <td>0.213619</td>\n",
       "      <td>1.702797e-02</td>\n",
       "      <td>1.806127e-02</td>\n",
       "      <td>1.802021e-02</td>\n",
       "      <td>0.805430</td>\n",
       "      <td>0.819517</td>\n",
       "      <td>1.450034e-01</td>\n",
       "      <td>0.711798</td>\n",
       "      <td>0.787832</td>\n",
       "      <td>0.797455</td>\n",
       "      <td>1.535762e-01</td>\n",
       "      <td>0.695825</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 14
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:58.055171Z",
     "start_time": "2024-12-07T21:26:58.033406Z"
    }
   },
   "cell_type": "code",
   "source": "lyr18_eval_except_test_df.describe()",
   "id": "bdb1a724c721cd13",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "          tf_sep  train_recon_improv  val_recon_improv  \\\n",
       "count  60.000000        6.000000e+01      6.000000e+01   \n",
       "mean    0.330176        1.372271e-01      1.364919e-01   \n",
       "std     0.345483        1.515392e-01      1.602267e-01   \n",
       "min     0.000000       -1.319963e-07     -2.674763e-07   \n",
       "25%     0.053991        2.440788e-02      2.047025e-02   \n",
       "50%     0.273841        5.709103e-02      5.012590e-02   \n",
       "75%     0.466425        2.443653e-01      2.149331e-01   \n",
       "max     1.690080        6.311205e-01      6.175942e-01   \n",
       "\n",
       "       val_recon_improv_w_val_mean    val_acc     val_f1     val_brier  \\\n",
       "count                 6.000000e+01  60.000000  58.000000  6.000000e+01   \n",
       "mean                  1.329416e-01   0.836446   0.855520  1.083910e-01   \n",
       "std                   1.534566e-01   0.122719   0.104755  7.179553e-02   \n",
       "min                  -7.574867e-08   0.500000   0.631579  9.240658e-19   \n",
       "25%                   2.076840e-02   0.746090   0.764090  3.616558e-02   \n",
       "50%                   5.049721e-02   0.828377   0.835687  1.234197e-01   \n",
       "75%                   2.190491e-01   0.954912   0.962054  1.621842e-01   \n",
       "max                   6.166543e-01   1.000000   1.000000  2.412132e-01   \n",
       "\n",
       "       val_soft_f1  train_acc   train_f1   train_brier  train_soft_f1  \n",
       "count    58.000000  60.000000  58.000000  6.000000e+01      58.000000  \n",
       "mean      0.803688   0.824882   0.848516  1.098008e-01       0.804177  \n",
       "std       0.132683   0.151645   0.117282  7.665211e-02       0.129689  \n",
       "min       0.581925   0.250000   0.400000  6.743369e-25       0.569212  \n",
       "25%       0.699319   0.747367   0.762519  4.509263e-02       0.683053  \n",
       "50%       0.796039   0.842470   0.850049  1.084775e-01       0.783369  \n",
       "75%       0.931945   0.946431   0.955738  1.628457e-01       0.918990  \n",
       "max       1.000000   1.000000   1.000000  2.680697e-01       1.000000  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tf_sep</th>\n",
       "      <th>train_recon_improv</th>\n",
       "      <th>val_recon_improv</th>\n",
       "      <th>val_recon_improv_w_val_mean</th>\n",
       "      <th>val_acc</th>\n",
       "      <th>val_f1</th>\n",
       "      <th>val_brier</th>\n",
       "      <th>val_soft_f1</th>\n",
       "      <th>train_acc</th>\n",
       "      <th>train_f1</th>\n",
       "      <th>train_brier</th>\n",
       "      <th>train_soft_f1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>60.000000</td>\n",
       "      <td>6.000000e+01</td>\n",
       "      <td>6.000000e+01</td>\n",
       "      <td>6.000000e+01</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>58.000000</td>\n",
       "      <td>6.000000e+01</td>\n",
       "      <td>58.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>58.000000</td>\n",
       "      <td>6.000000e+01</td>\n",
       "      <td>58.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.330176</td>\n",
       "      <td>1.372271e-01</td>\n",
       "      <td>1.364919e-01</td>\n",
       "      <td>1.329416e-01</td>\n",
       "      <td>0.836446</td>\n",
       "      <td>0.855520</td>\n",
       "      <td>1.083910e-01</td>\n",
       "      <td>0.803688</td>\n",
       "      <td>0.824882</td>\n",
       "      <td>0.848516</td>\n",
       "      <td>1.098008e-01</td>\n",
       "      <td>0.804177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.345483</td>\n",
       "      <td>1.515392e-01</td>\n",
       "      <td>1.602267e-01</td>\n",
       "      <td>1.534566e-01</td>\n",
       "      <td>0.122719</td>\n",
       "      <td>0.104755</td>\n",
       "      <td>7.179553e-02</td>\n",
       "      <td>0.132683</td>\n",
       "      <td>0.151645</td>\n",
       "      <td>0.117282</td>\n",
       "      <td>7.665211e-02</td>\n",
       "      <td>0.129689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-1.319963e-07</td>\n",
       "      <td>-2.674763e-07</td>\n",
       "      <td>-7.574867e-08</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.631579</td>\n",
       "      <td>9.240658e-19</td>\n",
       "      <td>0.581925</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>6.743369e-25</td>\n",
       "      <td>0.569212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.053991</td>\n",
       "      <td>2.440788e-02</td>\n",
       "      <td>2.047025e-02</td>\n",
       "      <td>2.076840e-02</td>\n",
       "      <td>0.746090</td>\n",
       "      <td>0.764090</td>\n",
       "      <td>3.616558e-02</td>\n",
       "      <td>0.699319</td>\n",
       "      <td>0.747367</td>\n",
       "      <td>0.762519</td>\n",
       "      <td>4.509263e-02</td>\n",
       "      <td>0.683053</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.273841</td>\n",
       "      <td>5.709103e-02</td>\n",
       "      <td>5.012590e-02</td>\n",
       "      <td>5.049721e-02</td>\n",
       "      <td>0.828377</td>\n",
       "      <td>0.835687</td>\n",
       "      <td>1.234197e-01</td>\n",
       "      <td>0.796039</td>\n",
       "      <td>0.842470</td>\n",
       "      <td>0.850049</td>\n",
       "      <td>1.084775e-01</td>\n",
       "      <td>0.783369</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.466425</td>\n",
       "      <td>2.443653e-01</td>\n",
       "      <td>2.149331e-01</td>\n",
       "      <td>2.190491e-01</td>\n",
       "      <td>0.954912</td>\n",
       "      <td>0.962054</td>\n",
       "      <td>1.621842e-01</td>\n",
       "      <td>0.931945</td>\n",
       "      <td>0.946431</td>\n",
       "      <td>0.955738</td>\n",
       "      <td>1.628457e-01</td>\n",
       "      <td>0.918990</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.690080</td>\n",
       "      <td>6.311205e-01</td>\n",
       "      <td>6.175942e-01</td>\n",
       "      <td>6.166543e-01</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.412132e-01</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.680697e-01</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 15
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-08T17:33:26.041799Z",
     "start_time": "2024-12-08T17:33:26.027652Z"
    }
   },
   "cell_type": "code",
   "source": [
    "lyr18_eval_except_test_affirmative_scenarios_df = lyr18_eval_except_test_df[lyr18_eval_except_test_df['scenario'].str.endswith(\"__affirmative\")]\n",
    "lyr18_eval_except_test_negated_scenarios_df = lyr18_eval_except_test_df[lyr18_eval_except_test_df['scenario'].str.endswith(\"__negated\")]\n",
    "lyr18_eval_except_test_conjunction_scenarios_df = lyr18_eval_except_test_df[lyr18_eval_except_test_df['scenario'].str.endswith(\"__conjunction\")]\n",
    "lyr18_eval_except_test_disjunction_scenarios_df = lyr18_eval_except_test_df[lyr18_eval_except_test_df['scenario'].str.endswith(\"__disjunction\")]\n",
    "lyr18_eval_except_test_affirmative_and_negated_scenarios_df = lyr18_eval_except_test_df[lyr18_eval_except_test_df['scenario'].str.endswith(\"__affirmative+negated\")]\n",
    "lyr18_eval_except_test_affirmative_and_disjunction_scenarios_df = lyr18_eval_except_test_df[lyr18_eval_except_test_df['scenario'].str.endswith(\"__affirmative+disjunction\")]\n",
    "lyr18_eval_except_test_negated_and_conjunction_scenarios_df = lyr18_eval_except_test_df[lyr18_eval_except_test_df['scenario'].str.endswith(\"__negated+conjunction\")]\n",
    "lyr18_eval_except_test_all_4_variants_scenarios_df = lyr18_eval_except_test_df[lyr18_eval_except_test_df['scenario'].str.endswith(\"__affirmative+negated+conjunction+disjunction\")]\n",
    "# Using mathematically equivalent difference of means (rather than mean of differences) to get around pandas confusion around mismatched row indexes\n",
    "affirm_baseline_avg_train_acc = lyr18_eval_except_test_affirmative_scenarios_df[\"train_acc\"].mean()\n",
    "affirm_baseline_avg_val_acc = lyr18_eval_except_test_affirmative_scenarios_df[\"val_acc\"].mean()\n",
    "lyr18_affirm_to_neg_train_acc_loss = affirm_baseline_avg_train_acc - lyr18_eval_except_test_negated_scenarios_df[\"train_acc\"].mean()\n",
    "lyr18_affirm_to_neg_val_acc_loss = affirm_baseline_avg_val_acc - lyr18_eval_except_test_negated_scenarios_df[\"val_acc\"].mean()\n",
    "lyr18_affirm_to_conj_train_acc_loss = affirm_baseline_avg_train_acc - lyr18_eval_except_test_conjunction_scenarios_df[\"train_acc\"].mean()\n",
    "lyr18_affirm_to_conj_val_acc_loss = affirm_baseline_avg_val_acc - lyr18_eval_except_test_conjunction_scenarios_df[\"val_acc\"].mean()\n",
    "lyr18_affirm_to_disj_train_acc_loss = affirm_baseline_avg_train_acc - lyr18_eval_except_test_disjunction_scenarios_df[\"train_acc\"].mean()\n",
    "lyr18_affirm_to_disj_val_acc_loss = affirm_baseline_avg_val_acc - lyr18_eval_except_test_disjunction_scenarios_df[\"val_acc\"].mean()\n",
    "lyr18_affirm_to_affirm_neg_train_acc_loss = affirm_baseline_avg_train_acc - lyr18_eval_except_test_affirmative_and_negated_scenarios_df[\"train_acc\"].mean()\n",
    "lyr18_affirm_to_affirm_neg_val_acc_loss = affirm_baseline_avg_val_acc - lyr18_eval_except_test_affirmative_and_negated_scenarios_df[\"val_acc\"].mean()\n",
    "lyr18_affirm_to_affirm_disj_train_acc_loss = affirm_baseline_avg_train_acc - lyr18_eval_except_test_affirmative_and_disjunction_scenarios_df[\"train_acc\"].mean()\n",
    "lyr18_affirm_to_affirm_disj_val_acc_loss = affirm_baseline_avg_val_acc - lyr18_eval_except_test_affirmative_and_disjunction_scenarios_df[\"val_acc\"].mean()\n",
    "lyr18_affirm_to_neg_conj_train_acc_loss = affirm_baseline_avg_train_acc - lyr18_eval_except_test_negated_and_conjunction_scenarios_df[\"train_acc\"].mean()\n",
    "lyr18_affirm_to_neg_conj_val_acc_loss = affirm_baseline_avg_val_acc - lyr18_eval_except_test_negated_and_conjunction_scenarios_df[\"val_acc\"].mean()\n",
    "lyr18_affirm_to_affirm_neg_conj_disj_train_acc_loss = affirm_baseline_avg_train_acc - lyr18_eval_except_test_all_4_variants_scenarios_df[\"train_acc\"].mean()\n",
    "lyr18_affirm_to_affirm_neg_conj_disj_val_acc_loss = affirm_baseline_avg_val_acc - lyr18_eval_except_test_all_4_variants_scenarios_df[\"val_acc\"].mean()\n",
    "\n",
    "pd.DataFrame({\n",
    "    \"scenario\": [\"negated\", \"conjunction\", \"disjunction\", \"affirmative+negated\", \"affirmative+disjunction\", \"negated+conjunction\", \"affirmative+negated+conjunction+disjunction\"],\n",
    "    \"train_acc_loss\": [lyr18_affirm_to_neg_train_acc_loss, lyr18_affirm_to_conj_train_acc_loss, lyr18_affirm_to_disj_train_acc_loss, lyr18_affirm_to_affirm_neg_train_acc_loss, lyr18_affirm_to_affirm_disj_train_acc_loss, lyr18_affirm_to_neg_conj_train_acc_loss, lyr18_affirm_to_affirm_neg_conj_disj_train_acc_loss],\n",
    "    \"val_acc_loss\": [lyr18_affirm_to_neg_val_acc_loss, lyr18_affirm_to_conj_val_acc_loss, lyr18_affirm_to_disj_val_acc_loss, lyr18_affirm_to_affirm_neg_val_acc_loss, lyr18_affirm_to_affirm_disj_val_acc_loss, lyr18_affirm_to_neg_conj_val_acc_loss, lyr18_affirm_to_affirm_neg_conj_disj_val_acc_loss]\n",
    "})"
   ],
   "id": "ede945c69be9af33",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "                                      scenario  train_acc_loss  val_acc_loss\n",
       "0                                      negated       -0.007461      0.026196\n",
       "1                                  conjunction        0.030802      0.088120\n",
       "2                                  disjunction        0.147588      0.164787\n",
       "3                          affirmative+negated       -0.006603      0.011639\n",
       "4                      affirmative+disjunction        0.106691      0.124164\n",
       "5                          negated+conjunction        0.070486      0.099852\n",
       "6  affirmative+negated+conjunction+disjunction        0.145237      0.159423"
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>scenario</th>\n",
       "      <th>train_acc_loss</th>\n",
       "      <th>val_acc_loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>negated</td>\n",
       "      <td>-0.007461</td>\n",
       "      <td>0.026196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>conjunction</td>\n",
       "      <td>0.030802</td>\n",
       "      <td>0.088120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>disjunction</td>\n",
       "      <td>0.147588</td>\n",
       "      <td>0.164787</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>affirmative+negated</td>\n",
       "      <td>-0.006603</td>\n",
       "      <td>0.011639</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>affirmative+disjunction</td>\n",
       "      <td>0.106691</td>\n",
       "      <td>0.124164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>negated+conjunction</td>\n",
       "      <td>0.070486</td>\n",
       "      <td>0.099852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>affirmative+negated+conjunction+disjunction</td>\n",
       "      <td>0.145237</td>\n",
       "      <td>0.159423</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 82
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-08T18:52:03.285372Z",
     "start_time": "2024-12-08T18:52:03.273685Z"
    }
   },
   "cell_type": "code",
   "source": [
    "lyr18_t_f_sep_and_t_p_dirs_correlations = lyr18_eval_except_test_df[['tf_sep', 'train_recon_improv', 'val_recon_improv', 'val_recon_improv_w_val_mean', \"val_acc\", \"train_acc\"]].corr()\n",
    "# fig1, ax = plt.subplots(figsize=(10, 6))\n",
    "# lyr18_t_f_sep_and_t_p_dirs_correlations_heatmap = ax.imshow(lyr18_t_f_sep_and_t_p_dirs_correlations, cmap=\"Greys\")\n",
    "# plt.show()\n",
    "lyr18_t_f_sep_and_t_p_dirs_correlations"
   ],
   "id": "b396c9b1dc83b047",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "                               tf_sep  train_recon_improv  val_recon_improv  \\\n",
       "tf_sep                       1.000000            0.836358          0.828997   \n",
       "train_recon_improv           0.836358            1.000000          0.990417   \n",
       "val_recon_improv             0.828997            0.990417          1.000000   \n",
       "val_recon_improv_w_val_mean  0.830357            0.990186          0.994927   \n",
       "val_acc                      0.682296            0.764079          0.758730   \n",
       "train_acc                    0.600867            0.689926          0.664601   \n",
       "\n",
       "                             val_recon_improv_w_val_mean   val_acc  train_acc  \n",
       "tf_sep                                          0.830357  0.682296   0.600867  \n",
       "train_recon_improv                              0.990186  0.764079   0.689926  \n",
       "val_recon_improv                                0.994927  0.758730   0.664601  \n",
       "val_recon_improv_w_val_mean                     1.000000  0.762526   0.670064  \n",
       "val_acc                                         0.762526  1.000000   0.704551  \n",
       "train_acc                                       0.670064  0.704551   1.000000  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tf_sep</th>\n",
       "      <th>train_recon_improv</th>\n",
       "      <th>val_recon_improv</th>\n",
       "      <th>val_recon_improv_w_val_mean</th>\n",
       "      <th>val_acc</th>\n",
       "      <th>train_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>tf_sep</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.836358</td>\n",
       "      <td>0.828997</td>\n",
       "      <td>0.830357</td>\n",
       "      <td>0.682296</td>\n",
       "      <td>0.600867</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train_recon_improv</th>\n",
       "      <td>0.836358</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.990417</td>\n",
       "      <td>0.990186</td>\n",
       "      <td>0.764079</td>\n",
       "      <td>0.689926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>val_recon_improv</th>\n",
       "      <td>0.828997</td>\n",
       "      <td>0.990417</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.994927</td>\n",
       "      <td>0.758730</td>\n",
       "      <td>0.664601</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>val_recon_improv_w_val_mean</th>\n",
       "      <td>0.830357</td>\n",
       "      <td>0.990186</td>\n",
       "      <td>0.994927</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.762526</td>\n",
       "      <td>0.670064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>val_acc</th>\n",
       "      <td>0.682296</td>\n",
       "      <td>0.764079</td>\n",
       "      <td>0.758730</td>\n",
       "      <td>0.762526</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.704551</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train_acc</th>\n",
       "      <td>0.600867</td>\n",
       "      <td>0.689926</td>\n",
       "      <td>0.664601</td>\n",
       "      <td>0.670064</td>\n",
       "      <td>0.704551</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 101
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:58.233965Z",
     "start_time": "2024-12-07T21:26:58.211792Z"
    }
   },
   "cell_type": "code",
   "source": "lyr25_eval_except_test_df",
   "id": "72358e781730e1c3",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "                                             scenario    tf_sep  \\\n",
       "0                           animal_class__affirmative  0.588167   \n",
       "1                           animal_class__conjunction  0.333727   \n",
       "2                           animal_class__disjunction  0.030490   \n",
       "3                               animal_class__negated  0.254506   \n",
       "4                                 cities__affirmative  1.406146   \n",
       "5                                 cities__conjunction  0.284483   \n",
       "6                                 cities__disjunction  0.077865   \n",
       "7                                     cities__negated  0.428149   \n",
       "8                           element_symb__affirmative  0.385225   \n",
       "9                           element_symb__conjunction  0.280702   \n",
       "10                          element_symb__disjunction  0.101869   \n",
       "11                              element_symb__negated  0.118579   \n",
       "12                                 facts__affirmative  0.060569   \n",
       "13                                 facts__conjunction  0.023507   \n",
       "14                                 facts__disjunction  0.015652   \n",
       "15                                     facts__negated  0.011196   \n",
       "16                             inventors__affirmative  0.028233   \n",
       "17                             inventors__conjunction  0.016118   \n",
       "18                             inventors__disjunction  0.006412   \n",
       "19                                 inventors__negated  0.032806   \n",
       "20                                      ambiguous_lie  0.053311   \n",
       "21                           ambiguous_truthful_reply  0.053311   \n",
       "22              honest_reply_despite_incentive_to_lie  0.000000   \n",
       "23                                    unambiguous_lie  0.071157   \n",
       "24                         unambiguous_truthful_reply  0.071157   \n",
       "25                                        larger_than  0.274207   \n",
       "26                                       smaller_than  0.234721   \n",
       "27                               sp_en_trans__negated  0.507973   \n",
       "28                           sp_en_trans__affirmative  0.766253   \n",
       "29                           sp_en_trans__conjunction  0.333990   \n",
       "30                           sp_en_trans__disjunction  0.046375   \n",
       "31                            common_claim_true_false  0.011256   \n",
       "32                             counterfact_true_false  0.006431   \n",
       "33                  animal_class__affirmative+negated  0.421336   \n",
       "34              animal_class__affirmative+disjunction  0.309329   \n",
       "35                  animal_class__negated+conjunction  0.294116   \n",
       "36  animal_class__affirmative+negated+conjunction+...  0.301722   \n",
       "37                        cities__affirmative+negated  0.917148   \n",
       "38                    cities__affirmative+disjunction  0.742006   \n",
       "39                        cities__negated+conjunction  0.356316   \n",
       "40  cities__affirmative+negated+conjunction+disjun...  0.549161   \n",
       "41                  element_symb__affirmative+negated  0.251902   \n",
       "42              element_symb__affirmative+disjunction  0.243547   \n",
       "43                  element_symb__negated+conjunction  0.199640   \n",
       "44  element_symb__affirmative+negated+conjunction+...  0.221594   \n",
       "45                         facts__affirmative+negated  0.035883   \n",
       "46                     facts__affirmative+disjunction  0.038110   \n",
       "47                         facts__negated+conjunction  0.017352   \n",
       "48  facts__affirmative+negated+conjunction+disjunc...  0.027731   \n",
       "49                     inventors__affirmative+negated  0.030519   \n",
       "50                 inventors__affirmative+disjunction  0.017323   \n",
       "51                     inventors__negated+conjunction  0.024462   \n",
       "52  inventors__affirmative+negated+conjunction+dis...  0.020892   \n",
       "53                   sp_en_trans__affirmative+negated  0.637113   \n",
       "54               sp_en_trans__affirmative+disjunction  0.406314   \n",
       "55                   sp_en_trans__negated+conjunction  0.420982   \n",
       "56  sp_en_trans__affirmative+negated+conjunction+d...  0.413648   \n",
       "57  ambiguous_truthful_reply+unambiguous_lie+unamb...  0.065208   \n",
       "58  animal_class+facts+inventors__affirmative+nega...  0.149870   \n",
       "59  animal_class+facts+inventors__affirmative+nega...  0.128825   \n",
       "\n",
       "    train_recon_improv  val_recon_improv  val_recon_improv_w_val_mean  \\\n",
       "0         3.649412e-01      4.011835e-01                 3.074009e-01   \n",
       "1         2.480434e-01      2.561188e-01                 2.589435e-01   \n",
       "2         2.746382e-02      3.780474e-02                 3.551936e-02   \n",
       "3         1.976709e-01      2.269427e-01                 2.214103e-01   \n",
       "4         5.851733e-01      5.821817e-01                 5.810651e-01   \n",
       "5         2.180753e-01      2.351196e-01                 2.340712e-01   \n",
       "6         7.738798e-02      4.914201e-02                 5.073094e-02   \n",
       "7         2.984820e-01      3.051164e-01                 2.961668e-01   \n",
       "8         2.695421e-01      3.163138e-01                 3.262626e-01   \n",
       "9         2.247273e-01      2.039327e-01                 2.055544e-01   \n",
       "10        9.261090e-02      9.222718e-02                 9.210908e-02   \n",
       "11        1.107324e-01      8.482091e-02                 8.756393e-02   \n",
       "12        6.017013e-02      4.325887e-02                 4.013136e-02   \n",
       "13        2.421945e-02      1.565483e-02                 1.487228e-02   \n",
       "14        1.430321e-02      1.807144e-02                 1.828032e-02   \n",
       "15        1.173250e-02      6.702047e-03                 6.779090e-03   \n",
       "16        2.957355e-02      1.377850e-02                 1.454122e-02   \n",
       "17        1.717506e-02      7.984143e-03                 8.650668e-03   \n",
       "18        6.618306e-03      4.956654e-03                 4.401128e-03   \n",
       "19        3.594561e-02      9.481063e-03                 1.112343e-02   \n",
       "20        6.500397e-08      5.883039e-07                -7.470586e-08   \n",
       "21        1.299436e-07     -2.124053e-07                 0.000000e+00   \n",
       "22       -8.239608e-08      0.000000e+00                 1.181275e-07   \n",
       "23        7.875588e-08     -8.695793e-08                 9.851908e-08   \n",
       "24        6.266302e-08      0.000000e+00                 8.009028e-08   \n",
       "25        2.141040e-01      2.191870e-01                 2.189815e-01   \n",
       "26        1.929585e-01      1.784094e-01                 1.752719e-01   \n",
       "27        3.407284e-01      3.245184e-01                 3.271965e-01   \n",
       "28        4.286215e-01      4.563926e-01                 4.038138e-01   \n",
       "29        2.493014e-01      2.580150e-01                 2.558037e-01   \n",
       "30        4.974986e-02      2.281293e-02                 2.155592e-02   \n",
       "31        1.066474e-02      1.269335e-02                 1.271016e-02   \n",
       "32        6.421208e-03      6.243058e-03                 6.242862e-03   \n",
       "33        2.310395e-01      2.937793e-01                 2.763326e-01   \n",
       "34        3.112807e-02      2.122509e-02                 1.699579e-02   \n",
       "35        1.182459e-01      1.149200e-01                 1.151456e-01   \n",
       "36        4.912116e-02      5.081535e-02                 5.054644e-02   \n",
       "37        3.440835e-01      3.494877e-01                 3.488409e-01   \n",
       "38        2.089672e-01      2.068474e-01                 2.060909e-01   \n",
       "39        9.978609e-02      1.028655e-01                 1.011307e-01   \n",
       "40        1.038561e-01      1.095188e-01                 1.092212e-01   \n",
       "41        1.212654e-01      1.339535e-01                 1.355060e-01   \n",
       "42        2.222870e-02      2.883926e-02                 2.845934e-02   \n",
       "43        9.875703e-02      7.888846e-02                 7.891204e-02   \n",
       "44        2.692761e-02      2.134994e-02                 2.142082e-02   \n",
       "45        3.404237e-02      2.308774e-02                 2.182429e-02   \n",
       "46        2.066776e-02      2.068299e-02                 2.064851e-02   \n",
       "47        1.202074e-02      8.362271e-03                 8.483914e-03   \n",
       "48        1.380154e-02      1.222099e-02                 1.243512e-02   \n",
       "49        2.606403e-02      1.239650e-02                 1.383055e-02   \n",
       "50        5.965794e-03      2.599023e-03                 2.713365e-03   \n",
       "51        2.622788e-02      2.300379e-02                 2.344006e-02   \n",
       "52        1.014814e-02      6.477528e-03                 6.785889e-03   \n",
       "53        1.738590e-01      1.621996e-01                 1.536014e-01   \n",
       "54        4.894242e-02      3.840607e-02                 3.760650e-02   \n",
       "55        8.498504e-02      6.820703e-02                 6.769840e-02   \n",
       "56        5.025428e-02      3.367015e-02                 3.370361e-02   \n",
       "57        6.084656e-02      1.764589e-02                 2.460461e-02   \n",
       "58        1.348219e-02      1.450066e-02                 1.446919e-02   \n",
       "59        8.884766e-03      9.284597e-03                 9.260300e-03   \n",
       "\n",
       "     val_acc    val_f1     val_brier  val_soft_f1  train_acc  train_f1  \\\n",
       "0   1.000000  1.000000  9.433798e-05     0.994576   0.977099  0.978723   \n",
       "1   0.910000  0.914286  7.965539e-02     0.828664   0.872500  0.874074   \n",
       "2   0.660000  0.613636  1.952025e-01     0.568776   0.695000  0.700980   \n",
       "3   1.000000  1.000000  6.929559e-03     0.978662   0.969466  0.970149   \n",
       "4   0.983333  0.982578  1.193676e-02     0.976074   0.986622  0.986864   \n",
       "5   0.930000  0.938416  5.544058e-02     0.897317   0.913189  0.917065   \n",
       "6   0.640000  0.632653  1.998048e-01     0.583376   0.747500  0.730667   \n",
       "7   0.953333  0.960000  2.757172e-02     0.944702   0.944816  0.945813   \n",
       "8   0.973684  0.975610  2.965148e-02     0.945144   0.912162  0.918239   \n",
       "9   0.850000  0.859813  9.269687e-02     0.821420   0.895000  0.910256   \n",
       "10  0.770000  0.785047  1.677429e-01     0.702310   0.757500  0.750643   \n",
       "11  1.000000  1.000000  3.790091e-27     1.000000   1.000000  1.000000   \n",
       "12  0.769912  0.816901  1.499897e-01     0.745699   0.783482  0.800000   \n",
       "13  0.780000  0.775510  1.641989e-01     0.644065   0.755000  0.775229   \n",
       "14  0.740000  0.704545  1.892708e-01     0.604824   0.665000  0.627778   \n",
       "15  0.743363  0.760331  1.692529e-01     0.680873   0.814732  0.819172   \n",
       "16  0.634146  0.651163  2.100852e-01     0.636837   0.706790  0.704050   \n",
       "17  0.650000  0.700855  2.187300e-01     0.626351   0.722500  0.753880   \n",
       "18  0.640000  0.666667  2.176724e-01     0.608711   0.642500  0.653753   \n",
       "19  0.621951  0.523077  2.440336e-01     0.493223   0.728395  0.685714   \n",
       "20  0.500000       NaN  2.003167e-01          NaN   0.571429       NaN   \n",
       "21  0.800000  0.888889  2.169776e-01     0.697194   0.444444  0.615385   \n",
       "22  0.500000  0.666667  2.486124e-01     0.669204   0.750000  0.857143   \n",
       "23  0.500000       NaN  2.471301e-01          NaN   0.435897       NaN   \n",
       "24  0.272727  0.428571  2.536726e-01     0.663523   0.431818  0.603175   \n",
       "25  0.954545  0.954774  3.357989e-02     0.936795   0.958333  0.960337   \n",
       "26  0.921717  0.930337  6.197086e-02     0.885670   0.928030  0.930147   \n",
       "27  1.000000  1.000000  9.884852e-06     0.999556   0.992933  0.992908   \n",
       "28  0.985915  0.988764  1.151348e-02     0.977109   0.971731  0.970803   \n",
       "29  0.920000  0.921569  5.175861e-02     0.904863   0.952500  0.957871   \n",
       "30  0.680000  0.600000  2.031341e-01     0.575317   0.745000  0.719780   \n",
       "31  0.738202  0.746464  1.733994e-01     0.647319   0.713764  0.715443   \n",
       "32  0.648365  0.669315  2.159329e-01     0.575354   0.644285  0.666838   \n",
       "33  0.984848  0.983051  1.678746e-02     0.967929   0.973282  0.974545   \n",
       "34  0.751880  0.666667  1.711332e-01     0.573242   0.693032  0.674651   \n",
       "35  0.894737  0.904110  7.056064e-02     0.876168   0.907721  0.910092   \n",
       "36  0.714286  0.672414  1.708738e-01     0.653357   0.700565  0.672840   \n",
       "37  0.953333  0.956790  3.580836e-02     0.933273   0.935619  0.938449   \n",
       "38  0.867500  0.852368  7.983817e-02     0.818254   0.879073  0.870383   \n",
       "39  0.916667  0.925816  6.428952e-02     0.883391   0.903091  0.904918   \n",
       "40  0.861000  0.871415  9.937463e-02     0.813847   0.838346  0.844691   \n",
       "41  0.947368  0.951220  3.593871e-02     0.931580   0.945946  0.948387   \n",
       "42  0.768116  0.786667  1.705870e-01     0.676340   0.704380  0.694340   \n",
       "43  0.818841  0.832215  1.130328e-01     0.781397   0.872263  0.889590   \n",
       "44  0.760870  0.772414  1.655345e-01     0.676647   0.758212  0.764444   \n",
       "45  0.747788  0.774704  1.733424e-01     0.676260   0.783482  0.792735   \n",
       "46  0.685446  0.673171  2.044741e-01     0.604037   0.675708  0.640523   \n",
       "47  0.741784  0.746544  1.909363e-01     0.619889   0.734670  0.746336   \n",
       "48  0.683099  0.685315  2.024529e-01     0.600473   0.702830  0.698925   \n",
       "49  0.731707  0.728395  1.826679e-01     0.631485   0.745370  0.725458   \n",
       "50  0.576923  0.620690  2.254370e-01     0.590835   0.643646  0.685366   \n",
       "51  0.681319  0.715686  2.271971e-01     0.561221   0.650552  0.684145   \n",
       "52  0.607143  0.641604  2.346848e-01     0.562086   0.593923  0.617188   \n",
       "53  0.943662  0.950617  4.520201e-02     0.919443   0.946996  0.947368   \n",
       "54  0.795322  0.816754  1.320057e-01     0.767865   0.806735  0.807018   \n",
       "55  0.959064  0.958580  2.222867e-02     0.951210   0.975110  0.976356   \n",
       "56  0.736842  0.711538  1.616360e-01     0.673184   0.751098  0.731013   \n",
       "57  0.807692  0.814815  1.462860e-01     0.712942   0.861386  0.877193   \n",
       "58  0.694444  0.719320  1.892812e-01     0.630639   0.699933  0.716352   \n",
       "59  0.713801  0.733404  2.026228e-01     0.629952   0.709687  0.720622   \n",
       "\n",
       "     train_brier  train_soft_f1  \n",
       "0   1.966778e-02       0.971843  \n",
       "1   9.711317e-02       0.810543  \n",
       "2   2.004607e-01       0.609305  \n",
       "3   3.020680e-02       0.955813  \n",
       "4   1.085203e-02       0.978052  \n",
       "5   6.533412e-02       0.876005  \n",
       "6   1.817614e-01       0.610717  \n",
       "7   4.119287e-02       0.920867  \n",
       "8   6.299112e-02       0.904907  \n",
       "9   7.776561e-02       0.858750  \n",
       "10  1.686156e-01       0.671521  \n",
       "11  1.161833e-14       1.000000  \n",
       "12  1.458184e-01       0.728273  \n",
       "13  1.622277e-01       0.680633  \n",
       "14  2.108675e-01       0.585649  \n",
       "15  1.370104e-01       0.713910  \n",
       "16  1.887885e-01       0.648495  \n",
       "17  1.943163e-01       0.631793  \n",
       "18  2.101950e-01       0.591201  \n",
       "19  1.885650e-01       0.597611  \n",
       "20  2.656892e-01            NaN  \n",
       "21  2.530501e-01       0.666735  \n",
       "22  2.516316e-01       0.666752  \n",
       "23  2.502127e-01            NaN  \n",
       "24  2.503446e-01       0.666657  \n",
       "25  2.941358e-02       0.944223  \n",
       "26  5.912147e-02       0.878934  \n",
       "27  6.999399e-03       0.992816  \n",
       "28  2.502351e-02       0.959947  \n",
       "29  3.470141e-02       0.927288  \n",
       "30  1.661407e-01       0.644891  \n",
       "31  1.879453e-01       0.629726  \n",
       "32  2.147054e-01       0.578437  \n",
       "33  1.937401e-02       0.967774  \n",
       "34  1.876073e-01       0.628148  \n",
       "35  6.750447e-02       0.874659  \n",
       "36  1.814867e-01       0.658516  \n",
       "37  4.553642e-02       0.916713  \n",
       "38  8.107951e-02       0.829360  \n",
       "39  7.278257e-02       0.857471  \n",
       "40  1.135132e-01       0.788325  \n",
       "41  4.882997e-02       0.915296  \n",
       "42  1.927492e-01       0.627430  \n",
       "43  8.977219e-02       0.831470  \n",
       "44  1.542000e-01       0.699045  \n",
       "45  1.559641e-01       0.676748  \n",
       "46  2.038644e-01       0.582256  \n",
       "47  1.754862e-01       0.650354  \n",
       "48  1.939521e-01       0.604142  \n",
       "49  1.740212e-01       0.641401  \n",
       "50  2.130755e-01       0.589188  \n",
       "51  2.320744e-01       0.544183  \n",
       "52  2.330289e-01       0.546984  \n",
       "53  4.428508e-02       0.908614  \n",
       "54  1.244082e-01       0.754493  \n",
       "55  1.856554e-02       0.962133  \n",
       "56  1.472147e-01       0.697281  \n",
       "57  1.149274e-01       0.779588  \n",
       "58  1.911026e-01       0.622918  \n",
       "59  2.093247e-01       0.616953  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>scenario</th>\n",
       "      <th>tf_sep</th>\n",
       "      <th>train_recon_improv</th>\n",
       "      <th>val_recon_improv</th>\n",
       "      <th>val_recon_improv_w_val_mean</th>\n",
       "      <th>val_acc</th>\n",
       "      <th>val_f1</th>\n",
       "      <th>val_brier</th>\n",
       "      <th>val_soft_f1</th>\n",
       "      <th>train_acc</th>\n",
       "      <th>train_f1</th>\n",
       "      <th>train_brier</th>\n",
       "      <th>train_soft_f1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>animal_class__affirmative</td>\n",
       "      <td>0.588167</td>\n",
       "      <td>3.649412e-01</td>\n",
       "      <td>4.011835e-01</td>\n",
       "      <td>3.074009e-01</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>9.433798e-05</td>\n",
       "      <td>0.994576</td>\n",
       "      <td>0.977099</td>\n",
       "      <td>0.978723</td>\n",
       "      <td>1.966778e-02</td>\n",
       "      <td>0.971843</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>animal_class__conjunction</td>\n",
       "      <td>0.333727</td>\n",
       "      <td>2.480434e-01</td>\n",
       "      <td>2.561188e-01</td>\n",
       "      <td>2.589435e-01</td>\n",
       "      <td>0.910000</td>\n",
       "      <td>0.914286</td>\n",
       "      <td>7.965539e-02</td>\n",
       "      <td>0.828664</td>\n",
       "      <td>0.872500</td>\n",
       "      <td>0.874074</td>\n",
       "      <td>9.711317e-02</td>\n",
       "      <td>0.810543</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>animal_class__disjunction</td>\n",
       "      <td>0.030490</td>\n",
       "      <td>2.746382e-02</td>\n",
       "      <td>3.780474e-02</td>\n",
       "      <td>3.551936e-02</td>\n",
       "      <td>0.660000</td>\n",
       "      <td>0.613636</td>\n",
       "      <td>1.952025e-01</td>\n",
       "      <td>0.568776</td>\n",
       "      <td>0.695000</td>\n",
       "      <td>0.700980</td>\n",
       "      <td>2.004607e-01</td>\n",
       "      <td>0.609305</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>animal_class__negated</td>\n",
       "      <td>0.254506</td>\n",
       "      <td>1.976709e-01</td>\n",
       "      <td>2.269427e-01</td>\n",
       "      <td>2.214103e-01</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.929559e-03</td>\n",
       "      <td>0.978662</td>\n",
       "      <td>0.969466</td>\n",
       "      <td>0.970149</td>\n",
       "      <td>3.020680e-02</td>\n",
       "      <td>0.955813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cities__affirmative</td>\n",
       "      <td>1.406146</td>\n",
       "      <td>5.851733e-01</td>\n",
       "      <td>5.821817e-01</td>\n",
       "      <td>5.810651e-01</td>\n",
       "      <td>0.983333</td>\n",
       "      <td>0.982578</td>\n",
       "      <td>1.193676e-02</td>\n",
       "      <td>0.976074</td>\n",
       "      <td>0.986622</td>\n",
       "      <td>0.986864</td>\n",
       "      <td>1.085203e-02</td>\n",
       "      <td>0.978052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>cities__conjunction</td>\n",
       "      <td>0.284483</td>\n",
       "      <td>2.180753e-01</td>\n",
       "      <td>2.351196e-01</td>\n",
       "      <td>2.340712e-01</td>\n",
       "      <td>0.930000</td>\n",
       "      <td>0.938416</td>\n",
       "      <td>5.544058e-02</td>\n",
       "      <td>0.897317</td>\n",
       "      <td>0.913189</td>\n",
       "      <td>0.917065</td>\n",
       "      <td>6.533412e-02</td>\n",
       "      <td>0.876005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>cities__disjunction</td>\n",
       "      <td>0.077865</td>\n",
       "      <td>7.738798e-02</td>\n",
       "      <td>4.914201e-02</td>\n",
       "      <td>5.073094e-02</td>\n",
       "      <td>0.640000</td>\n",
       "      <td>0.632653</td>\n",
       "      <td>1.998048e-01</td>\n",
       "      <td>0.583376</td>\n",
       "      <td>0.747500</td>\n",
       "      <td>0.730667</td>\n",
       "      <td>1.817614e-01</td>\n",
       "      <td>0.610717</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>cities__negated</td>\n",
       "      <td>0.428149</td>\n",
       "      <td>2.984820e-01</td>\n",
       "      <td>3.051164e-01</td>\n",
       "      <td>2.961668e-01</td>\n",
       "      <td>0.953333</td>\n",
       "      <td>0.960000</td>\n",
       "      <td>2.757172e-02</td>\n",
       "      <td>0.944702</td>\n",
       "      <td>0.944816</td>\n",
       "      <td>0.945813</td>\n",
       "      <td>4.119287e-02</td>\n",
       "      <td>0.920867</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>element_symb__affirmative</td>\n",
       "      <td>0.385225</td>\n",
       "      <td>2.695421e-01</td>\n",
       "      <td>3.163138e-01</td>\n",
       "      <td>3.262626e-01</td>\n",
       "      <td>0.973684</td>\n",
       "      <td>0.975610</td>\n",
       "      <td>2.965148e-02</td>\n",
       "      <td>0.945144</td>\n",
       "      <td>0.912162</td>\n",
       "      <td>0.918239</td>\n",
       "      <td>6.299112e-02</td>\n",
       "      <td>0.904907</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>element_symb__conjunction</td>\n",
       "      <td>0.280702</td>\n",
       "      <td>2.247273e-01</td>\n",
       "      <td>2.039327e-01</td>\n",
       "      <td>2.055544e-01</td>\n",
       "      <td>0.850000</td>\n",
       "      <td>0.859813</td>\n",
       "      <td>9.269687e-02</td>\n",
       "      <td>0.821420</td>\n",
       "      <td>0.895000</td>\n",
       "      <td>0.910256</td>\n",
       "      <td>7.776561e-02</td>\n",
       "      <td>0.858750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>element_symb__disjunction</td>\n",
       "      <td>0.101869</td>\n",
       "      <td>9.261090e-02</td>\n",
       "      <td>9.222718e-02</td>\n",
       "      <td>9.210908e-02</td>\n",
       "      <td>0.770000</td>\n",
       "      <td>0.785047</td>\n",
       "      <td>1.677429e-01</td>\n",
       "      <td>0.702310</td>\n",
       "      <td>0.757500</td>\n",
       "      <td>0.750643</td>\n",
       "      <td>1.686156e-01</td>\n",
       "      <td>0.671521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>element_symb__negated</td>\n",
       "      <td>0.118579</td>\n",
       "      <td>1.107324e-01</td>\n",
       "      <td>8.482091e-02</td>\n",
       "      <td>8.756393e-02</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.790091e-27</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.161833e-14</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>facts__affirmative</td>\n",
       "      <td>0.060569</td>\n",
       "      <td>6.017013e-02</td>\n",
       "      <td>4.325887e-02</td>\n",
       "      <td>4.013136e-02</td>\n",
       "      <td>0.769912</td>\n",
       "      <td>0.816901</td>\n",
       "      <td>1.499897e-01</td>\n",
       "      <td>0.745699</td>\n",
       "      <td>0.783482</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>1.458184e-01</td>\n",
       "      <td>0.728273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>facts__conjunction</td>\n",
       "      <td>0.023507</td>\n",
       "      <td>2.421945e-02</td>\n",
       "      <td>1.565483e-02</td>\n",
       "      <td>1.487228e-02</td>\n",
       "      <td>0.780000</td>\n",
       "      <td>0.775510</td>\n",
       "      <td>1.641989e-01</td>\n",
       "      <td>0.644065</td>\n",
       "      <td>0.755000</td>\n",
       "      <td>0.775229</td>\n",
       "      <td>1.622277e-01</td>\n",
       "      <td>0.680633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>facts__disjunction</td>\n",
       "      <td>0.015652</td>\n",
       "      <td>1.430321e-02</td>\n",
       "      <td>1.807144e-02</td>\n",
       "      <td>1.828032e-02</td>\n",
       "      <td>0.740000</td>\n",
       "      <td>0.704545</td>\n",
       "      <td>1.892708e-01</td>\n",
       "      <td>0.604824</td>\n",
       "      <td>0.665000</td>\n",
       "      <td>0.627778</td>\n",
       "      <td>2.108675e-01</td>\n",
       "      <td>0.585649</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>facts__negated</td>\n",
       "      <td>0.011196</td>\n",
       "      <td>1.173250e-02</td>\n",
       "      <td>6.702047e-03</td>\n",
       "      <td>6.779090e-03</td>\n",
       "      <td>0.743363</td>\n",
       "      <td>0.760331</td>\n",
       "      <td>1.692529e-01</td>\n",
       "      <td>0.680873</td>\n",
       "      <td>0.814732</td>\n",
       "      <td>0.819172</td>\n",
       "      <td>1.370104e-01</td>\n",
       "      <td>0.713910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>inventors__affirmative</td>\n",
       "      <td>0.028233</td>\n",
       "      <td>2.957355e-02</td>\n",
       "      <td>1.377850e-02</td>\n",
       "      <td>1.454122e-02</td>\n",
       "      <td>0.634146</td>\n",
       "      <td>0.651163</td>\n",
       "      <td>2.100852e-01</td>\n",
       "      <td>0.636837</td>\n",
       "      <td>0.706790</td>\n",
       "      <td>0.704050</td>\n",
       "      <td>1.887885e-01</td>\n",
       "      <td>0.648495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>inventors__conjunction</td>\n",
       "      <td>0.016118</td>\n",
       "      <td>1.717506e-02</td>\n",
       "      <td>7.984143e-03</td>\n",
       "      <td>8.650668e-03</td>\n",
       "      <td>0.650000</td>\n",
       "      <td>0.700855</td>\n",
       "      <td>2.187300e-01</td>\n",
       "      <td>0.626351</td>\n",
       "      <td>0.722500</td>\n",
       "      <td>0.753880</td>\n",
       "      <td>1.943163e-01</td>\n",
       "      <td>0.631793</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>inventors__disjunction</td>\n",
       "      <td>0.006412</td>\n",
       "      <td>6.618306e-03</td>\n",
       "      <td>4.956654e-03</td>\n",
       "      <td>4.401128e-03</td>\n",
       "      <td>0.640000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>2.176724e-01</td>\n",
       "      <td>0.608711</td>\n",
       "      <td>0.642500</td>\n",
       "      <td>0.653753</td>\n",
       "      <td>2.101950e-01</td>\n",
       "      <td>0.591201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>inventors__negated</td>\n",
       "      <td>0.032806</td>\n",
       "      <td>3.594561e-02</td>\n",
       "      <td>9.481063e-03</td>\n",
       "      <td>1.112343e-02</td>\n",
       "      <td>0.621951</td>\n",
       "      <td>0.523077</td>\n",
       "      <td>2.440336e-01</td>\n",
       "      <td>0.493223</td>\n",
       "      <td>0.728395</td>\n",
       "      <td>0.685714</td>\n",
       "      <td>1.885650e-01</td>\n",
       "      <td>0.597611</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>ambiguous_lie</td>\n",
       "      <td>0.053311</td>\n",
       "      <td>6.500397e-08</td>\n",
       "      <td>5.883039e-07</td>\n",
       "      <td>-7.470586e-08</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.003167e-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.571429</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.656892e-01</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>ambiguous_truthful_reply</td>\n",
       "      <td>0.053311</td>\n",
       "      <td>1.299436e-07</td>\n",
       "      <td>-2.124053e-07</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>2.169776e-01</td>\n",
       "      <td>0.697194</td>\n",
       "      <td>0.444444</td>\n",
       "      <td>0.615385</td>\n",
       "      <td>2.530501e-01</td>\n",
       "      <td>0.666735</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>honest_reply_despite_incentive_to_lie</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-8.239608e-08</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.181275e-07</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>2.486124e-01</td>\n",
       "      <td>0.669204</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>2.516316e-01</td>\n",
       "      <td>0.666752</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>unambiguous_lie</td>\n",
       "      <td>0.071157</td>\n",
       "      <td>7.875588e-08</td>\n",
       "      <td>-8.695793e-08</td>\n",
       "      <td>9.851908e-08</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.471301e-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.435897</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.502127e-01</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>unambiguous_truthful_reply</td>\n",
       "      <td>0.071157</td>\n",
       "      <td>6.266302e-08</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>8.009028e-08</td>\n",
       "      <td>0.272727</td>\n",
       "      <td>0.428571</td>\n",
       "      <td>2.536726e-01</td>\n",
       "      <td>0.663523</td>\n",
       "      <td>0.431818</td>\n",
       "      <td>0.603175</td>\n",
       "      <td>2.503446e-01</td>\n",
       "      <td>0.666657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>larger_than</td>\n",
       "      <td>0.274207</td>\n",
       "      <td>2.141040e-01</td>\n",
       "      <td>2.191870e-01</td>\n",
       "      <td>2.189815e-01</td>\n",
       "      <td>0.954545</td>\n",
       "      <td>0.954774</td>\n",
       "      <td>3.357989e-02</td>\n",
       "      <td>0.936795</td>\n",
       "      <td>0.958333</td>\n",
       "      <td>0.960337</td>\n",
       "      <td>2.941358e-02</td>\n",
       "      <td>0.944223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>smaller_than</td>\n",
       "      <td>0.234721</td>\n",
       "      <td>1.929585e-01</td>\n",
       "      <td>1.784094e-01</td>\n",
       "      <td>1.752719e-01</td>\n",
       "      <td>0.921717</td>\n",
       "      <td>0.930337</td>\n",
       "      <td>6.197086e-02</td>\n",
       "      <td>0.885670</td>\n",
       "      <td>0.928030</td>\n",
       "      <td>0.930147</td>\n",
       "      <td>5.912147e-02</td>\n",
       "      <td>0.878934</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>sp_en_trans__negated</td>\n",
       "      <td>0.507973</td>\n",
       "      <td>3.407284e-01</td>\n",
       "      <td>3.245184e-01</td>\n",
       "      <td>3.271965e-01</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>9.884852e-06</td>\n",
       "      <td>0.999556</td>\n",
       "      <td>0.992933</td>\n",
       "      <td>0.992908</td>\n",
       "      <td>6.999399e-03</td>\n",
       "      <td>0.992816</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>sp_en_trans__affirmative</td>\n",
       "      <td>0.766253</td>\n",
       "      <td>4.286215e-01</td>\n",
       "      <td>4.563926e-01</td>\n",
       "      <td>4.038138e-01</td>\n",
       "      <td>0.985915</td>\n",
       "      <td>0.988764</td>\n",
       "      <td>1.151348e-02</td>\n",
       "      <td>0.977109</td>\n",
       "      <td>0.971731</td>\n",
       "      <td>0.970803</td>\n",
       "      <td>2.502351e-02</td>\n",
       "      <td>0.959947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>sp_en_trans__conjunction</td>\n",
       "      <td>0.333990</td>\n",
       "      <td>2.493014e-01</td>\n",
       "      <td>2.580150e-01</td>\n",
       "      <td>2.558037e-01</td>\n",
       "      <td>0.920000</td>\n",
       "      <td>0.921569</td>\n",
       "      <td>5.175861e-02</td>\n",
       "      <td>0.904863</td>\n",
       "      <td>0.952500</td>\n",
       "      <td>0.957871</td>\n",
       "      <td>3.470141e-02</td>\n",
       "      <td>0.927288</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>sp_en_trans__disjunction</td>\n",
       "      <td>0.046375</td>\n",
       "      <td>4.974986e-02</td>\n",
       "      <td>2.281293e-02</td>\n",
       "      <td>2.155592e-02</td>\n",
       "      <td>0.680000</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>2.031341e-01</td>\n",
       "      <td>0.575317</td>\n",
       "      <td>0.745000</td>\n",
       "      <td>0.719780</td>\n",
       "      <td>1.661407e-01</td>\n",
       "      <td>0.644891</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>common_claim_true_false</td>\n",
       "      <td>0.011256</td>\n",
       "      <td>1.066474e-02</td>\n",
       "      <td>1.269335e-02</td>\n",
       "      <td>1.271016e-02</td>\n",
       "      <td>0.738202</td>\n",
       "      <td>0.746464</td>\n",
       "      <td>1.733994e-01</td>\n",
       "      <td>0.647319</td>\n",
       "      <td>0.713764</td>\n",
       "      <td>0.715443</td>\n",
       "      <td>1.879453e-01</td>\n",
       "      <td>0.629726</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>counterfact_true_false</td>\n",
       "      <td>0.006431</td>\n",
       "      <td>6.421208e-03</td>\n",
       "      <td>6.243058e-03</td>\n",
       "      <td>6.242862e-03</td>\n",
       "      <td>0.648365</td>\n",
       "      <td>0.669315</td>\n",
       "      <td>2.159329e-01</td>\n",
       "      <td>0.575354</td>\n",
       "      <td>0.644285</td>\n",
       "      <td>0.666838</td>\n",
       "      <td>2.147054e-01</td>\n",
       "      <td>0.578437</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>animal_class__affirmative+negated</td>\n",
       "      <td>0.421336</td>\n",
       "      <td>2.310395e-01</td>\n",
       "      <td>2.937793e-01</td>\n",
       "      <td>2.763326e-01</td>\n",
       "      <td>0.984848</td>\n",
       "      <td>0.983051</td>\n",
       "      <td>1.678746e-02</td>\n",
       "      <td>0.967929</td>\n",
       "      <td>0.973282</td>\n",
       "      <td>0.974545</td>\n",
       "      <td>1.937401e-02</td>\n",
       "      <td>0.967774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>animal_class__affirmative+disjunction</td>\n",
       "      <td>0.309329</td>\n",
       "      <td>3.112807e-02</td>\n",
       "      <td>2.122509e-02</td>\n",
       "      <td>1.699579e-02</td>\n",
       "      <td>0.751880</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>1.711332e-01</td>\n",
       "      <td>0.573242</td>\n",
       "      <td>0.693032</td>\n",
       "      <td>0.674651</td>\n",
       "      <td>1.876073e-01</td>\n",
       "      <td>0.628148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>animal_class__negated+conjunction</td>\n",
       "      <td>0.294116</td>\n",
       "      <td>1.182459e-01</td>\n",
       "      <td>1.149200e-01</td>\n",
       "      <td>1.151456e-01</td>\n",
       "      <td>0.894737</td>\n",
       "      <td>0.904110</td>\n",
       "      <td>7.056064e-02</td>\n",
       "      <td>0.876168</td>\n",
       "      <td>0.907721</td>\n",
       "      <td>0.910092</td>\n",
       "      <td>6.750447e-02</td>\n",
       "      <td>0.874659</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>animal_class__affirmative+negated+conjunction+...</td>\n",
       "      <td>0.301722</td>\n",
       "      <td>4.912116e-02</td>\n",
       "      <td>5.081535e-02</td>\n",
       "      <td>5.054644e-02</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>0.672414</td>\n",
       "      <td>1.708738e-01</td>\n",
       "      <td>0.653357</td>\n",
       "      <td>0.700565</td>\n",
       "      <td>0.672840</td>\n",
       "      <td>1.814867e-01</td>\n",
       "      <td>0.658516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>cities__affirmative+negated</td>\n",
       "      <td>0.917148</td>\n",
       "      <td>3.440835e-01</td>\n",
       "      <td>3.494877e-01</td>\n",
       "      <td>3.488409e-01</td>\n",
       "      <td>0.953333</td>\n",
       "      <td>0.956790</td>\n",
       "      <td>3.580836e-02</td>\n",
       "      <td>0.933273</td>\n",
       "      <td>0.935619</td>\n",
       "      <td>0.938449</td>\n",
       "      <td>4.553642e-02</td>\n",
       "      <td>0.916713</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>cities__affirmative+disjunction</td>\n",
       "      <td>0.742006</td>\n",
       "      <td>2.089672e-01</td>\n",
       "      <td>2.068474e-01</td>\n",
       "      <td>2.060909e-01</td>\n",
       "      <td>0.867500</td>\n",
       "      <td>0.852368</td>\n",
       "      <td>7.983817e-02</td>\n",
       "      <td>0.818254</td>\n",
       "      <td>0.879073</td>\n",
       "      <td>0.870383</td>\n",
       "      <td>8.107951e-02</td>\n",
       "      <td>0.829360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>cities__negated+conjunction</td>\n",
       "      <td>0.356316</td>\n",
       "      <td>9.978609e-02</td>\n",
       "      <td>1.028655e-01</td>\n",
       "      <td>1.011307e-01</td>\n",
       "      <td>0.916667</td>\n",
       "      <td>0.925816</td>\n",
       "      <td>6.428952e-02</td>\n",
       "      <td>0.883391</td>\n",
       "      <td>0.903091</td>\n",
       "      <td>0.904918</td>\n",
       "      <td>7.278257e-02</td>\n",
       "      <td>0.857471</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>cities__affirmative+negated+conjunction+disjun...</td>\n",
       "      <td>0.549161</td>\n",
       "      <td>1.038561e-01</td>\n",
       "      <td>1.095188e-01</td>\n",
       "      <td>1.092212e-01</td>\n",
       "      <td>0.861000</td>\n",
       "      <td>0.871415</td>\n",
       "      <td>9.937463e-02</td>\n",
       "      <td>0.813847</td>\n",
       "      <td>0.838346</td>\n",
       "      <td>0.844691</td>\n",
       "      <td>1.135132e-01</td>\n",
       "      <td>0.788325</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>element_symb__affirmative+negated</td>\n",
       "      <td>0.251902</td>\n",
       "      <td>1.212654e-01</td>\n",
       "      <td>1.339535e-01</td>\n",
       "      <td>1.355060e-01</td>\n",
       "      <td>0.947368</td>\n",
       "      <td>0.951220</td>\n",
       "      <td>3.593871e-02</td>\n",
       "      <td>0.931580</td>\n",
       "      <td>0.945946</td>\n",
       "      <td>0.948387</td>\n",
       "      <td>4.882997e-02</td>\n",
       "      <td>0.915296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>element_symb__affirmative+disjunction</td>\n",
       "      <td>0.243547</td>\n",
       "      <td>2.222870e-02</td>\n",
       "      <td>2.883926e-02</td>\n",
       "      <td>2.845934e-02</td>\n",
       "      <td>0.768116</td>\n",
       "      <td>0.786667</td>\n",
       "      <td>1.705870e-01</td>\n",
       "      <td>0.676340</td>\n",
       "      <td>0.704380</td>\n",
       "      <td>0.694340</td>\n",
       "      <td>1.927492e-01</td>\n",
       "      <td>0.627430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>element_symb__negated+conjunction</td>\n",
       "      <td>0.199640</td>\n",
       "      <td>9.875703e-02</td>\n",
       "      <td>7.888846e-02</td>\n",
       "      <td>7.891204e-02</td>\n",
       "      <td>0.818841</td>\n",
       "      <td>0.832215</td>\n",
       "      <td>1.130328e-01</td>\n",
       "      <td>0.781397</td>\n",
       "      <td>0.872263</td>\n",
       "      <td>0.889590</td>\n",
       "      <td>8.977219e-02</td>\n",
       "      <td>0.831470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>element_symb__affirmative+negated+conjunction+...</td>\n",
       "      <td>0.221594</td>\n",
       "      <td>2.692761e-02</td>\n",
       "      <td>2.134994e-02</td>\n",
       "      <td>2.142082e-02</td>\n",
       "      <td>0.760870</td>\n",
       "      <td>0.772414</td>\n",
       "      <td>1.655345e-01</td>\n",
       "      <td>0.676647</td>\n",
       "      <td>0.758212</td>\n",
       "      <td>0.764444</td>\n",
       "      <td>1.542000e-01</td>\n",
       "      <td>0.699045</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>facts__affirmative+negated</td>\n",
       "      <td>0.035883</td>\n",
       "      <td>3.404237e-02</td>\n",
       "      <td>2.308774e-02</td>\n",
       "      <td>2.182429e-02</td>\n",
       "      <td>0.747788</td>\n",
       "      <td>0.774704</td>\n",
       "      <td>1.733424e-01</td>\n",
       "      <td>0.676260</td>\n",
       "      <td>0.783482</td>\n",
       "      <td>0.792735</td>\n",
       "      <td>1.559641e-01</td>\n",
       "      <td>0.676748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>facts__affirmative+disjunction</td>\n",
       "      <td>0.038110</td>\n",
       "      <td>2.066776e-02</td>\n",
       "      <td>2.068299e-02</td>\n",
       "      <td>2.064851e-02</td>\n",
       "      <td>0.685446</td>\n",
       "      <td>0.673171</td>\n",
       "      <td>2.044741e-01</td>\n",
       "      <td>0.604037</td>\n",
       "      <td>0.675708</td>\n",
       "      <td>0.640523</td>\n",
       "      <td>2.038644e-01</td>\n",
       "      <td>0.582256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>facts__negated+conjunction</td>\n",
       "      <td>0.017352</td>\n",
       "      <td>1.202074e-02</td>\n",
       "      <td>8.362271e-03</td>\n",
       "      <td>8.483914e-03</td>\n",
       "      <td>0.741784</td>\n",
       "      <td>0.746544</td>\n",
       "      <td>1.909363e-01</td>\n",
       "      <td>0.619889</td>\n",
       "      <td>0.734670</td>\n",
       "      <td>0.746336</td>\n",
       "      <td>1.754862e-01</td>\n",
       "      <td>0.650354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>facts__affirmative+negated+conjunction+disjunc...</td>\n",
       "      <td>0.027731</td>\n",
       "      <td>1.380154e-02</td>\n",
       "      <td>1.222099e-02</td>\n",
       "      <td>1.243512e-02</td>\n",
       "      <td>0.683099</td>\n",
       "      <td>0.685315</td>\n",
       "      <td>2.024529e-01</td>\n",
       "      <td>0.600473</td>\n",
       "      <td>0.702830</td>\n",
       "      <td>0.698925</td>\n",
       "      <td>1.939521e-01</td>\n",
       "      <td>0.604142</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>inventors__affirmative+negated</td>\n",
       "      <td>0.030519</td>\n",
       "      <td>2.606403e-02</td>\n",
       "      <td>1.239650e-02</td>\n",
       "      <td>1.383055e-02</td>\n",
       "      <td>0.731707</td>\n",
       "      <td>0.728395</td>\n",
       "      <td>1.826679e-01</td>\n",
       "      <td>0.631485</td>\n",
       "      <td>0.745370</td>\n",
       "      <td>0.725458</td>\n",
       "      <td>1.740212e-01</td>\n",
       "      <td>0.641401</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>inventors__affirmative+disjunction</td>\n",
       "      <td>0.017323</td>\n",
       "      <td>5.965794e-03</td>\n",
       "      <td>2.599023e-03</td>\n",
       "      <td>2.713365e-03</td>\n",
       "      <td>0.576923</td>\n",
       "      <td>0.620690</td>\n",
       "      <td>2.254370e-01</td>\n",
       "      <td>0.590835</td>\n",
       "      <td>0.643646</td>\n",
       "      <td>0.685366</td>\n",
       "      <td>2.130755e-01</td>\n",
       "      <td>0.589188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>inventors__negated+conjunction</td>\n",
       "      <td>0.024462</td>\n",
       "      <td>2.622788e-02</td>\n",
       "      <td>2.300379e-02</td>\n",
       "      <td>2.344006e-02</td>\n",
       "      <td>0.681319</td>\n",
       "      <td>0.715686</td>\n",
       "      <td>2.271971e-01</td>\n",
       "      <td>0.561221</td>\n",
       "      <td>0.650552</td>\n",
       "      <td>0.684145</td>\n",
       "      <td>2.320744e-01</td>\n",
       "      <td>0.544183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>inventors__affirmative+negated+conjunction+dis...</td>\n",
       "      <td>0.020892</td>\n",
       "      <td>1.014814e-02</td>\n",
       "      <td>6.477528e-03</td>\n",
       "      <td>6.785889e-03</td>\n",
       "      <td>0.607143</td>\n",
       "      <td>0.641604</td>\n",
       "      <td>2.346848e-01</td>\n",
       "      <td>0.562086</td>\n",
       "      <td>0.593923</td>\n",
       "      <td>0.617188</td>\n",
       "      <td>2.330289e-01</td>\n",
       "      <td>0.546984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>sp_en_trans__affirmative+negated</td>\n",
       "      <td>0.637113</td>\n",
       "      <td>1.738590e-01</td>\n",
       "      <td>1.621996e-01</td>\n",
       "      <td>1.536014e-01</td>\n",
       "      <td>0.943662</td>\n",
       "      <td>0.950617</td>\n",
       "      <td>4.520201e-02</td>\n",
       "      <td>0.919443</td>\n",
       "      <td>0.946996</td>\n",
       "      <td>0.947368</td>\n",
       "      <td>4.428508e-02</td>\n",
       "      <td>0.908614</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>sp_en_trans__affirmative+disjunction</td>\n",
       "      <td>0.406314</td>\n",
       "      <td>4.894242e-02</td>\n",
       "      <td>3.840607e-02</td>\n",
       "      <td>3.760650e-02</td>\n",
       "      <td>0.795322</td>\n",
       "      <td>0.816754</td>\n",
       "      <td>1.320057e-01</td>\n",
       "      <td>0.767865</td>\n",
       "      <td>0.806735</td>\n",
       "      <td>0.807018</td>\n",
       "      <td>1.244082e-01</td>\n",
       "      <td>0.754493</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>sp_en_trans__negated+conjunction</td>\n",
       "      <td>0.420982</td>\n",
       "      <td>8.498504e-02</td>\n",
       "      <td>6.820703e-02</td>\n",
       "      <td>6.769840e-02</td>\n",
       "      <td>0.959064</td>\n",
       "      <td>0.958580</td>\n",
       "      <td>2.222867e-02</td>\n",
       "      <td>0.951210</td>\n",
       "      <td>0.975110</td>\n",
       "      <td>0.976356</td>\n",
       "      <td>1.856554e-02</td>\n",
       "      <td>0.962133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>sp_en_trans__affirmative+negated+conjunction+d...</td>\n",
       "      <td>0.413648</td>\n",
       "      <td>5.025428e-02</td>\n",
       "      <td>3.367015e-02</td>\n",
       "      <td>3.370361e-02</td>\n",
       "      <td>0.736842</td>\n",
       "      <td>0.711538</td>\n",
       "      <td>1.616360e-01</td>\n",
       "      <td>0.673184</td>\n",
       "      <td>0.751098</td>\n",
       "      <td>0.731013</td>\n",
       "      <td>1.472147e-01</td>\n",
       "      <td>0.697281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>ambiguous_truthful_reply+unambiguous_lie+unamb...</td>\n",
       "      <td>0.065208</td>\n",
       "      <td>6.084656e-02</td>\n",
       "      <td>1.764589e-02</td>\n",
       "      <td>2.460461e-02</td>\n",
       "      <td>0.807692</td>\n",
       "      <td>0.814815</td>\n",
       "      <td>1.462860e-01</td>\n",
       "      <td>0.712942</td>\n",
       "      <td>0.861386</td>\n",
       "      <td>0.877193</td>\n",
       "      <td>1.149274e-01</td>\n",
       "      <td>0.779588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>animal_class+facts+inventors__affirmative+nega...</td>\n",
       "      <td>0.149870</td>\n",
       "      <td>1.348219e-02</td>\n",
       "      <td>1.450066e-02</td>\n",
       "      <td>1.446919e-02</td>\n",
       "      <td>0.694444</td>\n",
       "      <td>0.719320</td>\n",
       "      <td>1.892812e-01</td>\n",
       "      <td>0.630639</td>\n",
       "      <td>0.699933</td>\n",
       "      <td>0.716352</td>\n",
       "      <td>1.911026e-01</td>\n",
       "      <td>0.622918</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>animal_class+facts+inventors__affirmative+nega...</td>\n",
       "      <td>0.128825</td>\n",
       "      <td>8.884766e-03</td>\n",
       "      <td>9.284597e-03</td>\n",
       "      <td>9.260300e-03</td>\n",
       "      <td>0.713801</td>\n",
       "      <td>0.733404</td>\n",
       "      <td>2.026228e-01</td>\n",
       "      <td>0.629952</td>\n",
       "      <td>0.709687</td>\n",
       "      <td>0.720622</td>\n",
       "      <td>2.093247e-01</td>\n",
       "      <td>0.616953</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 17
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:58.259389Z",
     "start_time": "2024-12-07T21:26:58.236970Z"
    }
   },
   "cell_type": "code",
   "source": "lyr25_eval_except_test_df.describe()",
   "id": "e0511946c58e7f0b",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "          tf_sep  train_recon_improv  val_recon_improv  \\\n",
       "count  60.000000        6.000000e+01      6.000000e+01   \n",
       "mean    0.235943        1.075456e-01      1.059050e-01   \n",
       "std     0.268247        1.271151e-01      1.345939e-01   \n",
       "min     0.000000       -8.239608e-08     -2.124053e-07   \n",
       "25%     0.030512        1.417779e-02      1.235262e-02   \n",
       "50%     0.139348        4.943551e-02      3.573745e-02   \n",
       "75%     0.339572        1.941366e-01      1.847902e-01   \n",
       "max     1.406146        5.851733e-01      5.821817e-01   \n",
       "\n",
       "       val_recon_improv_w_val_mean    val_acc     val_f1     val_brier  \\\n",
       "count                 6.000000e+01  60.000000  58.000000  6.000000e+01   \n",
       "mean                  1.029477e-01   0.783611   0.802116  1.347030e-01   \n",
       "std                   1.288299e-01   0.155743   0.140303  8.139284e-02   \n",
       "min                  -7.470586e-08   0.272727   0.428571  3.790091e-27   \n",
       "25%                   1.264140e-02   0.682654   0.689200  5.452009e-02   \n",
       "50%                   3.461149e-02   0.769014   0.785857  1.666387e-01   \n",
       "75%                   1.828425e-01   0.923788   0.936397  2.024954e-01   \n",
       "max                   5.810651e-01   1.000000   1.000000  2.536726e-01   \n",
       "\n",
       "       val_soft_f1  train_acc   train_f1   train_brier  train_soft_f1  \n",
       "count    58.000000  60.000000  58.000000  6.000000e+01      58.000000  \n",
       "mean      0.750525   0.790368   0.809428  1.345743e-01       0.752996  \n",
       "std       0.152109   0.143905   0.123342  7.883947e-02       0.144177  \n",
       "min       0.493223   0.431818   0.603175  1.161833e-14       0.544183  \n",
       "25%       0.627251   0.702264   0.701748  6.202370e-02       0.628542  \n",
       "50%       0.689034   0.757856   0.796368  1.550820e-01       0.698163  \n",
       "75%       0.902976   0.916899   0.927170  1.940431e-01       0.898414  \n",
       "max       1.000000   1.000000   1.000000  2.656892e-01       1.000000  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tf_sep</th>\n",
       "      <th>train_recon_improv</th>\n",
       "      <th>val_recon_improv</th>\n",
       "      <th>val_recon_improv_w_val_mean</th>\n",
       "      <th>val_acc</th>\n",
       "      <th>val_f1</th>\n",
       "      <th>val_brier</th>\n",
       "      <th>val_soft_f1</th>\n",
       "      <th>train_acc</th>\n",
       "      <th>train_f1</th>\n",
       "      <th>train_brier</th>\n",
       "      <th>train_soft_f1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>60.000000</td>\n",
       "      <td>6.000000e+01</td>\n",
       "      <td>6.000000e+01</td>\n",
       "      <td>6.000000e+01</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>58.000000</td>\n",
       "      <td>6.000000e+01</td>\n",
       "      <td>58.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>58.000000</td>\n",
       "      <td>6.000000e+01</td>\n",
       "      <td>58.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.235943</td>\n",
       "      <td>1.075456e-01</td>\n",
       "      <td>1.059050e-01</td>\n",
       "      <td>1.029477e-01</td>\n",
       "      <td>0.783611</td>\n",
       "      <td>0.802116</td>\n",
       "      <td>1.347030e-01</td>\n",
       "      <td>0.750525</td>\n",
       "      <td>0.790368</td>\n",
       "      <td>0.809428</td>\n",
       "      <td>1.345743e-01</td>\n",
       "      <td>0.752996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.268247</td>\n",
       "      <td>1.271151e-01</td>\n",
       "      <td>1.345939e-01</td>\n",
       "      <td>1.288299e-01</td>\n",
       "      <td>0.155743</td>\n",
       "      <td>0.140303</td>\n",
       "      <td>8.139284e-02</td>\n",
       "      <td>0.152109</td>\n",
       "      <td>0.143905</td>\n",
       "      <td>0.123342</td>\n",
       "      <td>7.883947e-02</td>\n",
       "      <td>0.144177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-8.239608e-08</td>\n",
       "      <td>-2.124053e-07</td>\n",
       "      <td>-7.470586e-08</td>\n",
       "      <td>0.272727</td>\n",
       "      <td>0.428571</td>\n",
       "      <td>3.790091e-27</td>\n",
       "      <td>0.493223</td>\n",
       "      <td>0.431818</td>\n",
       "      <td>0.603175</td>\n",
       "      <td>1.161833e-14</td>\n",
       "      <td>0.544183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.030512</td>\n",
       "      <td>1.417779e-02</td>\n",
       "      <td>1.235262e-02</td>\n",
       "      <td>1.264140e-02</td>\n",
       "      <td>0.682654</td>\n",
       "      <td>0.689200</td>\n",
       "      <td>5.452009e-02</td>\n",
       "      <td>0.627251</td>\n",
       "      <td>0.702264</td>\n",
       "      <td>0.701748</td>\n",
       "      <td>6.202370e-02</td>\n",
       "      <td>0.628542</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.139348</td>\n",
       "      <td>4.943551e-02</td>\n",
       "      <td>3.573745e-02</td>\n",
       "      <td>3.461149e-02</td>\n",
       "      <td>0.769014</td>\n",
       "      <td>0.785857</td>\n",
       "      <td>1.666387e-01</td>\n",
       "      <td>0.689034</td>\n",
       "      <td>0.757856</td>\n",
       "      <td>0.796368</td>\n",
       "      <td>1.550820e-01</td>\n",
       "      <td>0.698163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.339572</td>\n",
       "      <td>1.941366e-01</td>\n",
       "      <td>1.847902e-01</td>\n",
       "      <td>1.828425e-01</td>\n",
       "      <td>0.923788</td>\n",
       "      <td>0.936397</td>\n",
       "      <td>2.024954e-01</td>\n",
       "      <td>0.902976</td>\n",
       "      <td>0.916899</td>\n",
       "      <td>0.927170</td>\n",
       "      <td>1.940431e-01</td>\n",
       "      <td>0.898414</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.406146</td>\n",
       "      <td>5.851733e-01</td>\n",
       "      <td>5.821817e-01</td>\n",
       "      <td>5.810651e-01</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.536726e-01</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.656892e-01</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 18
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:42:20.462133Z",
     "start_time": "2024-12-07T21:42:20.452449Z"
    }
   },
   "cell_type": "code",
   "source": [
    "lyr25_t_f_sep_and_t_p_dirs_correlations = lyr25_eval_except_test_df[['tf_sep', 'train_recon_improv', 'val_recon_improv', 'val_recon_improv_w_val_mean']].corr()\n",
    "lyr25_t_f_sep_and_t_p_dirs_correlations"
   ],
   "id": "75ed7ffd350225fd",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "                               tf_sep  train_recon_improv  val_recon_improv  \\\n",
       "tf_sep                       1.000000            0.851259          0.839678   \n",
       "train_recon_improv           0.851259            1.000000          0.992763   \n",
       "val_recon_improv             0.839678            0.992763          1.000000   \n",
       "val_recon_improv_w_val_mean  0.842292            0.991248          0.995207   \n",
       "\n",
       "                             val_recon_improv_w_val_mean  \n",
       "tf_sep                                          0.842292  \n",
       "train_recon_improv                              0.991248  \n",
       "val_recon_improv                                0.995207  \n",
       "val_recon_improv_w_val_mean                     1.000000  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tf_sep</th>\n",
       "      <th>train_recon_improv</th>\n",
       "      <th>val_recon_improv</th>\n",
       "      <th>val_recon_improv_w_val_mean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>tf_sep</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.851259</td>\n",
       "      <td>0.839678</td>\n",
       "      <td>0.842292</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train_recon_improv</th>\n",
       "      <td>0.851259</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.992763</td>\n",
       "      <td>0.991248</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>val_recon_improv</th>\n",
       "      <td>0.839678</td>\n",
       "      <td>0.992763</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.995207</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>val_recon_improv_w_val_mean</th>\n",
       "      <td>0.842292</td>\n",
       "      <td>0.991248</td>\n",
       "      <td>0.995207</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 66
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:58.402020Z",
     "start_time": "2024-12-07T21:26:58.379454Z"
    }
   },
   "cell_type": "code",
   "source": "lyrs18_and_25_eval_except_test_df",
   "id": "f202f39b90325119",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "                                             scenario    tf_sep  \\\n",
       "0                           animal_class__affirmative  0.730812   \n",
       "1                           animal_class__conjunction  0.324344   \n",
       "2                           animal_class__disjunction  0.029893   \n",
       "3                               animal_class__negated  0.416986   \n",
       "4                                 cities__affirmative  1.548113   \n",
       "5                                 cities__conjunction  0.336618   \n",
       "6                                 cities__disjunction  0.068194   \n",
       "7                                     cities__negated  0.648312   \n",
       "8                           element_symb__affirmative  0.481369   \n",
       "9                           element_symb__conjunction  0.353086   \n",
       "10                          element_symb__disjunction  0.084360   \n",
       "11                              element_symb__negated  0.241868   \n",
       "12                                 facts__affirmative  0.076891   \n",
       "13                                 facts__conjunction  0.029567   \n",
       "14                                 facts__disjunction  0.016764   \n",
       "15                                     facts__negated  0.019656   \n",
       "16                             inventors__affirmative  0.044059   \n",
       "17                             inventors__conjunction  0.020307   \n",
       "18                             inventors__disjunction  0.008869   \n",
       "19                                 inventors__negated  0.059040   \n",
       "20                                      ambiguous_lie  0.052309   \n",
       "21                           ambiguous_truthful_reply  0.052309   \n",
       "22              honest_reply_despite_incentive_to_lie  0.000000   \n",
       "23                                    unambiguous_lie  0.089243   \n",
       "24                         unambiguous_truthful_reply  0.089243   \n",
       "25                                        larger_than  0.365530   \n",
       "26                                       smaller_than  0.291107   \n",
       "27                               sp_en_trans__negated  0.593574   \n",
       "28                           sp_en_trans__affirmative  0.764889   \n",
       "29                           sp_en_trans__conjunction  0.347323   \n",
       "30                           sp_en_trans__disjunction  0.051085   \n",
       "31                            common_claim_true_false  0.014099   \n",
       "32                             counterfact_true_false  0.009540   \n",
       "33                  animal_class__affirmative+negated  0.573899   \n",
       "34              animal_class__affirmative+disjunction  0.380353   \n",
       "35                  animal_class__negated+conjunction  0.370665   \n",
       "36  animal_class__affirmative+negated+conjunction+...  0.375509   \n",
       "37                        cities__affirmative+negated  1.098213   \n",
       "38                    cities__affirmative+disjunction  0.808154   \n",
       "39                        cities__negated+conjunction  0.492465   \n",
       "40  cities__affirmative+negated+conjunction+disjun...  0.650309   \n",
       "41                  element_symb__affirmative+negated  0.361619   \n",
       "42              element_symb__affirmative+disjunction  0.282865   \n",
       "43                  element_symb__negated+conjunction  0.297477   \n",
       "44  element_symb__affirmative+negated+conjunction+...  0.290171   \n",
       "45                         facts__affirmative+negated  0.048273   \n",
       "46                     facts__affirmative+disjunction  0.046827   \n",
       "47                         facts__negated+conjunction  0.024611   \n",
       "48  facts__affirmative+negated+conjunction+disjunc...  0.035719   \n",
       "49                     inventors__affirmative+negated  0.051550   \n",
       "50                 inventors__affirmative+disjunction  0.026464   \n",
       "51                     inventors__negated+conjunction  0.039674   \n",
       "52  inventors__affirmative+negated+conjunction+dis...  0.033069   \n",
       "53                   sp_en_trans__affirmative+negated  0.679232   \n",
       "54               sp_en_trans__affirmative+disjunction  0.407987   \n",
       "55                   sp_en_trans__negated+conjunction  0.470449   \n",
       "56  sp_en_trans__affirmative+negated+conjunction+d...  0.439218   \n",
       "57  ambiguous_truthful_reply+unambiguous_lie+unamb...  0.076932   \n",
       "58  animal_class+facts+inventors__affirmative+nega...  0.191296   \n",
       "59  animal_class+facts+inventors__affirmative+nega...  0.171222   \n",
       "\n",
       "    train_recon_improv  val_recon_improv  val_recon_improv_w_val_mean  \\\n",
       "0             0.376432      4.167493e-01                 3.190759e-01   \n",
       "1             0.247242      2.536601e-01                 2.566432e-01   \n",
       "2             0.027450      3.721342e-02                 3.495434e-02   \n",
       "3             0.219127      2.507378e-01                 2.446228e-01   \n",
       "4             0.590800      5.865278e-01                 5.854327e-01   \n",
       "5             0.224548      2.425223e-01                 2.414631e-01   \n",
       "6             0.074966      4.874871e-02                 5.021835e-02   \n",
       "7             0.315151      3.227825e-01                 3.135470e-01   \n",
       "8             0.279506      3.310533e-01                 3.412581e-01   \n",
       "9             0.236666      2.145218e-01                 2.158086e-01   \n",
       "10            0.088819      8.905924e-02                 8.895693e-02   \n",
       "11            0.127870      9.768481e-02                 1.007462e-01   \n",
       "12            0.063410      4.671065e-02                 4.342653e-02   \n",
       "13            0.025842      1.640895e-02                 1.576277e-02   \n",
       "14            0.014617      1.832306e-02                 1.855864e-02   \n",
       "15            0.013732      8.300493e-03                 8.401126e-03   \n",
       "16            0.032038      1.724613e-02                 1.796282e-02   \n",
       "17            0.018100      9.087952e-03                 9.740391e-03   \n",
       "18            0.007250      5.037946e-03                 4.478343e-03   \n",
       "19            0.040392      1.316852e-02                 1.465443e-02   \n",
       "20            0.000000     -3.878537e-07                 6.746208e-08   \n",
       "21            0.000000      9.594764e-08                 1.294894e-07   \n",
       "22            0.000000     -8.140490e-08                 1.066376e-07   \n",
       "23            0.000000     -3.078042e-07                 0.000000e+00   \n",
       "24            0.000000      6.545556e-08                 0.000000e+00   \n",
       "25            0.227083      2.309441e-01                 2.307898e-01   \n",
       "26            0.201326      1.889082e-01                 1.857684e-01   \n",
       "27            0.352762      3.367341e-01                 3.394803e-01   \n",
       "28            0.428075      4.579815e-01                 4.057482e-01   \n",
       "29            0.252539      2.609013e-01                 2.587486e-01   \n",
       "30            0.050818      2.492399e-02                 2.368459e-02   \n",
       "31            0.011225      1.334179e-02                 1.335970e-02   \n",
       "32            0.007145      6.934422e-03                 6.933936e-03   \n",
       "33            0.243376      3.042069e-01                 2.857524e-01   \n",
       "34            0.031573      2.324250e-02                 1.888703e-02   \n",
       "35            0.119292      1.154142e-01                 1.155958e-01   \n",
       "36            0.049828      5.133375e-02                 5.104955e-02   \n",
       "37            0.348623      3.530811e-01                 3.523741e-01   \n",
       "38            0.208314      2.062135e-01                 2.054593e-01   \n",
       "39            0.102916      1.058432e-01                 1.040960e-01   \n",
       "40            0.105041      1.106453e-01                 1.103473e-01   \n",
       "41            0.128200      1.417082e-01                 1.432608e-01   \n",
       "42            0.022589      2.904992e-02                 2.869749e-02   \n",
       "43            0.106490      8.543135e-02                 8.536934e-02   \n",
       "44            0.028869      2.349190e-02                 2.356160e-02   \n",
       "45            0.036374      2.546617e-02                 2.419861e-02   \n",
       "46            0.021649      2.148968e-02                 2.145191e-02   \n",
       "47            0.013329      8.919930e-03                 9.054577e-03   \n",
       "48            0.014869      1.278493e-02                 1.300787e-02   \n",
       "49            0.028507      1.579594e-02                 1.715225e-02   \n",
       "50            0.006453      3.337304e-03                 3.427257e-03   \n",
       "51            0.027966      2.529709e-02                 2.569605e-02   \n",
       "52            0.011005      7.604439e-03                 7.885243e-03   \n",
       "53            0.180261      1.693949e-01                 1.607487e-01   \n",
       "54            0.050064      4.018475e-02                 3.936839e-02   \n",
       "55            0.091876      7.510325e-02                 7.457987e-02   \n",
       "56            0.053440      3.738981e-02                 3.742623e-02   \n",
       "57            0.063350      2.100165e-02                 2.843561e-02   \n",
       "58            0.014626      1.542448e-02                 1.539118e-02   \n",
       "59            0.009766      1.022900e-02                 1.020268e-02   \n",
       "\n",
       "     val_acc    val_f1     val_brier  val_soft_f1  train_acc  train_f1  \\\n",
       "0   1.000000  1.000000  1.048590e-05     0.999071   0.961832  0.965035   \n",
       "1   0.890000  0.899083  7.981698e-02     0.856313   0.892500  0.896882   \n",
       "2   0.690000  0.666667  2.067646e-01     0.549148   0.637500  0.650602   \n",
       "3   0.969697  0.974359  1.444203e-02     0.974813   0.969466  0.970149   \n",
       "4   0.980000  0.979167  1.512385e-02     0.970972   0.988294  0.988543   \n",
       "5   0.940000  0.947059  5.112365e-02     0.910267   0.913189  0.917722   \n",
       "6   0.740000  0.723404  1.693977e-01     0.655712   0.750000  0.740933   \n",
       "7   0.976667  0.979592  1.335644e-02     0.972899   0.967391  0.967033   \n",
       "8   0.973684  0.975610  2.586430e-02     0.950555   0.918919  0.924051   \n",
       "9   0.870000  0.876190  8.621970e-02     0.839991   0.897500  0.911447   \n",
       "10  0.780000  0.792453  1.731925e-01     0.661480   0.752500  0.750630   \n",
       "11  1.000000  1.000000  3.631470e-22     1.000000   1.000000  1.000000   \n",
       "12  0.814159  0.846715  1.204582e-01     0.775369   0.814732  0.827443   \n",
       "13  0.710000  0.723810  1.850697e-01     0.611237   0.780000  0.804444   \n",
       "14  0.720000  0.674419  1.893128e-01     0.594273   0.692500  0.645533   \n",
       "15  0.752212  0.762712  1.481725e-01     0.729915   0.848214  0.851528   \n",
       "16  0.646341  0.666667  1.977272e-01     0.644556   0.679012  0.666667   \n",
       "17  0.670000  0.727273  2.217227e-01     0.637477   0.712500  0.742729   \n",
       "18  0.660000  0.696429  2.114068e-01     0.615934   0.675000  0.693396   \n",
       "19  0.560976  0.538462  2.276440e-01     0.523383   0.679012  0.653333   \n",
       "20  0.500000       NaN  2.491555e-01          NaN   0.357143       NaN   \n",
       "21  0.400000  0.571429  2.472415e-01     0.673252   0.444444  0.615385   \n",
       "22  1.000000  1.000000  1.027950e-01     0.817399   0.500000  0.666667   \n",
       "23  0.800000       NaN  2.487790e-01          NaN   0.512821       NaN   \n",
       "24  0.636364  0.777778  2.480648e-01     0.668427   0.522727  0.686567   \n",
       "25  0.967172  0.966921  2.803952e-02     0.944715   0.960859  0.962696   \n",
       "26  0.944444  0.951111  4.447663e-02     0.920616   0.946338  0.948076   \n",
       "27  1.000000  1.000000  0.000000e+00     1.000000   0.992933  0.992908   \n",
       "28  0.985915  0.988764  8.304221e-03     0.983670   0.975265  0.974170   \n",
       "29  0.910000  0.912621  5.324913e-02     0.908213   0.950000  0.955752   \n",
       "30  0.720000  0.681818  1.853800e-01     0.620151   0.765000  0.744565   \n",
       "31  0.746067  0.753813  1.705093e-01     0.647091   0.716573  0.719020   \n",
       "32  0.667449  0.677879  2.070386e-01     0.589363   0.664777  0.677210   \n",
       "33  0.984848  0.983051  1.641124e-02     0.973967   0.977099  0.978102   \n",
       "34  0.676692  0.505747  1.928114e-01     0.527331   0.651601  0.570766   \n",
       "35  0.909774  0.918919  6.692390e-02     0.881437   0.903955  0.907104   \n",
       "36  0.721805  0.678261  1.653051e-01     0.662606   0.701507  0.673532   \n",
       "37  0.963333  0.965625  3.053823e-02     0.943546   0.948579  0.950263   \n",
       "38  0.857500  0.840336  8.387524e-02     0.817318   0.879699  0.871658   \n",
       "39  0.926667  0.935860  5.163525e-02     0.909466   0.918129  0.920776   \n",
       "40  0.871000  0.879552  9.344633e-02     0.825923   0.846867  0.852165   \n",
       "41  0.947368  0.951220  3.112758e-02     0.941691   0.952703  0.954545   \n",
       "42  0.753623  0.753623  1.749970e-01     0.671189   0.757299  0.747628   \n",
       "43  0.833333  0.845638  1.084417e-01     0.797521   0.879562  0.895899   \n",
       "44  0.742754  0.743682  1.682252e-01     0.677976   0.746350  0.743068   \n",
       "45  0.694690  0.711297  2.078405e-01     0.636289   0.674107  0.680525   \n",
       "46  0.690141  0.670000  1.980803e-01     0.618786   0.683962  0.642667   \n",
       "47  0.723005  0.725581  1.838209e-01     0.633874   0.773585  0.786192   \n",
       "48  0.706573  0.708625  1.954480e-01     0.618070   0.719929  0.717430   \n",
       "49  0.756098  0.753086  1.698943e-01     0.656334   0.765432  0.745819   \n",
       "50  0.637363  0.685714  2.101495e-01     0.621952   0.696133  0.738095   \n",
       "51  0.664835  0.693467  2.251347e-01     0.566759   0.629834  0.658163   \n",
       "52  0.609890  0.646766  2.317923e-01     0.567178   0.596685  0.621762   \n",
       "53  0.943662  0.950617  3.961941e-02     0.930347   0.950530  0.950704   \n",
       "54  0.742690  0.752809  1.502128e-01     0.731752   0.802343  0.791345   \n",
       "55  0.964912  0.964286  2.174961e-02     0.954880   0.973646  0.974930   \n",
       "56  0.754386  0.732484  1.590001e-01     0.685282   0.762079  0.745497   \n",
       "57  0.846154  0.857143  1.347483e-01     0.762924   0.900990  0.915254   \n",
       "58  0.720899  0.746089  1.812848e-01     0.648037   0.717232  0.736679   \n",
       "59  0.657805  0.697046  2.217134e-01     0.648042   0.660190  0.693528   \n",
       "\n",
       "     train_brier  train_soft_f1  \n",
       "0   3.823655e-02       0.963365  \n",
       "1   7.976444e-02       0.846770  \n",
       "2   2.167152e-01       0.576844  \n",
       "3   2.588530e-02       0.959980  \n",
       "4   1.162213e-02       0.975314  \n",
       "5   6.478866e-02       0.885512  \n",
       "6   1.643413e-01       0.666838  \n",
       "7   2.275182e-02       0.956171  \n",
       "8   5.816379e-02       0.905209  \n",
       "9   7.613172e-02       0.870363  \n",
       "10  1.713014e-01       0.637860  \n",
       "11  4.758516e-22       1.000000  \n",
       "12  1.325435e-01       0.743538  \n",
       "13  1.579917e-01       0.671479  \n",
       "14  2.043912e-01       0.585283  \n",
       "15  1.157284e-01       0.767327  \n",
       "16  1.850799e-01       0.649348  \n",
       "17  1.870549e-01       0.649201  \n",
       "18  2.074786e-01       0.594165  \n",
       "19  2.032593e-01       0.552610  \n",
       "20  2.500906e-01            NaN  \n",
       "21  2.513282e-01       0.666654  \n",
       "22  3.327580e-01       0.664517  \n",
       "23  2.500563e-01            NaN  \n",
       "24  2.500595e-01       0.666667  \n",
       "25  3.030396e-02       0.944751  \n",
       "26  3.711441e-02       0.921277  \n",
       "27  7.067138e-03       0.992908  \n",
       "28  1.586142e-02       0.967787  \n",
       "29  3.266276e-02       0.933669  \n",
       "30  1.523729e-01       0.676091  \n",
       "31  1.849938e-01       0.630260  \n",
       "32  2.067304e-01       0.591837  \n",
       "33  1.733593e-02       0.975200  \n",
       "34  1.996672e-01       0.601834  \n",
       "35  6.366233e-02       0.879970  \n",
       "36  1.749482e-01       0.667494  \n",
       "37  3.909004e-02       0.928093  \n",
       "38  7.819608e-02       0.831894  \n",
       "39  6.175595e-02       0.885123  \n",
       "40  1.069703e-01       0.801253  \n",
       "41  4.448311e-02       0.926113  \n",
       "42  1.621564e-01       0.680277  \n",
       "43  8.495046e-02       0.845944  \n",
       "44  1.567635e-01       0.700789  \n",
       "45  2.165161e-01       0.627211  \n",
       "46  1.971686e-01       0.597043  \n",
       "47  1.634185e-01       0.670525  \n",
       "48  1.850207e-01       0.624169  \n",
       "49  1.655079e-01       0.662123  \n",
       "50  1.957986e-01       0.623715  \n",
       "51  2.314752e-01       0.548751  \n",
       "52  2.306985e-01       0.551868  \n",
       "53  3.741187e-02       0.921340  \n",
       "54  1.260725e-01       0.734614  \n",
       "55  1.923370e-02       0.965082  \n",
       "56  1.441755e-01       0.710124  \n",
       "57  8.372632e-02       0.843558  \n",
       "58  1.832371e-01       0.640655  \n",
       "59  2.297797e-01       0.634425  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>scenario</th>\n",
       "      <th>tf_sep</th>\n",
       "      <th>train_recon_improv</th>\n",
       "      <th>val_recon_improv</th>\n",
       "      <th>val_recon_improv_w_val_mean</th>\n",
       "      <th>val_acc</th>\n",
       "      <th>val_f1</th>\n",
       "      <th>val_brier</th>\n",
       "      <th>val_soft_f1</th>\n",
       "      <th>train_acc</th>\n",
       "      <th>train_f1</th>\n",
       "      <th>train_brier</th>\n",
       "      <th>train_soft_f1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>animal_class__affirmative</td>\n",
       "      <td>0.730812</td>\n",
       "      <td>0.376432</td>\n",
       "      <td>4.167493e-01</td>\n",
       "      <td>3.190759e-01</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.048590e-05</td>\n",
       "      <td>0.999071</td>\n",
       "      <td>0.961832</td>\n",
       "      <td>0.965035</td>\n",
       "      <td>3.823655e-02</td>\n",
       "      <td>0.963365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>animal_class__conjunction</td>\n",
       "      <td>0.324344</td>\n",
       "      <td>0.247242</td>\n",
       "      <td>2.536601e-01</td>\n",
       "      <td>2.566432e-01</td>\n",
       "      <td>0.890000</td>\n",
       "      <td>0.899083</td>\n",
       "      <td>7.981698e-02</td>\n",
       "      <td>0.856313</td>\n",
       "      <td>0.892500</td>\n",
       "      <td>0.896882</td>\n",
       "      <td>7.976444e-02</td>\n",
       "      <td>0.846770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>animal_class__disjunction</td>\n",
       "      <td>0.029893</td>\n",
       "      <td>0.027450</td>\n",
       "      <td>3.721342e-02</td>\n",
       "      <td>3.495434e-02</td>\n",
       "      <td>0.690000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>2.067646e-01</td>\n",
       "      <td>0.549148</td>\n",
       "      <td>0.637500</td>\n",
       "      <td>0.650602</td>\n",
       "      <td>2.167152e-01</td>\n",
       "      <td>0.576844</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>animal_class__negated</td>\n",
       "      <td>0.416986</td>\n",
       "      <td>0.219127</td>\n",
       "      <td>2.507378e-01</td>\n",
       "      <td>2.446228e-01</td>\n",
       "      <td>0.969697</td>\n",
       "      <td>0.974359</td>\n",
       "      <td>1.444203e-02</td>\n",
       "      <td>0.974813</td>\n",
       "      <td>0.969466</td>\n",
       "      <td>0.970149</td>\n",
       "      <td>2.588530e-02</td>\n",
       "      <td>0.959980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cities__affirmative</td>\n",
       "      <td>1.548113</td>\n",
       "      <td>0.590800</td>\n",
       "      <td>5.865278e-01</td>\n",
       "      <td>5.854327e-01</td>\n",
       "      <td>0.980000</td>\n",
       "      <td>0.979167</td>\n",
       "      <td>1.512385e-02</td>\n",
       "      <td>0.970972</td>\n",
       "      <td>0.988294</td>\n",
       "      <td>0.988543</td>\n",
       "      <td>1.162213e-02</td>\n",
       "      <td>0.975314</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>cities__conjunction</td>\n",
       "      <td>0.336618</td>\n",
       "      <td>0.224548</td>\n",
       "      <td>2.425223e-01</td>\n",
       "      <td>2.414631e-01</td>\n",
       "      <td>0.940000</td>\n",
       "      <td>0.947059</td>\n",
       "      <td>5.112365e-02</td>\n",
       "      <td>0.910267</td>\n",
       "      <td>0.913189</td>\n",
       "      <td>0.917722</td>\n",
       "      <td>6.478866e-02</td>\n",
       "      <td>0.885512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>cities__disjunction</td>\n",
       "      <td>0.068194</td>\n",
       "      <td>0.074966</td>\n",
       "      <td>4.874871e-02</td>\n",
       "      <td>5.021835e-02</td>\n",
       "      <td>0.740000</td>\n",
       "      <td>0.723404</td>\n",
       "      <td>1.693977e-01</td>\n",
       "      <td>0.655712</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.740933</td>\n",
       "      <td>1.643413e-01</td>\n",
       "      <td>0.666838</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>cities__negated</td>\n",
       "      <td>0.648312</td>\n",
       "      <td>0.315151</td>\n",
       "      <td>3.227825e-01</td>\n",
       "      <td>3.135470e-01</td>\n",
       "      <td>0.976667</td>\n",
       "      <td>0.979592</td>\n",
       "      <td>1.335644e-02</td>\n",
       "      <td>0.972899</td>\n",
       "      <td>0.967391</td>\n",
       "      <td>0.967033</td>\n",
       "      <td>2.275182e-02</td>\n",
       "      <td>0.956171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>element_symb__affirmative</td>\n",
       "      <td>0.481369</td>\n",
       "      <td>0.279506</td>\n",
       "      <td>3.310533e-01</td>\n",
       "      <td>3.412581e-01</td>\n",
       "      <td>0.973684</td>\n",
       "      <td>0.975610</td>\n",
       "      <td>2.586430e-02</td>\n",
       "      <td>0.950555</td>\n",
       "      <td>0.918919</td>\n",
       "      <td>0.924051</td>\n",
       "      <td>5.816379e-02</td>\n",
       "      <td>0.905209</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>element_symb__conjunction</td>\n",
       "      <td>0.353086</td>\n",
       "      <td>0.236666</td>\n",
       "      <td>2.145218e-01</td>\n",
       "      <td>2.158086e-01</td>\n",
       "      <td>0.870000</td>\n",
       "      <td>0.876190</td>\n",
       "      <td>8.621970e-02</td>\n",
       "      <td>0.839991</td>\n",
       "      <td>0.897500</td>\n",
       "      <td>0.911447</td>\n",
       "      <td>7.613172e-02</td>\n",
       "      <td>0.870363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>element_symb__disjunction</td>\n",
       "      <td>0.084360</td>\n",
       "      <td>0.088819</td>\n",
       "      <td>8.905924e-02</td>\n",
       "      <td>8.895693e-02</td>\n",
       "      <td>0.780000</td>\n",
       "      <td>0.792453</td>\n",
       "      <td>1.731925e-01</td>\n",
       "      <td>0.661480</td>\n",
       "      <td>0.752500</td>\n",
       "      <td>0.750630</td>\n",
       "      <td>1.713014e-01</td>\n",
       "      <td>0.637860</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>element_symb__negated</td>\n",
       "      <td>0.241868</td>\n",
       "      <td>0.127870</td>\n",
       "      <td>9.768481e-02</td>\n",
       "      <td>1.007462e-01</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.631470e-22</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.758516e-22</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>facts__affirmative</td>\n",
       "      <td>0.076891</td>\n",
       "      <td>0.063410</td>\n",
       "      <td>4.671065e-02</td>\n",
       "      <td>4.342653e-02</td>\n",
       "      <td>0.814159</td>\n",
       "      <td>0.846715</td>\n",
       "      <td>1.204582e-01</td>\n",
       "      <td>0.775369</td>\n",
       "      <td>0.814732</td>\n",
       "      <td>0.827443</td>\n",
       "      <td>1.325435e-01</td>\n",
       "      <td>0.743538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>facts__conjunction</td>\n",
       "      <td>0.029567</td>\n",
       "      <td>0.025842</td>\n",
       "      <td>1.640895e-02</td>\n",
       "      <td>1.576277e-02</td>\n",
       "      <td>0.710000</td>\n",
       "      <td>0.723810</td>\n",
       "      <td>1.850697e-01</td>\n",
       "      <td>0.611237</td>\n",
       "      <td>0.780000</td>\n",
       "      <td>0.804444</td>\n",
       "      <td>1.579917e-01</td>\n",
       "      <td>0.671479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>facts__disjunction</td>\n",
       "      <td>0.016764</td>\n",
       "      <td>0.014617</td>\n",
       "      <td>1.832306e-02</td>\n",
       "      <td>1.855864e-02</td>\n",
       "      <td>0.720000</td>\n",
       "      <td>0.674419</td>\n",
       "      <td>1.893128e-01</td>\n",
       "      <td>0.594273</td>\n",
       "      <td>0.692500</td>\n",
       "      <td>0.645533</td>\n",
       "      <td>2.043912e-01</td>\n",
       "      <td>0.585283</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>facts__negated</td>\n",
       "      <td>0.019656</td>\n",
       "      <td>0.013732</td>\n",
       "      <td>8.300493e-03</td>\n",
       "      <td>8.401126e-03</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>0.762712</td>\n",
       "      <td>1.481725e-01</td>\n",
       "      <td>0.729915</td>\n",
       "      <td>0.848214</td>\n",
       "      <td>0.851528</td>\n",
       "      <td>1.157284e-01</td>\n",
       "      <td>0.767327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>inventors__affirmative</td>\n",
       "      <td>0.044059</td>\n",
       "      <td>0.032038</td>\n",
       "      <td>1.724613e-02</td>\n",
       "      <td>1.796282e-02</td>\n",
       "      <td>0.646341</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>1.977272e-01</td>\n",
       "      <td>0.644556</td>\n",
       "      <td>0.679012</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>1.850799e-01</td>\n",
       "      <td>0.649348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>inventors__conjunction</td>\n",
       "      <td>0.020307</td>\n",
       "      <td>0.018100</td>\n",
       "      <td>9.087952e-03</td>\n",
       "      <td>9.740391e-03</td>\n",
       "      <td>0.670000</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>2.217227e-01</td>\n",
       "      <td>0.637477</td>\n",
       "      <td>0.712500</td>\n",
       "      <td>0.742729</td>\n",
       "      <td>1.870549e-01</td>\n",
       "      <td>0.649201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>inventors__disjunction</td>\n",
       "      <td>0.008869</td>\n",
       "      <td>0.007250</td>\n",
       "      <td>5.037946e-03</td>\n",
       "      <td>4.478343e-03</td>\n",
       "      <td>0.660000</td>\n",
       "      <td>0.696429</td>\n",
       "      <td>2.114068e-01</td>\n",
       "      <td>0.615934</td>\n",
       "      <td>0.675000</td>\n",
       "      <td>0.693396</td>\n",
       "      <td>2.074786e-01</td>\n",
       "      <td>0.594165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>inventors__negated</td>\n",
       "      <td>0.059040</td>\n",
       "      <td>0.040392</td>\n",
       "      <td>1.316852e-02</td>\n",
       "      <td>1.465443e-02</td>\n",
       "      <td>0.560976</td>\n",
       "      <td>0.538462</td>\n",
       "      <td>2.276440e-01</td>\n",
       "      <td>0.523383</td>\n",
       "      <td>0.679012</td>\n",
       "      <td>0.653333</td>\n",
       "      <td>2.032593e-01</td>\n",
       "      <td>0.552610</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>ambiguous_lie</td>\n",
       "      <td>0.052309</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-3.878537e-07</td>\n",
       "      <td>6.746208e-08</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.491555e-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.357143</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.500906e-01</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>ambiguous_truthful_reply</td>\n",
       "      <td>0.052309</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>9.594764e-08</td>\n",
       "      <td>1.294894e-07</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.571429</td>\n",
       "      <td>2.472415e-01</td>\n",
       "      <td>0.673252</td>\n",
       "      <td>0.444444</td>\n",
       "      <td>0.615385</td>\n",
       "      <td>2.513282e-01</td>\n",
       "      <td>0.666654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>honest_reply_despite_incentive_to_lie</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-8.140490e-08</td>\n",
       "      <td>1.066376e-07</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.027950e-01</td>\n",
       "      <td>0.817399</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>3.327580e-01</td>\n",
       "      <td>0.664517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>unambiguous_lie</td>\n",
       "      <td>0.089243</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-3.078042e-07</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.487790e-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.512821</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.500563e-01</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>unambiguous_truthful_reply</td>\n",
       "      <td>0.089243</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.545556e-08</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.636364</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>2.480648e-01</td>\n",
       "      <td>0.668427</td>\n",
       "      <td>0.522727</td>\n",
       "      <td>0.686567</td>\n",
       "      <td>2.500595e-01</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>larger_than</td>\n",
       "      <td>0.365530</td>\n",
       "      <td>0.227083</td>\n",
       "      <td>2.309441e-01</td>\n",
       "      <td>2.307898e-01</td>\n",
       "      <td>0.967172</td>\n",
       "      <td>0.966921</td>\n",
       "      <td>2.803952e-02</td>\n",
       "      <td>0.944715</td>\n",
       "      <td>0.960859</td>\n",
       "      <td>0.962696</td>\n",
       "      <td>3.030396e-02</td>\n",
       "      <td>0.944751</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>smaller_than</td>\n",
       "      <td>0.291107</td>\n",
       "      <td>0.201326</td>\n",
       "      <td>1.889082e-01</td>\n",
       "      <td>1.857684e-01</td>\n",
       "      <td>0.944444</td>\n",
       "      <td>0.951111</td>\n",
       "      <td>4.447663e-02</td>\n",
       "      <td>0.920616</td>\n",
       "      <td>0.946338</td>\n",
       "      <td>0.948076</td>\n",
       "      <td>3.711441e-02</td>\n",
       "      <td>0.921277</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>sp_en_trans__negated</td>\n",
       "      <td>0.593574</td>\n",
       "      <td>0.352762</td>\n",
       "      <td>3.367341e-01</td>\n",
       "      <td>3.394803e-01</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.992933</td>\n",
       "      <td>0.992908</td>\n",
       "      <td>7.067138e-03</td>\n",
       "      <td>0.992908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>sp_en_trans__affirmative</td>\n",
       "      <td>0.764889</td>\n",
       "      <td>0.428075</td>\n",
       "      <td>4.579815e-01</td>\n",
       "      <td>4.057482e-01</td>\n",
       "      <td>0.985915</td>\n",
       "      <td>0.988764</td>\n",
       "      <td>8.304221e-03</td>\n",
       "      <td>0.983670</td>\n",
       "      <td>0.975265</td>\n",
       "      <td>0.974170</td>\n",
       "      <td>1.586142e-02</td>\n",
       "      <td>0.967787</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>sp_en_trans__conjunction</td>\n",
       "      <td>0.347323</td>\n",
       "      <td>0.252539</td>\n",
       "      <td>2.609013e-01</td>\n",
       "      <td>2.587486e-01</td>\n",
       "      <td>0.910000</td>\n",
       "      <td>0.912621</td>\n",
       "      <td>5.324913e-02</td>\n",
       "      <td>0.908213</td>\n",
       "      <td>0.950000</td>\n",
       "      <td>0.955752</td>\n",
       "      <td>3.266276e-02</td>\n",
       "      <td>0.933669</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>sp_en_trans__disjunction</td>\n",
       "      <td>0.051085</td>\n",
       "      <td>0.050818</td>\n",
       "      <td>2.492399e-02</td>\n",
       "      <td>2.368459e-02</td>\n",
       "      <td>0.720000</td>\n",
       "      <td>0.681818</td>\n",
       "      <td>1.853800e-01</td>\n",
       "      <td>0.620151</td>\n",
       "      <td>0.765000</td>\n",
       "      <td>0.744565</td>\n",
       "      <td>1.523729e-01</td>\n",
       "      <td>0.676091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>common_claim_true_false</td>\n",
       "      <td>0.014099</td>\n",
       "      <td>0.011225</td>\n",
       "      <td>1.334179e-02</td>\n",
       "      <td>1.335970e-02</td>\n",
       "      <td>0.746067</td>\n",
       "      <td>0.753813</td>\n",
       "      <td>1.705093e-01</td>\n",
       "      <td>0.647091</td>\n",
       "      <td>0.716573</td>\n",
       "      <td>0.719020</td>\n",
       "      <td>1.849938e-01</td>\n",
       "      <td>0.630260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>counterfact_true_false</td>\n",
       "      <td>0.009540</td>\n",
       "      <td>0.007145</td>\n",
       "      <td>6.934422e-03</td>\n",
       "      <td>6.933936e-03</td>\n",
       "      <td>0.667449</td>\n",
       "      <td>0.677879</td>\n",
       "      <td>2.070386e-01</td>\n",
       "      <td>0.589363</td>\n",
       "      <td>0.664777</td>\n",
       "      <td>0.677210</td>\n",
       "      <td>2.067304e-01</td>\n",
       "      <td>0.591837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>animal_class__affirmative+negated</td>\n",
       "      <td>0.573899</td>\n",
       "      <td>0.243376</td>\n",
       "      <td>3.042069e-01</td>\n",
       "      <td>2.857524e-01</td>\n",
       "      <td>0.984848</td>\n",
       "      <td>0.983051</td>\n",
       "      <td>1.641124e-02</td>\n",
       "      <td>0.973967</td>\n",
       "      <td>0.977099</td>\n",
       "      <td>0.978102</td>\n",
       "      <td>1.733593e-02</td>\n",
       "      <td>0.975200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>animal_class__affirmative+disjunction</td>\n",
       "      <td>0.380353</td>\n",
       "      <td>0.031573</td>\n",
       "      <td>2.324250e-02</td>\n",
       "      <td>1.888703e-02</td>\n",
       "      <td>0.676692</td>\n",
       "      <td>0.505747</td>\n",
       "      <td>1.928114e-01</td>\n",
       "      <td>0.527331</td>\n",
       "      <td>0.651601</td>\n",
       "      <td>0.570766</td>\n",
       "      <td>1.996672e-01</td>\n",
       "      <td>0.601834</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>animal_class__negated+conjunction</td>\n",
       "      <td>0.370665</td>\n",
       "      <td>0.119292</td>\n",
       "      <td>1.154142e-01</td>\n",
       "      <td>1.155958e-01</td>\n",
       "      <td>0.909774</td>\n",
       "      <td>0.918919</td>\n",
       "      <td>6.692390e-02</td>\n",
       "      <td>0.881437</td>\n",
       "      <td>0.903955</td>\n",
       "      <td>0.907104</td>\n",
       "      <td>6.366233e-02</td>\n",
       "      <td>0.879970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>animal_class__affirmative+negated+conjunction+...</td>\n",
       "      <td>0.375509</td>\n",
       "      <td>0.049828</td>\n",
       "      <td>5.133375e-02</td>\n",
       "      <td>5.104955e-02</td>\n",
       "      <td>0.721805</td>\n",
       "      <td>0.678261</td>\n",
       "      <td>1.653051e-01</td>\n",
       "      <td>0.662606</td>\n",
       "      <td>0.701507</td>\n",
       "      <td>0.673532</td>\n",
       "      <td>1.749482e-01</td>\n",
       "      <td>0.667494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>cities__affirmative+negated</td>\n",
       "      <td>1.098213</td>\n",
       "      <td>0.348623</td>\n",
       "      <td>3.530811e-01</td>\n",
       "      <td>3.523741e-01</td>\n",
       "      <td>0.963333</td>\n",
       "      <td>0.965625</td>\n",
       "      <td>3.053823e-02</td>\n",
       "      <td>0.943546</td>\n",
       "      <td>0.948579</td>\n",
       "      <td>0.950263</td>\n",
       "      <td>3.909004e-02</td>\n",
       "      <td>0.928093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>cities__affirmative+disjunction</td>\n",
       "      <td>0.808154</td>\n",
       "      <td>0.208314</td>\n",
       "      <td>2.062135e-01</td>\n",
       "      <td>2.054593e-01</td>\n",
       "      <td>0.857500</td>\n",
       "      <td>0.840336</td>\n",
       "      <td>8.387524e-02</td>\n",
       "      <td>0.817318</td>\n",
       "      <td>0.879699</td>\n",
       "      <td>0.871658</td>\n",
       "      <td>7.819608e-02</td>\n",
       "      <td>0.831894</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>cities__negated+conjunction</td>\n",
       "      <td>0.492465</td>\n",
       "      <td>0.102916</td>\n",
       "      <td>1.058432e-01</td>\n",
       "      <td>1.040960e-01</td>\n",
       "      <td>0.926667</td>\n",
       "      <td>0.935860</td>\n",
       "      <td>5.163525e-02</td>\n",
       "      <td>0.909466</td>\n",
       "      <td>0.918129</td>\n",
       "      <td>0.920776</td>\n",
       "      <td>6.175595e-02</td>\n",
       "      <td>0.885123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>cities__affirmative+negated+conjunction+disjun...</td>\n",
       "      <td>0.650309</td>\n",
       "      <td>0.105041</td>\n",
       "      <td>1.106453e-01</td>\n",
       "      <td>1.103473e-01</td>\n",
       "      <td>0.871000</td>\n",
       "      <td>0.879552</td>\n",
       "      <td>9.344633e-02</td>\n",
       "      <td>0.825923</td>\n",
       "      <td>0.846867</td>\n",
       "      <td>0.852165</td>\n",
       "      <td>1.069703e-01</td>\n",
       "      <td>0.801253</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>element_symb__affirmative+negated</td>\n",
       "      <td>0.361619</td>\n",
       "      <td>0.128200</td>\n",
       "      <td>1.417082e-01</td>\n",
       "      <td>1.432608e-01</td>\n",
       "      <td>0.947368</td>\n",
       "      <td>0.951220</td>\n",
       "      <td>3.112758e-02</td>\n",
       "      <td>0.941691</td>\n",
       "      <td>0.952703</td>\n",
       "      <td>0.954545</td>\n",
       "      <td>4.448311e-02</td>\n",
       "      <td>0.926113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>element_symb__affirmative+disjunction</td>\n",
       "      <td>0.282865</td>\n",
       "      <td>0.022589</td>\n",
       "      <td>2.904992e-02</td>\n",
       "      <td>2.869749e-02</td>\n",
       "      <td>0.753623</td>\n",
       "      <td>0.753623</td>\n",
       "      <td>1.749970e-01</td>\n",
       "      <td>0.671189</td>\n",
       "      <td>0.757299</td>\n",
       "      <td>0.747628</td>\n",
       "      <td>1.621564e-01</td>\n",
       "      <td>0.680277</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>element_symb__negated+conjunction</td>\n",
       "      <td>0.297477</td>\n",
       "      <td>0.106490</td>\n",
       "      <td>8.543135e-02</td>\n",
       "      <td>8.536934e-02</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>0.845638</td>\n",
       "      <td>1.084417e-01</td>\n",
       "      <td>0.797521</td>\n",
       "      <td>0.879562</td>\n",
       "      <td>0.895899</td>\n",
       "      <td>8.495046e-02</td>\n",
       "      <td>0.845944</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>element_symb__affirmative+negated+conjunction+...</td>\n",
       "      <td>0.290171</td>\n",
       "      <td>0.028869</td>\n",
       "      <td>2.349190e-02</td>\n",
       "      <td>2.356160e-02</td>\n",
       "      <td>0.742754</td>\n",
       "      <td>0.743682</td>\n",
       "      <td>1.682252e-01</td>\n",
       "      <td>0.677976</td>\n",
       "      <td>0.746350</td>\n",
       "      <td>0.743068</td>\n",
       "      <td>1.567635e-01</td>\n",
       "      <td>0.700789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>facts__affirmative+negated</td>\n",
       "      <td>0.048273</td>\n",
       "      <td>0.036374</td>\n",
       "      <td>2.546617e-02</td>\n",
       "      <td>2.419861e-02</td>\n",
       "      <td>0.694690</td>\n",
       "      <td>0.711297</td>\n",
       "      <td>2.078405e-01</td>\n",
       "      <td>0.636289</td>\n",
       "      <td>0.674107</td>\n",
       "      <td>0.680525</td>\n",
       "      <td>2.165161e-01</td>\n",
       "      <td>0.627211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>facts__affirmative+disjunction</td>\n",
       "      <td>0.046827</td>\n",
       "      <td>0.021649</td>\n",
       "      <td>2.148968e-02</td>\n",
       "      <td>2.145191e-02</td>\n",
       "      <td>0.690141</td>\n",
       "      <td>0.670000</td>\n",
       "      <td>1.980803e-01</td>\n",
       "      <td>0.618786</td>\n",
       "      <td>0.683962</td>\n",
       "      <td>0.642667</td>\n",
       "      <td>1.971686e-01</td>\n",
       "      <td>0.597043</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>facts__negated+conjunction</td>\n",
       "      <td>0.024611</td>\n",
       "      <td>0.013329</td>\n",
       "      <td>8.919930e-03</td>\n",
       "      <td>9.054577e-03</td>\n",
       "      <td>0.723005</td>\n",
       "      <td>0.725581</td>\n",
       "      <td>1.838209e-01</td>\n",
       "      <td>0.633874</td>\n",
       "      <td>0.773585</td>\n",
       "      <td>0.786192</td>\n",
       "      <td>1.634185e-01</td>\n",
       "      <td>0.670525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>facts__affirmative+negated+conjunction+disjunc...</td>\n",
       "      <td>0.035719</td>\n",
       "      <td>0.014869</td>\n",
       "      <td>1.278493e-02</td>\n",
       "      <td>1.300787e-02</td>\n",
       "      <td>0.706573</td>\n",
       "      <td>0.708625</td>\n",
       "      <td>1.954480e-01</td>\n",
       "      <td>0.618070</td>\n",
       "      <td>0.719929</td>\n",
       "      <td>0.717430</td>\n",
       "      <td>1.850207e-01</td>\n",
       "      <td>0.624169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>inventors__affirmative+negated</td>\n",
       "      <td>0.051550</td>\n",
       "      <td>0.028507</td>\n",
       "      <td>1.579594e-02</td>\n",
       "      <td>1.715225e-02</td>\n",
       "      <td>0.756098</td>\n",
       "      <td>0.753086</td>\n",
       "      <td>1.698943e-01</td>\n",
       "      <td>0.656334</td>\n",
       "      <td>0.765432</td>\n",
       "      <td>0.745819</td>\n",
       "      <td>1.655079e-01</td>\n",
       "      <td>0.662123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>inventors__affirmative+disjunction</td>\n",
       "      <td>0.026464</td>\n",
       "      <td>0.006453</td>\n",
       "      <td>3.337304e-03</td>\n",
       "      <td>3.427257e-03</td>\n",
       "      <td>0.637363</td>\n",
       "      <td>0.685714</td>\n",
       "      <td>2.101495e-01</td>\n",
       "      <td>0.621952</td>\n",
       "      <td>0.696133</td>\n",
       "      <td>0.738095</td>\n",
       "      <td>1.957986e-01</td>\n",
       "      <td>0.623715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>inventors__negated+conjunction</td>\n",
       "      <td>0.039674</td>\n",
       "      <td>0.027966</td>\n",
       "      <td>2.529709e-02</td>\n",
       "      <td>2.569605e-02</td>\n",
       "      <td>0.664835</td>\n",
       "      <td>0.693467</td>\n",
       "      <td>2.251347e-01</td>\n",
       "      <td>0.566759</td>\n",
       "      <td>0.629834</td>\n",
       "      <td>0.658163</td>\n",
       "      <td>2.314752e-01</td>\n",
       "      <td>0.548751</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>inventors__affirmative+negated+conjunction+dis...</td>\n",
       "      <td>0.033069</td>\n",
       "      <td>0.011005</td>\n",
       "      <td>7.604439e-03</td>\n",
       "      <td>7.885243e-03</td>\n",
       "      <td>0.609890</td>\n",
       "      <td>0.646766</td>\n",
       "      <td>2.317923e-01</td>\n",
       "      <td>0.567178</td>\n",
       "      <td>0.596685</td>\n",
       "      <td>0.621762</td>\n",
       "      <td>2.306985e-01</td>\n",
       "      <td>0.551868</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>sp_en_trans__affirmative+negated</td>\n",
       "      <td>0.679232</td>\n",
       "      <td>0.180261</td>\n",
       "      <td>1.693949e-01</td>\n",
       "      <td>1.607487e-01</td>\n",
       "      <td>0.943662</td>\n",
       "      <td>0.950617</td>\n",
       "      <td>3.961941e-02</td>\n",
       "      <td>0.930347</td>\n",
       "      <td>0.950530</td>\n",
       "      <td>0.950704</td>\n",
       "      <td>3.741187e-02</td>\n",
       "      <td>0.921340</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>sp_en_trans__affirmative+disjunction</td>\n",
       "      <td>0.407987</td>\n",
       "      <td>0.050064</td>\n",
       "      <td>4.018475e-02</td>\n",
       "      <td>3.936839e-02</td>\n",
       "      <td>0.742690</td>\n",
       "      <td>0.752809</td>\n",
       "      <td>1.502128e-01</td>\n",
       "      <td>0.731752</td>\n",
       "      <td>0.802343</td>\n",
       "      <td>0.791345</td>\n",
       "      <td>1.260725e-01</td>\n",
       "      <td>0.734614</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>sp_en_trans__negated+conjunction</td>\n",
       "      <td>0.470449</td>\n",
       "      <td>0.091876</td>\n",
       "      <td>7.510325e-02</td>\n",
       "      <td>7.457987e-02</td>\n",
       "      <td>0.964912</td>\n",
       "      <td>0.964286</td>\n",
       "      <td>2.174961e-02</td>\n",
       "      <td>0.954880</td>\n",
       "      <td>0.973646</td>\n",
       "      <td>0.974930</td>\n",
       "      <td>1.923370e-02</td>\n",
       "      <td>0.965082</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>sp_en_trans__affirmative+negated+conjunction+d...</td>\n",
       "      <td>0.439218</td>\n",
       "      <td>0.053440</td>\n",
       "      <td>3.738981e-02</td>\n",
       "      <td>3.742623e-02</td>\n",
       "      <td>0.754386</td>\n",
       "      <td>0.732484</td>\n",
       "      <td>1.590001e-01</td>\n",
       "      <td>0.685282</td>\n",
       "      <td>0.762079</td>\n",
       "      <td>0.745497</td>\n",
       "      <td>1.441755e-01</td>\n",
       "      <td>0.710124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>ambiguous_truthful_reply+unambiguous_lie+unamb...</td>\n",
       "      <td>0.076932</td>\n",
       "      <td>0.063350</td>\n",
       "      <td>2.100165e-02</td>\n",
       "      <td>2.843561e-02</td>\n",
       "      <td>0.846154</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>1.347483e-01</td>\n",
       "      <td>0.762924</td>\n",
       "      <td>0.900990</td>\n",
       "      <td>0.915254</td>\n",
       "      <td>8.372632e-02</td>\n",
       "      <td>0.843558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>animal_class+facts+inventors__affirmative+nega...</td>\n",
       "      <td>0.191296</td>\n",
       "      <td>0.014626</td>\n",
       "      <td>1.542448e-02</td>\n",
       "      <td>1.539118e-02</td>\n",
       "      <td>0.720899</td>\n",
       "      <td>0.746089</td>\n",
       "      <td>1.812848e-01</td>\n",
       "      <td>0.648037</td>\n",
       "      <td>0.717232</td>\n",
       "      <td>0.736679</td>\n",
       "      <td>1.832371e-01</td>\n",
       "      <td>0.640655</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>animal_class+facts+inventors__affirmative+nega...</td>\n",
       "      <td>0.171222</td>\n",
       "      <td>0.009766</td>\n",
       "      <td>1.022900e-02</td>\n",
       "      <td>1.020268e-02</td>\n",
       "      <td>0.657805</td>\n",
       "      <td>0.697046</td>\n",
       "      <td>2.217134e-01</td>\n",
       "      <td>0.648042</td>\n",
       "      <td>0.660190</td>\n",
       "      <td>0.693528</td>\n",
       "      <td>2.297797e-01</td>\n",
       "      <td>0.634425</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 20
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:58.423925Z",
     "start_time": "2024-12-07T21:26:58.403025Z"
    }
   },
   "cell_type": "code",
   "source": "lyrs18_and_25_eval_except_test_df.describe()",
   "id": "2f5a438099ad3b18",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "          tf_sep  train_recon_improv  val_recon_improv  \\\n",
       "count  60.000000           60.000000      6.000000e+01   \n",
       "mean    0.283060            0.111226      1.097550e-01   \n",
       "std     0.304332            0.129781      1.374956e-01   \n",
       "min     0.000000            0.000000     -3.878537e-07   \n",
       "25%     0.046135            0.014808      1.329847e-02   \n",
       "50%     0.216582            0.050441      3.730162e-02   \n",
       "75%     0.410237            0.203073      1.932345e-01   \n",
       "max     1.548113            0.590800      5.865278e-01   \n",
       "\n",
       "       val_recon_improv_w_val_mean    val_acc     val_f1  val_brier  \\\n",
       "count                    60.000000  60.000000  58.000000  60.000000   \n",
       "mean                      0.106712   0.798716   0.810840   0.130735   \n",
       "std                       0.131577   0.143433   0.135400   0.081962   \n",
       "min                       0.000000   0.400000   0.505747   0.000000   \n",
       "25%                       0.014331   0.693553   0.699940   0.049462   \n",
       "50%                       0.036190   0.755242   0.770245   0.162153   \n",
       "75%                       0.190691   0.943858   0.950988   0.197815   \n",
       "max                       0.585433   1.000000   1.000000   0.249156   \n",
       "\n",
       "       val_soft_f1  train_acc   train_f1   train_brier  train_soft_f1  \n",
       "count    58.000000  60.000000  58.000000  6.000000e+01      58.000000  \n",
       "mean      0.761323   0.788483   0.809917  1.320642e-01       0.761945  \n",
       "std       0.152587   0.153608   0.127705  8.283937e-02       0.146435  \n",
       "min       0.523383   0.357143   0.570766  4.758516e-22       0.548751  \n",
       "25%       0.636586   0.682725   0.693429  5.474362e-02       0.638559  \n",
       "50%       0.707598   0.769509   0.788768  1.545682e-01       0.705456  \n",
       "75%       0.918028   0.925774   0.942069  1.977933e-01       0.917260  \n",
       "max       1.000000   1.000000   1.000000  3.327580e-01       1.000000  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tf_sep</th>\n",
       "      <th>train_recon_improv</th>\n",
       "      <th>val_recon_improv</th>\n",
       "      <th>val_recon_improv_w_val_mean</th>\n",
       "      <th>val_acc</th>\n",
       "      <th>val_f1</th>\n",
       "      <th>val_brier</th>\n",
       "      <th>val_soft_f1</th>\n",
       "      <th>train_acc</th>\n",
       "      <th>train_f1</th>\n",
       "      <th>train_brier</th>\n",
       "      <th>train_soft_f1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>6.000000e+01</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>58.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>58.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>58.000000</td>\n",
       "      <td>6.000000e+01</td>\n",
       "      <td>58.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.283060</td>\n",
       "      <td>0.111226</td>\n",
       "      <td>1.097550e-01</td>\n",
       "      <td>0.106712</td>\n",
       "      <td>0.798716</td>\n",
       "      <td>0.810840</td>\n",
       "      <td>0.130735</td>\n",
       "      <td>0.761323</td>\n",
       "      <td>0.788483</td>\n",
       "      <td>0.809917</td>\n",
       "      <td>1.320642e-01</td>\n",
       "      <td>0.761945</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.304332</td>\n",
       "      <td>0.129781</td>\n",
       "      <td>1.374956e-01</td>\n",
       "      <td>0.131577</td>\n",
       "      <td>0.143433</td>\n",
       "      <td>0.135400</td>\n",
       "      <td>0.081962</td>\n",
       "      <td>0.152587</td>\n",
       "      <td>0.153608</td>\n",
       "      <td>0.127705</td>\n",
       "      <td>8.283937e-02</td>\n",
       "      <td>0.146435</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-3.878537e-07</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.505747</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.523383</td>\n",
       "      <td>0.357143</td>\n",
       "      <td>0.570766</td>\n",
       "      <td>4.758516e-22</td>\n",
       "      <td>0.548751</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.046135</td>\n",
       "      <td>0.014808</td>\n",
       "      <td>1.329847e-02</td>\n",
       "      <td>0.014331</td>\n",
       "      <td>0.693553</td>\n",
       "      <td>0.699940</td>\n",
       "      <td>0.049462</td>\n",
       "      <td>0.636586</td>\n",
       "      <td>0.682725</td>\n",
       "      <td>0.693429</td>\n",
       "      <td>5.474362e-02</td>\n",
       "      <td>0.638559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.216582</td>\n",
       "      <td>0.050441</td>\n",
       "      <td>3.730162e-02</td>\n",
       "      <td>0.036190</td>\n",
       "      <td>0.755242</td>\n",
       "      <td>0.770245</td>\n",
       "      <td>0.162153</td>\n",
       "      <td>0.707598</td>\n",
       "      <td>0.769509</td>\n",
       "      <td>0.788768</td>\n",
       "      <td>1.545682e-01</td>\n",
       "      <td>0.705456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.410237</td>\n",
       "      <td>0.203073</td>\n",
       "      <td>1.932345e-01</td>\n",
       "      <td>0.190691</td>\n",
       "      <td>0.943858</td>\n",
       "      <td>0.950988</td>\n",
       "      <td>0.197815</td>\n",
       "      <td>0.918028</td>\n",
       "      <td>0.925774</td>\n",
       "      <td>0.942069</td>\n",
       "      <td>1.977933e-01</td>\n",
       "      <td>0.917260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.548113</td>\n",
       "      <td>0.590800</td>\n",
       "      <td>5.865278e-01</td>\n",
       "      <td>0.585433</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.249156</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.327580e-01</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 21
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:42:38.880764Z",
     "start_time": "2024-12-07T21:42:38.871560Z"
    }
   },
   "cell_type": "code",
   "source": [
    "lyrs18_and_25_t_f_sep_and_t_p_dirs_correlations = lyrs18_and_25_eval_except_test_df[['tf_sep', 'train_recon_improv', 'val_recon_improv', 'val_recon_improv_w_val_mean']].corr()\n",
    "lyrs18_and_25_t_f_sep_and_t_p_dirs_correlations"
   ],
   "id": "58ce0aa2abbd52f9",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "                               tf_sep  train_recon_improv  val_recon_improv  \\\n",
       "tf_sep                       1.000000            0.857080          0.848689   \n",
       "train_recon_improv           0.857080            1.000000          0.992773   \n",
       "val_recon_improv             0.848689            0.992773          1.000000   \n",
       "val_recon_improv_w_val_mean  0.851512            0.991497          0.995109   \n",
       "\n",
       "                             val_recon_improv_w_val_mean  \n",
       "tf_sep                                          0.851512  \n",
       "train_recon_improv                              0.991497  \n",
       "val_recon_improv                                0.995109  \n",
       "val_recon_improv_w_val_mean                     1.000000  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tf_sep</th>\n",
       "      <th>train_recon_improv</th>\n",
       "      <th>val_recon_improv</th>\n",
       "      <th>val_recon_improv_w_val_mean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>tf_sep</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.857080</td>\n",
       "      <td>0.848689</td>\n",
       "      <td>0.851512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train_recon_improv</th>\n",
       "      <td>0.857080</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.992773</td>\n",
       "      <td>0.991497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>val_recon_improv</th>\n",
       "      <td>0.848689</td>\n",
       "      <td>0.992773</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.995109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>val_recon_improv_w_val_mean</th>\n",
       "      <td>0.851512</td>\n",
       "      <td>0.991497</td>\n",
       "      <td>0.995109</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 67
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:58.546326Z",
     "start_time": "2024-12-07T21:26:58.542740Z"
    }
   },
   "cell_type": "code",
   "source": "# lyr25_rel_to_18_eval_except_test_df",
   "id": "b8eee00eb81e6ce4",
   "outputs": [],
   "execution_count": 23
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:38:35.072488Z",
     "start_time": "2024-12-07T21:38:35.059827Z"
    }
   },
   "cell_type": "code",
   "source": [
    "numeric_cols_in_25_rel_to_18_df = lyr25_rel_to_18_eval_except_test_df.columns[1:]\n",
    "numeric_higher_better_cols_in_25_rel_to_18_df = numeric_cols_in_25_rel_to_18_df[~numeric_cols_in_25_rel_to_18_df.str.contains(\"brier\")]\n",
    "numeric_lower_better_cols_in_25_rel_to_18_df = numeric_cols_in_25_rel_to_18_df[numeric_cols_in_25_rel_to_18_df.str.contains(\"brier\")]\n",
    "noteworthy_rows_mask_25_rel_to_18_df = pd.concat([(lyr25_rel_to_18_eval_except_test_df[numeric_higher_better_cols_in_25_rel_to_18_df] > 0.05), (lyr25_rel_to_18_eval_except_test_df[numeric_lower_better_cols_in_25_rel_to_18_df] < -0.05)], axis=1).any(axis=1)\n",
    "lyr25_rel_to_18_eval_except_test_df[noteworthy_rows_mask_25_rel_to_18_df]"
   ],
   "id": "862aa2316a6edd38",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "                                 scenario    tf_sep  train_recon_improv  \\\n",
       "13                     facts__conjunction -0.012120       -1.251708e-02   \n",
       "20                          ambiguous_lie  0.002003        1.139251e-08   \n",
       "21               ambiguous_truthful_reply  0.002003        1.953885e-07   \n",
       "22  honest_reply_despite_incentive_to_lie  0.000000        4.960024e-08   \n",
       "47             facts__negated+conjunction -0.014520       -1.049237e-02   \n",
       "\n",
       "    val_recon_improv  val_recon_improv_w_val_mean   val_acc    val_f1  \\\n",
       "13     -5.795836e-03                -6.845959e-03  0.060000  0.044741   \n",
       "20      2.253305e-07                -1.442802e-07  0.000000       NaN   \n",
       "21     -3.117704e-07                 0.000000e+00  0.200000  0.138889   \n",
       "22     -2.438925e-07                 1.181275e-07 -0.500000 -0.333333   \n",
       "47     -4.586252e-03                -4.692217e-03  0.079812  0.076819   \n",
       "\n",
       "    val_brier  val_soft_f1  train_acc  train_f1  train_brier  train_soft_f1  \n",
       "13  -0.022460    -0.006590  -0.035000 -0.039749     0.020944      -0.047058  \n",
       "20  -0.040896          NaN   0.142857       NaN     0.013283            NaN  \n",
       "21   0.008279    -0.013446  -0.111111 -0.098901    -0.010221      -0.000752  \n",
       "22   0.178510    -0.178256   0.500000  0.457143    -0.016438       0.002470  \n",
       "47  -0.010443     0.037964  -0.008255 -0.013046     0.003908       0.020419  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>scenario</th>\n",
       "      <th>tf_sep</th>\n",
       "      <th>train_recon_improv</th>\n",
       "      <th>val_recon_improv</th>\n",
       "      <th>val_recon_improv_w_val_mean</th>\n",
       "      <th>val_acc</th>\n",
       "      <th>val_f1</th>\n",
       "      <th>val_brier</th>\n",
       "      <th>val_soft_f1</th>\n",
       "      <th>train_acc</th>\n",
       "      <th>train_f1</th>\n",
       "      <th>train_brier</th>\n",
       "      <th>train_soft_f1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>facts__conjunction</td>\n",
       "      <td>-0.012120</td>\n",
       "      <td>-1.251708e-02</td>\n",
       "      <td>-5.795836e-03</td>\n",
       "      <td>-6.845959e-03</td>\n",
       "      <td>0.060000</td>\n",
       "      <td>0.044741</td>\n",
       "      <td>-0.022460</td>\n",
       "      <td>-0.006590</td>\n",
       "      <td>-0.035000</td>\n",
       "      <td>-0.039749</td>\n",
       "      <td>0.020944</td>\n",
       "      <td>-0.047058</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>ambiguous_lie</td>\n",
       "      <td>0.002003</td>\n",
       "      <td>1.139251e-08</td>\n",
       "      <td>2.253305e-07</td>\n",
       "      <td>-1.442802e-07</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.040896</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.013283</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>ambiguous_truthful_reply</td>\n",
       "      <td>0.002003</td>\n",
       "      <td>1.953885e-07</td>\n",
       "      <td>-3.117704e-07</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.138889</td>\n",
       "      <td>0.008279</td>\n",
       "      <td>-0.013446</td>\n",
       "      <td>-0.111111</td>\n",
       "      <td>-0.098901</td>\n",
       "      <td>-0.010221</td>\n",
       "      <td>-0.000752</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>honest_reply_despite_incentive_to_lie</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.960024e-08</td>\n",
       "      <td>-2.438925e-07</td>\n",
       "      <td>1.181275e-07</td>\n",
       "      <td>-0.500000</td>\n",
       "      <td>-0.333333</td>\n",
       "      <td>0.178510</td>\n",
       "      <td>-0.178256</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.457143</td>\n",
       "      <td>-0.016438</td>\n",
       "      <td>0.002470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>facts__negated+conjunction</td>\n",
       "      <td>-0.014520</td>\n",
       "      <td>-1.049237e-02</td>\n",
       "      <td>-4.586252e-03</td>\n",
       "      <td>-4.692217e-03</td>\n",
       "      <td>0.079812</td>\n",
       "      <td>0.076819</td>\n",
       "      <td>-0.010443</td>\n",
       "      <td>0.037964</td>\n",
       "      <td>-0.008255</td>\n",
       "      <td>-0.013046</td>\n",
       "      <td>0.003908</td>\n",
       "      <td>0.020419</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 60
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:58.595949Z",
     "start_time": "2024-12-07T21:26:58.568241Z"
    }
   },
   "cell_type": "code",
   "source": "lyr25_rel_to_18_eval_except_test_df.describe()",
   "id": "459fbbaa895e4f7",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "          tf_sep  train_recon_improv  val_recon_improv  \\\n",
       "count  60.000000           60.000000         60.000000   \n",
       "mean   -0.094233           -0.029681         -0.030587   \n",
       "std     0.110259            0.042556          0.043171   \n",
       "min    -0.440327           -0.173719         -0.181603   \n",
       "25%    -0.148954           -0.046525         -0.041938   \n",
       "50%    -0.039116           -0.011505         -0.014278   \n",
       "75%    -0.011445           -0.005064         -0.004510   \n",
       "max     0.035019            0.031118          0.026668   \n",
       "\n",
       "       val_recon_improv_w_val_mean    val_acc     val_f1  val_brier  \\\n",
       "count                    60.000000  60.000000  58.000000  60.000000   \n",
       "mean                     -0.029994  -0.052835  -0.053404   0.026312   \n",
       "std                       0.042016   0.099461   0.085328   0.032261   \n",
       "min                      -0.178093  -0.500000  -0.413534  -0.040896   \n",
       "25%                      -0.040575  -0.074259  -0.067979   0.009289   \n",
       "50%                      -0.014233  -0.031917  -0.033001   0.021637   \n",
       "75%                      -0.004486  -0.004386  -0.009317   0.033742   \n",
       "max                       0.026374   0.200000   0.138889   0.178510   \n",
       "\n",
       "       val_soft_f1  train_acc   train_f1  train_brier  train_soft_f1  \n",
       "count    58.000000  60.000000  58.000000    60.000000      58.000000  \n",
       "mean     -0.053163  -0.034514  -0.039088     0.024774      -0.051182  \n",
       "std       0.047940   0.083837   0.077403     0.021355       0.041399  \n",
       "min      -0.186958  -0.138686  -0.153717    -0.016438      -0.156660  \n",
       "25%      -0.080155  -0.079828  -0.075010     0.012993      -0.068526  \n",
       "50%      -0.042362  -0.036250  -0.038453     0.020911      -0.048650  \n",
       "75%      -0.020141  -0.015200  -0.014313     0.035630      -0.021611  \n",
       "max       0.037964   0.500000   0.457143     0.084870       0.020419  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tf_sep</th>\n",
       "      <th>train_recon_improv</th>\n",
       "      <th>val_recon_improv</th>\n",
       "      <th>val_recon_improv_w_val_mean</th>\n",
       "      <th>val_acc</th>\n",
       "      <th>val_f1</th>\n",
       "      <th>val_brier</th>\n",
       "      <th>val_soft_f1</th>\n",
       "      <th>train_acc</th>\n",
       "      <th>train_f1</th>\n",
       "      <th>train_brier</th>\n",
       "      <th>train_soft_f1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>58.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>58.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>58.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>58.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>-0.094233</td>\n",
       "      <td>-0.029681</td>\n",
       "      <td>-0.030587</td>\n",
       "      <td>-0.029994</td>\n",
       "      <td>-0.052835</td>\n",
       "      <td>-0.053404</td>\n",
       "      <td>0.026312</td>\n",
       "      <td>-0.053163</td>\n",
       "      <td>-0.034514</td>\n",
       "      <td>-0.039088</td>\n",
       "      <td>0.024774</td>\n",
       "      <td>-0.051182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.110259</td>\n",
       "      <td>0.042556</td>\n",
       "      <td>0.043171</td>\n",
       "      <td>0.042016</td>\n",
       "      <td>0.099461</td>\n",
       "      <td>0.085328</td>\n",
       "      <td>0.032261</td>\n",
       "      <td>0.047940</td>\n",
       "      <td>0.083837</td>\n",
       "      <td>0.077403</td>\n",
       "      <td>0.021355</td>\n",
       "      <td>0.041399</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-0.440327</td>\n",
       "      <td>-0.173719</td>\n",
       "      <td>-0.181603</td>\n",
       "      <td>-0.178093</td>\n",
       "      <td>-0.500000</td>\n",
       "      <td>-0.413534</td>\n",
       "      <td>-0.040896</td>\n",
       "      <td>-0.186958</td>\n",
       "      <td>-0.138686</td>\n",
       "      <td>-0.153717</td>\n",
       "      <td>-0.016438</td>\n",
       "      <td>-0.156660</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>-0.148954</td>\n",
       "      <td>-0.046525</td>\n",
       "      <td>-0.041938</td>\n",
       "      <td>-0.040575</td>\n",
       "      <td>-0.074259</td>\n",
       "      <td>-0.067979</td>\n",
       "      <td>0.009289</td>\n",
       "      <td>-0.080155</td>\n",
       "      <td>-0.079828</td>\n",
       "      <td>-0.075010</td>\n",
       "      <td>0.012993</td>\n",
       "      <td>-0.068526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>-0.039116</td>\n",
       "      <td>-0.011505</td>\n",
       "      <td>-0.014278</td>\n",
       "      <td>-0.014233</td>\n",
       "      <td>-0.031917</td>\n",
       "      <td>-0.033001</td>\n",
       "      <td>0.021637</td>\n",
       "      <td>-0.042362</td>\n",
       "      <td>-0.036250</td>\n",
       "      <td>-0.038453</td>\n",
       "      <td>0.020911</td>\n",
       "      <td>-0.048650</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>-0.011445</td>\n",
       "      <td>-0.005064</td>\n",
       "      <td>-0.004510</td>\n",
       "      <td>-0.004486</td>\n",
       "      <td>-0.004386</td>\n",
       "      <td>-0.009317</td>\n",
       "      <td>0.033742</td>\n",
       "      <td>-0.020141</td>\n",
       "      <td>-0.015200</td>\n",
       "      <td>-0.014313</td>\n",
       "      <td>0.035630</td>\n",
       "      <td>-0.021611</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.035019</td>\n",
       "      <td>0.031118</td>\n",
       "      <td>0.026668</td>\n",
       "      <td>0.026374</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.138889</td>\n",
       "      <td>0.178510</td>\n",
       "      <td>0.037964</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.457143</td>\n",
       "      <td>0.084870</td>\n",
       "      <td>0.020419</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 25
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:34:45.829308Z",
     "start_time": "2024-12-07T21:34:45.817353Z"
    }
   },
   "cell_type": "code",
   "source": [
    "numeric_cols_in_18_and_25_rel_to_just_18_df = lyrs18_and_25_rel_to_18_eval_except_test_df.columns[1:]\n",
    "numeric_higher_better_cols_in_18_and_25_rel_to_just_18_df = numeric_cols_in_18_and_25_rel_to_just_18_df[~numeric_cols_in_18_and_25_rel_to_just_18_df.str.contains(\"brier\")]\n",
    "numeric_lower_better_cols_in_18_and_25_rel_to_just_18_df = numeric_cols_in_18_and_25_rel_to_just_18_df[numeric_cols_in_18_and_25_rel_to_just_18_df.str.contains(\"brier\")]\n",
    "noteworthy_rows_mask_18_and_25_rel_to_just_18_df = pd.concat([(lyrs18_and_25_rel_to_18_eval_except_test_df[numeric_higher_better_cols_in_18_and_25_rel_to_just_18_df] > 0.05), (lyrs18_and_25_rel_to_18_eval_except_test_df[numeric_lower_better_cols_in_18_and_25_rel_to_just_18_df] < -0.05)], axis=1).any(axis=1)\n",
    "lyrs18_and_25_rel_to_18_eval_except_test_df[noteworthy_rows_mask_18_and_25_rel_to_just_18_df]"
   ],
   "id": "27c167ec6d665b6f",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "                                 scenario    tf_sep  train_recon_improv  \\\n",
       "22  honest_reply_despite_incentive_to_lie  0.000000        1.319963e-07   \n",
       "23                        unambiguous_lie -0.018086       -1.258997e-07   \n",
       "47             facts__negated+conjunction -0.007260       -9.184187e-03   \n",
       "\n",
       "    val_recon_improv  val_recon_improv_w_val_mean   val_acc    val_f1  \\\n",
       "22     -3.252974e-07                 1.066376e-07  0.000000  0.000000   \n",
       "23     -4.032789e-08                 0.000000e+00  0.200000       NaN   \n",
       "47     -4.028593e-03                -4.121554e-03  0.061033  0.055857   \n",
       "\n",
       "    val_brier  val_soft_f1  train_acc  train_f1  train_brier  train_soft_f1  \n",
       "22   0.032692    -0.030060   0.250000  0.266667     0.064688       0.000234  \n",
       "23   0.031118          NaN  -0.025641       NaN    -0.004977            NaN  \n",
       "47  -0.017558     0.051949   0.030660  0.026810    -0.008160       0.040590  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>scenario</th>\n",
       "      <th>tf_sep</th>\n",
       "      <th>train_recon_improv</th>\n",
       "      <th>val_recon_improv</th>\n",
       "      <th>val_recon_improv_w_val_mean</th>\n",
       "      <th>val_acc</th>\n",
       "      <th>val_f1</th>\n",
       "      <th>val_brier</th>\n",
       "      <th>val_soft_f1</th>\n",
       "      <th>train_acc</th>\n",
       "      <th>train_f1</th>\n",
       "      <th>train_brier</th>\n",
       "      <th>train_soft_f1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>honest_reply_despite_incentive_to_lie</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.319963e-07</td>\n",
       "      <td>-3.252974e-07</td>\n",
       "      <td>1.066376e-07</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.032692</td>\n",
       "      <td>-0.030060</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.266667</td>\n",
       "      <td>0.064688</td>\n",
       "      <td>0.000234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>unambiguous_lie</td>\n",
       "      <td>-0.018086</td>\n",
       "      <td>-1.258997e-07</td>\n",
       "      <td>-4.032789e-08</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.031118</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.025641</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.004977</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>facts__negated+conjunction</td>\n",
       "      <td>-0.007260</td>\n",
       "      <td>-9.184187e-03</td>\n",
       "      <td>-4.028593e-03</td>\n",
       "      <td>-4.121554e-03</td>\n",
       "      <td>0.061033</td>\n",
       "      <td>0.055857</td>\n",
       "      <td>-0.017558</td>\n",
       "      <td>0.051949</td>\n",
       "      <td>0.030660</td>\n",
       "      <td>0.026810</td>\n",
       "      <td>-0.008160</td>\n",
       "      <td>0.040590</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 59
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:58.656783Z",
     "start_time": "2024-12-07T21:26:58.631271Z"
    }
   },
   "cell_type": "code",
   "source": "lyrs18_and_25_rel_to_18_eval_except_test_df.describe()",
   "id": "f35cca782195d437",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "          tf_sep  train_recon_improv  val_recon_improv  \\\n",
       "count  60.000000           60.000000         60.000000   \n",
       "mean   -0.047116           -0.026001         -0.026737   \n",
       "std     0.055130            0.037530          0.037942   \n",
       "min    -0.220163           -0.156582         -0.157808   \n",
       "25%    -0.074477           -0.040943         -0.037373   \n",
       "50%    -0.019558           -0.010039         -0.012036   \n",
       "75%    -0.005722           -0.004522         -0.003963   \n",
       "max     0.017509            0.027326          0.023500   \n",
       "\n",
       "       val_recon_improv_w_val_mean    val_acc     val_f1  val_brier  \\\n",
       "count                    60.000000  60.000000  58.000000  60.000000   \n",
       "mean                     -0.026230  -0.037730  -0.044680   0.022344   \n",
       "std                       0.036970   0.058757   0.055994   0.022806   \n",
       "min                      -0.154881  -0.200000  -0.255315  -0.017558   \n",
       "25%                      -0.036140  -0.064574  -0.071590   0.007865   \n",
       "50%                      -0.012194  -0.028635  -0.029060   0.017034   \n",
       "75%                      -0.003930  -0.004621  -0.006428   0.031975   \n",
       "max                       0.023222   0.200000   0.055857   0.090627   \n",
       "\n",
       "       val_soft_f1  train_acc   train_f1  train_brier  train_soft_f1  \n",
       "count    58.000000  60.000000  58.000000    60.000000      58.000000  \n",
       "mean     -0.042365  -0.036399  -0.038598     0.022263      -0.042233  \n",
       "std       0.041406   0.056553   0.063225     0.022227       0.038076  \n",
       "min      -0.160064  -0.170759  -0.218424    -0.012093      -0.155436  \n",
       "25%      -0.059580  -0.065625  -0.066569     0.010372      -0.059556  \n",
       "50%      -0.037295  -0.027575  -0.031504     0.017936      -0.039227  \n",
       "75%      -0.014967  -0.013976  -0.011575     0.032574      -0.015916  \n",
       "max       0.051949   0.250000   0.266667     0.108816       0.040590  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tf_sep</th>\n",
       "      <th>train_recon_improv</th>\n",
       "      <th>val_recon_improv</th>\n",
       "      <th>val_recon_improv_w_val_mean</th>\n",
       "      <th>val_acc</th>\n",
       "      <th>val_f1</th>\n",
       "      <th>val_brier</th>\n",
       "      <th>val_soft_f1</th>\n",
       "      <th>train_acc</th>\n",
       "      <th>train_f1</th>\n",
       "      <th>train_brier</th>\n",
       "      <th>train_soft_f1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>58.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>58.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>58.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>58.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>-0.047116</td>\n",
       "      <td>-0.026001</td>\n",
       "      <td>-0.026737</td>\n",
       "      <td>-0.026230</td>\n",
       "      <td>-0.037730</td>\n",
       "      <td>-0.044680</td>\n",
       "      <td>0.022344</td>\n",
       "      <td>-0.042365</td>\n",
       "      <td>-0.036399</td>\n",
       "      <td>-0.038598</td>\n",
       "      <td>0.022263</td>\n",
       "      <td>-0.042233</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.055130</td>\n",
       "      <td>0.037530</td>\n",
       "      <td>0.037942</td>\n",
       "      <td>0.036970</td>\n",
       "      <td>0.058757</td>\n",
       "      <td>0.055994</td>\n",
       "      <td>0.022806</td>\n",
       "      <td>0.041406</td>\n",
       "      <td>0.056553</td>\n",
       "      <td>0.063225</td>\n",
       "      <td>0.022227</td>\n",
       "      <td>0.038076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-0.220163</td>\n",
       "      <td>-0.156582</td>\n",
       "      <td>-0.157808</td>\n",
       "      <td>-0.154881</td>\n",
       "      <td>-0.200000</td>\n",
       "      <td>-0.255315</td>\n",
       "      <td>-0.017558</td>\n",
       "      <td>-0.160064</td>\n",
       "      <td>-0.170759</td>\n",
       "      <td>-0.218424</td>\n",
       "      <td>-0.012093</td>\n",
       "      <td>-0.155436</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>-0.074477</td>\n",
       "      <td>-0.040943</td>\n",
       "      <td>-0.037373</td>\n",
       "      <td>-0.036140</td>\n",
       "      <td>-0.064574</td>\n",
       "      <td>-0.071590</td>\n",
       "      <td>0.007865</td>\n",
       "      <td>-0.059580</td>\n",
       "      <td>-0.065625</td>\n",
       "      <td>-0.066569</td>\n",
       "      <td>0.010372</td>\n",
       "      <td>-0.059556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>-0.019558</td>\n",
       "      <td>-0.010039</td>\n",
       "      <td>-0.012036</td>\n",
       "      <td>-0.012194</td>\n",
       "      <td>-0.028635</td>\n",
       "      <td>-0.029060</td>\n",
       "      <td>0.017034</td>\n",
       "      <td>-0.037295</td>\n",
       "      <td>-0.027575</td>\n",
       "      <td>-0.031504</td>\n",
       "      <td>0.017936</td>\n",
       "      <td>-0.039227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>-0.005722</td>\n",
       "      <td>-0.004522</td>\n",
       "      <td>-0.003963</td>\n",
       "      <td>-0.003930</td>\n",
       "      <td>-0.004621</td>\n",
       "      <td>-0.006428</td>\n",
       "      <td>0.031975</td>\n",
       "      <td>-0.014967</td>\n",
       "      <td>-0.013976</td>\n",
       "      <td>-0.011575</td>\n",
       "      <td>0.032574</td>\n",
       "      <td>-0.015916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.017509</td>\n",
       "      <td>0.027326</td>\n",
       "      <td>0.023500</td>\n",
       "      <td>0.023222</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.055857</td>\n",
       "      <td>0.090627</td>\n",
       "      <td>0.051949</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.266667</td>\n",
       "      <td>0.108816</td>\n",
       "      <td>0.040590</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 28
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "For layer 25 relative to layer 18: with 3 scenarios out of 60 showing a 5+ percentage-point-improvement in validation accuracy while most scenarios show large negative effects on validation accuracy and most statistics, it seems likely that those 3 cases are probably just a result of noise/randomness. This seems especially likely because in those cases the layer25 probe did worse than the layer18 probe on the training set (and in 2 of the 3 cases did worse on the confidence-adjusted f1-like metric).  \n",
    "For layers 18 & 25 relative to just layer 18: with just 2 scenarios out of 60 showing a positive change in validation accuracy at all while most scenarios show large negative effects on validation accuracy and most statistics, it seems likely that those 2 cases are probably just a result of noise/randomness.  \n",
    "\n",
    "Incidentally, it's interesting that, for the \"layers 18 and 25\" probe, gradient descent didn't find the solution of \"recover most or all of the performance of the only-layer-18 probe by setting the weights for layer 25 activations to ~0\".\n",
    "\n",
    "Going forward, the layer25 and layers18&25 directions/probes will be largely ignored.   "
   ],
   "id": "3911e3b475c84e96"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:58.674916Z",
     "start_time": "2024-12-07T21:26:58.657792Z"
    }
   },
   "cell_type": "code",
   "source": "lyr18_baseline_linear_probes_eval_except_test_df",
   "id": "c329dbf8c20b75d5",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "                                             scenario   val_acc    val_f1  \\\n",
       "0                           animal_class__affirmative  1.000000  1.000000   \n",
       "1                           animal_class__conjunction  0.980000  0.980000   \n",
       "2                           animal_class__disjunction  0.790000  0.778947   \n",
       "3                               animal_class__negated  1.000000  1.000000   \n",
       "4                                 cities__affirmative  0.986667  0.986014   \n",
       "5                                 cities__conjunction  0.986667  0.987730   \n",
       "6                                 cities__disjunction  0.870000  0.868687   \n",
       "7                                     cities__negated  0.986667  0.988095   \n",
       "8                           element_symb__affirmative  1.000000  1.000000   \n",
       "9                           element_symb__conjunction  1.000000  1.000000   \n",
       "10                          element_symb__disjunction  0.980000  0.980769   \n",
       "11                              element_symb__negated  1.000000  1.000000   \n",
       "12                                 facts__affirmative  0.955752  0.960000   \n",
       "13                                 facts__conjunction  0.820000  0.790698   \n",
       "14                                 facts__disjunction  0.820000  0.812500   \n",
       "15                                     facts__negated  0.858407  0.859649   \n",
       "16                             inventors__affirmative  0.914634  0.929293   \n",
       "17                             inventors__conjunction  0.890000  0.910569   \n",
       "18                             inventors__disjunction  0.800000  0.843750   \n",
       "19                                 inventors__negated  0.853659  0.812500   \n",
       "20                                      ambiguous_lie  1.000000       NaN   \n",
       "21                           ambiguous_truthful_reply  1.000000  1.000000   \n",
       "22              honest_reply_despite_incentive_to_lie  1.000000  1.000000   \n",
       "23                                    unambiguous_lie  1.000000       NaN   \n",
       "24                         unambiguous_truthful_reply  1.000000  1.000000   \n",
       "25                                        larger_than  1.000000  1.000000   \n",
       "26                                       smaller_than  1.000000  1.000000   \n",
       "27                               sp_en_trans__negated  1.000000  1.000000   \n",
       "28                           sp_en_trans__affirmative  1.000000  1.000000   \n",
       "29                           sp_en_trans__conjunction  1.000000  1.000000   \n",
       "30                           sp_en_trans__disjunction  0.990000  0.989011   \n",
       "31                            common_claim_true_false  0.811236  0.814159   \n",
       "32                             counterfact_true_false  0.826216  0.826920   \n",
       "33                  animal_class__affirmative+negated  1.000000  1.000000   \n",
       "34              animal_class__affirmative+disjunction  0.842105  0.803738   \n",
       "35                  animal_class__negated+conjunction  0.977444  0.978417   \n",
       "36  animal_class__affirmative+negated+conjunction+...  0.909774  0.904000   \n",
       "37                        cities__affirmative+negated  0.985000  0.985554   \n",
       "38                    cities__affirmative+disjunction  0.955000  0.953125   \n",
       "39                        cities__negated+conjunction  0.986667  0.987952   \n",
       "40  cities__affirmative+negated+conjunction+disjun...  0.971000  0.972302   \n",
       "41                  element_symb__affirmative+negated  1.000000  1.000000   \n",
       "42              element_symb__affirmative+disjunction  0.978261  0.979310   \n",
       "43                  element_symb__negated+conjunction  0.992754  0.992366   \n",
       "44  element_symb__affirmative+negated+conjunction+...  0.978261  0.978417   \n",
       "45                         facts__affirmative+negated  0.871681  0.876596   \n",
       "46                     facts__affirmative+disjunction  0.887324  0.891892   \n",
       "47                         facts__negated+conjunction  0.830986  0.820000   \n",
       "48  facts__affirmative+negated+conjunction+disjunc...  0.861502  0.858513   \n",
       "49                     inventors__affirmative+negated  0.871951  0.871166   \n",
       "50                 inventors__affirmative+disjunction  0.846154  0.878261   \n",
       "51                     inventors__negated+conjunction  0.873626  0.877005   \n",
       "52  inventors__affirmative+negated+conjunction+dis...  0.857143  0.875598   \n",
       "53                   sp_en_trans__affirmative+negated  1.000000  1.000000   \n",
       "54               sp_en_trans__affirmative+disjunction  0.994152  0.994413   \n",
       "55                   sp_en_trans__negated+conjunction  1.000000  1.000000   \n",
       "56  sp_en_trans__affirmative+negated+conjunction+d...  0.997076  0.997085   \n",
       "57  ambiguous_truthful_reply+unambiguous_lie+unamb...  0.923077  0.933333   \n",
       "58  animal_class+facts+inventors__affirmative+nega...  0.896825  0.897906   \n",
       "59  animal_class+facts+inventors__affirmative+nega...  0.867081  0.872074   \n",
       "\n",
       "       val_brier  val_soft_f1  train_acc  train_f1   train_brier  \\\n",
       "0   4.737931e-15     1.000000   1.000000  1.000000  2.171720e-16   \n",
       "1   1.438196e-02     0.974614   1.000000  1.000000  4.181327e-04   \n",
       "2   1.331134e-01     0.719436   0.860000  0.876106  8.851917e-02   \n",
       "3   4.370021e-10     0.999997   1.000000  1.000000  7.619170e-16   \n",
       "4   1.022535e-02     0.983910   0.992475  0.992605  4.630021e-03   \n",
       "5   1.043765e-02     0.982741   1.000000  1.000000  6.578032e-04   \n",
       "6   1.144866e-01     0.742273   0.930000  0.930000  6.681901e-02   \n",
       "7   9.586872e-03     0.985873   0.994983  0.994819  3.803460e-03   \n",
       "8   1.891266e-11     0.999999   1.000000  1.000000  1.771124e-17   \n",
       "9   8.702983e-10     0.999995   1.000000  1.000000  1.617376e-16   \n",
       "10  2.572177e-02     0.948162   1.000000  1.000000  1.200666e-03   \n",
       "11  3.210423e-04     0.995895   1.000000  1.000000  1.460392e-17   \n",
       "12  2.697033e-02     0.955003   0.997768  0.997691  1.257249e-03   \n",
       "13  1.233484e-01     0.746431   0.977500  0.977444  3.169205e-02   \n",
       "14  1.180682e-01     0.742128   0.952500  0.948509  5.228021e-02   \n",
       "15  1.030634e-01     0.813018   0.968750  0.968889  3.144237e-02   \n",
       "16  6.569698e-02     0.898393   0.938272  0.944444  4.398387e-02   \n",
       "17  7.758649e-02     0.871089   0.985000  0.986784  2.356241e-02   \n",
       "18  1.551561e-01     0.754390   0.907500  0.919740  9.533650e-02   \n",
       "19  9.715707e-02     0.772859   0.919753  0.909091  5.893136e-02   \n",
       "20  1.050118e-24          NaN   1.000000       NaN  1.446902e-24   \n",
       "21  0.000000e+00     1.000000   1.000000  1.000000  0.000000e+00   \n",
       "22  1.421085e-14     1.000000   1.000000  1.000000  7.105427e-15   \n",
       "23  1.518638e-24          NaN   1.000000       NaN  1.157558e-24   \n",
       "24  0.000000e+00     1.000000   1.000000  1.000000  0.000000e+00   \n",
       "25  1.426665e-05     0.999800   1.000000  1.000000  1.502934e-15   \n",
       "26  3.349663e-05     0.999626   1.000000  1.000000  8.750799e-15   \n",
       "27  2.562758e-12     1.000000   1.000000  1.000000  2.515887e-16   \n",
       "28  8.378279e-08     0.999972   1.000000  1.000000  2.523146e-16   \n",
       "29  2.664824e-04     0.997272   1.000000  1.000000  3.091911e-17   \n",
       "30  3.743603e-03     0.993305   1.000000  1.000000  2.741257e-15   \n",
       "31  1.324474e-01     0.732995   0.838764  0.839306  1.165958e-01   \n",
       "32  1.185529e-01     0.766909   0.842400  0.844281  1.073910e-01   \n",
       "33  3.641339e-07     0.999915   1.000000  1.000000  6.039329e-16   \n",
       "34  1.066825e-01     0.748289   0.890772  0.898246  7.368915e-02   \n",
       "35  1.068789e-02     0.981544   1.000000  1.000000  2.953050e-04   \n",
       "36  6.324344e-02     0.875548   0.944444  0.945822  3.911059e-02   \n",
       "37  9.284142e-03     0.985693   0.992893  0.992830  4.429694e-03   \n",
       "38  3.823327e-02     0.921924   0.974937  0.975217  2.049703e-02   \n",
       "39  1.243303e-02     0.981832   0.997076  0.997005  2.545326e-03   \n",
       "40  2.320118e-02     0.957636   0.980451  0.980233  1.428717e-02   \n",
       "41  1.849128e-03     0.995199   1.000000  1.000000  1.763436e-15   \n",
       "42  1.921232e-02     0.960651   1.000000  1.000000  1.620967e-03   \n",
       "43  8.176259e-03     0.975870   1.000000  1.000000  1.753441e-03   \n",
       "44  1.467793e-02     0.969567   1.000000  1.000000  9.719951e-04   \n",
       "45  1.043555e-01     0.811190   0.944196  0.943946  4.724095e-02   \n",
       "46  7.251821e-02     0.857505   0.987028  0.986233  1.909521e-02   \n",
       "47  1.242275e-01     0.755510   0.932783  0.933020  5.948954e-02   \n",
       "48  1.085148e-01     0.778201   0.920991  0.919082  6.407186e-02   \n",
       "49  8.386829e-02     0.834489   0.899691  0.898911  6.930428e-02   \n",
       "50  1.095920e-01     0.826457   0.915746  0.926061  6.958865e-02   \n",
       "51  8.702927e-02     0.826027   0.955801  0.956757  4.640511e-02   \n",
       "52  1.007724e-01     0.822551   0.912293  0.918953  6.555394e-02   \n",
       "53  1.670350e-07     0.999967   1.000000  1.000000  3.035009e-16   \n",
       "54  2.782293e-03     0.994445   1.000000  1.000000  3.170199e-06   \n",
       "55  1.023078e-04     0.998664   1.000000  1.000000  7.771587e-16   \n",
       "56  2.586771e-03     0.994640   1.000000  1.000000  7.300621e-06   \n",
       "57  4.773707e-02     0.927686   1.000000  1.000000  1.206968e-04   \n",
       "58  8.068992e-02     0.848357   0.942448  0.942885  4.567364e-02   \n",
       "59  9.542867e-02     0.820643   0.895192  0.896047  7.606133e-02   \n",
       "\n",
       "    train_soft_f1  \n",
       "0        1.000000  \n",
       "1        0.994526  \n",
       "2        0.811538  \n",
       "3        1.000000  \n",
       "4        0.988487  \n",
       "5        0.994488  \n",
       "6        0.806922  \n",
       "7        0.989490  \n",
       "8        1.000000  \n",
       "9        1.000000  \n",
       "10       0.986966  \n",
       "11       1.000000  \n",
       "12       0.993596  \n",
       "13       0.885599  \n",
       "14       0.825039  \n",
       "15       0.899969  \n",
       "16       0.899879  \n",
       "17       0.924587  \n",
       "18       0.792131  \n",
       "19       0.846446  \n",
       "20            NaN  \n",
       "21       1.000000  \n",
       "22       1.000000  \n",
       "23            NaN  \n",
       "24       1.000000  \n",
       "25       1.000000  \n",
       "26       1.000000  \n",
       "27       1.000000  \n",
       "28       1.000000  \n",
       "29       1.000000  \n",
       "30       1.000000  \n",
       "31       0.743862  \n",
       "32       0.778638  \n",
       "33       1.000000  \n",
       "34       0.839471  \n",
       "35       0.996060  \n",
       "36       0.914783  \n",
       "37       0.988979  \n",
       "38       0.944502  \n",
       "39       0.990748  \n",
       "40       0.963197  \n",
       "41       1.000000  \n",
       "42       0.986654  \n",
       "43       0.988917  \n",
       "44       0.991544  \n",
       "45       0.872309  \n",
       "46       0.918500  \n",
       "47       0.838461  \n",
       "48       0.831850  \n",
       "49       0.848420  \n",
       "50       0.847062  \n",
       "51       0.877578  \n",
       "52       0.850173  \n",
       "53       1.000000  \n",
       "54       0.999439  \n",
       "55       1.000000  \n",
       "56       0.999334  \n",
       "57       0.995401  \n",
       "58       0.887364  \n",
       "59       0.831885  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>scenario</th>\n",
       "      <th>val_acc</th>\n",
       "      <th>val_f1</th>\n",
       "      <th>val_brier</th>\n",
       "      <th>val_soft_f1</th>\n",
       "      <th>train_acc</th>\n",
       "      <th>train_f1</th>\n",
       "      <th>train_brier</th>\n",
       "      <th>train_soft_f1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>animal_class__affirmative</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.737931e-15</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.171720e-16</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>animal_class__conjunction</td>\n",
       "      <td>0.980000</td>\n",
       "      <td>0.980000</td>\n",
       "      <td>1.438196e-02</td>\n",
       "      <td>0.974614</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.181327e-04</td>\n",
       "      <td>0.994526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>animal_class__disjunction</td>\n",
       "      <td>0.790000</td>\n",
       "      <td>0.778947</td>\n",
       "      <td>1.331134e-01</td>\n",
       "      <td>0.719436</td>\n",
       "      <td>0.860000</td>\n",
       "      <td>0.876106</td>\n",
       "      <td>8.851917e-02</td>\n",
       "      <td>0.811538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>animal_class__negated</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.370021e-10</td>\n",
       "      <td>0.999997</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>7.619170e-16</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cities__affirmative</td>\n",
       "      <td>0.986667</td>\n",
       "      <td>0.986014</td>\n",
       "      <td>1.022535e-02</td>\n",
       "      <td>0.983910</td>\n",
       "      <td>0.992475</td>\n",
       "      <td>0.992605</td>\n",
       "      <td>4.630021e-03</td>\n",
       "      <td>0.988487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>cities__conjunction</td>\n",
       "      <td>0.986667</td>\n",
       "      <td>0.987730</td>\n",
       "      <td>1.043765e-02</td>\n",
       "      <td>0.982741</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.578032e-04</td>\n",
       "      <td>0.994488</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>cities__disjunction</td>\n",
       "      <td>0.870000</td>\n",
       "      <td>0.868687</td>\n",
       "      <td>1.144866e-01</td>\n",
       "      <td>0.742273</td>\n",
       "      <td>0.930000</td>\n",
       "      <td>0.930000</td>\n",
       "      <td>6.681901e-02</td>\n",
       "      <td>0.806922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>cities__negated</td>\n",
       "      <td>0.986667</td>\n",
       "      <td>0.988095</td>\n",
       "      <td>9.586872e-03</td>\n",
       "      <td>0.985873</td>\n",
       "      <td>0.994983</td>\n",
       "      <td>0.994819</td>\n",
       "      <td>3.803460e-03</td>\n",
       "      <td>0.989490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>element_symb__affirmative</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.891266e-11</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.771124e-17</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>element_symb__conjunction</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>8.702983e-10</td>\n",
       "      <td>0.999995</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.617376e-16</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>element_symb__disjunction</td>\n",
       "      <td>0.980000</td>\n",
       "      <td>0.980769</td>\n",
       "      <td>2.572177e-02</td>\n",
       "      <td>0.948162</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.200666e-03</td>\n",
       "      <td>0.986966</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>element_symb__negated</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.210423e-04</td>\n",
       "      <td>0.995895</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.460392e-17</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>facts__affirmative</td>\n",
       "      <td>0.955752</td>\n",
       "      <td>0.960000</td>\n",
       "      <td>2.697033e-02</td>\n",
       "      <td>0.955003</td>\n",
       "      <td>0.997768</td>\n",
       "      <td>0.997691</td>\n",
       "      <td>1.257249e-03</td>\n",
       "      <td>0.993596</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>facts__conjunction</td>\n",
       "      <td>0.820000</td>\n",
       "      <td>0.790698</td>\n",
       "      <td>1.233484e-01</td>\n",
       "      <td>0.746431</td>\n",
       "      <td>0.977500</td>\n",
       "      <td>0.977444</td>\n",
       "      <td>3.169205e-02</td>\n",
       "      <td>0.885599</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>facts__disjunction</td>\n",
       "      <td>0.820000</td>\n",
       "      <td>0.812500</td>\n",
       "      <td>1.180682e-01</td>\n",
       "      <td>0.742128</td>\n",
       "      <td>0.952500</td>\n",
       "      <td>0.948509</td>\n",
       "      <td>5.228021e-02</td>\n",
       "      <td>0.825039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>facts__negated</td>\n",
       "      <td>0.858407</td>\n",
       "      <td>0.859649</td>\n",
       "      <td>1.030634e-01</td>\n",
       "      <td>0.813018</td>\n",
       "      <td>0.968750</td>\n",
       "      <td>0.968889</td>\n",
       "      <td>3.144237e-02</td>\n",
       "      <td>0.899969</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>inventors__affirmative</td>\n",
       "      <td>0.914634</td>\n",
       "      <td>0.929293</td>\n",
       "      <td>6.569698e-02</td>\n",
       "      <td>0.898393</td>\n",
       "      <td>0.938272</td>\n",
       "      <td>0.944444</td>\n",
       "      <td>4.398387e-02</td>\n",
       "      <td>0.899879</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>inventors__conjunction</td>\n",
       "      <td>0.890000</td>\n",
       "      <td>0.910569</td>\n",
       "      <td>7.758649e-02</td>\n",
       "      <td>0.871089</td>\n",
       "      <td>0.985000</td>\n",
       "      <td>0.986784</td>\n",
       "      <td>2.356241e-02</td>\n",
       "      <td>0.924587</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>inventors__disjunction</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.843750</td>\n",
       "      <td>1.551561e-01</td>\n",
       "      <td>0.754390</td>\n",
       "      <td>0.907500</td>\n",
       "      <td>0.919740</td>\n",
       "      <td>9.533650e-02</td>\n",
       "      <td>0.792131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>inventors__negated</td>\n",
       "      <td>0.853659</td>\n",
       "      <td>0.812500</td>\n",
       "      <td>9.715707e-02</td>\n",
       "      <td>0.772859</td>\n",
       "      <td>0.919753</td>\n",
       "      <td>0.909091</td>\n",
       "      <td>5.893136e-02</td>\n",
       "      <td>0.846446</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>ambiguous_lie</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.050118e-24</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.446902e-24</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>ambiguous_truthful_reply</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>honest_reply_despite_incentive_to_lie</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.421085e-14</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>7.105427e-15</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>unambiguous_lie</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.518638e-24</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.157558e-24</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>unambiguous_truthful_reply</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>larger_than</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.426665e-05</td>\n",
       "      <td>0.999800</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.502934e-15</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>smaller_than</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.349663e-05</td>\n",
       "      <td>0.999626</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>8.750799e-15</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>sp_en_trans__negated</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.562758e-12</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.515887e-16</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>sp_en_trans__affirmative</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>8.378279e-08</td>\n",
       "      <td>0.999972</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.523146e-16</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>sp_en_trans__conjunction</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.664824e-04</td>\n",
       "      <td>0.997272</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.091911e-17</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>sp_en_trans__disjunction</td>\n",
       "      <td>0.990000</td>\n",
       "      <td>0.989011</td>\n",
       "      <td>3.743603e-03</td>\n",
       "      <td>0.993305</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.741257e-15</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>common_claim_true_false</td>\n",
       "      <td>0.811236</td>\n",
       "      <td>0.814159</td>\n",
       "      <td>1.324474e-01</td>\n",
       "      <td>0.732995</td>\n",
       "      <td>0.838764</td>\n",
       "      <td>0.839306</td>\n",
       "      <td>1.165958e-01</td>\n",
       "      <td>0.743862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>counterfact_true_false</td>\n",
       "      <td>0.826216</td>\n",
       "      <td>0.826920</td>\n",
       "      <td>1.185529e-01</td>\n",
       "      <td>0.766909</td>\n",
       "      <td>0.842400</td>\n",
       "      <td>0.844281</td>\n",
       "      <td>1.073910e-01</td>\n",
       "      <td>0.778638</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>animal_class__affirmative+negated</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.641339e-07</td>\n",
       "      <td>0.999915</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.039329e-16</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>animal_class__affirmative+disjunction</td>\n",
       "      <td>0.842105</td>\n",
       "      <td>0.803738</td>\n",
       "      <td>1.066825e-01</td>\n",
       "      <td>0.748289</td>\n",
       "      <td>0.890772</td>\n",
       "      <td>0.898246</td>\n",
       "      <td>7.368915e-02</td>\n",
       "      <td>0.839471</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>animal_class__negated+conjunction</td>\n",
       "      <td>0.977444</td>\n",
       "      <td>0.978417</td>\n",
       "      <td>1.068789e-02</td>\n",
       "      <td>0.981544</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.953050e-04</td>\n",
       "      <td>0.996060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>animal_class__affirmative+negated+conjunction+...</td>\n",
       "      <td>0.909774</td>\n",
       "      <td>0.904000</td>\n",
       "      <td>6.324344e-02</td>\n",
       "      <td>0.875548</td>\n",
       "      <td>0.944444</td>\n",
       "      <td>0.945822</td>\n",
       "      <td>3.911059e-02</td>\n",
       "      <td>0.914783</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>cities__affirmative+negated</td>\n",
       "      <td>0.985000</td>\n",
       "      <td>0.985554</td>\n",
       "      <td>9.284142e-03</td>\n",
       "      <td>0.985693</td>\n",
       "      <td>0.992893</td>\n",
       "      <td>0.992830</td>\n",
       "      <td>4.429694e-03</td>\n",
       "      <td>0.988979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>cities__affirmative+disjunction</td>\n",
       "      <td>0.955000</td>\n",
       "      <td>0.953125</td>\n",
       "      <td>3.823327e-02</td>\n",
       "      <td>0.921924</td>\n",
       "      <td>0.974937</td>\n",
       "      <td>0.975217</td>\n",
       "      <td>2.049703e-02</td>\n",
       "      <td>0.944502</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>cities__negated+conjunction</td>\n",
       "      <td>0.986667</td>\n",
       "      <td>0.987952</td>\n",
       "      <td>1.243303e-02</td>\n",
       "      <td>0.981832</td>\n",
       "      <td>0.997076</td>\n",
       "      <td>0.997005</td>\n",
       "      <td>2.545326e-03</td>\n",
       "      <td>0.990748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>cities__affirmative+negated+conjunction+disjun...</td>\n",
       "      <td>0.971000</td>\n",
       "      <td>0.972302</td>\n",
       "      <td>2.320118e-02</td>\n",
       "      <td>0.957636</td>\n",
       "      <td>0.980451</td>\n",
       "      <td>0.980233</td>\n",
       "      <td>1.428717e-02</td>\n",
       "      <td>0.963197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>element_symb__affirmative+negated</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.849128e-03</td>\n",
       "      <td>0.995199</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.763436e-15</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>element_symb__affirmative+disjunction</td>\n",
       "      <td>0.978261</td>\n",
       "      <td>0.979310</td>\n",
       "      <td>1.921232e-02</td>\n",
       "      <td>0.960651</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.620967e-03</td>\n",
       "      <td>0.986654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>element_symb__negated+conjunction</td>\n",
       "      <td>0.992754</td>\n",
       "      <td>0.992366</td>\n",
       "      <td>8.176259e-03</td>\n",
       "      <td>0.975870</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.753441e-03</td>\n",
       "      <td>0.988917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>element_symb__affirmative+negated+conjunction+...</td>\n",
       "      <td>0.978261</td>\n",
       "      <td>0.978417</td>\n",
       "      <td>1.467793e-02</td>\n",
       "      <td>0.969567</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>9.719951e-04</td>\n",
       "      <td>0.991544</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>facts__affirmative+negated</td>\n",
       "      <td>0.871681</td>\n",
       "      <td>0.876596</td>\n",
       "      <td>1.043555e-01</td>\n",
       "      <td>0.811190</td>\n",
       "      <td>0.944196</td>\n",
       "      <td>0.943946</td>\n",
       "      <td>4.724095e-02</td>\n",
       "      <td>0.872309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>facts__affirmative+disjunction</td>\n",
       "      <td>0.887324</td>\n",
       "      <td>0.891892</td>\n",
       "      <td>7.251821e-02</td>\n",
       "      <td>0.857505</td>\n",
       "      <td>0.987028</td>\n",
       "      <td>0.986233</td>\n",
       "      <td>1.909521e-02</td>\n",
       "      <td>0.918500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>facts__negated+conjunction</td>\n",
       "      <td>0.830986</td>\n",
       "      <td>0.820000</td>\n",
       "      <td>1.242275e-01</td>\n",
       "      <td>0.755510</td>\n",
       "      <td>0.932783</td>\n",
       "      <td>0.933020</td>\n",
       "      <td>5.948954e-02</td>\n",
       "      <td>0.838461</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>facts__affirmative+negated+conjunction+disjunc...</td>\n",
       "      <td>0.861502</td>\n",
       "      <td>0.858513</td>\n",
       "      <td>1.085148e-01</td>\n",
       "      <td>0.778201</td>\n",
       "      <td>0.920991</td>\n",
       "      <td>0.919082</td>\n",
       "      <td>6.407186e-02</td>\n",
       "      <td>0.831850</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>inventors__affirmative+negated</td>\n",
       "      <td>0.871951</td>\n",
       "      <td>0.871166</td>\n",
       "      <td>8.386829e-02</td>\n",
       "      <td>0.834489</td>\n",
       "      <td>0.899691</td>\n",
       "      <td>0.898911</td>\n",
       "      <td>6.930428e-02</td>\n",
       "      <td>0.848420</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>inventors__affirmative+disjunction</td>\n",
       "      <td>0.846154</td>\n",
       "      <td>0.878261</td>\n",
       "      <td>1.095920e-01</td>\n",
       "      <td>0.826457</td>\n",
       "      <td>0.915746</td>\n",
       "      <td>0.926061</td>\n",
       "      <td>6.958865e-02</td>\n",
       "      <td>0.847062</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>inventors__negated+conjunction</td>\n",
       "      <td>0.873626</td>\n",
       "      <td>0.877005</td>\n",
       "      <td>8.702927e-02</td>\n",
       "      <td>0.826027</td>\n",
       "      <td>0.955801</td>\n",
       "      <td>0.956757</td>\n",
       "      <td>4.640511e-02</td>\n",
       "      <td>0.877578</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>inventors__affirmative+negated+conjunction+dis...</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>0.875598</td>\n",
       "      <td>1.007724e-01</td>\n",
       "      <td>0.822551</td>\n",
       "      <td>0.912293</td>\n",
       "      <td>0.918953</td>\n",
       "      <td>6.555394e-02</td>\n",
       "      <td>0.850173</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>sp_en_trans__affirmative+negated</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.670350e-07</td>\n",
       "      <td>0.999967</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.035009e-16</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>sp_en_trans__affirmative+disjunction</td>\n",
       "      <td>0.994152</td>\n",
       "      <td>0.994413</td>\n",
       "      <td>2.782293e-03</td>\n",
       "      <td>0.994445</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.170199e-06</td>\n",
       "      <td>0.999439</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>sp_en_trans__negated+conjunction</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.023078e-04</td>\n",
       "      <td>0.998664</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>7.771587e-16</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>sp_en_trans__affirmative+negated+conjunction+d...</td>\n",
       "      <td>0.997076</td>\n",
       "      <td>0.997085</td>\n",
       "      <td>2.586771e-03</td>\n",
       "      <td>0.994640</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>7.300621e-06</td>\n",
       "      <td>0.999334</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>ambiguous_truthful_reply+unambiguous_lie+unamb...</td>\n",
       "      <td>0.923077</td>\n",
       "      <td>0.933333</td>\n",
       "      <td>4.773707e-02</td>\n",
       "      <td>0.927686</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.206968e-04</td>\n",
       "      <td>0.995401</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>animal_class+facts+inventors__affirmative+nega...</td>\n",
       "      <td>0.896825</td>\n",
       "      <td>0.897906</td>\n",
       "      <td>8.068992e-02</td>\n",
       "      <td>0.848357</td>\n",
       "      <td>0.942448</td>\n",
       "      <td>0.942885</td>\n",
       "      <td>4.567364e-02</td>\n",
       "      <td>0.887364</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>animal_class+facts+inventors__affirmative+nega...</td>\n",
       "      <td>0.867081</td>\n",
       "      <td>0.872074</td>\n",
       "      <td>9.542867e-02</td>\n",
       "      <td>0.820643</td>\n",
       "      <td>0.895192</td>\n",
       "      <td>0.896047</td>\n",
       "      <td>7.606133e-02</td>\n",
       "      <td>0.831885</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 29
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:58.695678Z",
     "start_time": "2024-12-07T21:26:58.676925Z"
    }
   },
   "cell_type": "code",
   "source": "lyr18_ttpd_probe_to_baseline_linear_probe_eval_except_test_df",
   "id": "162bbd602e761c44",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "                                             scenario   val_acc    val_f1  \\\n",
       "0                           animal_class__affirmative  0.000000  0.000000   \n",
       "1                           animal_class__conjunction -0.090000 -0.080917   \n",
       "2                           animal_class__disjunction -0.050000 -0.081273   \n",
       "3                               animal_class__negated  0.000000  0.000000   \n",
       "4                                 cities__affirmative -0.003333 -0.003436   \n",
       "5                                 cities__conjunction -0.026667 -0.023444   \n",
       "6                                 cities__disjunction -0.070000 -0.072769   \n",
       "7                                     cities__negated  0.003333  0.003055   \n",
       "8                           element_symb__affirmative -0.026316 -0.024390   \n",
       "9                           element_symb__conjunction -0.120000 -0.115385   \n",
       "10                          element_symb__disjunction -0.180000 -0.162587   \n",
       "11                              element_symb__negated  0.000000  0.000000   \n",
       "12                                 facts__affirmative -0.097345 -0.074286   \n",
       "13                                 facts__conjunction -0.100000 -0.059928   \n",
       "14                                 facts__disjunction -0.060000 -0.085227   \n",
       "15                                     facts__negated -0.035398 -0.041467   \n",
       "16                             inventors__affirmative -0.073171 -0.069078   \n",
       "17                             inventors__conjunction -0.170000 -0.156183   \n",
       "18                             inventors__disjunction -0.100000 -0.111607   \n",
       "19                                 inventors__negated -0.195122 -0.180921   \n",
       "20                                      ambiguous_lie -0.500000       NaN   \n",
       "21                           ambiguous_truthful_reply -0.400000 -0.250000   \n",
       "22              honest_reply_despite_incentive_to_lie  0.000000  0.000000   \n",
       "23                                    unambiguous_lie -0.400000       NaN   \n",
       "24                         unambiguous_truthful_reply -0.272727 -0.157895   \n",
       "25                                        larger_than -0.027778 -0.028133   \n",
       "26                                       smaller_than -0.050505 -0.044643   \n",
       "27                               sp_en_trans__negated  0.000000  0.000000   \n",
       "28                           sp_en_trans__affirmative -0.028169 -0.022222   \n",
       "29                           sp_en_trans__conjunction -0.070000 -0.069307   \n",
       "30                           sp_en_trans__disjunction -0.150000 -0.170829   \n",
       "31                            common_claim_true_false -0.029213 -0.027346   \n",
       "32                             counterfact_true_false -0.118254 -0.097069   \n",
       "33                  animal_class__affirmative+negated  0.000000  0.000000   \n",
       "34              animal_class__affirmative+disjunction -0.045113 -0.042676   \n",
       "35                  animal_class__negated+conjunction -0.060150 -0.053247   \n",
       "36  animal_class__affirmative+negated+conjunction+... -0.161654 -0.191554   \n",
       "37                        cities__affirmative+negated -0.001667 -0.001477   \n",
       "38                    cities__affirmative+disjunction -0.035000 -0.040082   \n",
       "39                        cities__negated+conjunction -0.056667 -0.048124   \n",
       "40  cities__affirmative+negated+conjunction+disjun... -0.080000 -0.072578   \n",
       "41                  element_symb__affirmative+negated -0.013158 -0.012658   \n",
       "42              element_symb__affirmative+disjunction -0.137681 -0.127959   \n",
       "43                  element_symb__negated+conjunction -0.101449 -0.095815   \n",
       "44  element_symb__affirmative+negated+conjunction+... -0.173913 -0.165917   \n",
       "45                         facts__affirmative+negated -0.084071 -0.065572   \n",
       "46                     facts__affirmative+disjunction -0.098592 -0.093686   \n",
       "47                         facts__negated+conjunction -0.169014 -0.150275   \n",
       "48  facts__affirmative+negated+conjunction+disjunc... -0.152582 -0.140331   \n",
       "49                     inventors__affirmative+negated -0.042683 -0.041897   \n",
       "50                 inventors__affirmative+disjunction -0.109890 -0.115885   \n",
       "51                     inventors__negated+conjunction -0.197802 -0.151424   \n",
       "52  inventors__affirmative+negated+conjunction+dis... -0.164835 -0.137280   \n",
       "53                   sp_en_trans__affirmative+negated -0.028169 -0.024691   \n",
       "54               sp_en_trans__affirmative+disjunction -0.192982 -0.177209   \n",
       "55                   sp_en_trans__negated+conjunction -0.046784 -0.046512   \n",
       "56  sp_en_trans__affirmative+negated+conjunction+d... -0.169591 -0.174262   \n",
       "57  ambiguous_truthful_reply+unambiguous_lie+unamb... -0.115385 -0.118519   \n",
       "58  animal_class+facts+inventors__affirmative+nega... -0.146825 -0.128675   \n",
       "59  animal_class+facts+inventors__affirmative+nega... -0.061652 -0.052557   \n",
       "\n",
       "       val_brier  val_soft_f1  train_acc  train_f1   train_brier  \\\n",
       "0   2.445273e-08    -0.000057  -0.007634 -0.007092  8.771814e-03   \n",
       "1   5.421183e-02    -0.098470  -0.082500 -0.079903  6.150439e-02   \n",
       "2   5.247253e-02    -0.132715  -0.142500 -0.147089  9.893978e-02   \n",
       "3   6.577484e-04    -0.004059  -0.015267 -0.015152  1.469421e-02   \n",
       "4   5.138021e-03    -0.009379  -0.009197 -0.009079  7.408913e-03   \n",
       "5   2.352697e-02    -0.047931  -0.069282 -0.066241  4.671761e-02   \n",
       "6   2.219895e-02    -0.025689  -0.075000 -0.077959  4.837384e-02   \n",
       "7  -1.191099e-03    -0.001395  -0.009197 -0.009336  6.372243e-03   \n",
       "8   2.134477e-02    -0.034914  -0.094595 -0.087500  6.313708e-02   \n",
       "9   7.882613e-02    -0.146866  -0.067500 -0.059341  5.868069e-02   \n",
       "10  1.156770e-01    -0.195731  -0.210000 -0.212121  1.341573e-01   \n",
       "11 -3.210423e-04     0.004105   0.000000  0.000000 -1.460392e-17   \n",
       "12  7.851715e-02    -0.134223  -0.147321 -0.134147  1.059934e-01   \n",
       "13  6.331087e-02    -0.095777  -0.187500 -0.162466  1.095918e-01   \n",
       "14  5.012951e-02    -0.108559  -0.240000 -0.268844  1.358451e-01   \n",
       "15  2.084527e-02    -0.039522  -0.064732 -0.065086  4.646199e-02   \n",
       "16  4.140351e-02    -0.093774  -0.104938 -0.099617  8.918222e-02   \n",
       "17  8.208182e-02    -0.137317  -0.175000 -0.157188  1.068195e-01   \n",
       "18  4.861680e-02    -0.118081  -0.167500 -0.165023  8.372878e-02   \n",
       "19  9.796846e-02    -0.166351  -0.212963 -0.222622  1.125483e-01   \n",
       "20  2.412132e-01          NaN  -0.571429       NaN  2.524060e-01   \n",
       "21  2.086981e-01    -0.289361  -0.444444 -0.285714  2.632712e-01   \n",
       "22  7.010247e-02    -0.152540  -0.750000 -0.600000  2.680697e-01   \n",
       "23  2.176609e-01          NaN  -0.461538       NaN  2.550336e-01   \n",
       "24  2.202386e-01    -0.302403  -0.500000 -0.333333  2.593466e-01   \n",
       "25  2.689818e-02    -0.053652  -0.029040 -0.027981  2.098595e-02   \n",
       "26  3.480716e-02    -0.067332  -0.054924 -0.053081  3.824428e-02   \n",
       "27  2.294588e-04    -0.002374  -0.003534 -0.003534  2.715710e-03   \n",
       "28  1.804549e-02    -0.025808  -0.031802 -0.032727  1.792615e-02   \n",
       "29  5.257753e-02    -0.078916  -0.032500 -0.029213  2.128471e-02   \n",
       "30  1.053350e-01    -0.231029  -0.167500 -0.186630  1.089132e-01   \n",
       "31  2.038309e-02    -0.048703  -0.091292 -0.090213  5.257300e-02   \n",
       "32  6.895546e-02    -0.132472  -0.132181 -0.108981  7.850540e-02   \n",
       "33  3.723638e-03    -0.011150  -0.015267 -0.014706  9.318734e-03   \n",
       "34  3.366106e-02    -0.091190  -0.111111 -0.109056  7.697122e-02   \n",
       "35  4.756792e-02    -0.084634  -0.069680 -0.067151  5.264326e-02   \n",
       "36  7.256425e-02    -0.157480  -0.204331 -0.231536  1.003570e-01   \n",
       "37  4.505215e-03    -0.009584  -0.011288 -0.011209  1.013268e-02   \n",
       "38  2.377561e-02    -0.057143  -0.068922 -0.073642  4.192433e-02   \n",
       "39  3.695871e-02    -0.072555  -0.071011 -0.067833  5.502248e-02   \n",
       "40  5.333573e-02    -0.094180  -0.115288 -0.109250  7.560212e-02   \n",
       "41  1.291110e-02    -0.023026  -0.040541 -0.039216  3.085962e-02   \n",
       "42  8.470442e-02    -0.153955  -0.156934 -0.151943  1.062579e-01   \n",
       "43  7.986078e-02    -0.128377  -0.093066 -0.081862  6.419752e-02   \n",
       "44  1.178062e-01    -0.230123  -0.185219 -0.180444  1.233812e-01   \n",
       "45  3.432238e-02    -0.045978  -0.099330 -0.091661  6.045906e-02   \n",
       "46  8.472200e-02    -0.153020  -0.218160 -0.232464  1.422455e-01   \n",
       "47  7.715170e-02    -0.173585  -0.189858 -0.173638  1.120891e-01   \n",
       "48  7.475991e-02    -0.128504  -0.173939 -0.168066  9.911590e-02   \n",
       "49  4.072750e-02    -0.089523  -0.069444 -0.072413  6.662170e-02   \n",
       "50  5.125949e-02    -0.130888  -0.149171 -0.154131  9.314296e-02   \n",
       "51  1.271030e-01    -0.242914  -0.308011 -0.258532  1.727039e-01   \n",
       "52  1.090399e-01    -0.218577  -0.259669 -0.228873  1.493716e-01   \n",
       "53  2.340480e-02    -0.039710  -0.028269 -0.028674  1.558442e-02   \n",
       "54  1.216646e-01    -0.206986  -0.171303 -0.177005  1.080387e-01   \n",
       "55  3.650491e-02    -0.067766  -0.043924 -0.040650  3.132679e-02   \n",
       "56  1.323373e-01    -0.262572  -0.158126 -0.162651  1.220621e-01   \n",
       "57  7.519370e-02    -0.098123  -0.049505 -0.041322  4.783866e-02   \n",
       "58  8.549259e-02    -0.160522  -0.177977 -0.163309  1.147210e-01   \n",
       "59  4.957473e-02    -0.108845  -0.107361 -0.098593  7.751492e-02   \n",
       "\n",
       "    train_soft_f1  \n",
       "0   -1.052990e-02  \n",
       "1   -1.148468e-01  \n",
       "2   -1.794198e-01  \n",
       "3   -2.405677e-02  \n",
       "4   -9.576901e-03  \n",
       "5   -8.151176e-02  \n",
       "6   -6.483336e-02  \n",
       "7   -1.042434e-02  \n",
       "8   -8.674779e-02  \n",
       "9   -1.073186e-01  \n",
       "10  -2.495769e-01  \n",
       "11   8.294688e-10  \n",
       "12  -1.898896e-01  \n",
       "13  -1.579075e-01  \n",
       "14  -2.071611e-01  \n",
       "15  -6.755548e-02  \n",
       "16  -1.224995e-01  \n",
       "17  -1.667652e-01  \n",
       "18  -1.478823e-01  \n",
       "19  -1.935640e-01  \n",
       "20            NaN  \n",
       "21  -3.325137e-01  \n",
       "22  -3.357172e-01  \n",
       "23            NaN  \n",
       "24  -3.330134e-01  \n",
       "25  -4.465499e-02  \n",
       "26  -7.909786e-02  \n",
       "27  -4.754704e-03  \n",
       "28  -3.349019e-02  \n",
       "29  -4.480009e-02  \n",
       "30  -2.295333e-01  \n",
       "31  -7.833020e-02  \n",
       "32  -1.404222e-01  \n",
       "33  -1.559329e-02  \n",
       "34  -1.291663e-01  \n",
       "35  -1.001897e-01  \n",
       "36  -1.895712e-01  \n",
       "37  -1.737827e-02  \n",
       "38  -7.360984e-02  \n",
       "39  -1.041479e-01  \n",
       "40  -1.224877e-01  \n",
       "41  -4.579331e-02  \n",
       "42  -2.025634e-01  \n",
       "43  -1.009118e-01  \n",
       "44  -2.359114e-01  \n",
       "45  -8.966229e-02  \n",
       "46  -2.392702e-01  \n",
       "47  -2.085259e-01  \n",
       "48  -1.633740e-01  \n",
       "49  -1.167077e-01  \n",
       "50  -1.658311e-01  \n",
       "51  -3.083661e-01  \n",
       "52  -2.664578e-01  \n",
       "53  -4.114472e-02  \n",
       "54  -2.137683e-01  \n",
       "55  -5.802527e-02  \n",
       "56  -2.467094e-01  \n",
       "57  -8.404473e-02  \n",
       "58  -1.988424e-01  \n",
       "59  -1.360602e-01  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>scenario</th>\n",
       "      <th>val_acc</th>\n",
       "      <th>val_f1</th>\n",
       "      <th>val_brier</th>\n",
       "      <th>val_soft_f1</th>\n",
       "      <th>train_acc</th>\n",
       "      <th>train_f1</th>\n",
       "      <th>train_brier</th>\n",
       "      <th>train_soft_f1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>animal_class__affirmative</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.445273e-08</td>\n",
       "      <td>-0.000057</td>\n",
       "      <td>-0.007634</td>\n",
       "      <td>-0.007092</td>\n",
       "      <td>8.771814e-03</td>\n",
       "      <td>-1.052990e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>animal_class__conjunction</td>\n",
       "      <td>-0.090000</td>\n",
       "      <td>-0.080917</td>\n",
       "      <td>5.421183e-02</td>\n",
       "      <td>-0.098470</td>\n",
       "      <td>-0.082500</td>\n",
       "      <td>-0.079903</td>\n",
       "      <td>6.150439e-02</td>\n",
       "      <td>-1.148468e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>animal_class__disjunction</td>\n",
       "      <td>-0.050000</td>\n",
       "      <td>-0.081273</td>\n",
       "      <td>5.247253e-02</td>\n",
       "      <td>-0.132715</td>\n",
       "      <td>-0.142500</td>\n",
       "      <td>-0.147089</td>\n",
       "      <td>9.893978e-02</td>\n",
       "      <td>-1.794198e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>animal_class__negated</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.577484e-04</td>\n",
       "      <td>-0.004059</td>\n",
       "      <td>-0.015267</td>\n",
       "      <td>-0.015152</td>\n",
       "      <td>1.469421e-02</td>\n",
       "      <td>-2.405677e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cities__affirmative</td>\n",
       "      <td>-0.003333</td>\n",
       "      <td>-0.003436</td>\n",
       "      <td>5.138021e-03</td>\n",
       "      <td>-0.009379</td>\n",
       "      <td>-0.009197</td>\n",
       "      <td>-0.009079</td>\n",
       "      <td>7.408913e-03</td>\n",
       "      <td>-9.576901e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>cities__conjunction</td>\n",
       "      <td>-0.026667</td>\n",
       "      <td>-0.023444</td>\n",
       "      <td>2.352697e-02</td>\n",
       "      <td>-0.047931</td>\n",
       "      <td>-0.069282</td>\n",
       "      <td>-0.066241</td>\n",
       "      <td>4.671761e-02</td>\n",
       "      <td>-8.151176e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>cities__disjunction</td>\n",
       "      <td>-0.070000</td>\n",
       "      <td>-0.072769</td>\n",
       "      <td>2.219895e-02</td>\n",
       "      <td>-0.025689</td>\n",
       "      <td>-0.075000</td>\n",
       "      <td>-0.077959</td>\n",
       "      <td>4.837384e-02</td>\n",
       "      <td>-6.483336e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>cities__negated</td>\n",
       "      <td>0.003333</td>\n",
       "      <td>0.003055</td>\n",
       "      <td>-1.191099e-03</td>\n",
       "      <td>-0.001395</td>\n",
       "      <td>-0.009197</td>\n",
       "      <td>-0.009336</td>\n",
       "      <td>6.372243e-03</td>\n",
       "      <td>-1.042434e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>element_symb__affirmative</td>\n",
       "      <td>-0.026316</td>\n",
       "      <td>-0.024390</td>\n",
       "      <td>2.134477e-02</td>\n",
       "      <td>-0.034914</td>\n",
       "      <td>-0.094595</td>\n",
       "      <td>-0.087500</td>\n",
       "      <td>6.313708e-02</td>\n",
       "      <td>-8.674779e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>element_symb__conjunction</td>\n",
       "      <td>-0.120000</td>\n",
       "      <td>-0.115385</td>\n",
       "      <td>7.882613e-02</td>\n",
       "      <td>-0.146866</td>\n",
       "      <td>-0.067500</td>\n",
       "      <td>-0.059341</td>\n",
       "      <td>5.868069e-02</td>\n",
       "      <td>-1.073186e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>element_symb__disjunction</td>\n",
       "      <td>-0.180000</td>\n",
       "      <td>-0.162587</td>\n",
       "      <td>1.156770e-01</td>\n",
       "      <td>-0.195731</td>\n",
       "      <td>-0.210000</td>\n",
       "      <td>-0.212121</td>\n",
       "      <td>1.341573e-01</td>\n",
       "      <td>-2.495769e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>element_symb__negated</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-3.210423e-04</td>\n",
       "      <td>0.004105</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-1.460392e-17</td>\n",
       "      <td>8.294688e-10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>facts__affirmative</td>\n",
       "      <td>-0.097345</td>\n",
       "      <td>-0.074286</td>\n",
       "      <td>7.851715e-02</td>\n",
       "      <td>-0.134223</td>\n",
       "      <td>-0.147321</td>\n",
       "      <td>-0.134147</td>\n",
       "      <td>1.059934e-01</td>\n",
       "      <td>-1.898896e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>facts__conjunction</td>\n",
       "      <td>-0.100000</td>\n",
       "      <td>-0.059928</td>\n",
       "      <td>6.331087e-02</td>\n",
       "      <td>-0.095777</td>\n",
       "      <td>-0.187500</td>\n",
       "      <td>-0.162466</td>\n",
       "      <td>1.095918e-01</td>\n",
       "      <td>-1.579075e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>facts__disjunction</td>\n",
       "      <td>-0.060000</td>\n",
       "      <td>-0.085227</td>\n",
       "      <td>5.012951e-02</td>\n",
       "      <td>-0.108559</td>\n",
       "      <td>-0.240000</td>\n",
       "      <td>-0.268844</td>\n",
       "      <td>1.358451e-01</td>\n",
       "      <td>-2.071611e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>facts__negated</td>\n",
       "      <td>-0.035398</td>\n",
       "      <td>-0.041467</td>\n",
       "      <td>2.084527e-02</td>\n",
       "      <td>-0.039522</td>\n",
       "      <td>-0.064732</td>\n",
       "      <td>-0.065086</td>\n",
       "      <td>4.646199e-02</td>\n",
       "      <td>-6.755548e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>inventors__affirmative</td>\n",
       "      <td>-0.073171</td>\n",
       "      <td>-0.069078</td>\n",
       "      <td>4.140351e-02</td>\n",
       "      <td>-0.093774</td>\n",
       "      <td>-0.104938</td>\n",
       "      <td>-0.099617</td>\n",
       "      <td>8.918222e-02</td>\n",
       "      <td>-1.224995e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>inventors__conjunction</td>\n",
       "      <td>-0.170000</td>\n",
       "      <td>-0.156183</td>\n",
       "      <td>8.208182e-02</td>\n",
       "      <td>-0.137317</td>\n",
       "      <td>-0.175000</td>\n",
       "      <td>-0.157188</td>\n",
       "      <td>1.068195e-01</td>\n",
       "      <td>-1.667652e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>inventors__disjunction</td>\n",
       "      <td>-0.100000</td>\n",
       "      <td>-0.111607</td>\n",
       "      <td>4.861680e-02</td>\n",
       "      <td>-0.118081</td>\n",
       "      <td>-0.167500</td>\n",
       "      <td>-0.165023</td>\n",
       "      <td>8.372878e-02</td>\n",
       "      <td>-1.478823e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>inventors__negated</td>\n",
       "      <td>-0.195122</td>\n",
       "      <td>-0.180921</td>\n",
       "      <td>9.796846e-02</td>\n",
       "      <td>-0.166351</td>\n",
       "      <td>-0.212963</td>\n",
       "      <td>-0.222622</td>\n",
       "      <td>1.125483e-01</td>\n",
       "      <td>-1.935640e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>ambiguous_lie</td>\n",
       "      <td>-0.500000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.412132e-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.571429</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.524060e-01</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>ambiguous_truthful_reply</td>\n",
       "      <td>-0.400000</td>\n",
       "      <td>-0.250000</td>\n",
       "      <td>2.086981e-01</td>\n",
       "      <td>-0.289361</td>\n",
       "      <td>-0.444444</td>\n",
       "      <td>-0.285714</td>\n",
       "      <td>2.632712e-01</td>\n",
       "      <td>-3.325137e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>honest_reply_despite_incentive_to_lie</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.010247e-02</td>\n",
       "      <td>-0.152540</td>\n",
       "      <td>-0.750000</td>\n",
       "      <td>-0.600000</td>\n",
       "      <td>2.680697e-01</td>\n",
       "      <td>-3.357172e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>unambiguous_lie</td>\n",
       "      <td>-0.400000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.176609e-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.461538</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.550336e-01</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>unambiguous_truthful_reply</td>\n",
       "      <td>-0.272727</td>\n",
       "      <td>-0.157895</td>\n",
       "      <td>2.202386e-01</td>\n",
       "      <td>-0.302403</td>\n",
       "      <td>-0.500000</td>\n",
       "      <td>-0.333333</td>\n",
       "      <td>2.593466e-01</td>\n",
       "      <td>-3.330134e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>larger_than</td>\n",
       "      <td>-0.027778</td>\n",
       "      <td>-0.028133</td>\n",
       "      <td>2.689818e-02</td>\n",
       "      <td>-0.053652</td>\n",
       "      <td>-0.029040</td>\n",
       "      <td>-0.027981</td>\n",
       "      <td>2.098595e-02</td>\n",
       "      <td>-4.465499e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>smaller_than</td>\n",
       "      <td>-0.050505</td>\n",
       "      <td>-0.044643</td>\n",
       "      <td>3.480716e-02</td>\n",
       "      <td>-0.067332</td>\n",
       "      <td>-0.054924</td>\n",
       "      <td>-0.053081</td>\n",
       "      <td>3.824428e-02</td>\n",
       "      <td>-7.909786e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>sp_en_trans__negated</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.294588e-04</td>\n",
       "      <td>-0.002374</td>\n",
       "      <td>-0.003534</td>\n",
       "      <td>-0.003534</td>\n",
       "      <td>2.715710e-03</td>\n",
       "      <td>-4.754704e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>sp_en_trans__affirmative</td>\n",
       "      <td>-0.028169</td>\n",
       "      <td>-0.022222</td>\n",
       "      <td>1.804549e-02</td>\n",
       "      <td>-0.025808</td>\n",
       "      <td>-0.031802</td>\n",
       "      <td>-0.032727</td>\n",
       "      <td>1.792615e-02</td>\n",
       "      <td>-3.349019e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>sp_en_trans__conjunction</td>\n",
       "      <td>-0.070000</td>\n",
       "      <td>-0.069307</td>\n",
       "      <td>5.257753e-02</td>\n",
       "      <td>-0.078916</td>\n",
       "      <td>-0.032500</td>\n",
       "      <td>-0.029213</td>\n",
       "      <td>2.128471e-02</td>\n",
       "      <td>-4.480009e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>sp_en_trans__disjunction</td>\n",
       "      <td>-0.150000</td>\n",
       "      <td>-0.170829</td>\n",
       "      <td>1.053350e-01</td>\n",
       "      <td>-0.231029</td>\n",
       "      <td>-0.167500</td>\n",
       "      <td>-0.186630</td>\n",
       "      <td>1.089132e-01</td>\n",
       "      <td>-2.295333e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>common_claim_true_false</td>\n",
       "      <td>-0.029213</td>\n",
       "      <td>-0.027346</td>\n",
       "      <td>2.038309e-02</td>\n",
       "      <td>-0.048703</td>\n",
       "      <td>-0.091292</td>\n",
       "      <td>-0.090213</td>\n",
       "      <td>5.257300e-02</td>\n",
       "      <td>-7.833020e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>counterfact_true_false</td>\n",
       "      <td>-0.118254</td>\n",
       "      <td>-0.097069</td>\n",
       "      <td>6.895546e-02</td>\n",
       "      <td>-0.132472</td>\n",
       "      <td>-0.132181</td>\n",
       "      <td>-0.108981</td>\n",
       "      <td>7.850540e-02</td>\n",
       "      <td>-1.404222e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>animal_class__affirmative+negated</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.723638e-03</td>\n",
       "      <td>-0.011150</td>\n",
       "      <td>-0.015267</td>\n",
       "      <td>-0.014706</td>\n",
       "      <td>9.318734e-03</td>\n",
       "      <td>-1.559329e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>animal_class__affirmative+disjunction</td>\n",
       "      <td>-0.045113</td>\n",
       "      <td>-0.042676</td>\n",
       "      <td>3.366106e-02</td>\n",
       "      <td>-0.091190</td>\n",
       "      <td>-0.111111</td>\n",
       "      <td>-0.109056</td>\n",
       "      <td>7.697122e-02</td>\n",
       "      <td>-1.291663e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>animal_class__negated+conjunction</td>\n",
       "      <td>-0.060150</td>\n",
       "      <td>-0.053247</td>\n",
       "      <td>4.756792e-02</td>\n",
       "      <td>-0.084634</td>\n",
       "      <td>-0.069680</td>\n",
       "      <td>-0.067151</td>\n",
       "      <td>5.264326e-02</td>\n",
       "      <td>-1.001897e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>animal_class__affirmative+negated+conjunction+...</td>\n",
       "      <td>-0.161654</td>\n",
       "      <td>-0.191554</td>\n",
       "      <td>7.256425e-02</td>\n",
       "      <td>-0.157480</td>\n",
       "      <td>-0.204331</td>\n",
       "      <td>-0.231536</td>\n",
       "      <td>1.003570e-01</td>\n",
       "      <td>-1.895712e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>cities__affirmative+negated</td>\n",
       "      <td>-0.001667</td>\n",
       "      <td>-0.001477</td>\n",
       "      <td>4.505215e-03</td>\n",
       "      <td>-0.009584</td>\n",
       "      <td>-0.011288</td>\n",
       "      <td>-0.011209</td>\n",
       "      <td>1.013268e-02</td>\n",
       "      <td>-1.737827e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>cities__affirmative+disjunction</td>\n",
       "      <td>-0.035000</td>\n",
       "      <td>-0.040082</td>\n",
       "      <td>2.377561e-02</td>\n",
       "      <td>-0.057143</td>\n",
       "      <td>-0.068922</td>\n",
       "      <td>-0.073642</td>\n",
       "      <td>4.192433e-02</td>\n",
       "      <td>-7.360984e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>cities__negated+conjunction</td>\n",
       "      <td>-0.056667</td>\n",
       "      <td>-0.048124</td>\n",
       "      <td>3.695871e-02</td>\n",
       "      <td>-0.072555</td>\n",
       "      <td>-0.071011</td>\n",
       "      <td>-0.067833</td>\n",
       "      <td>5.502248e-02</td>\n",
       "      <td>-1.041479e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>cities__affirmative+negated+conjunction+disjun...</td>\n",
       "      <td>-0.080000</td>\n",
       "      <td>-0.072578</td>\n",
       "      <td>5.333573e-02</td>\n",
       "      <td>-0.094180</td>\n",
       "      <td>-0.115288</td>\n",
       "      <td>-0.109250</td>\n",
       "      <td>7.560212e-02</td>\n",
       "      <td>-1.224877e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>element_symb__affirmative+negated</td>\n",
       "      <td>-0.013158</td>\n",
       "      <td>-0.012658</td>\n",
       "      <td>1.291110e-02</td>\n",
       "      <td>-0.023026</td>\n",
       "      <td>-0.040541</td>\n",
       "      <td>-0.039216</td>\n",
       "      <td>3.085962e-02</td>\n",
       "      <td>-4.579331e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>element_symb__affirmative+disjunction</td>\n",
       "      <td>-0.137681</td>\n",
       "      <td>-0.127959</td>\n",
       "      <td>8.470442e-02</td>\n",
       "      <td>-0.153955</td>\n",
       "      <td>-0.156934</td>\n",
       "      <td>-0.151943</td>\n",
       "      <td>1.062579e-01</td>\n",
       "      <td>-2.025634e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>element_symb__negated+conjunction</td>\n",
       "      <td>-0.101449</td>\n",
       "      <td>-0.095815</td>\n",
       "      <td>7.986078e-02</td>\n",
       "      <td>-0.128377</td>\n",
       "      <td>-0.093066</td>\n",
       "      <td>-0.081862</td>\n",
       "      <td>6.419752e-02</td>\n",
       "      <td>-1.009118e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>element_symb__affirmative+negated+conjunction+...</td>\n",
       "      <td>-0.173913</td>\n",
       "      <td>-0.165917</td>\n",
       "      <td>1.178062e-01</td>\n",
       "      <td>-0.230123</td>\n",
       "      <td>-0.185219</td>\n",
       "      <td>-0.180444</td>\n",
       "      <td>1.233812e-01</td>\n",
       "      <td>-2.359114e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>facts__affirmative+negated</td>\n",
       "      <td>-0.084071</td>\n",
       "      <td>-0.065572</td>\n",
       "      <td>3.432238e-02</td>\n",
       "      <td>-0.045978</td>\n",
       "      <td>-0.099330</td>\n",
       "      <td>-0.091661</td>\n",
       "      <td>6.045906e-02</td>\n",
       "      <td>-8.966229e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>facts__affirmative+disjunction</td>\n",
       "      <td>-0.098592</td>\n",
       "      <td>-0.093686</td>\n",
       "      <td>8.472200e-02</td>\n",
       "      <td>-0.153020</td>\n",
       "      <td>-0.218160</td>\n",
       "      <td>-0.232464</td>\n",
       "      <td>1.422455e-01</td>\n",
       "      <td>-2.392702e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>facts__negated+conjunction</td>\n",
       "      <td>-0.169014</td>\n",
       "      <td>-0.150275</td>\n",
       "      <td>7.715170e-02</td>\n",
       "      <td>-0.173585</td>\n",
       "      <td>-0.189858</td>\n",
       "      <td>-0.173638</td>\n",
       "      <td>1.120891e-01</td>\n",
       "      <td>-2.085259e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>facts__affirmative+negated+conjunction+disjunc...</td>\n",
       "      <td>-0.152582</td>\n",
       "      <td>-0.140331</td>\n",
       "      <td>7.475991e-02</td>\n",
       "      <td>-0.128504</td>\n",
       "      <td>-0.173939</td>\n",
       "      <td>-0.168066</td>\n",
       "      <td>9.911590e-02</td>\n",
       "      <td>-1.633740e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>inventors__affirmative+negated</td>\n",
       "      <td>-0.042683</td>\n",
       "      <td>-0.041897</td>\n",
       "      <td>4.072750e-02</td>\n",
       "      <td>-0.089523</td>\n",
       "      <td>-0.069444</td>\n",
       "      <td>-0.072413</td>\n",
       "      <td>6.662170e-02</td>\n",
       "      <td>-1.167077e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>inventors__affirmative+disjunction</td>\n",
       "      <td>-0.109890</td>\n",
       "      <td>-0.115885</td>\n",
       "      <td>5.125949e-02</td>\n",
       "      <td>-0.130888</td>\n",
       "      <td>-0.149171</td>\n",
       "      <td>-0.154131</td>\n",
       "      <td>9.314296e-02</td>\n",
       "      <td>-1.658311e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>inventors__negated+conjunction</td>\n",
       "      <td>-0.197802</td>\n",
       "      <td>-0.151424</td>\n",
       "      <td>1.271030e-01</td>\n",
       "      <td>-0.242914</td>\n",
       "      <td>-0.308011</td>\n",
       "      <td>-0.258532</td>\n",
       "      <td>1.727039e-01</td>\n",
       "      <td>-3.083661e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>inventors__affirmative+negated+conjunction+dis...</td>\n",
       "      <td>-0.164835</td>\n",
       "      <td>-0.137280</td>\n",
       "      <td>1.090399e-01</td>\n",
       "      <td>-0.218577</td>\n",
       "      <td>-0.259669</td>\n",
       "      <td>-0.228873</td>\n",
       "      <td>1.493716e-01</td>\n",
       "      <td>-2.664578e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>sp_en_trans__affirmative+negated</td>\n",
       "      <td>-0.028169</td>\n",
       "      <td>-0.024691</td>\n",
       "      <td>2.340480e-02</td>\n",
       "      <td>-0.039710</td>\n",
       "      <td>-0.028269</td>\n",
       "      <td>-0.028674</td>\n",
       "      <td>1.558442e-02</td>\n",
       "      <td>-4.114472e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>sp_en_trans__affirmative+disjunction</td>\n",
       "      <td>-0.192982</td>\n",
       "      <td>-0.177209</td>\n",
       "      <td>1.216646e-01</td>\n",
       "      <td>-0.206986</td>\n",
       "      <td>-0.171303</td>\n",
       "      <td>-0.177005</td>\n",
       "      <td>1.080387e-01</td>\n",
       "      <td>-2.137683e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>sp_en_trans__negated+conjunction</td>\n",
       "      <td>-0.046784</td>\n",
       "      <td>-0.046512</td>\n",
       "      <td>3.650491e-02</td>\n",
       "      <td>-0.067766</td>\n",
       "      <td>-0.043924</td>\n",
       "      <td>-0.040650</td>\n",
       "      <td>3.132679e-02</td>\n",
       "      <td>-5.802527e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>sp_en_trans__affirmative+negated+conjunction+d...</td>\n",
       "      <td>-0.169591</td>\n",
       "      <td>-0.174262</td>\n",
       "      <td>1.323373e-01</td>\n",
       "      <td>-0.262572</td>\n",
       "      <td>-0.158126</td>\n",
       "      <td>-0.162651</td>\n",
       "      <td>1.220621e-01</td>\n",
       "      <td>-2.467094e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>ambiguous_truthful_reply+unambiguous_lie+unamb...</td>\n",
       "      <td>-0.115385</td>\n",
       "      <td>-0.118519</td>\n",
       "      <td>7.519370e-02</td>\n",
       "      <td>-0.098123</td>\n",
       "      <td>-0.049505</td>\n",
       "      <td>-0.041322</td>\n",
       "      <td>4.783866e-02</td>\n",
       "      <td>-8.404473e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>animal_class+facts+inventors__affirmative+nega...</td>\n",
       "      <td>-0.146825</td>\n",
       "      <td>-0.128675</td>\n",
       "      <td>8.549259e-02</td>\n",
       "      <td>-0.160522</td>\n",
       "      <td>-0.177977</td>\n",
       "      <td>-0.163309</td>\n",
       "      <td>1.147210e-01</td>\n",
       "      <td>-1.988424e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>animal_class+facts+inventors__affirmative+nega...</td>\n",
       "      <td>-0.061652</td>\n",
       "      <td>-0.052557</td>\n",
       "      <td>4.957473e-02</td>\n",
       "      <td>-0.108845</td>\n",
       "      <td>-0.107361</td>\n",
       "      <td>-0.098593</td>\n",
       "      <td>7.751492e-02</td>\n",
       "      <td>-1.360602e-01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 30
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:58.722068Z",
     "start_time": "2024-12-07T21:26:58.696687Z"
    }
   },
   "cell_type": "code",
   "source": "lyr18_ttpd_probe_to_baseline_linear_probe_eval_except_test_df.describe()",
   "id": "7b934300eaacef93",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "         val_acc     val_f1  val_brier  val_soft_f1  train_acc   train_f1  \\\n",
       "count  60.000000  58.000000  60.000000    58.000000  60.000000  58.000000   \n",
       "mean   -0.101466  -0.080658   0.063920    -0.107107  -0.144992  -0.121156   \n",
       "std     0.101026   0.062597   0.055532     0.077555   0.145254   0.103688   \n",
       "min    -0.500000  -0.250000  -0.001191    -0.302403  -0.750000  -0.600000   \n",
       "25%    -0.150646  -0.128496   0.023496    -0.152900  -0.179788  -0.167306   \n",
       "50%    -0.076585  -0.070942   0.051866    -0.096950  -0.106149  -0.095127   \n",
       "75%    -0.028952  -0.027543   0.082737    -0.046466  -0.053569  -0.044262   \n",
       "max     0.003333   0.003055   0.241213     0.004105   0.000000   0.000000   \n",
       "\n",
       "        train_brier  train_soft_f1  \n",
       "count  6.000000e+01   5.800000e+01  \n",
       "mean   8.562848e-02  -1.331467e-01  \n",
       "std    6.756823e-02   9.059338e-02  \n",
       "min   -1.460392e-17  -3.357172e-01  \n",
       "25%    4.100432e-02  -1.975228e-01  \n",
       "50%    7.628667e-02  -1.195977e-01  \n",
       "75%    1.102161e-01  -6.551389e-02  \n",
       "max    2.680697e-01   8.294688e-10  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>val_acc</th>\n",
       "      <th>val_f1</th>\n",
       "      <th>val_brier</th>\n",
       "      <th>val_soft_f1</th>\n",
       "      <th>train_acc</th>\n",
       "      <th>train_f1</th>\n",
       "      <th>train_brier</th>\n",
       "      <th>train_soft_f1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>60.000000</td>\n",
       "      <td>58.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>58.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>58.000000</td>\n",
       "      <td>6.000000e+01</td>\n",
       "      <td>5.800000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>-0.101466</td>\n",
       "      <td>-0.080658</td>\n",
       "      <td>0.063920</td>\n",
       "      <td>-0.107107</td>\n",
       "      <td>-0.144992</td>\n",
       "      <td>-0.121156</td>\n",
       "      <td>8.562848e-02</td>\n",
       "      <td>-1.331467e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.101026</td>\n",
       "      <td>0.062597</td>\n",
       "      <td>0.055532</td>\n",
       "      <td>0.077555</td>\n",
       "      <td>0.145254</td>\n",
       "      <td>0.103688</td>\n",
       "      <td>6.756823e-02</td>\n",
       "      <td>9.059338e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-0.500000</td>\n",
       "      <td>-0.250000</td>\n",
       "      <td>-0.001191</td>\n",
       "      <td>-0.302403</td>\n",
       "      <td>-0.750000</td>\n",
       "      <td>-0.600000</td>\n",
       "      <td>-1.460392e-17</td>\n",
       "      <td>-3.357172e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>-0.150646</td>\n",
       "      <td>-0.128496</td>\n",
       "      <td>0.023496</td>\n",
       "      <td>-0.152900</td>\n",
       "      <td>-0.179788</td>\n",
       "      <td>-0.167306</td>\n",
       "      <td>4.100432e-02</td>\n",
       "      <td>-1.975228e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>-0.076585</td>\n",
       "      <td>-0.070942</td>\n",
       "      <td>0.051866</td>\n",
       "      <td>-0.096950</td>\n",
       "      <td>-0.106149</td>\n",
       "      <td>-0.095127</td>\n",
       "      <td>7.628667e-02</td>\n",
       "      <td>-1.195977e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>-0.028952</td>\n",
       "      <td>-0.027543</td>\n",
       "      <td>0.082737</td>\n",
       "      <td>-0.046466</td>\n",
       "      <td>-0.053569</td>\n",
       "      <td>-0.044262</td>\n",
       "      <td>1.102161e-01</td>\n",
       "      <td>-6.551389e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.003333</td>\n",
       "      <td>0.003055</td>\n",
       "      <td>0.241213</td>\n",
       "      <td>0.004105</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.680697e-01</td>\n",
       "      <td>8.294688e-10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 31
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "It is slightly interesting that the simple linear probes seem to be doing markedly better within topic and data variant than the TTPD probes, given that Bürger et al found them generalizing similarly well.  ",
   "id": "77db17c9172fc26d"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:58.727940Z",
     "start_time": "2024-12-07T21:26:58.724096Z"
    }
   },
   "cell_type": "code",
   "source": [
    "lyr18_probe_metrics_on_test_dsets: dict[tuple[int,...], dict[int, ConfusionMetrics]] = {\n",
    "    scenario_id: {\n",
    "        dset_idx_in_scenario: test_dset_metrics.lyr18_probe_metrics for dset_idx_in_scenario, test_dset_metrics in test_dsets_metrics.items()\n",
    "    } \n",
    "    for scenario_id, test_dsets_metrics in probes_metrics_on_test_dsets.items()\n",
    "}"
   ],
   "id": "47537f4f62c29e0f",
   "outputs": [],
   "execution_count": 32
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-08T18:49:46.016937Z",
     "start_time": "2024-12-08T18:49:45.881992Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# Just using accuracy because there are going to be _so_ many columns in this analysis's dataframe and examination of validation accuracy vs f1 vs soft-f1 for layer 18 probes showed that they were generally very similar for the standard topics' datasets (because those datasets were specifically constructed to be balanced between true and false statements) \n",
    "class GeneralizationAccuracyWithinVsAcrossTopics(NamedTuple):\n",
    "    affirm_within_topics: float\n",
    "    affirm_across_topics: float\n",
    "    neg_within_topics: float\n",
    "    neg_across_topics: float\n",
    "    conj_within_topics: float\n",
    "    conj_across_topics: float\n",
    "    disj_within_topics: float\n",
    "    disj_across_topics: float\n",
    "    de_affirm_within_topics: float\n",
    "    de_affirm_across_topics: float\n",
    "    de_neg_within_topics: float\n",
    "    de_neg_across_topics: float\n",
    "\n",
    "class TopicGeneralizationSummary(NamedTuple):\n",
    "    avg_across_topic_penalty: float\n",
    "    affirm_across_topic_penalty: float\n",
    "    neg_across_topic_penalty: float\n",
    "    conj_across_topic_penalty: float\n",
    "    disj_across_topic_penalty: float\n",
    "    de_affirm_across_topic_penalty: float\n",
    "    de_neg_across_topic_penalty: float\n",
    "\n",
    "lyr18_probe_generalization_within_vs_across_topics: dict[tuple[int,...], GeneralizationAccuracyWithinVsAcrossTopics] = {}\n",
    "lyr18_probe_topic_generalization_summary: dict[tuple[int,...], TopicGeneralizationSummary] = {}\n",
    "\n",
    "avg_cross_topic_acc_by_source_and_target_data_variants = pd.DataFrame({\n",
    "    \"source_data_variants\": [\"affirm\", \"neg\", \"conj\", \"disj\", \"affirm+neg\", \"affirm+disj\", \"neg+conj\", \"affirm+neg+conj+disj\", \"multitopic_affirm+neg+conj\"],\n",
    "    \"affirm\": [0.0]*9, \"neg\": [0.0]*9, \"conj\": [0.0]*9, \"disj\": [0.0]*9\n",
    "})\n",
    "\n",
    "for scenario_id, test_dsets_metrics in lyr18_probe_metrics_on_test_dsets.items():\n",
    "    curr_standard_categs = scenario_standard_categs[scenario_id]\n",
    "    curr_data_variants = scenario_data_variants[scenario_id]\n",
    "    if not curr_standard_categs:\n",
    "        continue\n",
    "    unseen_standard_categs = list(set(dset_idxs_for_4way_topics.keys()) - set(curr_standard_categs))\n",
    "    metrics_on_test_dsets = lyr18_probe_metrics_on_test_dsets[scenario_id]\n",
    "    \n",
    "    affirm_within_topics_acc = np.nan\n",
    "    if \"affirm\" not in curr_data_variants:\n",
    "        same_topics_affirm_test_metrics = ConfusionMetrics.combine(*[\n",
    "            metrics_on_test_dsets[dset_idxs_for_4way_topics[topic_nm][\"affirm\"]] for topic_nm in curr_standard_categs\n",
    "        ])\n",
    "        affirm_within_topics_acc = same_topics_affirm_test_metrics.get_traditional_metrics().accuracy\n",
    "    across_topics_affirm_test_metrics = ConfusionMetrics.combine(*[\n",
    "        metrics_on_test_dsets[dset_idxs_for_4way_topics[topic_nm][\"affirm\"]] for topic_nm in unseen_standard_categs\n",
    "    ])\n",
    "    affirm_across_topics_acc = across_topics_affirm_test_metrics.get_traditional_metrics().accuracy\n",
    "    neg_within_topics_acc = np.nan\n",
    "    if \"neg\" not in curr_data_variants:\n",
    "        same_topics_neg_test_metrics = ConfusionMetrics.combine(*[\n",
    "            metrics_on_test_dsets[dset_idxs_for_4way_topics[topic_nm][\"neg\"]] for topic_nm in curr_standard_categs\n",
    "        ])\n",
    "        neg_within_topics_acc = same_topics_neg_test_metrics.get_traditional_metrics().accuracy\n",
    "    across_topics_neg_test_metrics = ConfusionMetrics.combine(*[\n",
    "        metrics_on_test_dsets[dset_idxs_for_4way_topics[topic_nm][\"neg\"]] for topic_nm in unseen_standard_categs\n",
    "    ])\n",
    "    neg_across_topics_acc = across_topics_neg_test_metrics.get_traditional_metrics().accuracy\n",
    "    conj_within_topics_acc = np.nan\n",
    "    if \"conj\" not in curr_data_variants:\n",
    "        same_topics_conj_test_metrics = ConfusionMetrics.combine(*[\n",
    "            metrics_on_test_dsets[dset_idxs_for_4way_topics[topic_nm][\"conj\"]] for topic_nm in curr_standard_categs\n",
    "        ])\n",
    "        conj_within_topics_acc = same_topics_conj_test_metrics.get_traditional_metrics().accuracy\n",
    "    across_topics_conj_test_metrics = ConfusionMetrics.combine(*[\n",
    "        metrics_on_test_dsets[dset_idxs_for_4way_topics[topic_nm][\"conj\"]] for topic_nm in unseen_standard_categs\n",
    "    ])\n",
    "    conj_across_topics_acc = across_topics_conj_test_metrics.get_traditional_metrics().accuracy\n",
    "    disj_within_topics_acc = np.nan\n",
    "    if \"disj\" not in curr_data_variants:\n",
    "        same_topics_disj_test_metrics = ConfusionMetrics.combine(*[\n",
    "            metrics_on_test_dsets[dset_idxs_for_4way_topics[topic_nm][\"disj\"]] for topic_nm in curr_standard_categs\n",
    "        ])\n",
    "        disj_within_topics_acc = same_topics_disj_test_metrics.get_traditional_metrics().accuracy\n",
    "    across_topics_disj_test_metrics = ConfusionMetrics.combine(*[\n",
    "        metrics_on_test_dsets[dset_idxs_for_4way_topics[topic_nm][\"disj\"]] for topic_nm in unseen_standard_categs\n",
    "    ])\n",
    "    disj_across_topics_acc = across_topics_disj_test_metrics.get_traditional_metrics().accuracy\n",
    "    assert \"de_affirm\" not in curr_data_variants\n",
    "    same_topics_de_affirm_test_metrics = ConfusionMetrics.combine(*[\n",
    "        metrics_on_test_dsets[dset_idxs_for_4way_topics[topic_nm][\"de_affirm\"]] for topic_nm in curr_standard_categs\n",
    "    ])\n",
    "    de_affirm_within_topics_acc = same_topics_de_affirm_test_metrics.get_traditional_metrics().accuracy\n",
    "    across_topics_de_affirm_test_metrics = ConfusionMetrics.combine(*[\n",
    "        metrics_on_test_dsets[dset_idxs_for_4way_topics[topic_nm][\"de_affirm\"]] for topic_nm in unseen_standard_categs\n",
    "    ])\n",
    "    de_affirm_across_topics_acc = across_topics_de_affirm_test_metrics.get_traditional_metrics().accuracy\n",
    "    assert \"de_neg\" not in curr_data_variants\n",
    "    same_topics_de_neg_test_metrics = ConfusionMetrics.combine(*[\n",
    "        metrics_on_test_dsets[dset_idxs_for_4way_topics[topic_nm][\"de_neg\"]] for topic_nm in curr_standard_categs\n",
    "    ])\n",
    "    de_neg_within_topics_acc = same_topics_de_neg_test_metrics.get_traditional_metrics().accuracy\n",
    "    across_topics_de_neg_test_metrics = ConfusionMetrics.combine(*[\n",
    "        metrics_on_test_dsets[dset_idxs_for_4way_topics[topic_nm][\"de_neg\"]] for topic_nm in unseen_standard_categs\n",
    "    ])\n",
    "    de_neg_across_topics_acc = across_topics_de_neg_test_metrics.get_traditional_metrics().accuracy\n",
    "    \n",
    "    if \"other\" not in curr_data_variants:\n",
    "        avg_cross_topic_acc_source_row_selector: pd.Series\n",
    "        match curr_data_variants:\n",
    "            case _ if curr_data_variants in [{\"affirm\"}, {\"neg\"}, {\"conj\"}, {\"disj\"}]:\n",
    "                avg_cross_topic_acc_source_row_selector = avg_cross_topic_acc_by_source_and_target_data_variants[\"source_data_variants\"] == str(list(curr_data_variants)[0])\n",
    "            case _ if curr_data_variants == {\"affirm\", \"neg\"}:\n",
    "                avg_cross_topic_acc_source_row_selector = avg_cross_topic_acc_by_source_and_target_data_variants[\"source_data_variants\"] == \"affirm+neg\"\n",
    "            case _ if curr_data_variants == {\"affirm\", \"disj\"}:\n",
    "                avg_cross_topic_acc_source_row_selector = avg_cross_topic_acc_by_source_and_target_data_variants[\"source_data_variants\"] == \"affirm+disj\"\n",
    "            case _ if curr_data_variants == {\"neg\", \"conj\"}:\n",
    "                avg_cross_topic_acc_source_row_selector = avg_cross_topic_acc_by_source_and_target_data_variants[\"source_data_variants\"] == \"neg+conj\"\n",
    "            case _ if curr_data_variants == {\"affirm\", \"neg\", \"conj\", \"disj\"}:\n",
    "                avg_cross_topic_acc_source_row_selector = avg_cross_topic_acc_by_source_and_target_data_variants[\"source_data_variants\"] == \"affirm+neg+conj+disj\"\n",
    "            case _ if curr_data_variants == {\"affirm\", \"neg\", \"conj\"}:\n",
    "                avg_cross_topic_acc_source_row_selector = avg_cross_topic_acc_by_source_and_target_data_variants[\"source_data_variants\"] == \"multitopic_affirm+neg+conj\"\n",
    "            case _:\n",
    "                raise ValueError(f\"invalid value {curr_data_variants} for curr_data_variants for scenario {scenario_id}\")\n",
    "        avg_cross_topic_acc_by_source_and_target_data_variants.loc[avg_cross_topic_acc_source_row_selector, \"affirm\"] += affirm_across_topics_acc\n",
    "        avg_cross_topic_acc_by_source_and_target_data_variants.loc[avg_cross_topic_acc_source_row_selector, \"neg\"] += neg_across_topics_acc\n",
    "        avg_cross_topic_acc_by_source_and_target_data_variants.loc[avg_cross_topic_acc_source_row_selector, \"conj\"] += conj_across_topics_acc\n",
    "        avg_cross_topic_acc_by_source_and_target_data_variants.loc[avg_cross_topic_acc_source_row_selector, \"disj\"] += disj_across_topics_acc\n",
    "    \n",
    "    lyr18_probe_generalization_within_vs_across_topics[scenario_id] = GeneralizationAccuracyWithinVsAcrossTopics(\n",
    "        affirm_within_topics=affirm_within_topics_acc, affirm_across_topics=affirm_across_topics_acc, neg_within_topics=neg_within_topics_acc,\n",
    "        neg_across_topics=neg_across_topics_acc, conj_within_topics=conj_within_topics_acc, conj_across_topics=conj_across_topics_acc,\n",
    "        disj_within_topics=disj_within_topics_acc, disj_across_topics=disj_across_topics_acc, de_affirm_within_topics=de_affirm_within_topics_acc,\n",
    "        de_affirm_across_topics=de_affirm_across_topics_acc, de_neg_within_topics=de_neg_within_topics_acc, de_neg_across_topics=de_neg_across_topics_acc\n",
    "    )\n",
    "    \n",
    "    affirm_across_topic_acc_penalty = affirm_within_topics_acc-affirm_across_topics_acc\n",
    "    neg_across_topic_acc_penalty = neg_within_topics_acc-neg_across_topics_acc\n",
    "    conj_across_topic_acc_penalty = conj_within_topics_acc-conj_across_topics_acc\n",
    "    disj_across_topic_acc_penalty = disj_within_topics_acc-disj_across_topics_acc\n",
    "    de_affirm_across_topic_acc_penalty = de_affirm_within_topics_acc-de_affirm_across_topics_acc\n",
    "    de_neg_across_topic_acc_penalty = de_neg_within_topics_acc-de_neg_across_topics_acc\n",
    "    avg_across_topic_acc_penalty = float(np.nanmean([affirm_across_topic_acc_penalty, neg_across_topic_acc_penalty, conj_across_topic_acc_penalty, disj_across_topic_acc_penalty, de_affirm_across_topic_acc_penalty, de_neg_across_topic_acc_penalty]))\n",
    "    \n",
    "    lyr18_probe_topic_generalization_summary[scenario_id] = TopicGeneralizationSummary(\n",
    "        avg_across_topic_penalty=avg_across_topic_acc_penalty, affirm_across_topic_penalty=affirm_across_topic_acc_penalty, \n",
    "        neg_across_topic_penalty=neg_across_topic_acc_penalty, conj_across_topic_penalty=conj_across_topic_acc_penalty,\n",
    "        disj_across_topic_penalty=disj_across_topic_acc_penalty, de_affirm_across_topic_penalty=de_affirm_across_topic_acc_penalty,\n",
    "        de_neg_across_topic_penalty=de_neg_across_topic_acc_penalty\n",
    "    )\n",
    "\n",
    "avg_cross_topic_acc_by_source_and_target_data_variants.loc[~avg_cross_topic_acc_by_source_and_target_data_variants[\"source_data_variants\"].str.startswith(\"multitopic_\"), [\"affirm\", \"neg\", \"conj\", \"disj\"]] /= num_standard_topics\n",
    "\n",
    "\n",
    "generalization_within_vs_across_topics_df = pd.DataFrame([\n",
    "    { \"scenario\": scenario_labels[scenario_id], **generalization_info._asdict()  } for scenario_id, generalization_info in lyr18_probe_generalization_within_vs_across_topics.items()\n",
    "])\n",
    "topic_generalization_summary_df = pd.DataFrame([\n",
    "    { \"scenario\": scenario_labels[scenario_id], **generalization_summary._asdict()  } for scenario_id, generalization_summary in lyr18_probe_topic_generalization_summary.items()\n",
    "])"
   ],
   "id": "99a414f67e57c59d",
   "outputs": [],
   "execution_count": 95
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-08T18:49:46.073825Z",
     "start_time": "2024-12-08T18:49:46.055967Z"
    }
   },
   "cell_type": "code",
   "source": "generalization_within_vs_across_topics_df",
   "id": "c4e6ba7cd6f7259d",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "                                             scenario  affirm_within_topics  \\\n",
       "0                           animal_class__affirmative                   NaN   \n",
       "1                           animal_class__conjunction              0.993902   \n",
       "2                           animal_class__disjunction              0.829268   \n",
       "3                               animal_class__negated              0.859756   \n",
       "4                                 cities__affirmative                   NaN   \n",
       "5                                 cities__conjunction              0.982620   \n",
       "6                                 cities__disjunction              0.951203   \n",
       "7                                     cities__negated              0.691845   \n",
       "8                           element_symb__affirmative                   NaN   \n",
       "9                           element_symb__conjunction              0.715054   \n",
       "10                          element_symb__disjunction              0.645161   \n",
       "11                              element_symb__negated              0.392473   \n",
       "12                                 facts__affirmative                   NaN   \n",
       "13                                 facts__conjunction              0.775401   \n",
       "14                                 facts__disjunction              0.648841   \n",
       "15                                     facts__negated              0.713012   \n",
       "16                             inventors__affirmative                   NaN   \n",
       "17                             inventors__conjunction              0.605911   \n",
       "18                             inventors__disjunction              0.854680   \n",
       "19                                 inventors__negated              0.416256   \n",
       "20                               sp_en_trans__negated              0.497175   \n",
       "21                           sp_en_trans__affirmative                   NaN   \n",
       "22                           sp_en_trans__conjunction              0.550847   \n",
       "23                           sp_en_trans__disjunction              0.508475   \n",
       "24                  animal_class__affirmative+negated                   NaN   \n",
       "25              animal_class__affirmative+disjunction                   NaN   \n",
       "26                  animal_class__negated+conjunction              0.993902   \n",
       "27  animal_class__affirmative+negated+conjunction+...                   NaN   \n",
       "28                        cities__affirmative+negated                   NaN   \n",
       "29                    cities__affirmative+disjunction                   NaN   \n",
       "30                        cities__negated+conjunction              0.981283   \n",
       "31  cities__affirmative+negated+conjunction+disjun...                   NaN   \n",
       "32                  element_symb__affirmative+negated                   NaN   \n",
       "33              element_symb__affirmative+disjunction                   NaN   \n",
       "34                  element_symb__negated+conjunction              0.500000   \n",
       "35  element_symb__affirmative+negated+conjunction+...                   NaN   \n",
       "36                         facts__affirmative+negated                   NaN   \n",
       "37                     facts__affirmative+disjunction                   NaN   \n",
       "38                         facts__negated+conjunction              0.871658   \n",
       "39  facts__affirmative+negated+conjunction+disjunc...                   NaN   \n",
       "40                     inventors__affirmative+negated                   NaN   \n",
       "41                 inventors__affirmative+disjunction                   NaN   \n",
       "42                     inventors__negated+conjunction              0.435961   \n",
       "43  inventors__affirmative+negated+conjunction+dis...                   NaN   \n",
       "44                   sp_en_trans__affirmative+negated                   NaN   \n",
       "45               sp_en_trans__affirmative+disjunction                   NaN   \n",
       "46                   sp_en_trans__negated+conjunction              0.500000   \n",
       "47  sp_en_trans__affirmative+negated+conjunction+d...                   NaN   \n",
       "48  animal_class+facts+inventors__affirmative+nega...                   NaN   \n",
       "49  animal_class+facts+inventors__affirmative+nega...                   NaN   \n",
       "\n",
       "    affirm_across_topics  neg_within_topics  neg_across_topics  \\\n",
       "0               0.840160           0.835366           0.629704   \n",
       "1               0.829504           0.932927           0.645688   \n",
       "2               0.857476           0.487805           0.511156   \n",
       "3               0.877456                NaN           0.837829   \n",
       "4               0.683423           0.804813           0.511670   \n",
       "5               0.792340           0.969251           0.551765   \n",
       "6               0.667265           0.514037           0.471574   \n",
       "7               0.597247                NaN           0.779174   \n",
       "8               0.903388           0.500000           0.527340   \n",
       "9               0.693391           0.500000           0.525663   \n",
       "10              0.621603           0.360215           0.577994   \n",
       "11              0.286146                NaN           0.875210   \n",
       "12              0.924021           0.552585           0.513047   \n",
       "13              0.589025           0.504456           0.509210   \n",
       "14              0.776285           0.506239           0.508442   \n",
       "15              0.727168                NaN           0.869148   \n",
       "16              0.830858           0.576355           0.729084   \n",
       "17              0.594712           0.610837           0.500181   \n",
       "18              0.569721           0.554187           0.499457   \n",
       "19              0.181094                NaN           0.649402   \n",
       "20              0.403128                NaN           0.821898   \n",
       "21              0.879844           0.274011           0.750800   \n",
       "22              0.774973           0.500000           0.652328   \n",
       "23              0.643086           0.497175           0.562033   \n",
       "24              0.909424                NaN           0.850150   \n",
       "25              0.858142           0.865854           0.643024   \n",
       "26              0.858142                NaN           0.704296   \n",
       "27              0.887779                NaN           0.779887   \n",
       "28              0.845003                NaN           0.828247   \n",
       "29              0.719330           0.790775           0.543986   \n",
       "30              0.847995                NaN           0.675643   \n",
       "31              0.816278                NaN           0.704967   \n",
       "32              0.909091                NaN           0.902382   \n",
       "33              0.899363           0.736559           0.693056   \n",
       "34              0.638376                NaN           0.885273   \n",
       "35              0.921503                NaN           0.904394   \n",
       "36              0.909823                NaN           0.920568   \n",
       "37              0.926708           0.607843           0.779739   \n",
       "38              0.936685                NaN           0.954720   \n",
       "39              0.930929                NaN           0.945510   \n",
       "40              0.845346                NaN           0.876494   \n",
       "41              0.850054           0.593596           0.725462   \n",
       "42              0.497646                NaN           0.874683   \n",
       "43              0.870337                NaN           0.506338   \n",
       "44              0.910060                NaN           0.853893   \n",
       "45              0.854604           0.429379           0.600071   \n",
       "46              0.479559                NaN           0.878066   \n",
       "47              0.910060                NaN           0.889797   \n",
       "48              0.976424                NaN           0.877210   \n",
       "49              0.976916                NaN           0.943517   \n",
       "\n",
       "    conj_within_topics  conj_across_topics  disj_within_topics  \\\n",
       "0             0.596000            0.648942            0.522000   \n",
       "1                  NaN            0.749285            0.502000   \n",
       "2             0.882000            0.692396                 NaN   \n",
       "3             0.560000            0.644940            0.498000   \n",
       "4             0.618825            0.578400            0.476000   \n",
       "5                  NaN            0.782400            0.518000   \n",
       "6             0.640854            0.695200                 NaN   \n",
       "7             0.502003            0.545600            0.484000   \n",
       "8             0.960000            0.802173            0.482000   \n",
       "9                  NaN            0.817896            0.696000   \n",
       "10            0.642000            0.576901                 NaN   \n",
       "11            0.272000            0.506861            0.448000   \n",
       "12            0.680000            0.869354            0.596000   \n",
       "13                 NaN            0.830475            0.574000   \n",
       "14            0.524000            0.681246                 NaN   \n",
       "15            0.474000            0.518296            0.462000   \n",
       "16            0.756000            0.744140            0.652000   \n",
       "17                 NaN            0.713265            0.446000   \n",
       "18            0.682000            0.779017                 NaN   \n",
       "19            0.576000            0.500858            0.544000   \n",
       "20            0.440000            0.462836            0.536000   \n",
       "21            0.834000            0.675815            0.730000   \n",
       "22                 NaN            0.825043            0.574000   \n",
       "23            0.874000            0.554031                 NaN   \n",
       "24            0.622000            0.687250            0.500000   \n",
       "25            0.668000            0.679531                 NaN   \n",
       "26                 NaN            0.765009            0.502000   \n",
       "27                 NaN            0.738708                 NaN   \n",
       "28            0.584780            0.636000            0.474000   \n",
       "29            0.678238            0.626400                 NaN   \n",
       "30                 NaN            0.755200            0.560000   \n",
       "31                 NaN            0.686400                 NaN   \n",
       "32            0.810000            0.613493            0.544000   \n",
       "33            0.778000            0.629503                 NaN   \n",
       "34                 NaN            0.672956            0.566000   \n",
       "35                 NaN            0.668382                 NaN   \n",
       "36            0.546000            0.583762            0.482000   \n",
       "37            0.714000            0.882218                 NaN   \n",
       "38                 NaN            0.750429            0.712000   \n",
       "39                 NaN            0.736707                 NaN   \n",
       "40            0.580000            0.569468            0.572000   \n",
       "41            0.752000            0.803888                 NaN   \n",
       "42                 NaN            0.503716            0.538000   \n",
       "43                 NaN            0.794454                 NaN   \n",
       "44            0.922000            0.601487            0.532000   \n",
       "45            0.796000            0.570898                 NaN   \n",
       "46                 NaN            0.580046            0.528000   \n",
       "47                 NaN            0.559748                 NaN   \n",
       "48                 NaN            0.845476            0.611333   \n",
       "49            0.753333            0.895116            0.507333   \n",
       "\n",
       "    disj_across_topics  de_affirm_within_topics  de_affirm_across_topics  \\\n",
       "0             0.592400                 0.900000                 0.688000   \n",
       "1             0.612400                 0.880000                 0.656000   \n",
       "2             0.598400                 0.500000                 0.664000   \n",
       "3             0.513600                 0.760000                 0.696000   \n",
       "4             0.492400                 0.940000                 0.600000   \n",
       "5             0.555200                 0.980000                 0.704000   \n",
       "6             0.668800                 0.660000                 0.568000   \n",
       "7             0.489200                 0.640000                 0.552000   \n",
       "8             0.506800                 0.660000                 0.852000   \n",
       "9             0.585600                 0.740000                 0.792000   \n",
       "10            0.609200                 0.640000                 0.672000   \n",
       "11            0.506800                 0.300000                 0.488000   \n",
       "12            0.527600                 0.640000                 0.820000   \n",
       "13            0.494800                 0.620000                 0.656000   \n",
       "14            0.569600                 0.540000                 0.620000   \n",
       "15            0.505200                 0.540000                 0.576000   \n",
       "16            0.619200                 0.820000                 0.804000   \n",
       "17            0.514400                 0.480000                 0.672000   \n",
       "18            0.573600                 0.640000                 0.504000   \n",
       "19            0.462400                 0.400000                 0.396000   \n",
       "20            0.490400                 0.500000                 0.420000   \n",
       "21            0.612400                 0.860000                 0.744000   \n",
       "22            0.590400                 0.540000                 0.788000   \n",
       "23            0.519200                 0.540000                 0.584000   \n",
       "24            0.630800                 0.900000                 0.736000   \n",
       "25            0.610400                 0.900000                 0.700000   \n",
       "26            0.611200                 0.880000                 0.692000   \n",
       "27            0.644400                 0.880000                 0.716000   \n",
       "28            0.596800                 0.960000                 0.696000   \n",
       "29            0.589600                 0.920000                 0.664000   \n",
       "30            0.614000                 0.980000                 0.724000   \n",
       "31            0.662000                 0.980000                 0.696000   \n",
       "32            0.538400                 0.580000                 0.772000   \n",
       "33            0.628800                 0.740000                 0.768000   \n",
       "34            0.579600                 0.400000                 0.796000   \n",
       "35            0.643200                 0.620000                 0.832000   \n",
       "36            0.544800                 0.600000                 0.776000   \n",
       "37            0.553200                 0.580000                 0.788000   \n",
       "38            0.574800                 0.640000                 0.804000   \n",
       "39            0.548800                 0.660000                 0.792000   \n",
       "40            0.522400                 0.820000                 0.656000   \n",
       "41            0.540000                 0.820000                 0.828000   \n",
       "42            0.487600                 0.420000                 0.540000   \n",
       "43            0.620400                 0.700000                 0.816000   \n",
       "44            0.567200                 0.880000                 0.788000   \n",
       "45            0.504400                 0.840000                 0.656000   \n",
       "46            0.498400                 0.740000                 0.480000   \n",
       "47            0.513200                 0.980000                 0.784000   \n",
       "48            0.526000                 0.800000                 0.813333   \n",
       "49            0.510667                 0.766667                 0.826667   \n",
       "\n",
       "    de_neg_within_topics  de_neg_across_topics  \n",
       "0               0.520000              0.473896  \n",
       "1               0.540000              0.489960  \n",
       "2               0.540000              0.550201  \n",
       "3               0.860000              0.638554  \n",
       "4               0.500000              0.481928  \n",
       "5               0.840000              0.526104  \n",
       "6               0.580000              0.497992  \n",
       "7               1.000000              0.650602  \n",
       "8               0.520000              0.546185  \n",
       "9               0.520000              0.514056  \n",
       "10              0.240000              0.481928  \n",
       "11              0.920000              0.730924  \n",
       "12              0.489796              0.428000  \n",
       "13              0.489796              0.524000  \n",
       "14              0.510204              0.480000  \n",
       "15              0.714286              0.664000  \n",
       "16              0.600000              0.534137  \n",
       "17              0.600000              0.473896  \n",
       "18              0.500000              0.502008  \n",
       "19              0.740000              0.614458  \n",
       "20              0.960000              0.811245  \n",
       "21              0.500000              0.534137  \n",
       "22              0.460000              0.530120  \n",
       "23              0.460000              0.538153  \n",
       "24              0.760000              0.706827  \n",
       "25              0.540000              0.457831  \n",
       "26              0.640000              0.522088  \n",
       "27              0.740000              0.578313  \n",
       "28              1.000000              0.718876  \n",
       "29              0.500000              0.489960  \n",
       "30              0.980000              0.610442  \n",
       "31              0.960000              0.626506  \n",
       "32              0.860000              0.730924  \n",
       "33              0.520000              0.550201  \n",
       "34              0.780000              0.835341  \n",
       "35              0.760000              0.783133  \n",
       "36              0.714286              0.768000  \n",
       "37              0.510204              0.608000  \n",
       "38              0.714286              0.828000  \n",
       "39              0.755102              0.780000  \n",
       "40              0.840000              0.742972  \n",
       "41              0.640000              0.562249  \n",
       "42              0.840000              0.722892  \n",
       "43              0.600000              0.502008  \n",
       "44              0.540000              0.738956  \n",
       "45              0.520000              0.465863  \n",
       "46              0.920000              0.835341  \n",
       "47              0.540000              0.767068  \n",
       "48              0.765101              0.660000  \n",
       "49              0.778523              0.686667  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>scenario</th>\n",
       "      <th>affirm_within_topics</th>\n",
       "      <th>affirm_across_topics</th>\n",
       "      <th>neg_within_topics</th>\n",
       "      <th>neg_across_topics</th>\n",
       "      <th>conj_within_topics</th>\n",
       "      <th>conj_across_topics</th>\n",
       "      <th>disj_within_topics</th>\n",
       "      <th>disj_across_topics</th>\n",
       "      <th>de_affirm_within_topics</th>\n",
       "      <th>de_affirm_across_topics</th>\n",
       "      <th>de_neg_within_topics</th>\n",
       "      <th>de_neg_across_topics</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>animal_class__affirmative</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.840160</td>\n",
       "      <td>0.835366</td>\n",
       "      <td>0.629704</td>\n",
       "      <td>0.596000</td>\n",
       "      <td>0.648942</td>\n",
       "      <td>0.522000</td>\n",
       "      <td>0.592400</td>\n",
       "      <td>0.900000</td>\n",
       "      <td>0.688000</td>\n",
       "      <td>0.520000</td>\n",
       "      <td>0.473896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>animal_class__conjunction</td>\n",
       "      <td>0.993902</td>\n",
       "      <td>0.829504</td>\n",
       "      <td>0.932927</td>\n",
       "      <td>0.645688</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.749285</td>\n",
       "      <td>0.502000</td>\n",
       "      <td>0.612400</td>\n",
       "      <td>0.880000</td>\n",
       "      <td>0.656000</td>\n",
       "      <td>0.540000</td>\n",
       "      <td>0.489960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>animal_class__disjunction</td>\n",
       "      <td>0.829268</td>\n",
       "      <td>0.857476</td>\n",
       "      <td>0.487805</td>\n",
       "      <td>0.511156</td>\n",
       "      <td>0.882000</td>\n",
       "      <td>0.692396</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.598400</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.664000</td>\n",
       "      <td>0.540000</td>\n",
       "      <td>0.550201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>animal_class__negated</td>\n",
       "      <td>0.859756</td>\n",
       "      <td>0.877456</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.837829</td>\n",
       "      <td>0.560000</td>\n",
       "      <td>0.644940</td>\n",
       "      <td>0.498000</td>\n",
       "      <td>0.513600</td>\n",
       "      <td>0.760000</td>\n",
       "      <td>0.696000</td>\n",
       "      <td>0.860000</td>\n",
       "      <td>0.638554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cities__affirmative</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.683423</td>\n",
       "      <td>0.804813</td>\n",
       "      <td>0.511670</td>\n",
       "      <td>0.618825</td>\n",
       "      <td>0.578400</td>\n",
       "      <td>0.476000</td>\n",
       "      <td>0.492400</td>\n",
       "      <td>0.940000</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.481928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>cities__conjunction</td>\n",
       "      <td>0.982620</td>\n",
       "      <td>0.792340</td>\n",
       "      <td>0.969251</td>\n",
       "      <td>0.551765</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.782400</td>\n",
       "      <td>0.518000</td>\n",
       "      <td>0.555200</td>\n",
       "      <td>0.980000</td>\n",
       "      <td>0.704000</td>\n",
       "      <td>0.840000</td>\n",
       "      <td>0.526104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>cities__disjunction</td>\n",
       "      <td>0.951203</td>\n",
       "      <td>0.667265</td>\n",
       "      <td>0.514037</td>\n",
       "      <td>0.471574</td>\n",
       "      <td>0.640854</td>\n",
       "      <td>0.695200</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.668800</td>\n",
       "      <td>0.660000</td>\n",
       "      <td>0.568000</td>\n",
       "      <td>0.580000</td>\n",
       "      <td>0.497992</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>cities__negated</td>\n",
       "      <td>0.691845</td>\n",
       "      <td>0.597247</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.779174</td>\n",
       "      <td>0.502003</td>\n",
       "      <td>0.545600</td>\n",
       "      <td>0.484000</td>\n",
       "      <td>0.489200</td>\n",
       "      <td>0.640000</td>\n",
       "      <td>0.552000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.650602</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>element_symb__affirmative</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.903388</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.527340</td>\n",
       "      <td>0.960000</td>\n",
       "      <td>0.802173</td>\n",
       "      <td>0.482000</td>\n",
       "      <td>0.506800</td>\n",
       "      <td>0.660000</td>\n",
       "      <td>0.852000</td>\n",
       "      <td>0.520000</td>\n",
       "      <td>0.546185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>element_symb__conjunction</td>\n",
       "      <td>0.715054</td>\n",
       "      <td>0.693391</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.525663</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.817896</td>\n",
       "      <td>0.696000</td>\n",
       "      <td>0.585600</td>\n",
       "      <td>0.740000</td>\n",
       "      <td>0.792000</td>\n",
       "      <td>0.520000</td>\n",
       "      <td>0.514056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>element_symb__disjunction</td>\n",
       "      <td>0.645161</td>\n",
       "      <td>0.621603</td>\n",
       "      <td>0.360215</td>\n",
       "      <td>0.577994</td>\n",
       "      <td>0.642000</td>\n",
       "      <td>0.576901</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.609200</td>\n",
       "      <td>0.640000</td>\n",
       "      <td>0.672000</td>\n",
       "      <td>0.240000</td>\n",
       "      <td>0.481928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>element_symb__negated</td>\n",
       "      <td>0.392473</td>\n",
       "      <td>0.286146</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.875210</td>\n",
       "      <td>0.272000</td>\n",
       "      <td>0.506861</td>\n",
       "      <td>0.448000</td>\n",
       "      <td>0.506800</td>\n",
       "      <td>0.300000</td>\n",
       "      <td>0.488000</td>\n",
       "      <td>0.920000</td>\n",
       "      <td>0.730924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>facts__affirmative</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.924021</td>\n",
       "      <td>0.552585</td>\n",
       "      <td>0.513047</td>\n",
       "      <td>0.680000</td>\n",
       "      <td>0.869354</td>\n",
       "      <td>0.596000</td>\n",
       "      <td>0.527600</td>\n",
       "      <td>0.640000</td>\n",
       "      <td>0.820000</td>\n",
       "      <td>0.489796</td>\n",
       "      <td>0.428000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>facts__conjunction</td>\n",
       "      <td>0.775401</td>\n",
       "      <td>0.589025</td>\n",
       "      <td>0.504456</td>\n",
       "      <td>0.509210</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.830475</td>\n",
       "      <td>0.574000</td>\n",
       "      <td>0.494800</td>\n",
       "      <td>0.620000</td>\n",
       "      <td>0.656000</td>\n",
       "      <td>0.489796</td>\n",
       "      <td>0.524000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>facts__disjunction</td>\n",
       "      <td>0.648841</td>\n",
       "      <td>0.776285</td>\n",
       "      <td>0.506239</td>\n",
       "      <td>0.508442</td>\n",
       "      <td>0.524000</td>\n",
       "      <td>0.681246</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.569600</td>\n",
       "      <td>0.540000</td>\n",
       "      <td>0.620000</td>\n",
       "      <td>0.510204</td>\n",
       "      <td>0.480000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>facts__negated</td>\n",
       "      <td>0.713012</td>\n",
       "      <td>0.727168</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.869148</td>\n",
       "      <td>0.474000</td>\n",
       "      <td>0.518296</td>\n",
       "      <td>0.462000</td>\n",
       "      <td>0.505200</td>\n",
       "      <td>0.540000</td>\n",
       "      <td>0.576000</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>0.664000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>inventors__affirmative</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.830858</td>\n",
       "      <td>0.576355</td>\n",
       "      <td>0.729084</td>\n",
       "      <td>0.756000</td>\n",
       "      <td>0.744140</td>\n",
       "      <td>0.652000</td>\n",
       "      <td>0.619200</td>\n",
       "      <td>0.820000</td>\n",
       "      <td>0.804000</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.534137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>inventors__conjunction</td>\n",
       "      <td>0.605911</td>\n",
       "      <td>0.594712</td>\n",
       "      <td>0.610837</td>\n",
       "      <td>0.500181</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.713265</td>\n",
       "      <td>0.446000</td>\n",
       "      <td>0.514400</td>\n",
       "      <td>0.480000</td>\n",
       "      <td>0.672000</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.473896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>inventors__disjunction</td>\n",
       "      <td>0.854680</td>\n",
       "      <td>0.569721</td>\n",
       "      <td>0.554187</td>\n",
       "      <td>0.499457</td>\n",
       "      <td>0.682000</td>\n",
       "      <td>0.779017</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.573600</td>\n",
       "      <td>0.640000</td>\n",
       "      <td>0.504000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.502008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>inventors__negated</td>\n",
       "      <td>0.416256</td>\n",
       "      <td>0.181094</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.649402</td>\n",
       "      <td>0.576000</td>\n",
       "      <td>0.500858</td>\n",
       "      <td>0.544000</td>\n",
       "      <td>0.462400</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.396000</td>\n",
       "      <td>0.740000</td>\n",
       "      <td>0.614458</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>sp_en_trans__negated</td>\n",
       "      <td>0.497175</td>\n",
       "      <td>0.403128</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.821898</td>\n",
       "      <td>0.440000</td>\n",
       "      <td>0.462836</td>\n",
       "      <td>0.536000</td>\n",
       "      <td>0.490400</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.420000</td>\n",
       "      <td>0.960000</td>\n",
       "      <td>0.811245</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>sp_en_trans__affirmative</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.879844</td>\n",
       "      <td>0.274011</td>\n",
       "      <td>0.750800</td>\n",
       "      <td>0.834000</td>\n",
       "      <td>0.675815</td>\n",
       "      <td>0.730000</td>\n",
       "      <td>0.612400</td>\n",
       "      <td>0.860000</td>\n",
       "      <td>0.744000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.534137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>sp_en_trans__conjunction</td>\n",
       "      <td>0.550847</td>\n",
       "      <td>0.774973</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.652328</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.825043</td>\n",
       "      <td>0.574000</td>\n",
       "      <td>0.590400</td>\n",
       "      <td>0.540000</td>\n",
       "      <td>0.788000</td>\n",
       "      <td>0.460000</td>\n",
       "      <td>0.530120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>sp_en_trans__disjunction</td>\n",
       "      <td>0.508475</td>\n",
       "      <td>0.643086</td>\n",
       "      <td>0.497175</td>\n",
       "      <td>0.562033</td>\n",
       "      <td>0.874000</td>\n",
       "      <td>0.554031</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.519200</td>\n",
       "      <td>0.540000</td>\n",
       "      <td>0.584000</td>\n",
       "      <td>0.460000</td>\n",
       "      <td>0.538153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>animal_class__affirmative+negated</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.909424</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.850150</td>\n",
       "      <td>0.622000</td>\n",
       "      <td>0.687250</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.630800</td>\n",
       "      <td>0.900000</td>\n",
       "      <td>0.736000</td>\n",
       "      <td>0.760000</td>\n",
       "      <td>0.706827</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>animal_class__affirmative+disjunction</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.858142</td>\n",
       "      <td>0.865854</td>\n",
       "      <td>0.643024</td>\n",
       "      <td>0.668000</td>\n",
       "      <td>0.679531</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.610400</td>\n",
       "      <td>0.900000</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>0.540000</td>\n",
       "      <td>0.457831</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>animal_class__negated+conjunction</td>\n",
       "      <td>0.993902</td>\n",
       "      <td>0.858142</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.704296</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.765009</td>\n",
       "      <td>0.502000</td>\n",
       "      <td>0.611200</td>\n",
       "      <td>0.880000</td>\n",
       "      <td>0.692000</td>\n",
       "      <td>0.640000</td>\n",
       "      <td>0.522088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>animal_class__affirmative+negated+conjunction+...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.887779</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.779887</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.738708</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.644400</td>\n",
       "      <td>0.880000</td>\n",
       "      <td>0.716000</td>\n",
       "      <td>0.740000</td>\n",
       "      <td>0.578313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>cities__affirmative+negated</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.845003</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.828247</td>\n",
       "      <td>0.584780</td>\n",
       "      <td>0.636000</td>\n",
       "      <td>0.474000</td>\n",
       "      <td>0.596800</td>\n",
       "      <td>0.960000</td>\n",
       "      <td>0.696000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.718876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>cities__affirmative+disjunction</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.719330</td>\n",
       "      <td>0.790775</td>\n",
       "      <td>0.543986</td>\n",
       "      <td>0.678238</td>\n",
       "      <td>0.626400</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.589600</td>\n",
       "      <td>0.920000</td>\n",
       "      <td>0.664000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.489960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>cities__negated+conjunction</td>\n",
       "      <td>0.981283</td>\n",
       "      <td>0.847995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.675643</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.755200</td>\n",
       "      <td>0.560000</td>\n",
       "      <td>0.614000</td>\n",
       "      <td>0.980000</td>\n",
       "      <td>0.724000</td>\n",
       "      <td>0.980000</td>\n",
       "      <td>0.610442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>cities__affirmative+negated+conjunction+disjun...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.816278</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.704967</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.686400</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.662000</td>\n",
       "      <td>0.980000</td>\n",
       "      <td>0.696000</td>\n",
       "      <td>0.960000</td>\n",
       "      <td>0.626506</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>element_symb__affirmative+negated</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.909091</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.902382</td>\n",
       "      <td>0.810000</td>\n",
       "      <td>0.613493</td>\n",
       "      <td>0.544000</td>\n",
       "      <td>0.538400</td>\n",
       "      <td>0.580000</td>\n",
       "      <td>0.772000</td>\n",
       "      <td>0.860000</td>\n",
       "      <td>0.730924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>element_symb__affirmative+disjunction</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.899363</td>\n",
       "      <td>0.736559</td>\n",
       "      <td>0.693056</td>\n",
       "      <td>0.778000</td>\n",
       "      <td>0.629503</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.628800</td>\n",
       "      <td>0.740000</td>\n",
       "      <td>0.768000</td>\n",
       "      <td>0.520000</td>\n",
       "      <td>0.550201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>element_symb__negated+conjunction</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.638376</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.885273</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.672956</td>\n",
       "      <td>0.566000</td>\n",
       "      <td>0.579600</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.796000</td>\n",
       "      <td>0.780000</td>\n",
       "      <td>0.835341</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>element_symb__affirmative+negated+conjunction+...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.921503</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.904394</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.668382</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.643200</td>\n",
       "      <td>0.620000</td>\n",
       "      <td>0.832000</td>\n",
       "      <td>0.760000</td>\n",
       "      <td>0.783133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>facts__affirmative+negated</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.909823</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.920568</td>\n",
       "      <td>0.546000</td>\n",
       "      <td>0.583762</td>\n",
       "      <td>0.482000</td>\n",
       "      <td>0.544800</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.776000</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>0.768000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>facts__affirmative+disjunction</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.926708</td>\n",
       "      <td>0.607843</td>\n",
       "      <td>0.779739</td>\n",
       "      <td>0.714000</td>\n",
       "      <td>0.882218</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.553200</td>\n",
       "      <td>0.580000</td>\n",
       "      <td>0.788000</td>\n",
       "      <td>0.510204</td>\n",
       "      <td>0.608000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>facts__negated+conjunction</td>\n",
       "      <td>0.871658</td>\n",
       "      <td>0.936685</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.954720</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.750429</td>\n",
       "      <td>0.712000</td>\n",
       "      <td>0.574800</td>\n",
       "      <td>0.640000</td>\n",
       "      <td>0.804000</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>0.828000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>facts__affirmative+negated+conjunction+disjunc...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.930929</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.945510</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.736707</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.548800</td>\n",
       "      <td>0.660000</td>\n",
       "      <td>0.792000</td>\n",
       "      <td>0.755102</td>\n",
       "      <td>0.780000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>inventors__affirmative+negated</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.845346</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.876494</td>\n",
       "      <td>0.580000</td>\n",
       "      <td>0.569468</td>\n",
       "      <td>0.572000</td>\n",
       "      <td>0.522400</td>\n",
       "      <td>0.820000</td>\n",
       "      <td>0.656000</td>\n",
       "      <td>0.840000</td>\n",
       "      <td>0.742972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>inventors__affirmative+disjunction</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.850054</td>\n",
       "      <td>0.593596</td>\n",
       "      <td>0.725462</td>\n",
       "      <td>0.752000</td>\n",
       "      <td>0.803888</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.540000</td>\n",
       "      <td>0.820000</td>\n",
       "      <td>0.828000</td>\n",
       "      <td>0.640000</td>\n",
       "      <td>0.562249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>inventors__negated+conjunction</td>\n",
       "      <td>0.435961</td>\n",
       "      <td>0.497646</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.874683</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.503716</td>\n",
       "      <td>0.538000</td>\n",
       "      <td>0.487600</td>\n",
       "      <td>0.420000</td>\n",
       "      <td>0.540000</td>\n",
       "      <td>0.840000</td>\n",
       "      <td>0.722892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>inventors__affirmative+negated+conjunction+dis...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.870337</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.506338</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.794454</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.620400</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>0.816000</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.502008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>sp_en_trans__affirmative+negated</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.910060</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.853893</td>\n",
       "      <td>0.922000</td>\n",
       "      <td>0.601487</td>\n",
       "      <td>0.532000</td>\n",
       "      <td>0.567200</td>\n",
       "      <td>0.880000</td>\n",
       "      <td>0.788000</td>\n",
       "      <td>0.540000</td>\n",
       "      <td>0.738956</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>sp_en_trans__affirmative+disjunction</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.854604</td>\n",
       "      <td>0.429379</td>\n",
       "      <td>0.600071</td>\n",
       "      <td>0.796000</td>\n",
       "      <td>0.570898</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.504400</td>\n",
       "      <td>0.840000</td>\n",
       "      <td>0.656000</td>\n",
       "      <td>0.520000</td>\n",
       "      <td>0.465863</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>sp_en_trans__negated+conjunction</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.479559</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.878066</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.580046</td>\n",
       "      <td>0.528000</td>\n",
       "      <td>0.498400</td>\n",
       "      <td>0.740000</td>\n",
       "      <td>0.480000</td>\n",
       "      <td>0.920000</td>\n",
       "      <td>0.835341</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>sp_en_trans__affirmative+negated+conjunction+d...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.910060</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.889797</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.559748</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.513200</td>\n",
       "      <td>0.980000</td>\n",
       "      <td>0.784000</td>\n",
       "      <td>0.540000</td>\n",
       "      <td>0.767068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>animal_class+facts+inventors__affirmative+nega...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.976424</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.877210</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.845476</td>\n",
       "      <td>0.611333</td>\n",
       "      <td>0.526000</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.813333</td>\n",
       "      <td>0.765101</td>\n",
       "      <td>0.660000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>animal_class+facts+inventors__affirmative+nega...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.976916</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.943517</td>\n",
       "      <td>0.753333</td>\n",
       "      <td>0.895116</td>\n",
       "      <td>0.507333</td>\n",
       "      <td>0.510667</td>\n",
       "      <td>0.766667</td>\n",
       "      <td>0.826667</td>\n",
       "      <td>0.778523</td>\n",
       "      <td>0.686667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 96
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-08T18:49:46.374115Z",
     "start_time": "2024-12-08T18:49:46.359111Z"
    }
   },
   "cell_type": "code",
   "source": "topic_generalization_summary_df",
   "id": "ca0c66af1ab7331c",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "                                             scenario  \\\n",
       "0                           animal_class__affirmative   \n",
       "1                           animal_class__conjunction   \n",
       "2                           animal_class__disjunction   \n",
       "3                               animal_class__negated   \n",
       "4                                 cities__affirmative   \n",
       "5                                 cities__conjunction   \n",
       "6                                 cities__disjunction   \n",
       "7                                     cities__negated   \n",
       "8                           element_symb__affirmative   \n",
       "9                           element_symb__conjunction   \n",
       "10                          element_symb__disjunction   \n",
       "11                              element_symb__negated   \n",
       "12                                 facts__affirmative   \n",
       "13                                 facts__conjunction   \n",
       "14                                 facts__disjunction   \n",
       "15                                     facts__negated   \n",
       "16                             inventors__affirmative   \n",
       "17                             inventors__conjunction   \n",
       "18                             inventors__disjunction   \n",
       "19                                 inventors__negated   \n",
       "20                               sp_en_trans__negated   \n",
       "21                           sp_en_trans__affirmative   \n",
       "22                           sp_en_trans__conjunction   \n",
       "23                           sp_en_trans__disjunction   \n",
       "24                  animal_class__affirmative+negated   \n",
       "25              animal_class__affirmative+disjunction   \n",
       "26                  animal_class__negated+conjunction   \n",
       "27  animal_class__affirmative+negated+conjunction+...   \n",
       "28                        cities__affirmative+negated   \n",
       "29                    cities__affirmative+disjunction   \n",
       "30                        cities__negated+conjunction   \n",
       "31  cities__affirmative+negated+conjunction+disjun...   \n",
       "32                  element_symb__affirmative+negated   \n",
       "33              element_symb__affirmative+disjunction   \n",
       "34                  element_symb__negated+conjunction   \n",
       "35  element_symb__affirmative+negated+conjunction+...   \n",
       "36                         facts__affirmative+negated   \n",
       "37                     facts__affirmative+disjunction   \n",
       "38                         facts__negated+conjunction   \n",
       "39  facts__affirmative+negated+conjunction+disjunc...   \n",
       "40                     inventors__affirmative+negated   \n",
       "41                 inventors__affirmative+disjunction   \n",
       "42                     inventors__negated+conjunction   \n",
       "43  inventors__affirmative+negated+conjunction+dis...   \n",
       "44                   sp_en_trans__affirmative+negated   \n",
       "45               sp_en_trans__affirmative+disjunction   \n",
       "46                   sp_en_trans__negated+conjunction   \n",
       "47  sp_en_trans__affirmative+negated+conjunction+d...   \n",
       "48  animal_class+facts+inventors__affirmative+nega...   \n",
       "49  animal_class+facts+inventors__affirmative+nega...   \n",
       "\n",
       "    avg_across_topic_penalty  affirm_across_topic_penalty  \\\n",
       "0                   0.068085                          NaN   \n",
       "1                   0.123056                     0.164399   \n",
       "2                  -0.007231                    -0.028208   \n",
       "3                   0.033441                    -0.017700   \n",
       "4                   0.135048                          NaN   \n",
       "5                   0.232092                     0.190280   \n",
       "6                   0.089213                     0.283938   \n",
       "7                   0.096640                     0.094598   \n",
       "8                  -0.022499                          NaN   \n",
       "9                   0.012069                     0.021662   \n",
       "10                 -0.080610                     0.023558   \n",
       "11                 -0.037251                     0.106328   \n",
       "12                 -0.039924                          NaN   \n",
       "13                  0.038124                     0.186376   \n",
       "14                 -0.067338                    -0.127444   \n",
       "15                 -0.017473                    -0.014156   \n",
       "16                 -0.005241                          NaN   \n",
       "17                 -0.002488                     0.011199   \n",
       "18                  0.075333                     0.284959   \n",
       "19                  0.104289                     0.235162   \n",
       "20                  0.069113                     0.094047   \n",
       "21                 -0.023828                          NaN   \n",
       "22                 -0.142195                    -0.224126   \n",
       "23                 -0.000331                    -0.134611   \n",
       "24                  0.005281                          NaN   \n",
       "25                  0.123367                          NaN   \n",
       "26                  0.083118                     0.135761   \n",
       "27                  0.162843                          NaN   \n",
       "28                  0.092776                          NaN   \n",
       "29                  0.141167                          NaN   \n",
       "30                  0.176212                     0.133288   \n",
       "31                  0.308747                          NaN   \n",
       "32                  0.034796                          NaN   \n",
       "33                  0.033450                          NaN   \n",
       "34                 -0.150829                    -0.138376   \n",
       "35                 -0.117566                          NaN   \n",
       "36                 -0.082569                          NaN   \n",
       "37                 -0.161478                          NaN   \n",
       "38                 -0.051385                    -0.065027   \n",
       "39                 -0.078449                          NaN   \n",
       "40                  0.080290                          NaN   \n",
       "41                 -0.028501                          NaN   \n",
       "42                 -0.003544                    -0.061685   \n",
       "43                 -0.009004                          NaN   \n",
       "44                  0.044589                          NaN   \n",
       "45                  0.073137                          NaN   \n",
       "46                  0.098675                     0.020441   \n",
       "47                 -0.015534                          NaN   \n",
       "48                  0.059034                          NaN   \n",
       "49                 -0.028315                          NaN   \n",
       "\n",
       "    neg_across_topic_penalty  conj_across_topic_penalty  \\\n",
       "0                   0.205662                  -0.052942   \n",
       "1                   0.287239                        NaN   \n",
       "2                  -0.023351                   0.189604   \n",
       "3                        NaN                  -0.084940   \n",
       "4                   0.293143                   0.040425   \n",
       "5                   0.417486                        NaN   \n",
       "6                   0.042464                  -0.054346   \n",
       "7                        NaN                  -0.043597   \n",
       "8                  -0.027340                   0.157827   \n",
       "9                  -0.025663                        NaN   \n",
       "10                 -0.217779                   0.065099   \n",
       "11                       NaN                  -0.234861   \n",
       "12                  0.039538                  -0.189354   \n",
       "13                 -0.004753                        NaN   \n",
       "14                 -0.002203                  -0.157246   \n",
       "15                       NaN                  -0.044296   \n",
       "16                 -0.152729                   0.011860   \n",
       "17                  0.110656                        NaN   \n",
       "18                  0.054730                  -0.097017   \n",
       "19                       NaN                   0.075142   \n",
       "20                       NaN                  -0.022836   \n",
       "21                 -0.476789                   0.158185   \n",
       "22                 -0.152328                        NaN   \n",
       "23                 -0.064858                   0.319969   \n",
       "24                       NaN                  -0.065250   \n",
       "25                  0.222830                  -0.011531   \n",
       "26                       NaN                        NaN   \n",
       "27                       NaN                        NaN   \n",
       "28                       NaN                  -0.051220   \n",
       "29                  0.246790                   0.051838   \n",
       "30                       NaN                        NaN   \n",
       "31                       NaN                        NaN   \n",
       "32                       NaN                   0.196507   \n",
       "33                  0.043503                   0.148497   \n",
       "34                       NaN                        NaN   \n",
       "35                       NaN                        NaN   \n",
       "36                       NaN                  -0.037762   \n",
       "37                 -0.171896                  -0.168218   \n",
       "38                       NaN                        NaN   \n",
       "39                       NaN                        NaN   \n",
       "40                       NaN                   0.010532   \n",
       "41                 -0.131866                  -0.051888   \n",
       "42                       NaN                        NaN   \n",
       "43                       NaN                        NaN   \n",
       "44                       NaN                   0.320513   \n",
       "45                 -0.170693                   0.225102   \n",
       "46                       NaN                        NaN   \n",
       "47                       NaN                        NaN   \n",
       "48                       NaN                        NaN   \n",
       "49                       NaN                  -0.141783   \n",
       "\n",
       "    disj_across_topic_penalty  de_affirm_across_topic_penalty  \\\n",
       "0                   -0.070400                        0.212000   \n",
       "1                   -0.110400                        0.224000   \n",
       "2                         NaN                       -0.164000   \n",
       "3                   -0.015600                        0.064000   \n",
       "4                   -0.016400                        0.340000   \n",
       "5                   -0.037200                        0.276000   \n",
       "6                         NaN                        0.092000   \n",
       "7                   -0.005200                        0.088000   \n",
       "8                   -0.024800                       -0.192000   \n",
       "9                    0.110400                       -0.052000   \n",
       "10                        NaN                       -0.032000   \n",
       "11                  -0.058800                       -0.188000   \n",
       "12                   0.068400                       -0.180000   \n",
       "13                   0.079200                       -0.036000   \n",
       "14                        NaN                       -0.080000   \n",
       "15                  -0.043200                       -0.036000   \n",
       "16                   0.032800                        0.016000   \n",
       "17                  -0.068400                       -0.192000   \n",
       "18                        NaN                        0.136000   \n",
       "19                   0.081600                        0.004000   \n",
       "20                   0.045600                        0.080000   \n",
       "21                   0.117600                        0.116000   \n",
       "22                  -0.016400                       -0.248000   \n",
       "23                        NaN                       -0.044000   \n",
       "24                  -0.130800                        0.164000   \n",
       "25                        NaN                        0.200000   \n",
       "26                  -0.109200                        0.188000   \n",
       "27                        NaN                        0.164000   \n",
       "28                  -0.122800                        0.264000   \n",
       "29                        NaN                        0.256000   \n",
       "30                  -0.054000                        0.256000   \n",
       "31                        NaN                        0.284000   \n",
       "32                   0.005600                       -0.192000   \n",
       "33                        NaN                       -0.028000   \n",
       "34                  -0.013600                       -0.396000   \n",
       "35                        NaN                       -0.212000   \n",
       "36                  -0.062800                       -0.176000   \n",
       "37                        NaN                       -0.208000   \n",
       "38                   0.137200                       -0.164000   \n",
       "39                        NaN                       -0.132000   \n",
       "40                   0.049600                        0.164000   \n",
       "41                        NaN                       -0.008000   \n",
       "42                   0.050400                       -0.120000   \n",
       "43                        NaN                       -0.116000   \n",
       "44                  -0.035200                        0.092000   \n",
       "45                        NaN                        0.184000   \n",
       "46                   0.029600                        0.260000   \n",
       "47                        NaN                        0.196000   \n",
       "48                   0.085333                       -0.013333   \n",
       "49                  -0.003333                       -0.060000   \n",
       "\n",
       "    de_neg_across_topic_penalty  \n",
       "0                      0.046104  \n",
       "1                      0.050040  \n",
       "2                     -0.010201  \n",
       "3                      0.221446  \n",
       "4                      0.018072  \n",
       "5                      0.313896  \n",
       "6                      0.082008  \n",
       "7                      0.349398  \n",
       "8                     -0.026185  \n",
       "9                      0.005944  \n",
       "10                    -0.241928  \n",
       "11                     0.189076  \n",
       "12                     0.061796  \n",
       "13                    -0.034204  \n",
       "14                     0.030204  \n",
       "15                     0.050286  \n",
       "16                     0.065863  \n",
       "17                     0.126104  \n",
       "18                    -0.002008  \n",
       "19                     0.125542  \n",
       "20                     0.148755  \n",
       "21                    -0.034137  \n",
       "22                    -0.070120  \n",
       "23                    -0.078153  \n",
       "24                     0.053173  \n",
       "25                     0.082169  \n",
       "26                     0.117912  \n",
       "27                     0.161687  \n",
       "28                     0.281124  \n",
       "29                     0.010040  \n",
       "30                     0.369558  \n",
       "31                     0.333494  \n",
       "32                     0.129076  \n",
       "33                    -0.030201  \n",
       "34                    -0.055341  \n",
       "35                    -0.023133  \n",
       "36                    -0.053714  \n",
       "37                    -0.097796  \n",
       "38                    -0.113714  \n",
       "39                    -0.024898  \n",
       "40                     0.097028  \n",
       "41                     0.077751  \n",
       "42                     0.117108  \n",
       "43                     0.097992  \n",
       "44                    -0.198956  \n",
       "45                     0.054137  \n",
       "46                     0.084659  \n",
       "47                    -0.227068  \n",
       "48                     0.105101  \n",
       "49                     0.091857  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>scenario</th>\n",
       "      <th>avg_across_topic_penalty</th>\n",
       "      <th>affirm_across_topic_penalty</th>\n",
       "      <th>neg_across_topic_penalty</th>\n",
       "      <th>conj_across_topic_penalty</th>\n",
       "      <th>disj_across_topic_penalty</th>\n",
       "      <th>de_affirm_across_topic_penalty</th>\n",
       "      <th>de_neg_across_topic_penalty</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>animal_class__affirmative</td>\n",
       "      <td>0.068085</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.205662</td>\n",
       "      <td>-0.052942</td>\n",
       "      <td>-0.070400</td>\n",
       "      <td>0.212000</td>\n",
       "      <td>0.046104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>animal_class__conjunction</td>\n",
       "      <td>0.123056</td>\n",
       "      <td>0.164399</td>\n",
       "      <td>0.287239</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.110400</td>\n",
       "      <td>0.224000</td>\n",
       "      <td>0.050040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>animal_class__disjunction</td>\n",
       "      <td>-0.007231</td>\n",
       "      <td>-0.028208</td>\n",
       "      <td>-0.023351</td>\n",
       "      <td>0.189604</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.164000</td>\n",
       "      <td>-0.010201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>animal_class__negated</td>\n",
       "      <td>0.033441</td>\n",
       "      <td>-0.017700</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.084940</td>\n",
       "      <td>-0.015600</td>\n",
       "      <td>0.064000</td>\n",
       "      <td>0.221446</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cities__affirmative</td>\n",
       "      <td>0.135048</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.293143</td>\n",
       "      <td>0.040425</td>\n",
       "      <td>-0.016400</td>\n",
       "      <td>0.340000</td>\n",
       "      <td>0.018072</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>cities__conjunction</td>\n",
       "      <td>0.232092</td>\n",
       "      <td>0.190280</td>\n",
       "      <td>0.417486</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.037200</td>\n",
       "      <td>0.276000</td>\n",
       "      <td>0.313896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>cities__disjunction</td>\n",
       "      <td>0.089213</td>\n",
       "      <td>0.283938</td>\n",
       "      <td>0.042464</td>\n",
       "      <td>-0.054346</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.092000</td>\n",
       "      <td>0.082008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>cities__negated</td>\n",
       "      <td>0.096640</td>\n",
       "      <td>0.094598</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.043597</td>\n",
       "      <td>-0.005200</td>\n",
       "      <td>0.088000</td>\n",
       "      <td>0.349398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>element_symb__affirmative</td>\n",
       "      <td>-0.022499</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.027340</td>\n",
       "      <td>0.157827</td>\n",
       "      <td>-0.024800</td>\n",
       "      <td>-0.192000</td>\n",
       "      <td>-0.026185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>element_symb__conjunction</td>\n",
       "      <td>0.012069</td>\n",
       "      <td>0.021662</td>\n",
       "      <td>-0.025663</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.110400</td>\n",
       "      <td>-0.052000</td>\n",
       "      <td>0.005944</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>element_symb__disjunction</td>\n",
       "      <td>-0.080610</td>\n",
       "      <td>0.023558</td>\n",
       "      <td>-0.217779</td>\n",
       "      <td>0.065099</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.032000</td>\n",
       "      <td>-0.241928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>element_symb__negated</td>\n",
       "      <td>-0.037251</td>\n",
       "      <td>0.106328</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.234861</td>\n",
       "      <td>-0.058800</td>\n",
       "      <td>-0.188000</td>\n",
       "      <td>0.189076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>facts__affirmative</td>\n",
       "      <td>-0.039924</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.039538</td>\n",
       "      <td>-0.189354</td>\n",
       "      <td>0.068400</td>\n",
       "      <td>-0.180000</td>\n",
       "      <td>0.061796</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>facts__conjunction</td>\n",
       "      <td>0.038124</td>\n",
       "      <td>0.186376</td>\n",
       "      <td>-0.004753</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.079200</td>\n",
       "      <td>-0.036000</td>\n",
       "      <td>-0.034204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>facts__disjunction</td>\n",
       "      <td>-0.067338</td>\n",
       "      <td>-0.127444</td>\n",
       "      <td>-0.002203</td>\n",
       "      <td>-0.157246</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.080000</td>\n",
       "      <td>0.030204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>facts__negated</td>\n",
       "      <td>-0.017473</td>\n",
       "      <td>-0.014156</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.044296</td>\n",
       "      <td>-0.043200</td>\n",
       "      <td>-0.036000</td>\n",
       "      <td>0.050286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>inventors__affirmative</td>\n",
       "      <td>-0.005241</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.152729</td>\n",
       "      <td>0.011860</td>\n",
       "      <td>0.032800</td>\n",
       "      <td>0.016000</td>\n",
       "      <td>0.065863</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>inventors__conjunction</td>\n",
       "      <td>-0.002488</td>\n",
       "      <td>0.011199</td>\n",
       "      <td>0.110656</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.068400</td>\n",
       "      <td>-0.192000</td>\n",
       "      <td>0.126104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>inventors__disjunction</td>\n",
       "      <td>0.075333</td>\n",
       "      <td>0.284959</td>\n",
       "      <td>0.054730</td>\n",
       "      <td>-0.097017</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.136000</td>\n",
       "      <td>-0.002008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>inventors__negated</td>\n",
       "      <td>0.104289</td>\n",
       "      <td>0.235162</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.075142</td>\n",
       "      <td>0.081600</td>\n",
       "      <td>0.004000</td>\n",
       "      <td>0.125542</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>sp_en_trans__negated</td>\n",
       "      <td>0.069113</td>\n",
       "      <td>0.094047</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.022836</td>\n",
       "      <td>0.045600</td>\n",
       "      <td>0.080000</td>\n",
       "      <td>0.148755</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>sp_en_trans__affirmative</td>\n",
       "      <td>-0.023828</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.476789</td>\n",
       "      <td>0.158185</td>\n",
       "      <td>0.117600</td>\n",
       "      <td>0.116000</td>\n",
       "      <td>-0.034137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>sp_en_trans__conjunction</td>\n",
       "      <td>-0.142195</td>\n",
       "      <td>-0.224126</td>\n",
       "      <td>-0.152328</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.016400</td>\n",
       "      <td>-0.248000</td>\n",
       "      <td>-0.070120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>sp_en_trans__disjunction</td>\n",
       "      <td>-0.000331</td>\n",
       "      <td>-0.134611</td>\n",
       "      <td>-0.064858</td>\n",
       "      <td>0.319969</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.044000</td>\n",
       "      <td>-0.078153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>animal_class__affirmative+negated</td>\n",
       "      <td>0.005281</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.065250</td>\n",
       "      <td>-0.130800</td>\n",
       "      <td>0.164000</td>\n",
       "      <td>0.053173</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>animal_class__affirmative+disjunction</td>\n",
       "      <td>0.123367</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.222830</td>\n",
       "      <td>-0.011531</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.082169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>animal_class__negated+conjunction</td>\n",
       "      <td>0.083118</td>\n",
       "      <td>0.135761</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.109200</td>\n",
       "      <td>0.188000</td>\n",
       "      <td>0.117912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>animal_class__affirmative+negated+conjunction+...</td>\n",
       "      <td>0.162843</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.164000</td>\n",
       "      <td>0.161687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>cities__affirmative+negated</td>\n",
       "      <td>0.092776</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.051220</td>\n",
       "      <td>-0.122800</td>\n",
       "      <td>0.264000</td>\n",
       "      <td>0.281124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>cities__affirmative+disjunction</td>\n",
       "      <td>0.141167</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.246790</td>\n",
       "      <td>0.051838</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.256000</td>\n",
       "      <td>0.010040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>cities__negated+conjunction</td>\n",
       "      <td>0.176212</td>\n",
       "      <td>0.133288</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.054000</td>\n",
       "      <td>0.256000</td>\n",
       "      <td>0.369558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>cities__affirmative+negated+conjunction+disjun...</td>\n",
       "      <td>0.308747</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.284000</td>\n",
       "      <td>0.333494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>element_symb__affirmative+negated</td>\n",
       "      <td>0.034796</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.196507</td>\n",
       "      <td>0.005600</td>\n",
       "      <td>-0.192000</td>\n",
       "      <td>0.129076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>element_symb__affirmative+disjunction</td>\n",
       "      <td>0.033450</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.043503</td>\n",
       "      <td>0.148497</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.028000</td>\n",
       "      <td>-0.030201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>element_symb__negated+conjunction</td>\n",
       "      <td>-0.150829</td>\n",
       "      <td>-0.138376</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.013600</td>\n",
       "      <td>-0.396000</td>\n",
       "      <td>-0.055341</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>element_symb__affirmative+negated+conjunction+...</td>\n",
       "      <td>-0.117566</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.212000</td>\n",
       "      <td>-0.023133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>facts__affirmative+negated</td>\n",
       "      <td>-0.082569</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.037762</td>\n",
       "      <td>-0.062800</td>\n",
       "      <td>-0.176000</td>\n",
       "      <td>-0.053714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>facts__affirmative+disjunction</td>\n",
       "      <td>-0.161478</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.171896</td>\n",
       "      <td>-0.168218</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.208000</td>\n",
       "      <td>-0.097796</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>facts__negated+conjunction</td>\n",
       "      <td>-0.051385</td>\n",
       "      <td>-0.065027</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.137200</td>\n",
       "      <td>-0.164000</td>\n",
       "      <td>-0.113714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>facts__affirmative+negated+conjunction+disjunc...</td>\n",
       "      <td>-0.078449</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.132000</td>\n",
       "      <td>-0.024898</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>inventors__affirmative+negated</td>\n",
       "      <td>0.080290</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.010532</td>\n",
       "      <td>0.049600</td>\n",
       "      <td>0.164000</td>\n",
       "      <td>0.097028</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>inventors__affirmative+disjunction</td>\n",
       "      <td>-0.028501</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.131866</td>\n",
       "      <td>-0.051888</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.008000</td>\n",
       "      <td>0.077751</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>inventors__negated+conjunction</td>\n",
       "      <td>-0.003544</td>\n",
       "      <td>-0.061685</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.050400</td>\n",
       "      <td>-0.120000</td>\n",
       "      <td>0.117108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>inventors__affirmative+negated+conjunction+dis...</td>\n",
       "      <td>-0.009004</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.116000</td>\n",
       "      <td>0.097992</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>sp_en_trans__affirmative+negated</td>\n",
       "      <td>0.044589</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.320513</td>\n",
       "      <td>-0.035200</td>\n",
       "      <td>0.092000</td>\n",
       "      <td>-0.198956</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>sp_en_trans__affirmative+disjunction</td>\n",
       "      <td>0.073137</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.170693</td>\n",
       "      <td>0.225102</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.184000</td>\n",
       "      <td>0.054137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>sp_en_trans__negated+conjunction</td>\n",
       "      <td>0.098675</td>\n",
       "      <td>0.020441</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.029600</td>\n",
       "      <td>0.260000</td>\n",
       "      <td>0.084659</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>sp_en_trans__affirmative+negated+conjunction+d...</td>\n",
       "      <td>-0.015534</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.196000</td>\n",
       "      <td>-0.227068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>animal_class+facts+inventors__affirmative+nega...</td>\n",
       "      <td>0.059034</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.085333</td>\n",
       "      <td>-0.013333</td>\n",
       "      <td>0.105101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>animal_class+facts+inventors__affirmative+nega...</td>\n",
       "      <td>-0.028315</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.141783</td>\n",
       "      <td>-0.003333</td>\n",
       "      <td>-0.060000</td>\n",
       "      <td>0.091857</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 97
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-08T18:49:46.984911Z",
     "start_time": "2024-12-08T18:49:46.978871Z"
    }
   },
   "cell_type": "code",
   "source": "(avg_topic_generalization_penalty_across_all_scenarios := topic_generalization_summary_df.avg_across_topic_penalty.mean().item())",
   "id": "dcde1cca383f38ff",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.028407982234862916"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 98
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "The cross-topic generalization penalty is surprisingly small (<3 percentage-points on average), so we will ignore the \"already-seen topic vs fully-unseen topic\" distinction for analysis beyond this point.\n",
    "\n",
    "In passing, it is surprising that the 3-topic affirmative+negated+conjunction scenario had worse cross-topic generalization than the average scenario."
   ],
   "id": "64e3fc8529a256ef"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-08T18:49:51.553965Z",
     "start_time": "2024-12-08T18:49:51.545985Z"
    }
   },
   "cell_type": "code",
   "source": "avg_cross_topic_acc_by_source_and_target_data_variants",
   "id": "63da60f97874cca9",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "         source_data_variants    affirm       neg      conj      disj\n",
       "0                      affirm  0.843616  0.610274  0.719804  0.558467\n",
       "1                         neg  0.512040  0.805444  0.529898  0.494600\n",
       "2                        conj  0.712324  0.564139  0.786394  0.558800\n",
       "3                        disj  0.689239  0.521776  0.663132  0.589800\n",
       "4                  affirm+neg  0.888125  0.871955  0.615243  0.566733\n",
       "5                 affirm+disj  0.851367  0.664223  0.698740  0.571067\n",
       "6                    neg+conj  0.709734  0.828780  0.671226  0.560933\n",
       "7        affirm+neg+conj+disj  0.889481  0.788482  0.697400  0.605333\n",
       "8  multitopic_affirm+neg+conj  0.976424  0.877210  0.845476  0.526000"
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_data_variants</th>\n",
       "      <th>affirm</th>\n",
       "      <th>neg</th>\n",
       "      <th>conj</th>\n",
       "      <th>disj</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>affirm</td>\n",
       "      <td>0.843616</td>\n",
       "      <td>0.610274</td>\n",
       "      <td>0.719804</td>\n",
       "      <td>0.558467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>neg</td>\n",
       "      <td>0.512040</td>\n",
       "      <td>0.805444</td>\n",
       "      <td>0.529898</td>\n",
       "      <td>0.494600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>conj</td>\n",
       "      <td>0.712324</td>\n",
       "      <td>0.564139</td>\n",
       "      <td>0.786394</td>\n",
       "      <td>0.558800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>disj</td>\n",
       "      <td>0.689239</td>\n",
       "      <td>0.521776</td>\n",
       "      <td>0.663132</td>\n",
       "      <td>0.589800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>affirm+neg</td>\n",
       "      <td>0.888125</td>\n",
       "      <td>0.871955</td>\n",
       "      <td>0.615243</td>\n",
       "      <td>0.566733</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>affirm+disj</td>\n",
       "      <td>0.851367</td>\n",
       "      <td>0.664223</td>\n",
       "      <td>0.698740</td>\n",
       "      <td>0.571067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>neg+conj</td>\n",
       "      <td>0.709734</td>\n",
       "      <td>0.828780</td>\n",
       "      <td>0.671226</td>\n",
       "      <td>0.560933</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>affirm+neg+conj+disj</td>\n",
       "      <td>0.889481</td>\n",
       "      <td>0.788482</td>\n",
       "      <td>0.697400</td>\n",
       "      <td>0.605333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>multitopic_affirm+neg+conj</td>\n",
       "      <td>0.976424</td>\n",
       "      <td>0.877210</td>\n",
       "      <td>0.845476</td>\n",
       "      <td>0.526000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 99
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "Unsurprisingly, the best generalization to unseen disjunctive datasets was achieved when the directions and probe were trained at least partially on disjunctive data. One interesting thing is that the affirm+neg+conj+disj scenarios generalized better to unseen topics' disjunctive data better than the only-disjunctive scenarios, while the scenario trained on affirmative, negated, and conjunctive data from multiple topics generalized terribly to unseen topics' disjunctive data.",
   "id": "d18bcba748e748f"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:58.797410Z",
     "start_time": "2024-12-07T21:26:58.792039Z"
    }
   },
   "cell_type": "code",
   "source": [
    "lyr18_baseline_linear_probe_metrics_on_test_dsets: dict[tuple[int,...], dict[int, ConfusionMetrics]] = {\n",
    "    scenario_id: {\n",
    "        dset_idx_in_scenario: test_dset_metrics.lyr18_baseline_linear_probe_metrics for dset_idx_in_scenario, test_dset_metrics in test_dsets_metrics.items()\n",
    "    } \n",
    "    for scenario_id, test_dsets_metrics in probes_metrics_on_test_dsets.items()\n",
    "}"
   ],
   "id": "bd1f4ff82cb49a5b",
   "outputs": [],
   "execution_count": 37
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:58.804139Z",
     "start_time": "2024-12-07T21:26:58.798416Z"
    }
   },
   "cell_type": "code",
   "source": [
    "#using more verbose phrasing in just this case to disambiguate from affirmative english texts in 'other' topics (real world scenarios, relative comparison, true false)\n",
    "is_dset_en_affirm_in_std_topic: dict[int, bool] = (~(dsets_index_df.is_negated | dsets_index_df.is_conj | dsets_index_df.is_disj | dsets_index_df.is_other | dsets_index_df.in_german)).to_dict()\n",
    "is_dset_en_negated: dict[int, bool] = (dsets_index_df.is_negated & ~dsets_index_df.in_german).to_dict()\n",
    "is_dset_conj: dict[int, bool] = dsets_index_df.is_conj.to_dict()\n",
    "is_dset_disj: dict[int, bool] = dsets_index_df.is_disj.to_dict()\n",
    "is_dset_de_affirm: dict[int, bool] = (~dsets_index_df.is_negated & dsets_index_df.in_german).to_dict()\n",
    "is_dset_de_negated: dict[int, bool] = (dsets_index_df.is_negated & dsets_index_df.in_german).to_dict()"
   ],
   "id": "306a9c2671adca46",
   "outputs": [],
   "execution_count": 38
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:58.810420Z",
     "start_time": "2024-12-07T21:26:58.805145Z"
    }
   },
   "cell_type": "code",
   "source": "dset_categ_lookup: dict[int, str] = dsets_index_df[\"Categ_Folder\"].to_dict()",
   "id": "d003b76309a68e65",
   "outputs": [],
   "execution_count": 39
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:58.816391Z",
     "start_time": "2024-12-07T21:26:58.811445Z"
    }
   },
   "cell_type": "code",
   "source": [
    "lyr18_probes_acc_on_unseen_multi_topic_data_groupings_row_dicts: list[dict[str, str | float]] = []\n",
    "lyr18_baseline_linear_probes_acc_on_unseen_multi_topic_data_groupings_row_dicts: list[dict[str, str | float]] = []\n",
    "lyr18_probes_acc_on_unseen_single_topic_data_groupings_row_dicts: list[dict[str, str|float]] = []\n",
    "lyr18_baseline_linear_probes_acc_on_unseen_single_topic_data_groupings_row_dicts: list[dict[str, str|float]] = []\n",
    "\n",
    "lyr18_ttpd_probes_rel_to_baseline_probes_acc_delta_on_unseen_multi_topic_data_groupings_row_dicts: list[dict[str, str | float]] = []\n",
    "lyr18_ttpd_probes_rel_to_baseline_probes_acc_delta_on_unseen_single_topic_data_groupings_row_dicts: list[dict[str, str|float]] = []"
   ],
   "id": "80a8e3e733edbc9f",
   "outputs": [],
   "execution_count": 40
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:58.821317Z",
     "start_time": "2024-12-07T21:26:58.817397Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def get_aggregate_trad_metrics(probe_dsets_metrics: dict[int, ConfusionMetrics], relevance_condition: Callable[[int], bool]) -> TraditionalMetrics:\n",
    "    relev_dsets_metrics = [metrics for curr_dset_idx, metrics in probe_dsets_metrics.items() if relevance_condition(curr_dset_idx)]\n",
    "    return ConfusionMetrics.combine(*relev_dsets_metrics).get_traditional_metrics()"
   ],
   "id": "348e1600bce845b7",
   "outputs": [],
   "execution_count": 41
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:58.854717Z",
     "start_time": "2024-12-07T21:26:58.822324Z"
    }
   },
   "cell_type": "code",
   "source": [
    "for scenario_id in dir_probe_scenarios_specs:\n",
    "    ttpd_probe_metrics = lyr18_probe_metrics_on_test_dsets[scenario_id]\n",
    "    baseline_probe_metrics = lyr18_baseline_linear_probe_metrics_on_test_dsets[scenario_id]\n",
    "    \n",
    "    ttpd_all_unseen_acc = get_aggregate_trad_metrics(ttpd_probe_metrics, lambda idx: True).accuracy\n",
    "    baseline_all_unseen_acc = get_aggregate_trad_metrics(baseline_probe_metrics, lambda idx: True).accuracy\n",
    "    \n",
    "    ttpd_all_unseen_en_affirm_acc = get_aggregate_trad_metrics(ttpd_probe_metrics, lambda idx: is_dset_en_affirm_in_std_topic[idx]).accuracy\n",
    "    baseline_all_unseen_en_affirm_acc = get_aggregate_trad_metrics(baseline_probe_metrics, lambda idx: is_dset_en_affirm_in_std_topic[idx]).accuracy\n",
    "    \n",
    "    ttpd_all_unseen_en_neg_acc = get_aggregate_trad_metrics(ttpd_probe_metrics, lambda idx: is_dset_en_negated[idx]).accuracy\n",
    "    baseline_all_unseen_en_neg_acc = get_aggregate_trad_metrics(baseline_probe_metrics, lambda idx: is_dset_en_negated[idx]).accuracy\n",
    "    \n",
    "    ttpd_all_unseen_conj_acc = get_aggregate_trad_metrics(ttpd_probe_metrics, lambda idx: is_dset_conj[idx]).accuracy\n",
    "    baseline_all_unseen_conj_acc = get_aggregate_trad_metrics(baseline_probe_metrics, lambda idx: is_dset_conj[idx]).accuracy\n",
    "    \n",
    "    ttpd_all_unseen_disj_acc = get_aggregate_trad_metrics(ttpd_probe_metrics, lambda idx: is_dset_disj[idx]).accuracy\n",
    "    baseline_all_unseen_disj_acc = get_aggregate_trad_metrics(baseline_probe_metrics, lambda idx: is_dset_disj[idx]).accuracy\n",
    "    \n",
    "    ttpd_all_unseen_de_affirm_acc = get_aggregate_trad_metrics(ttpd_probe_metrics, lambda idx: is_dset_de_affirm[idx]).accuracy\n",
    "    baseline_all_unseen_de_affirm_acc = get_aggregate_trad_metrics(baseline_probe_metrics, lambda idx: is_dset_de_affirm[idx]).accuracy\n",
    "    \n",
    "    ttpd_all_unseen_de_neg_acc = get_aggregate_trad_metrics(ttpd_probe_metrics, lambda idx: is_dset_de_negated[idx]).accuracy\n",
    "    baseline_all_unseen_de_neg_acc = get_aggregate_trad_metrics(baseline_probe_metrics, lambda idx: is_dset_de_negated[idx]).accuracy\n",
    "\n",
    "    animal_class_filter: Callable[[int], bool] = lambda idx: dset_categ_lookup[idx] == \"animal_class\"\n",
    "    ttpd_all_unseen_animal_class_acc = get_aggregate_trad_metrics(ttpd_probe_metrics, animal_class_filter).accuracy\n",
    "    baseline_all_unseen_animal_class_acc = get_aggregate_trad_metrics(baseline_probe_metrics, animal_class_filter).accuracy\n",
    "    \n",
    "    cities_filter: Callable[[int], bool] = lambda idx: dset_categ_lookup[idx] == \"cities\"\n",
    "    ttpd_all_unseen_cities_acc = get_aggregate_trad_metrics(ttpd_probe_metrics, cities_filter).accuracy\n",
    "    baseline_all_unseen_cities_acc = get_aggregate_trad_metrics(baseline_probe_metrics, cities_filter).accuracy\n",
    "    \n",
    "    element_symbol_filter: Callable[[int], bool] = lambda idx: dset_categ_lookup[idx] == \"element_symb\"\n",
    "    ttpd_all_unseen_element_symbol_acc = get_aggregate_trad_metrics(ttpd_probe_metrics, element_symbol_filter).accuracy\n",
    "    baseline_all_unseen_element_symbol_acc = get_aggregate_trad_metrics(baseline_probe_metrics, element_symbol_filter).accuracy\n",
    "    \n",
    "    facts_filter: Callable[[int], bool] = lambda idx: dset_categ_lookup[idx] == \"facts\"\n",
    "    ttpd_all_unseen_facts_acc = get_aggregate_trad_metrics(ttpd_probe_metrics, facts_filter).accuracy\n",
    "    baseline_all_unseen_facts_acc = get_aggregate_trad_metrics(baseline_probe_metrics, facts_filter).accuracy\n",
    "    \n",
    "    inventors_filter: Callable[[int], bool] = lambda idx: dset_categ_lookup[idx] == \"inventors\"\n",
    "    ttpd_all_unseen_inventors_acc = get_aggregate_trad_metrics(ttpd_probe_metrics, inventors_filter).accuracy\n",
    "    baseline_all_unseen_inventors_acc = get_aggregate_trad_metrics(baseline_probe_metrics, inventors_filter).accuracy\n",
    "    \n",
    "    sp_en_trans_filter: Callable[[int], bool] = lambda idx: dset_categ_lookup[idx] == \"sp_en_trans\"\n",
    "    ttpd_all_unseen_sp_en_trans_acc = get_aggregate_trad_metrics(ttpd_probe_metrics, sp_en_trans_filter).accuracy\n",
    "    baseline_all_unseen_sp_en_trans_acc = get_aggregate_trad_metrics(baseline_probe_metrics, sp_en_trans_filter).accuracy\n",
    "    \n",
    "    real_world_scenarios_filter: Callable[[int], bool] = lambda idx: dset_categ_lookup[idx] == \"real_world_scenarios\"\n",
    "    ttpd_all_unseen_real_world_scenarios_acc = get_aggregate_trad_metrics(ttpd_probe_metrics, real_world_scenarios_filter).accuracy\n",
    "    baseline_all_unseen_real_world_scenarios_acc = get_aggregate_trad_metrics(baseline_probe_metrics, real_world_scenarios_filter).accuracy\n",
    "    \n",
    "    relative_comparison_filter: Callable[[int], bool] = lambda idx: dset_categ_lookup[idx] == \"relative_comparison\"\n",
    "    ttpd_all_unseen_relative_comparison_acc = get_aggregate_trad_metrics(ttpd_probe_metrics, relative_comparison_filter).accuracy\n",
    "    baseline_all_unseen_relative_comparison_acc = get_aggregate_trad_metrics(baseline_probe_metrics, relative_comparison_filter).accuracy\n",
    "    \n",
    "    true_false_filter: Callable[[int], bool] = lambda idx: dset_categ_lookup[idx] == \"true_false\"\n",
    "    ttpd_all_unseen_true_false_acc = get_aggregate_trad_metrics(ttpd_probe_metrics, true_false_filter).accuracy\n",
    "    baseline_all_unseen_true_false_acc = get_aggregate_trad_metrics(baseline_probe_metrics, true_false_filter).accuracy\n",
    "    \n",
    "    ttpd_multi_topic_acc_row_dict = {\n",
    "        \"scenario\": scenario_labels[scenario_id], \"all\": ttpd_all_unseen_acc, \"en_affirm\": ttpd_all_unseen_en_affirm_acc, \"en_negated\": ttpd_all_unseen_en_neg_acc,\n",
    "        \"conj\": ttpd_all_unseen_conj_acc, \"disj\": ttpd_all_unseen_disj_acc, \"de_affirm\": ttpd_all_unseen_de_affirm_acc, \"de_negated\": ttpd_all_unseen_de_neg_acc\n",
    "    }\n",
    "    lyr18_probes_acc_on_unseen_multi_topic_data_groupings_row_dicts.append(ttpd_multi_topic_acc_row_dict)\n",
    "    baseline_multi_topic_acc_row_dict = {\n",
    "        \"scenario\": scenario_labels[scenario_id], \"all\": baseline_all_unseen_acc, \"en_affirm\": baseline_all_unseen_en_affirm_acc, \n",
    "        \"en_negated\": baseline_all_unseen_en_neg_acc, \"conj\": baseline_all_unseen_conj_acc, \"disj\": baseline_all_unseen_disj_acc, \n",
    "        \"de_affirm\": baseline_all_unseen_de_affirm_acc, \"de_negated\": baseline_all_unseen_de_neg_acc\n",
    "    }\n",
    "    lyr18_baseline_linear_probes_acc_on_unseen_multi_topic_data_groupings_row_dicts.append(baseline_multi_topic_acc_row_dict)\n",
    "    lyr18_ttpd_probes_rel_to_baseline_probes_acc_delta_on_unseen_multi_topic_data_groupings_row_dicts.append({\n",
    "        col_nm: (ttpd_multi_topic_acc_row_dict[col_nm] if col_nm == \"scenario\" else ttpd_multi_topic_acc_row_dict[col_nm] - baseline_multi_topic_acc_row_dict[col_nm]) for col_nm in ttpd_multi_topic_acc_row_dict.keys()\n",
    "    })\n",
    "    \n",
    "    ttpd_single_topic_acc_row_dict = {\n",
    "        \"scenario\": scenario_labels[scenario_id], \"animal class\": ttpd_all_unseen_animal_class_acc, \"cities\": ttpd_all_unseen_cities_acc,\n",
    "        \"element symbols\": ttpd_all_unseen_element_symbol_acc, \"facts\": ttpd_all_unseen_facts_acc,\n",
    "        \"inventors\": ttpd_all_unseen_inventors_acc, \"spanish english translation\": ttpd_all_unseen_sp_en_trans_acc, \n",
    "        \"real world scenarios\": ttpd_all_unseen_real_world_scenarios_acc, \"relative comparison\": ttpd_all_unseen_relative_comparison_acc,\n",
    "        \"true false\": ttpd_all_unseen_true_false_acc\n",
    "    }\n",
    "    lyr18_probes_acc_on_unseen_single_topic_data_groupings_row_dicts.append(ttpd_single_topic_acc_row_dict)\n",
    "    baseline_single_topic_acc_row_dict = {\n",
    "        \"scenario\": scenario_labels[scenario_id],  \"animal class\": baseline_all_unseen_animal_class_acc, \"cities\": baseline_all_unseen_cities_acc,\n",
    "        \"element symbols\": baseline_all_unseen_element_symbol_acc, \"facts\": baseline_all_unseen_facts_acc,\n",
    "        \"inventors\": baseline_all_unseen_inventors_acc, \"spanish english translation\": baseline_all_unseen_sp_en_trans_acc, \n",
    "        \"real world scenarios\": baseline_all_unseen_real_world_scenarios_acc, \"relative comparison\": baseline_all_unseen_relative_comparison_acc,\n",
    "        \"true false\": baseline_all_unseen_true_false_acc\n",
    "    }\n",
    "    lyr18_baseline_linear_probes_acc_on_unseen_single_topic_data_groupings_row_dicts.append(baseline_single_topic_acc_row_dict)\n",
    "    lyr18_ttpd_probes_rel_to_baseline_probes_acc_delta_on_unseen_single_topic_data_groupings_row_dicts.append({\n",
    "        col_nm: (ttpd_single_topic_acc_row_dict[col_nm] if col_nm == \"scenario\" else ttpd_single_topic_acc_row_dict[col_nm] - baseline_single_topic_acc_row_dict[col_nm]) for col_nm in ttpd_single_topic_acc_row_dict.keys()\n",
    "    })"
   ],
   "id": "95f96b1768f5db81",
   "outputs": [],
   "execution_count": 42
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:58.861404Z",
     "start_time": "2024-12-07T21:26:58.855725Z"
    }
   },
   "cell_type": "code",
   "source": [
    "lyr18_probes_acc_on_unseen_multi_topic_data_groupings_df = pd.DataFrame(lyr18_probes_acc_on_unseen_multi_topic_data_groupings_row_dicts)\n",
    "lyr18_baseline_linear_probes_acc_on_unseen_multi_topic_data_groupings_df = pd.DataFrame(lyr18_baseline_linear_probes_acc_on_unseen_multi_topic_data_groupings_row_dicts)\n",
    "lyr18_probes_acc_on_unseen_single_topic_data_groupings_df = pd.DataFrame(lyr18_probes_acc_on_unseen_single_topic_data_groupings_row_dicts)\n",
    "lyr18_baseline_linear_probes_acc_on_unseen_single_topic_data_groupings_df = pd.DataFrame(lyr18_baseline_linear_probes_acc_on_unseen_single_topic_data_groupings_row_dicts)\n",
    "\n",
    "lyr18_ttpd_probes_rel_to_baseline_probes_acc_delta_on_unseen_multi_topic_data_groupings_df = pd.DataFrame(lyr18_ttpd_probes_rel_to_baseline_probes_acc_delta_on_unseen_multi_topic_data_groupings_row_dicts)\n",
    "lyr18_ttpd_probes_rel_to_baseline_probes_acc_delta_on_unseen_single_topic_data_groupings_df = pd.DataFrame(lyr18_ttpd_probes_rel_to_baseline_probes_acc_delta_on_unseen_single_topic_data_groupings_row_dicts)"
   ],
   "id": "37c531ab2cc335c6",
   "outputs": [],
   "execution_count": 43
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:58.877064Z",
     "start_time": "2024-12-07T21:26:58.862409Z"
    }
   },
   "cell_type": "code",
   "source": "lyr18_probes_acc_on_unseen_multi_topic_data_groupings_df",
   "id": "2a66d8d8a499195c",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "                                             scenario       all  en_affirm  \\\n",
       "0                           animal_class__affirmative  0.598928   0.840160   \n",
       "1                           animal_class__conjunction  0.616058   0.838017   \n",
       "2                           animal_class__disjunction  0.646416   0.856015   \n",
       "3                               animal_class__negated  0.622431   0.876539   \n",
       "4                                 cities__affirmative  0.573471   0.683423   \n",
       "5                                 cities__conjunction  0.679085   0.882223   \n",
       "6                                 cities__disjunction  0.615186   0.801389   \n",
       "7                                     cities__negated  0.556986   0.641932   \n",
       "8                           element_symb__affirmative  0.701161   0.903388   \n",
       "9                           element_symb__conjunction  0.628308   0.694664   \n",
       "10                          element_symb__disjunction  0.575339   0.622987   \n",
       "11                              element_symb__negated  0.533518   0.292390   \n",
       "12                                 facts__affirmative  0.676092   0.924021   \n",
       "13                                 facts__conjunction  0.574209   0.622040   \n",
       "14                                 facts__disjunction  0.557769   0.753710   \n",
       "15                                     facts__negated  0.556267   0.724661   \n",
       "16                             inventors__affirmative  0.672174   0.830858   \n",
       "17                             inventors__conjunction  0.545092   0.596148   \n",
       "18                             inventors__disjunction  0.577730   0.606252   \n",
       "19                                 inventors__negated  0.465199   0.211241   \n",
       "20                                      ambiguous_lie  0.505988   0.501105   \n",
       "21                           ambiguous_truthful_reply  0.469807   0.428165   \n",
       "22              honest_reply_despite_incentive_to_lie  0.499688   0.507105   \n",
       "23                                    unambiguous_lie  0.534747   0.598990   \n",
       "24                         unambiguous_truthful_reply  0.524016   0.502684   \n",
       "25                                        larger_than  0.706914   0.937480   \n",
       "26                                       smaller_than  0.645057   0.872750   \n",
       "27                               sp_en_trans__negated  0.493253   0.413641   \n",
       "28                           sp_en_trans__affirmative  0.642989   0.879844   \n",
       "29                           sp_en_trans__conjunction  0.623397   0.749921   \n",
       "30                           sp_en_trans__disjunction  0.565090   0.628039   \n",
       "31                            common_claim_true_false  0.649682   0.898011   \n",
       "32                             counterfact_true_false  0.789570   0.948847   \n",
       "33                  animal_class__affirmative+negated  0.652084   0.909424   \n",
       "34              animal_class__affirmative+disjunction  0.616356   0.858142   \n",
       "35                  animal_class__negated+conjunction  0.630521   0.865172   \n",
       "36  animal_class__affirmative+negated+conjunction+...  0.646796   0.887779   \n",
       "37                        cities__affirmative+negated  0.623329   0.845003   \n",
       "38                    cities__affirmative+disjunction  0.587876   0.719330   \n",
       "39                        cities__negated+conjunction  0.675916   0.910957   \n",
       "40  cities__affirmative+negated+conjunction+disjun...  0.627967   0.816278   \n",
       "41                  element_symb__affirmative+negated  0.661379   0.909091   \n",
       "42              element_symb__affirmative+disjunction  0.643407   0.899363   \n",
       "43                  element_symb__negated+conjunction  0.647982   0.630249   \n",
       "44  element_symb__affirmative+negated+conjunction+...  0.706974   0.921503   \n",
       "45                         facts__affirmative+negated  0.625623   0.909823   \n",
       "46                     facts__affirmative+disjunction  0.711958   0.926708   \n",
       "47                         facts__negated+conjunction  0.693473   0.925166   \n",
       "48  facts__affirmative+negated+conjunction+disjunc...  0.676826   0.930929   \n",
       "49                     inventors__affirmative+negated  0.621356   0.845346   \n",
       "50                 inventors__affirmative+disjunction  0.681886   0.850054   \n",
       "51                     inventors__negated+conjunction  0.542446   0.489738   \n",
       "52  inventors__affirmative+negated+conjunction+dis...  0.661424   0.870337   \n",
       "53                   sp_en_trans__affirmative+negated  0.674909   0.910060   \n",
       "54               sp_en_trans__affirmative+disjunction  0.579624   0.854604   \n",
       "55                   sp_en_trans__negated+conjunction  0.570426   0.481844   \n",
       "56  sp_en_trans__affirmative+negated+conjunction+d...  0.690017   0.910060   \n",
       "57  ambiguous_truthful_reply+unambiguous_lie+unamb...  0.564910   0.821282   \n",
       "58  animal_class+facts+inventors__affirmative+nega...  0.728863   0.976424   \n",
       "59  animal_class+facts+inventors__affirmative+nega...  0.726494   0.976916   \n",
       "\n",
       "    en_negated      conj      disj  de_affirm  de_negated  \n",
       "0     0.640354  0.642321  0.580667   0.723333    0.481605  \n",
       "1     0.660562  0.749285  0.594000   0.693333    0.498328  \n",
       "2     0.509946  0.716108  0.598400   0.636667    0.548495  \n",
       "3     0.837829  0.634317  0.511000   0.706667    0.675585  \n",
       "4     0.650142  0.593547  0.489667   0.656667    0.484950  \n",
       "5     0.748974  0.782400  0.549000   0.750000    0.578595  \n",
       "6     0.491632  0.674837  0.668800   0.583333    0.511706  \n",
       "7     0.779174  0.529265  0.488333   0.566667    0.709030  \n",
       "8     0.525734  0.821911  0.502667   0.820000    0.541806  \n",
       "9     0.524155  0.817896  0.604000   0.783333    0.515050  \n",
       "10    0.565204  0.585043  0.609200   0.666667    0.441472  \n",
       "11    0.875210  0.477489  0.497000   0.456667    0.762542  \n",
       "12    0.520051  0.845673  0.539000   0.790000    0.438127  \n",
       "13    0.508368  0.830475  0.508000   0.650000    0.518395  \n",
       "14    0.508052  0.661581  0.569600   0.606667    0.484950  \n",
       "15    0.869148  0.512756  0.498000   0.570000    0.672241  \n",
       "16    0.709504  0.745623  0.624667   0.806667    0.545151  \n",
       "17    0.514367  0.713265  0.503000   0.640000    0.494983  \n",
       "18    0.506473  0.766883  0.573600   0.526667    0.501672  \n",
       "19    0.649402  0.510255  0.476000   0.396667    0.635452  \n",
       "20    0.511209  0.455978  0.499333   0.520000    0.508361  \n",
       "21    0.491632  0.467984  0.511000   0.426667    0.541806  \n",
       "22    0.493211  0.510255  0.499000   0.513333    0.481605  \n",
       "23    0.683612  0.513507  0.497000   0.593333    0.575251  \n",
       "24    0.502684  0.523512  0.502000   0.486667    0.508361  \n",
       "25    0.528576  0.854927  0.600667   0.843333    0.484950  \n",
       "26    0.683612  0.753627  0.633000   0.726667    0.528428  \n",
       "27    0.821898  0.459980  0.498000   0.433333    0.836120  \n",
       "28    0.697506  0.695598  0.632000   0.763333    0.528428  \n",
       "29    0.635302  0.825043  0.587667   0.746667    0.518395  \n",
       "30    0.554784  0.594047  0.519200   0.576667    0.525084  \n",
       "31    0.632775  0.649575  0.504667   0.783333    0.605351  \n",
       "32    0.662457  0.883942  0.507000   0.826667    0.618729  \n",
       "33    0.850150  0.679090  0.609000   0.763333    0.715719  \n",
       "34    0.654563  0.678089  0.610400   0.733333    0.471572  \n",
       "35    0.704296  0.765009  0.593000   0.723333    0.541806  \n",
       "36    0.779887  0.738708  0.644400   0.743333    0.605351  \n",
       "37    0.828247  0.616808  0.576333   0.740000    0.765886  \n",
       "38    0.660562  0.645823  0.589600   0.706667    0.491639  \n",
       "39    0.675643  0.755200  0.605000   0.766667    0.672241  \n",
       "40    0.704967  0.686400  0.662000   0.743333    0.682274  \n",
       "41    0.902382  0.638069  0.539333   0.740000    0.752508  \n",
       "42    0.695611  0.648074  0.628800   0.763333    0.545151  \n",
       "43    0.885273  0.672956  0.577333   0.730000    0.826087  \n",
       "44    0.904394  0.668382  0.643200   0.796667    0.779264  \n",
       "45    0.920568  0.579040  0.534333   0.746667    0.759197  \n",
       "46    0.749290  0.861181  0.553200   0.753333    0.591973  \n",
       "47    0.954720  0.750429  0.597667   0.776667    0.809365  \n",
       "48    0.945510  0.736707  0.548800   0.770000    0.775920  \n",
       "49    0.876494  0.570785  0.530667   0.683333    0.759197  \n",
       "50    0.708557  0.797399  0.540000   0.826667    0.575251  \n",
       "51    0.874683  0.503716  0.496000   0.520000    0.742475  \n",
       "52    0.506338  0.794454  0.620400   0.796667    0.518395  \n",
       "53    0.853893  0.641571  0.561333   0.803333    0.705686  \n",
       "54    0.580991  0.599050  0.504400   0.686667    0.474916  \n",
       "55    0.878066  0.580046  0.503333   0.523333    0.849498  \n",
       "56    0.889797  0.559748  0.513200   0.816667    0.729097  \n",
       "57    0.578781  0.523012  0.493667   0.660000    0.511706  \n",
       "58    0.877210  0.845476  0.568667   0.806667    0.712375  \n",
       "59    0.943517  0.841921  0.509000   0.796667    0.732441  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>scenario</th>\n",
       "      <th>all</th>\n",
       "      <th>en_affirm</th>\n",
       "      <th>en_negated</th>\n",
       "      <th>conj</th>\n",
       "      <th>disj</th>\n",
       "      <th>de_affirm</th>\n",
       "      <th>de_negated</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>animal_class__affirmative</td>\n",
       "      <td>0.598928</td>\n",
       "      <td>0.840160</td>\n",
       "      <td>0.640354</td>\n",
       "      <td>0.642321</td>\n",
       "      <td>0.580667</td>\n",
       "      <td>0.723333</td>\n",
       "      <td>0.481605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>animal_class__conjunction</td>\n",
       "      <td>0.616058</td>\n",
       "      <td>0.838017</td>\n",
       "      <td>0.660562</td>\n",
       "      <td>0.749285</td>\n",
       "      <td>0.594000</td>\n",
       "      <td>0.693333</td>\n",
       "      <td>0.498328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>animal_class__disjunction</td>\n",
       "      <td>0.646416</td>\n",
       "      <td>0.856015</td>\n",
       "      <td>0.509946</td>\n",
       "      <td>0.716108</td>\n",
       "      <td>0.598400</td>\n",
       "      <td>0.636667</td>\n",
       "      <td>0.548495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>animal_class__negated</td>\n",
       "      <td>0.622431</td>\n",
       "      <td>0.876539</td>\n",
       "      <td>0.837829</td>\n",
       "      <td>0.634317</td>\n",
       "      <td>0.511000</td>\n",
       "      <td>0.706667</td>\n",
       "      <td>0.675585</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cities__affirmative</td>\n",
       "      <td>0.573471</td>\n",
       "      <td>0.683423</td>\n",
       "      <td>0.650142</td>\n",
       "      <td>0.593547</td>\n",
       "      <td>0.489667</td>\n",
       "      <td>0.656667</td>\n",
       "      <td>0.484950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>cities__conjunction</td>\n",
       "      <td>0.679085</td>\n",
       "      <td>0.882223</td>\n",
       "      <td>0.748974</td>\n",
       "      <td>0.782400</td>\n",
       "      <td>0.549000</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.578595</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>cities__disjunction</td>\n",
       "      <td>0.615186</td>\n",
       "      <td>0.801389</td>\n",
       "      <td>0.491632</td>\n",
       "      <td>0.674837</td>\n",
       "      <td>0.668800</td>\n",
       "      <td>0.583333</td>\n",
       "      <td>0.511706</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>cities__negated</td>\n",
       "      <td>0.556986</td>\n",
       "      <td>0.641932</td>\n",
       "      <td>0.779174</td>\n",
       "      <td>0.529265</td>\n",
       "      <td>0.488333</td>\n",
       "      <td>0.566667</td>\n",
       "      <td>0.709030</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>element_symb__affirmative</td>\n",
       "      <td>0.701161</td>\n",
       "      <td>0.903388</td>\n",
       "      <td>0.525734</td>\n",
       "      <td>0.821911</td>\n",
       "      <td>0.502667</td>\n",
       "      <td>0.820000</td>\n",
       "      <td>0.541806</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>element_symb__conjunction</td>\n",
       "      <td>0.628308</td>\n",
       "      <td>0.694664</td>\n",
       "      <td>0.524155</td>\n",
       "      <td>0.817896</td>\n",
       "      <td>0.604000</td>\n",
       "      <td>0.783333</td>\n",
       "      <td>0.515050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>element_symb__disjunction</td>\n",
       "      <td>0.575339</td>\n",
       "      <td>0.622987</td>\n",
       "      <td>0.565204</td>\n",
       "      <td>0.585043</td>\n",
       "      <td>0.609200</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.441472</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>element_symb__negated</td>\n",
       "      <td>0.533518</td>\n",
       "      <td>0.292390</td>\n",
       "      <td>0.875210</td>\n",
       "      <td>0.477489</td>\n",
       "      <td>0.497000</td>\n",
       "      <td>0.456667</td>\n",
       "      <td>0.762542</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>facts__affirmative</td>\n",
       "      <td>0.676092</td>\n",
       "      <td>0.924021</td>\n",
       "      <td>0.520051</td>\n",
       "      <td>0.845673</td>\n",
       "      <td>0.539000</td>\n",
       "      <td>0.790000</td>\n",
       "      <td>0.438127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>facts__conjunction</td>\n",
       "      <td>0.574209</td>\n",
       "      <td>0.622040</td>\n",
       "      <td>0.508368</td>\n",
       "      <td>0.830475</td>\n",
       "      <td>0.508000</td>\n",
       "      <td>0.650000</td>\n",
       "      <td>0.518395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>facts__disjunction</td>\n",
       "      <td>0.557769</td>\n",
       "      <td>0.753710</td>\n",
       "      <td>0.508052</td>\n",
       "      <td>0.661581</td>\n",
       "      <td>0.569600</td>\n",
       "      <td>0.606667</td>\n",
       "      <td>0.484950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>facts__negated</td>\n",
       "      <td>0.556267</td>\n",
       "      <td>0.724661</td>\n",
       "      <td>0.869148</td>\n",
       "      <td>0.512756</td>\n",
       "      <td>0.498000</td>\n",
       "      <td>0.570000</td>\n",
       "      <td>0.672241</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>inventors__affirmative</td>\n",
       "      <td>0.672174</td>\n",
       "      <td>0.830858</td>\n",
       "      <td>0.709504</td>\n",
       "      <td>0.745623</td>\n",
       "      <td>0.624667</td>\n",
       "      <td>0.806667</td>\n",
       "      <td>0.545151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>inventors__conjunction</td>\n",
       "      <td>0.545092</td>\n",
       "      <td>0.596148</td>\n",
       "      <td>0.514367</td>\n",
       "      <td>0.713265</td>\n",
       "      <td>0.503000</td>\n",
       "      <td>0.640000</td>\n",
       "      <td>0.494983</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>inventors__disjunction</td>\n",
       "      <td>0.577730</td>\n",
       "      <td>0.606252</td>\n",
       "      <td>0.506473</td>\n",
       "      <td>0.766883</td>\n",
       "      <td>0.573600</td>\n",
       "      <td>0.526667</td>\n",
       "      <td>0.501672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>inventors__negated</td>\n",
       "      <td>0.465199</td>\n",
       "      <td>0.211241</td>\n",
       "      <td>0.649402</td>\n",
       "      <td>0.510255</td>\n",
       "      <td>0.476000</td>\n",
       "      <td>0.396667</td>\n",
       "      <td>0.635452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>ambiguous_lie</td>\n",
       "      <td>0.505988</td>\n",
       "      <td>0.501105</td>\n",
       "      <td>0.511209</td>\n",
       "      <td>0.455978</td>\n",
       "      <td>0.499333</td>\n",
       "      <td>0.520000</td>\n",
       "      <td>0.508361</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>ambiguous_truthful_reply</td>\n",
       "      <td>0.469807</td>\n",
       "      <td>0.428165</td>\n",
       "      <td>0.491632</td>\n",
       "      <td>0.467984</td>\n",
       "      <td>0.511000</td>\n",
       "      <td>0.426667</td>\n",
       "      <td>0.541806</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>honest_reply_despite_incentive_to_lie</td>\n",
       "      <td>0.499688</td>\n",
       "      <td>0.507105</td>\n",
       "      <td>0.493211</td>\n",
       "      <td>0.510255</td>\n",
       "      <td>0.499000</td>\n",
       "      <td>0.513333</td>\n",
       "      <td>0.481605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>unambiguous_lie</td>\n",
       "      <td>0.534747</td>\n",
       "      <td>0.598990</td>\n",
       "      <td>0.683612</td>\n",
       "      <td>0.513507</td>\n",
       "      <td>0.497000</td>\n",
       "      <td>0.593333</td>\n",
       "      <td>0.575251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>unambiguous_truthful_reply</td>\n",
       "      <td>0.524016</td>\n",
       "      <td>0.502684</td>\n",
       "      <td>0.502684</td>\n",
       "      <td>0.523512</td>\n",
       "      <td>0.502000</td>\n",
       "      <td>0.486667</td>\n",
       "      <td>0.508361</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>larger_than</td>\n",
       "      <td>0.706914</td>\n",
       "      <td>0.937480</td>\n",
       "      <td>0.528576</td>\n",
       "      <td>0.854927</td>\n",
       "      <td>0.600667</td>\n",
       "      <td>0.843333</td>\n",
       "      <td>0.484950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>smaller_than</td>\n",
       "      <td>0.645057</td>\n",
       "      <td>0.872750</td>\n",
       "      <td>0.683612</td>\n",
       "      <td>0.753627</td>\n",
       "      <td>0.633000</td>\n",
       "      <td>0.726667</td>\n",
       "      <td>0.528428</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>sp_en_trans__negated</td>\n",
       "      <td>0.493253</td>\n",
       "      <td>0.413641</td>\n",
       "      <td>0.821898</td>\n",
       "      <td>0.459980</td>\n",
       "      <td>0.498000</td>\n",
       "      <td>0.433333</td>\n",
       "      <td>0.836120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>sp_en_trans__affirmative</td>\n",
       "      <td>0.642989</td>\n",
       "      <td>0.879844</td>\n",
       "      <td>0.697506</td>\n",
       "      <td>0.695598</td>\n",
       "      <td>0.632000</td>\n",
       "      <td>0.763333</td>\n",
       "      <td>0.528428</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>sp_en_trans__conjunction</td>\n",
       "      <td>0.623397</td>\n",
       "      <td>0.749921</td>\n",
       "      <td>0.635302</td>\n",
       "      <td>0.825043</td>\n",
       "      <td>0.587667</td>\n",
       "      <td>0.746667</td>\n",
       "      <td>0.518395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>sp_en_trans__disjunction</td>\n",
       "      <td>0.565090</td>\n",
       "      <td>0.628039</td>\n",
       "      <td>0.554784</td>\n",
       "      <td>0.594047</td>\n",
       "      <td>0.519200</td>\n",
       "      <td>0.576667</td>\n",
       "      <td>0.525084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>common_claim_true_false</td>\n",
       "      <td>0.649682</td>\n",
       "      <td>0.898011</td>\n",
       "      <td>0.632775</td>\n",
       "      <td>0.649575</td>\n",
       "      <td>0.504667</td>\n",
       "      <td>0.783333</td>\n",
       "      <td>0.605351</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>counterfact_true_false</td>\n",
       "      <td>0.789570</td>\n",
       "      <td>0.948847</td>\n",
       "      <td>0.662457</td>\n",
       "      <td>0.883942</td>\n",
       "      <td>0.507000</td>\n",
       "      <td>0.826667</td>\n",
       "      <td>0.618729</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>animal_class__affirmative+negated</td>\n",
       "      <td>0.652084</td>\n",
       "      <td>0.909424</td>\n",
       "      <td>0.850150</td>\n",
       "      <td>0.679090</td>\n",
       "      <td>0.609000</td>\n",
       "      <td>0.763333</td>\n",
       "      <td>0.715719</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>animal_class__affirmative+disjunction</td>\n",
       "      <td>0.616356</td>\n",
       "      <td>0.858142</td>\n",
       "      <td>0.654563</td>\n",
       "      <td>0.678089</td>\n",
       "      <td>0.610400</td>\n",
       "      <td>0.733333</td>\n",
       "      <td>0.471572</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>animal_class__negated+conjunction</td>\n",
       "      <td>0.630521</td>\n",
       "      <td>0.865172</td>\n",
       "      <td>0.704296</td>\n",
       "      <td>0.765009</td>\n",
       "      <td>0.593000</td>\n",
       "      <td>0.723333</td>\n",
       "      <td>0.541806</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>animal_class__affirmative+negated+conjunction+...</td>\n",
       "      <td>0.646796</td>\n",
       "      <td>0.887779</td>\n",
       "      <td>0.779887</td>\n",
       "      <td>0.738708</td>\n",
       "      <td>0.644400</td>\n",
       "      <td>0.743333</td>\n",
       "      <td>0.605351</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>cities__affirmative+negated</td>\n",
       "      <td>0.623329</td>\n",
       "      <td>0.845003</td>\n",
       "      <td>0.828247</td>\n",
       "      <td>0.616808</td>\n",
       "      <td>0.576333</td>\n",
       "      <td>0.740000</td>\n",
       "      <td>0.765886</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>cities__affirmative+disjunction</td>\n",
       "      <td>0.587876</td>\n",
       "      <td>0.719330</td>\n",
       "      <td>0.660562</td>\n",
       "      <td>0.645823</td>\n",
       "      <td>0.589600</td>\n",
       "      <td>0.706667</td>\n",
       "      <td>0.491639</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>cities__negated+conjunction</td>\n",
       "      <td>0.675916</td>\n",
       "      <td>0.910957</td>\n",
       "      <td>0.675643</td>\n",
       "      <td>0.755200</td>\n",
       "      <td>0.605000</td>\n",
       "      <td>0.766667</td>\n",
       "      <td>0.672241</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>cities__affirmative+negated+conjunction+disjun...</td>\n",
       "      <td>0.627967</td>\n",
       "      <td>0.816278</td>\n",
       "      <td>0.704967</td>\n",
       "      <td>0.686400</td>\n",
       "      <td>0.662000</td>\n",
       "      <td>0.743333</td>\n",
       "      <td>0.682274</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>element_symb__affirmative+negated</td>\n",
       "      <td>0.661379</td>\n",
       "      <td>0.909091</td>\n",
       "      <td>0.902382</td>\n",
       "      <td>0.638069</td>\n",
       "      <td>0.539333</td>\n",
       "      <td>0.740000</td>\n",
       "      <td>0.752508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>element_symb__affirmative+disjunction</td>\n",
       "      <td>0.643407</td>\n",
       "      <td>0.899363</td>\n",
       "      <td>0.695611</td>\n",
       "      <td>0.648074</td>\n",
       "      <td>0.628800</td>\n",
       "      <td>0.763333</td>\n",
       "      <td>0.545151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>element_symb__negated+conjunction</td>\n",
       "      <td>0.647982</td>\n",
       "      <td>0.630249</td>\n",
       "      <td>0.885273</td>\n",
       "      <td>0.672956</td>\n",
       "      <td>0.577333</td>\n",
       "      <td>0.730000</td>\n",
       "      <td>0.826087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>element_symb__affirmative+negated+conjunction+...</td>\n",
       "      <td>0.706974</td>\n",
       "      <td>0.921503</td>\n",
       "      <td>0.904394</td>\n",
       "      <td>0.668382</td>\n",
       "      <td>0.643200</td>\n",
       "      <td>0.796667</td>\n",
       "      <td>0.779264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>facts__affirmative+negated</td>\n",
       "      <td>0.625623</td>\n",
       "      <td>0.909823</td>\n",
       "      <td>0.920568</td>\n",
       "      <td>0.579040</td>\n",
       "      <td>0.534333</td>\n",
       "      <td>0.746667</td>\n",
       "      <td>0.759197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>facts__affirmative+disjunction</td>\n",
       "      <td>0.711958</td>\n",
       "      <td>0.926708</td>\n",
       "      <td>0.749290</td>\n",
       "      <td>0.861181</td>\n",
       "      <td>0.553200</td>\n",
       "      <td>0.753333</td>\n",
       "      <td>0.591973</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>facts__negated+conjunction</td>\n",
       "      <td>0.693473</td>\n",
       "      <td>0.925166</td>\n",
       "      <td>0.954720</td>\n",
       "      <td>0.750429</td>\n",
       "      <td>0.597667</td>\n",
       "      <td>0.776667</td>\n",
       "      <td>0.809365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>facts__affirmative+negated+conjunction+disjunc...</td>\n",
       "      <td>0.676826</td>\n",
       "      <td>0.930929</td>\n",
       "      <td>0.945510</td>\n",
       "      <td>0.736707</td>\n",
       "      <td>0.548800</td>\n",
       "      <td>0.770000</td>\n",
       "      <td>0.775920</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>inventors__affirmative+negated</td>\n",
       "      <td>0.621356</td>\n",
       "      <td>0.845346</td>\n",
       "      <td>0.876494</td>\n",
       "      <td>0.570785</td>\n",
       "      <td>0.530667</td>\n",
       "      <td>0.683333</td>\n",
       "      <td>0.759197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>inventors__affirmative+disjunction</td>\n",
       "      <td>0.681886</td>\n",
       "      <td>0.850054</td>\n",
       "      <td>0.708557</td>\n",
       "      <td>0.797399</td>\n",
       "      <td>0.540000</td>\n",
       "      <td>0.826667</td>\n",
       "      <td>0.575251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>inventors__negated+conjunction</td>\n",
       "      <td>0.542446</td>\n",
       "      <td>0.489738</td>\n",
       "      <td>0.874683</td>\n",
       "      <td>0.503716</td>\n",
       "      <td>0.496000</td>\n",
       "      <td>0.520000</td>\n",
       "      <td>0.742475</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>inventors__affirmative+negated+conjunction+dis...</td>\n",
       "      <td>0.661424</td>\n",
       "      <td>0.870337</td>\n",
       "      <td>0.506338</td>\n",
       "      <td>0.794454</td>\n",
       "      <td>0.620400</td>\n",
       "      <td>0.796667</td>\n",
       "      <td>0.518395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>sp_en_trans__affirmative+negated</td>\n",
       "      <td>0.674909</td>\n",
       "      <td>0.910060</td>\n",
       "      <td>0.853893</td>\n",
       "      <td>0.641571</td>\n",
       "      <td>0.561333</td>\n",
       "      <td>0.803333</td>\n",
       "      <td>0.705686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>sp_en_trans__affirmative+disjunction</td>\n",
       "      <td>0.579624</td>\n",
       "      <td>0.854604</td>\n",
       "      <td>0.580991</td>\n",
       "      <td>0.599050</td>\n",
       "      <td>0.504400</td>\n",
       "      <td>0.686667</td>\n",
       "      <td>0.474916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>sp_en_trans__negated+conjunction</td>\n",
       "      <td>0.570426</td>\n",
       "      <td>0.481844</td>\n",
       "      <td>0.878066</td>\n",
       "      <td>0.580046</td>\n",
       "      <td>0.503333</td>\n",
       "      <td>0.523333</td>\n",
       "      <td>0.849498</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>sp_en_trans__affirmative+negated+conjunction+d...</td>\n",
       "      <td>0.690017</td>\n",
       "      <td>0.910060</td>\n",
       "      <td>0.889797</td>\n",
       "      <td>0.559748</td>\n",
       "      <td>0.513200</td>\n",
       "      <td>0.816667</td>\n",
       "      <td>0.729097</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>ambiguous_truthful_reply+unambiguous_lie+unamb...</td>\n",
       "      <td>0.564910</td>\n",
       "      <td>0.821282</td>\n",
       "      <td>0.578781</td>\n",
       "      <td>0.523012</td>\n",
       "      <td>0.493667</td>\n",
       "      <td>0.660000</td>\n",
       "      <td>0.511706</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>animal_class+facts+inventors__affirmative+nega...</td>\n",
       "      <td>0.728863</td>\n",
       "      <td>0.976424</td>\n",
       "      <td>0.877210</td>\n",
       "      <td>0.845476</td>\n",
       "      <td>0.568667</td>\n",
       "      <td>0.806667</td>\n",
       "      <td>0.712375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>animal_class+facts+inventors__affirmative+nega...</td>\n",
       "      <td>0.726494</td>\n",
       "      <td>0.976916</td>\n",
       "      <td>0.943517</td>\n",
       "      <td>0.841921</td>\n",
       "      <td>0.509000</td>\n",
       "      <td>0.796667</td>\n",
       "      <td>0.732441</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 44
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:58.892568Z",
     "start_time": "2024-12-07T21:26:58.878073Z"
    }
   },
   "cell_type": "code",
   "source": "lyr18_probes_acc_on_unseen_multi_topic_data_groupings_df.describe()",
   "id": "6d80d39050df21bf",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "             all  en_affirm  en_negated       conj       disj  de_affirm  \\\n",
       "count  60.000000  60.000000   60.000000  60.000000  60.000000  60.000000   \n",
       "mean    0.618207   0.765337    0.697965   0.669601   0.555470   0.686444   \n",
       "std     0.069084   0.181112    0.149788   0.119410   0.053205   0.116672   \n",
       "min     0.465199   0.211241    0.491632   0.455978   0.476000   0.396667   \n",
       "25%     0.569092   0.629697    0.548232   0.579794   0.504133   0.603333   \n",
       "50%     0.624510   0.845174    0.683612   0.670669   0.548900   0.728333   \n",
       "75%     0.672858   0.904814    0.851086   0.757652   0.598967   0.771667   \n",
       "max     0.789570   0.976916    0.954720   0.883942   0.668800   0.843333   \n",
       "\n",
       "       de_negated  \n",
       "count   60.000000  \n",
       "mean     0.606633  \n",
       "std      0.117852  \n",
       "min      0.438127  \n",
       "25%      0.510870  \n",
       "50%      0.561873  \n",
       "75%      0.713211  \n",
       "max      0.849498  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>all</th>\n",
       "      <th>en_affirm</th>\n",
       "      <th>en_negated</th>\n",
       "      <th>conj</th>\n",
       "      <th>disj</th>\n",
       "      <th>de_affirm</th>\n",
       "      <th>de_negated</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.618207</td>\n",
       "      <td>0.765337</td>\n",
       "      <td>0.697965</td>\n",
       "      <td>0.669601</td>\n",
       "      <td>0.555470</td>\n",
       "      <td>0.686444</td>\n",
       "      <td>0.606633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.069084</td>\n",
       "      <td>0.181112</td>\n",
       "      <td>0.149788</td>\n",
       "      <td>0.119410</td>\n",
       "      <td>0.053205</td>\n",
       "      <td>0.116672</td>\n",
       "      <td>0.117852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.465199</td>\n",
       "      <td>0.211241</td>\n",
       "      <td>0.491632</td>\n",
       "      <td>0.455978</td>\n",
       "      <td>0.476000</td>\n",
       "      <td>0.396667</td>\n",
       "      <td>0.438127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.569092</td>\n",
       "      <td>0.629697</td>\n",
       "      <td>0.548232</td>\n",
       "      <td>0.579794</td>\n",
       "      <td>0.504133</td>\n",
       "      <td>0.603333</td>\n",
       "      <td>0.510870</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.624510</td>\n",
       "      <td>0.845174</td>\n",
       "      <td>0.683612</td>\n",
       "      <td>0.670669</td>\n",
       "      <td>0.548900</td>\n",
       "      <td>0.728333</td>\n",
       "      <td>0.561873</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.672858</td>\n",
       "      <td>0.904814</td>\n",
       "      <td>0.851086</td>\n",
       "      <td>0.757652</td>\n",
       "      <td>0.598967</td>\n",
       "      <td>0.771667</td>\n",
       "      <td>0.713211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.789570</td>\n",
       "      <td>0.976916</td>\n",
       "      <td>0.954720</td>\n",
       "      <td>0.883942</td>\n",
       "      <td>0.668800</td>\n",
       "      <td>0.843333</td>\n",
       "      <td>0.849498</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 45
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:58.907624Z",
     "start_time": "2024-12-07T21:26:58.893566Z"
    }
   },
   "cell_type": "code",
   "source": "lyr18_baseline_linear_probes_acc_on_unseen_multi_topic_data_groupings_df",
   "id": "c8976d71bc7846d2",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "                                             scenario       all  en_affirm  \\\n",
       "0                           animal_class__affirmative  0.628325   0.864469   \n",
       "1                           animal_class__conjunction  0.727648   0.920429   \n",
       "2                           animal_class__disjunction  0.512028   0.525103   \n",
       "3                               animal_class__negated  0.663155   0.799179   \n",
       "4                                 cities__affirmative  0.631684   0.763615   \n",
       "5                                 cities__conjunction  0.705427   0.916009   \n",
       "6                                 cities__disjunction  0.628012   0.805494   \n",
       "7                                     cities__negated  0.633931   0.848121   \n",
       "8                           element_symb__affirmative  0.583103   0.796713   \n",
       "9                           element_symb__conjunction  0.680610   0.929586   \n",
       "10                          element_symb__disjunction  0.688209   0.830123   \n",
       "11                              element_symb__negated  0.484982   0.435744   \n",
       "12                                 facts__affirmative  0.585750   0.664620   \n",
       "13                                 facts__conjunction  0.699663   0.914746   \n",
       "14                                 facts__disjunction  0.648584   0.825703   \n",
       "15                                     facts__negated  0.546915   0.604989   \n",
       "16                             inventors__affirmative  0.730953   0.931184   \n",
       "17                             inventors__conjunction  0.562347   0.557626   \n",
       "18                             inventors__disjunction  0.521091   0.519103   \n",
       "19                                 inventors__negated  0.542572   0.530786   \n",
       "20                                      ambiguous_lie  0.499026   0.492895   \n",
       "21                           ambiguous_truthful_reply  0.500597   0.507105   \n",
       "22              honest_reply_despite_incentive_to_lie  0.500753   0.507105   \n",
       "23                                    unambiguous_lie  0.498741   0.492895   \n",
       "24                         unambiguous_truthful_reply  0.500303   0.507105   \n",
       "25                                        larger_than  0.639092   0.856331   \n",
       "26                                       smaller_than  0.692717   0.906536   \n",
       "27                               sp_en_trans__negated  0.412905   0.127881   \n",
       "28                           sp_en_trans__affirmative  0.702931   0.907216   \n",
       "29                           sp_en_trans__conjunction  0.599285   0.627408   \n",
       "30                           sp_en_trans__disjunction  0.564441   0.544048   \n",
       "31                            common_claim_true_false  0.739791   0.931481   \n",
       "32                             counterfact_true_false  0.710386   0.917272   \n",
       "33                  animal_class__affirmative+negated  0.651068   0.895438   \n",
       "34              animal_class__affirmative+disjunction  0.650506   0.914752   \n",
       "35                  animal_class__negated+conjunction  0.750502   0.946321   \n",
       "36  animal_class__affirmative+negated+conjunction+...  0.747384   0.930736   \n",
       "37                        cities__affirmative+negated  0.657081   0.845601   \n",
       "38                    cities__affirmative+disjunction  0.661342   0.749850   \n",
       "39                        cities__negated+conjunction  0.704151   0.893274   \n",
       "40  cities__affirmative+negated+conjunction+disjun...  0.736364   0.891083   \n",
       "41                  element_symb__affirmative+negated  0.672694   0.784636   \n",
       "42              element_symb__affirmative+disjunction  0.686703   0.827910   \n",
       "43                  element_symb__negated+conjunction  0.707569   0.904642   \n",
       "44  element_symb__affirmative+negated+conjunction+...  0.697404   0.739685   \n",
       "45                         facts__affirmative+negated  0.731372   0.955871   \n",
       "46                     facts__affirmative+disjunction  0.669407   0.754797   \n",
       "47                         facts__negated+conjunction  0.706040   0.904010   \n",
       "48  facts__affirmative+negated+conjunction+disjunc...  0.734150   0.938987   \n",
       "49                     inventors__affirmative+negated  0.706920   0.950742   \n",
       "50                 inventors__affirmative+disjunction  0.716153   0.944223   \n",
       "51                     inventors__negated+conjunction  0.710009   0.888538   \n",
       "52  inventors__affirmative+negated+conjunction+dis...  0.742345   0.916697   \n",
       "53                   sp_en_trans__affirmative+negated  0.623205   0.599716   \n",
       "54               sp_en_trans__affirmative+disjunction  0.683557   0.731959   \n",
       "55                   sp_en_trans__negated+conjunction  0.660740   0.606883   \n",
       "56  sp_en_trans__affirmative+negated+conjunction+d...  0.711913   0.890508   \n",
       "57  ambiguous_truthful_reply+unambiguous_lie+unamb...  0.501463   0.518472   \n",
       "58  animal_class+facts+inventors__affirmative+nega...  0.760918   0.977407   \n",
       "59  animal_class+facts+inventors__affirmative+nega...  0.768171   0.980354   \n",
       "\n",
       "    en_negated      conj      disj  de_affirm  de_negated  \n",
       "0     0.692138  0.674837  0.647667   0.730000    0.478261  \n",
       "1     0.735081  0.885363  0.515667   0.760000    0.628763  \n",
       "2     0.508052  0.514757  0.646000   0.526667    0.521739  \n",
       "3     0.864469  0.663332  0.504667   0.716667    0.782609  \n",
       "4     0.522577  0.685093  0.521333   0.740000    0.347826  \n",
       "5     0.520366  0.794800  0.577667   0.800000    0.468227  \n",
       "6     0.578781  0.564032  0.729600   0.720000    0.525084  \n",
       "7     0.828845  0.592796  0.541333   0.786667    0.789298  \n",
       "8     0.633723  0.571786  0.502000   0.763333    0.515050  \n",
       "9     0.500789  0.842767  0.530667   0.820000    0.461538  \n",
       "10    0.715819  0.819910  0.730000   0.706667    0.568562  \n",
       "11    0.828581  0.460230  0.408000   0.366667    0.839465  \n",
       "12    0.581307  0.490245  0.502000   0.743333    0.598662  \n",
       "13    0.518472  0.708405  0.515667   0.760000    0.608696  \n",
       "14    0.531418  0.707854  0.630800   0.670000    0.488294  \n",
       "15    0.884881  0.514257  0.498000   0.553333    0.685619  \n",
       "16    0.830123  0.731116  0.653333   0.833333    0.635452  \n",
       "17    0.508368  0.923957  0.538667   0.650000    0.521739  \n",
       "18    0.492895  0.578539  0.729200   0.533333    0.478261  \n",
       "19    0.907280  0.501501  0.505000   0.570000    0.802676  \n",
       "20    0.507105  0.489495  0.502000   0.483333    0.518395  \n",
       "21    0.492895  0.510505  0.498000   0.516667    0.481605  \n",
       "22    0.492895  0.510505  0.498000   0.516667    0.481605  \n",
       "23    0.507105  0.489495  0.502000   0.483333    0.518395  \n",
       "24    0.492895  0.510505  0.498000   0.516667    0.481605  \n",
       "25    0.457215  0.616558  0.538667   0.773333    0.461538  \n",
       "26    0.833281  0.689345  0.627667   0.760000    0.628763  \n",
       "27    0.769996  0.487994  0.476667   0.233333    0.799331  \n",
       "28    0.654247  0.765133  0.563333   0.786667    0.468227  \n",
       "29    0.511209  0.844768  0.604333   0.803333    0.561873  \n",
       "30    0.195769  0.674587  0.638800   0.526667    0.314381  \n",
       "31    0.796337  0.888444  0.525667   0.836667    0.612040  \n",
       "32    0.423429  0.793647  0.650333   0.843333    0.434783  \n",
       "33    0.808192  0.730615  0.579333   0.756667    0.729097  \n",
       "34    0.512472  0.788894  0.682400   0.803333    0.458194  \n",
       "35    0.888778  0.900515  0.512333   0.800000    0.762542  \n",
       "36    0.795871  0.865066  0.708800   0.796667    0.678930  \n",
       "37    0.839617  0.593297  0.585000   0.776667    0.772575  \n",
       "38    0.512157  0.633817  0.726000   0.756667    0.377926  \n",
       "39    0.706164  0.846000  0.577667   0.776667    0.722408  \n",
       "40    0.770796  0.871200  0.754800   0.833333    0.722408  \n",
       "41    0.919826  0.826413  0.523333   0.743333    0.836120  \n",
       "42    0.692769  0.857429  0.732000   0.793333    0.535117  \n",
       "43    0.895673  0.897656  0.549333   0.763333    0.765886  \n",
       "44    0.911439  0.879074  0.718000   0.750000    0.829431  \n",
       "45    0.934766  0.792146  0.532333   0.846667    0.752508  \n",
       "46    0.572782  0.861181  0.632800   0.726667    0.615385  \n",
       "47    0.927859  0.715266  0.510333   0.736667    0.775920  \n",
       "48    0.932847  0.786735  0.633200   0.783333    0.779264  \n",
       "49    0.903296  0.717609  0.617000   0.840000    0.765886  \n",
       "50    0.733186  0.644572  0.725600   0.836667    0.595318  \n",
       "51    0.884824  0.925386  0.520000   0.813333    0.759197  \n",
       "52    0.907642  0.899085  0.757600   0.826667    0.749164  \n",
       "53    0.890508  0.589045  0.552667   0.656667    0.799331  \n",
       "54    0.319230  0.806403  0.634400   0.736667    0.367893  \n",
       "55    0.871667  0.806747  0.588667   0.770000    0.812709  \n",
       "56    0.688233  0.838765  0.617200   0.790000    0.745819  \n",
       "57    0.492895  0.510505  0.498000   0.516667    0.481605  \n",
       "58    0.976424  0.939952  0.518667   0.836667    0.829431  \n",
       "59    0.982809  0.854927  0.595667   0.850000    0.822742  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>scenario</th>\n",
       "      <th>all</th>\n",
       "      <th>en_affirm</th>\n",
       "      <th>en_negated</th>\n",
       "      <th>conj</th>\n",
       "      <th>disj</th>\n",
       "      <th>de_affirm</th>\n",
       "      <th>de_negated</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>animal_class__affirmative</td>\n",
       "      <td>0.628325</td>\n",
       "      <td>0.864469</td>\n",
       "      <td>0.692138</td>\n",
       "      <td>0.674837</td>\n",
       "      <td>0.647667</td>\n",
       "      <td>0.730000</td>\n",
       "      <td>0.478261</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>animal_class__conjunction</td>\n",
       "      <td>0.727648</td>\n",
       "      <td>0.920429</td>\n",
       "      <td>0.735081</td>\n",
       "      <td>0.885363</td>\n",
       "      <td>0.515667</td>\n",
       "      <td>0.760000</td>\n",
       "      <td>0.628763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>animal_class__disjunction</td>\n",
       "      <td>0.512028</td>\n",
       "      <td>0.525103</td>\n",
       "      <td>0.508052</td>\n",
       "      <td>0.514757</td>\n",
       "      <td>0.646000</td>\n",
       "      <td>0.526667</td>\n",
       "      <td>0.521739</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>animal_class__negated</td>\n",
       "      <td>0.663155</td>\n",
       "      <td>0.799179</td>\n",
       "      <td>0.864469</td>\n",
       "      <td>0.663332</td>\n",
       "      <td>0.504667</td>\n",
       "      <td>0.716667</td>\n",
       "      <td>0.782609</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cities__affirmative</td>\n",
       "      <td>0.631684</td>\n",
       "      <td>0.763615</td>\n",
       "      <td>0.522577</td>\n",
       "      <td>0.685093</td>\n",
       "      <td>0.521333</td>\n",
       "      <td>0.740000</td>\n",
       "      <td>0.347826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>cities__conjunction</td>\n",
       "      <td>0.705427</td>\n",
       "      <td>0.916009</td>\n",
       "      <td>0.520366</td>\n",
       "      <td>0.794800</td>\n",
       "      <td>0.577667</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.468227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>cities__disjunction</td>\n",
       "      <td>0.628012</td>\n",
       "      <td>0.805494</td>\n",
       "      <td>0.578781</td>\n",
       "      <td>0.564032</td>\n",
       "      <td>0.729600</td>\n",
       "      <td>0.720000</td>\n",
       "      <td>0.525084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>cities__negated</td>\n",
       "      <td>0.633931</td>\n",
       "      <td>0.848121</td>\n",
       "      <td>0.828845</td>\n",
       "      <td>0.592796</td>\n",
       "      <td>0.541333</td>\n",
       "      <td>0.786667</td>\n",
       "      <td>0.789298</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>element_symb__affirmative</td>\n",
       "      <td>0.583103</td>\n",
       "      <td>0.796713</td>\n",
       "      <td>0.633723</td>\n",
       "      <td>0.571786</td>\n",
       "      <td>0.502000</td>\n",
       "      <td>0.763333</td>\n",
       "      <td>0.515050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>element_symb__conjunction</td>\n",
       "      <td>0.680610</td>\n",
       "      <td>0.929586</td>\n",
       "      <td>0.500789</td>\n",
       "      <td>0.842767</td>\n",
       "      <td>0.530667</td>\n",
       "      <td>0.820000</td>\n",
       "      <td>0.461538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>element_symb__disjunction</td>\n",
       "      <td>0.688209</td>\n",
       "      <td>0.830123</td>\n",
       "      <td>0.715819</td>\n",
       "      <td>0.819910</td>\n",
       "      <td>0.730000</td>\n",
       "      <td>0.706667</td>\n",
       "      <td>0.568562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>element_symb__negated</td>\n",
       "      <td>0.484982</td>\n",
       "      <td>0.435744</td>\n",
       "      <td>0.828581</td>\n",
       "      <td>0.460230</td>\n",
       "      <td>0.408000</td>\n",
       "      <td>0.366667</td>\n",
       "      <td>0.839465</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>facts__affirmative</td>\n",
       "      <td>0.585750</td>\n",
       "      <td>0.664620</td>\n",
       "      <td>0.581307</td>\n",
       "      <td>0.490245</td>\n",
       "      <td>0.502000</td>\n",
       "      <td>0.743333</td>\n",
       "      <td>0.598662</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>facts__conjunction</td>\n",
       "      <td>0.699663</td>\n",
       "      <td>0.914746</td>\n",
       "      <td>0.518472</td>\n",
       "      <td>0.708405</td>\n",
       "      <td>0.515667</td>\n",
       "      <td>0.760000</td>\n",
       "      <td>0.608696</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>facts__disjunction</td>\n",
       "      <td>0.648584</td>\n",
       "      <td>0.825703</td>\n",
       "      <td>0.531418</td>\n",
       "      <td>0.707854</td>\n",
       "      <td>0.630800</td>\n",
       "      <td>0.670000</td>\n",
       "      <td>0.488294</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>facts__negated</td>\n",
       "      <td>0.546915</td>\n",
       "      <td>0.604989</td>\n",
       "      <td>0.884881</td>\n",
       "      <td>0.514257</td>\n",
       "      <td>0.498000</td>\n",
       "      <td>0.553333</td>\n",
       "      <td>0.685619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>inventors__affirmative</td>\n",
       "      <td>0.730953</td>\n",
       "      <td>0.931184</td>\n",
       "      <td>0.830123</td>\n",
       "      <td>0.731116</td>\n",
       "      <td>0.653333</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>0.635452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>inventors__conjunction</td>\n",
       "      <td>0.562347</td>\n",
       "      <td>0.557626</td>\n",
       "      <td>0.508368</td>\n",
       "      <td>0.923957</td>\n",
       "      <td>0.538667</td>\n",
       "      <td>0.650000</td>\n",
       "      <td>0.521739</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>inventors__disjunction</td>\n",
       "      <td>0.521091</td>\n",
       "      <td>0.519103</td>\n",
       "      <td>0.492895</td>\n",
       "      <td>0.578539</td>\n",
       "      <td>0.729200</td>\n",
       "      <td>0.533333</td>\n",
       "      <td>0.478261</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>inventors__negated</td>\n",
       "      <td>0.542572</td>\n",
       "      <td>0.530786</td>\n",
       "      <td>0.907280</td>\n",
       "      <td>0.501501</td>\n",
       "      <td>0.505000</td>\n",
       "      <td>0.570000</td>\n",
       "      <td>0.802676</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>ambiguous_lie</td>\n",
       "      <td>0.499026</td>\n",
       "      <td>0.492895</td>\n",
       "      <td>0.507105</td>\n",
       "      <td>0.489495</td>\n",
       "      <td>0.502000</td>\n",
       "      <td>0.483333</td>\n",
       "      <td>0.518395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>ambiguous_truthful_reply</td>\n",
       "      <td>0.500597</td>\n",
       "      <td>0.507105</td>\n",
       "      <td>0.492895</td>\n",
       "      <td>0.510505</td>\n",
       "      <td>0.498000</td>\n",
       "      <td>0.516667</td>\n",
       "      <td>0.481605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>honest_reply_despite_incentive_to_lie</td>\n",
       "      <td>0.500753</td>\n",
       "      <td>0.507105</td>\n",
       "      <td>0.492895</td>\n",
       "      <td>0.510505</td>\n",
       "      <td>0.498000</td>\n",
       "      <td>0.516667</td>\n",
       "      <td>0.481605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>unambiguous_lie</td>\n",
       "      <td>0.498741</td>\n",
       "      <td>0.492895</td>\n",
       "      <td>0.507105</td>\n",
       "      <td>0.489495</td>\n",
       "      <td>0.502000</td>\n",
       "      <td>0.483333</td>\n",
       "      <td>0.518395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>unambiguous_truthful_reply</td>\n",
       "      <td>0.500303</td>\n",
       "      <td>0.507105</td>\n",
       "      <td>0.492895</td>\n",
       "      <td>0.510505</td>\n",
       "      <td>0.498000</td>\n",
       "      <td>0.516667</td>\n",
       "      <td>0.481605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>larger_than</td>\n",
       "      <td>0.639092</td>\n",
       "      <td>0.856331</td>\n",
       "      <td>0.457215</td>\n",
       "      <td>0.616558</td>\n",
       "      <td>0.538667</td>\n",
       "      <td>0.773333</td>\n",
       "      <td>0.461538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>smaller_than</td>\n",
       "      <td>0.692717</td>\n",
       "      <td>0.906536</td>\n",
       "      <td>0.833281</td>\n",
       "      <td>0.689345</td>\n",
       "      <td>0.627667</td>\n",
       "      <td>0.760000</td>\n",
       "      <td>0.628763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>sp_en_trans__negated</td>\n",
       "      <td>0.412905</td>\n",
       "      <td>0.127881</td>\n",
       "      <td>0.769996</td>\n",
       "      <td>0.487994</td>\n",
       "      <td>0.476667</td>\n",
       "      <td>0.233333</td>\n",
       "      <td>0.799331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>sp_en_trans__affirmative</td>\n",
       "      <td>0.702931</td>\n",
       "      <td>0.907216</td>\n",
       "      <td>0.654247</td>\n",
       "      <td>0.765133</td>\n",
       "      <td>0.563333</td>\n",
       "      <td>0.786667</td>\n",
       "      <td>0.468227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>sp_en_trans__conjunction</td>\n",
       "      <td>0.599285</td>\n",
       "      <td>0.627408</td>\n",
       "      <td>0.511209</td>\n",
       "      <td>0.844768</td>\n",
       "      <td>0.604333</td>\n",
       "      <td>0.803333</td>\n",
       "      <td>0.561873</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>sp_en_trans__disjunction</td>\n",
       "      <td>0.564441</td>\n",
       "      <td>0.544048</td>\n",
       "      <td>0.195769</td>\n",
       "      <td>0.674587</td>\n",
       "      <td>0.638800</td>\n",
       "      <td>0.526667</td>\n",
       "      <td>0.314381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>common_claim_true_false</td>\n",
       "      <td>0.739791</td>\n",
       "      <td>0.931481</td>\n",
       "      <td>0.796337</td>\n",
       "      <td>0.888444</td>\n",
       "      <td>0.525667</td>\n",
       "      <td>0.836667</td>\n",
       "      <td>0.612040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>counterfact_true_false</td>\n",
       "      <td>0.710386</td>\n",
       "      <td>0.917272</td>\n",
       "      <td>0.423429</td>\n",
       "      <td>0.793647</td>\n",
       "      <td>0.650333</td>\n",
       "      <td>0.843333</td>\n",
       "      <td>0.434783</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>animal_class__affirmative+negated</td>\n",
       "      <td>0.651068</td>\n",
       "      <td>0.895438</td>\n",
       "      <td>0.808192</td>\n",
       "      <td>0.730615</td>\n",
       "      <td>0.579333</td>\n",
       "      <td>0.756667</td>\n",
       "      <td>0.729097</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>animal_class__affirmative+disjunction</td>\n",
       "      <td>0.650506</td>\n",
       "      <td>0.914752</td>\n",
       "      <td>0.512472</td>\n",
       "      <td>0.788894</td>\n",
       "      <td>0.682400</td>\n",
       "      <td>0.803333</td>\n",
       "      <td>0.458194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>animal_class__negated+conjunction</td>\n",
       "      <td>0.750502</td>\n",
       "      <td>0.946321</td>\n",
       "      <td>0.888778</td>\n",
       "      <td>0.900515</td>\n",
       "      <td>0.512333</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.762542</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>animal_class__affirmative+negated+conjunction+...</td>\n",
       "      <td>0.747384</td>\n",
       "      <td>0.930736</td>\n",
       "      <td>0.795871</td>\n",
       "      <td>0.865066</td>\n",
       "      <td>0.708800</td>\n",
       "      <td>0.796667</td>\n",
       "      <td>0.678930</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>cities__affirmative+negated</td>\n",
       "      <td>0.657081</td>\n",
       "      <td>0.845601</td>\n",
       "      <td>0.839617</td>\n",
       "      <td>0.593297</td>\n",
       "      <td>0.585000</td>\n",
       "      <td>0.776667</td>\n",
       "      <td>0.772575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>cities__affirmative+disjunction</td>\n",
       "      <td>0.661342</td>\n",
       "      <td>0.749850</td>\n",
       "      <td>0.512157</td>\n",
       "      <td>0.633817</td>\n",
       "      <td>0.726000</td>\n",
       "      <td>0.756667</td>\n",
       "      <td>0.377926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>cities__negated+conjunction</td>\n",
       "      <td>0.704151</td>\n",
       "      <td>0.893274</td>\n",
       "      <td>0.706164</td>\n",
       "      <td>0.846000</td>\n",
       "      <td>0.577667</td>\n",
       "      <td>0.776667</td>\n",
       "      <td>0.722408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>cities__affirmative+negated+conjunction+disjun...</td>\n",
       "      <td>0.736364</td>\n",
       "      <td>0.891083</td>\n",
       "      <td>0.770796</td>\n",
       "      <td>0.871200</td>\n",
       "      <td>0.754800</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>0.722408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>element_symb__affirmative+negated</td>\n",
       "      <td>0.672694</td>\n",
       "      <td>0.784636</td>\n",
       "      <td>0.919826</td>\n",
       "      <td>0.826413</td>\n",
       "      <td>0.523333</td>\n",
       "      <td>0.743333</td>\n",
       "      <td>0.836120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>element_symb__affirmative+disjunction</td>\n",
       "      <td>0.686703</td>\n",
       "      <td>0.827910</td>\n",
       "      <td>0.692769</td>\n",
       "      <td>0.857429</td>\n",
       "      <td>0.732000</td>\n",
       "      <td>0.793333</td>\n",
       "      <td>0.535117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>element_symb__negated+conjunction</td>\n",
       "      <td>0.707569</td>\n",
       "      <td>0.904642</td>\n",
       "      <td>0.895673</td>\n",
       "      <td>0.897656</td>\n",
       "      <td>0.549333</td>\n",
       "      <td>0.763333</td>\n",
       "      <td>0.765886</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>element_symb__affirmative+negated+conjunction+...</td>\n",
       "      <td>0.697404</td>\n",
       "      <td>0.739685</td>\n",
       "      <td>0.911439</td>\n",
       "      <td>0.879074</td>\n",
       "      <td>0.718000</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.829431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>facts__affirmative+negated</td>\n",
       "      <td>0.731372</td>\n",
       "      <td>0.955871</td>\n",
       "      <td>0.934766</td>\n",
       "      <td>0.792146</td>\n",
       "      <td>0.532333</td>\n",
       "      <td>0.846667</td>\n",
       "      <td>0.752508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>facts__affirmative+disjunction</td>\n",
       "      <td>0.669407</td>\n",
       "      <td>0.754797</td>\n",
       "      <td>0.572782</td>\n",
       "      <td>0.861181</td>\n",
       "      <td>0.632800</td>\n",
       "      <td>0.726667</td>\n",
       "      <td>0.615385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>facts__negated+conjunction</td>\n",
       "      <td>0.706040</td>\n",
       "      <td>0.904010</td>\n",
       "      <td>0.927859</td>\n",
       "      <td>0.715266</td>\n",
       "      <td>0.510333</td>\n",
       "      <td>0.736667</td>\n",
       "      <td>0.775920</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>facts__affirmative+negated+conjunction+disjunc...</td>\n",
       "      <td>0.734150</td>\n",
       "      <td>0.938987</td>\n",
       "      <td>0.932847</td>\n",
       "      <td>0.786735</td>\n",
       "      <td>0.633200</td>\n",
       "      <td>0.783333</td>\n",
       "      <td>0.779264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>inventors__affirmative+negated</td>\n",
       "      <td>0.706920</td>\n",
       "      <td>0.950742</td>\n",
       "      <td>0.903296</td>\n",
       "      <td>0.717609</td>\n",
       "      <td>0.617000</td>\n",
       "      <td>0.840000</td>\n",
       "      <td>0.765886</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>inventors__affirmative+disjunction</td>\n",
       "      <td>0.716153</td>\n",
       "      <td>0.944223</td>\n",
       "      <td>0.733186</td>\n",
       "      <td>0.644572</td>\n",
       "      <td>0.725600</td>\n",
       "      <td>0.836667</td>\n",
       "      <td>0.595318</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>inventors__negated+conjunction</td>\n",
       "      <td>0.710009</td>\n",
       "      <td>0.888538</td>\n",
       "      <td>0.884824</td>\n",
       "      <td>0.925386</td>\n",
       "      <td>0.520000</td>\n",
       "      <td>0.813333</td>\n",
       "      <td>0.759197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>inventors__affirmative+negated+conjunction+dis...</td>\n",
       "      <td>0.742345</td>\n",
       "      <td>0.916697</td>\n",
       "      <td>0.907642</td>\n",
       "      <td>0.899085</td>\n",
       "      <td>0.757600</td>\n",
       "      <td>0.826667</td>\n",
       "      <td>0.749164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>sp_en_trans__affirmative+negated</td>\n",
       "      <td>0.623205</td>\n",
       "      <td>0.599716</td>\n",
       "      <td>0.890508</td>\n",
       "      <td>0.589045</td>\n",
       "      <td>0.552667</td>\n",
       "      <td>0.656667</td>\n",
       "      <td>0.799331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>sp_en_trans__affirmative+disjunction</td>\n",
       "      <td>0.683557</td>\n",
       "      <td>0.731959</td>\n",
       "      <td>0.319230</td>\n",
       "      <td>0.806403</td>\n",
       "      <td>0.634400</td>\n",
       "      <td>0.736667</td>\n",
       "      <td>0.367893</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>sp_en_trans__negated+conjunction</td>\n",
       "      <td>0.660740</td>\n",
       "      <td>0.606883</td>\n",
       "      <td>0.871667</td>\n",
       "      <td>0.806747</td>\n",
       "      <td>0.588667</td>\n",
       "      <td>0.770000</td>\n",
       "      <td>0.812709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>sp_en_trans__affirmative+negated+conjunction+d...</td>\n",
       "      <td>0.711913</td>\n",
       "      <td>0.890508</td>\n",
       "      <td>0.688233</td>\n",
       "      <td>0.838765</td>\n",
       "      <td>0.617200</td>\n",
       "      <td>0.790000</td>\n",
       "      <td>0.745819</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>ambiguous_truthful_reply+unambiguous_lie+unamb...</td>\n",
       "      <td>0.501463</td>\n",
       "      <td>0.518472</td>\n",
       "      <td>0.492895</td>\n",
       "      <td>0.510505</td>\n",
       "      <td>0.498000</td>\n",
       "      <td>0.516667</td>\n",
       "      <td>0.481605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>animal_class+facts+inventors__affirmative+nega...</td>\n",
       "      <td>0.760918</td>\n",
       "      <td>0.977407</td>\n",
       "      <td>0.976424</td>\n",
       "      <td>0.939952</td>\n",
       "      <td>0.518667</td>\n",
       "      <td>0.836667</td>\n",
       "      <td>0.829431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>animal_class+facts+inventors__affirmative+nega...</td>\n",
       "      <td>0.768171</td>\n",
       "      <td>0.980354</td>\n",
       "      <td>0.982809</td>\n",
       "      <td>0.854927</td>\n",
       "      <td>0.595667</td>\n",
       "      <td>0.850000</td>\n",
       "      <td>0.822742</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 46
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:58.922414Z",
     "start_time": "2024-12-07T21:26:58.908670Z"
    }
   },
   "cell_type": "code",
   "source": "lyr18_ttpd_probes_rel_to_baseline_probes_acc_delta_on_unseen_multi_topic_data_groupings_df",
   "id": "71e7daf644639d47",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "                                             scenario       all  en_affirm  \\\n",
       "0                           animal_class__affirmative -0.029397  -0.024309   \n",
       "1                           animal_class__conjunction -0.111591  -0.082412   \n",
       "2                           animal_class__disjunction  0.134387   0.330913   \n",
       "3                               animal_class__negated -0.040724   0.077360   \n",
       "4                                 cities__affirmative -0.058214  -0.080192   \n",
       "5                                 cities__conjunction -0.026342  -0.033786   \n",
       "6                                 cities__disjunction -0.012825  -0.004105   \n",
       "7                                     cities__negated -0.076945  -0.206189   \n",
       "8                           element_symb__affirmative  0.118058   0.106676   \n",
       "9                           element_symb__conjunction -0.052302  -0.234923   \n",
       "10                          element_symb__disjunction -0.112870  -0.207136   \n",
       "11                              element_symb__negated  0.048535  -0.143353   \n",
       "12                                 facts__affirmative  0.090342   0.259401   \n",
       "13                                 facts__conjunction -0.125454  -0.292706   \n",
       "14                                 facts__disjunction -0.090815  -0.071992   \n",
       "15                                     facts__negated  0.009352   0.119672   \n",
       "16                             inventors__affirmative -0.058779  -0.100326   \n",
       "17                             inventors__conjunction -0.017255   0.038522   \n",
       "18                             inventors__disjunction  0.056639   0.087149   \n",
       "19                                 inventors__negated -0.077373  -0.319545   \n",
       "20                                      ambiguous_lie  0.006962   0.008210   \n",
       "21                           ambiguous_truthful_reply -0.030790  -0.078939   \n",
       "22              honest_reply_despite_incentive_to_lie -0.001065   0.000000   \n",
       "23                                    unambiguous_lie  0.036006   0.106094   \n",
       "24                         unambiguous_truthful_reply  0.023713  -0.004421   \n",
       "25                                        larger_than  0.067821   0.081149   \n",
       "26                                       smaller_than -0.047660  -0.033786   \n",
       "27                               sp_en_trans__negated  0.080348   0.285759   \n",
       "28                           sp_en_trans__affirmative -0.059942  -0.027373   \n",
       "29                           sp_en_trans__conjunction  0.024112   0.122513   \n",
       "30                           sp_en_trans__disjunction  0.000649   0.083991   \n",
       "31                            common_claim_true_false -0.090109  -0.033470   \n",
       "32                             counterfact_true_false  0.079184   0.031576   \n",
       "33                  animal_class__affirmative+negated  0.001016   0.013986   \n",
       "34              animal_class__affirmative+disjunction -0.034150  -0.056610   \n",
       "35                  animal_class__negated+conjunction -0.119981  -0.081149   \n",
       "36  animal_class__affirmative+negated+conjunction+... -0.100587  -0.042957   \n",
       "37                        cities__affirmative+negated -0.033752  -0.000598   \n",
       "38                    cities__affirmative+disjunction -0.073465  -0.030521   \n",
       "39                        cities__negated+conjunction -0.028234   0.017682   \n",
       "40  cities__affirmative+negated+conjunction+disjun... -0.108398  -0.074806   \n",
       "41                  element_symb__affirmative+negated -0.011316   0.124455   \n",
       "42              element_symb__affirmative+disjunction -0.043296   0.071453   \n",
       "43                  element_symb__negated+conjunction -0.059587  -0.274392   \n",
       "44  element_symb__affirmative+negated+conjunction+...  0.009570   0.181818   \n",
       "45                         facts__affirmative+negated -0.105749  -0.046048   \n",
       "46                     facts__affirmative+disjunction  0.042551   0.171911   \n",
       "47                         facts__negated+conjunction -0.012567   0.021156   \n",
       "48  facts__affirmative+negated+conjunction+disjunc... -0.057324  -0.008058   \n",
       "49                     inventors__affirmative+negated -0.085564  -0.105397   \n",
       "50                 inventors__affirmative+disjunction -0.034267  -0.094169   \n",
       "51                     inventors__negated+conjunction -0.167563  -0.398800   \n",
       "52  inventors__affirmative+negated+conjunction+dis... -0.080921  -0.046360   \n",
       "53                   sp_en_trans__affirmative+negated  0.051704   0.310345   \n",
       "54               sp_en_trans__affirmative+disjunction -0.103933   0.122645   \n",
       "55                   sp_en_trans__negated+conjunction -0.090314  -0.125039   \n",
       "56  sp_en_trans__affirmative+negated+conjunction+d... -0.021897   0.019552   \n",
       "57  ambiguous_truthful_reply+unambiguous_lie+unamb...  0.063447   0.302810   \n",
       "58  animal_class+facts+inventors__affirmative+nega... -0.032055  -0.000982   \n",
       "59  animal_class+facts+inventors__affirmative+nega... -0.041677  -0.003438   \n",
       "\n",
       "    en_negated      conj      disj  de_affirm  de_negated  \n",
       "0    -0.051784 -0.032516 -0.067000  -0.006667    0.003344  \n",
       "1    -0.074518 -0.136078  0.078333  -0.066667   -0.130435  \n",
       "2     0.001895  0.201351 -0.047600   0.110000    0.026756  \n",
       "3    -0.026640 -0.029015  0.006333  -0.010000   -0.107023  \n",
       "4     0.127566 -0.091546 -0.031667  -0.083333    0.137124  \n",
       "5     0.228608 -0.012400 -0.028667  -0.050000    0.110368  \n",
       "6    -0.087149  0.110805 -0.060800  -0.136667   -0.013378  \n",
       "7    -0.049671 -0.063532 -0.053000  -0.220000   -0.080268  \n",
       "8    -0.107989  0.250125  0.000667   0.056667    0.026756  \n",
       "9     0.023366 -0.024871  0.073333  -0.036667    0.053512  \n",
       "10   -0.150616 -0.234867 -0.120800  -0.040000   -0.127090  \n",
       "11    0.046629  0.017259  0.089000   0.090000   -0.076923  \n",
       "12   -0.061257  0.355428  0.037000   0.046667   -0.160535  \n",
       "13   -0.010104  0.122070 -0.007667  -0.110000   -0.090301  \n",
       "14   -0.023366 -0.046273 -0.061200  -0.063333   -0.003344  \n",
       "15   -0.015733 -0.001501  0.000000   0.016667   -0.013378  \n",
       "16   -0.120619  0.014507 -0.028667  -0.026667   -0.090301  \n",
       "17    0.005999 -0.210692 -0.035667  -0.010000   -0.026756  \n",
       "18    0.013578  0.188344 -0.155600  -0.006667    0.023411  \n",
       "19   -0.257878  0.008754 -0.029000  -0.173333   -0.167224  \n",
       "20    0.004105 -0.033517 -0.002667   0.036667   -0.010033  \n",
       "21   -0.001263 -0.042521  0.013000  -0.090000    0.060201  \n",
       "22    0.000316 -0.000250  0.001000  -0.003333    0.000000  \n",
       "23    0.176508  0.024012 -0.005000   0.110000    0.056856  \n",
       "24    0.009788  0.013007  0.004000  -0.030000    0.026756  \n",
       "25    0.071361  0.238369  0.062000   0.070000    0.023411  \n",
       "26   -0.149668  0.064282  0.005333  -0.033333   -0.100334  \n",
       "27    0.051902 -0.028014  0.021333   0.200000    0.036789  \n",
       "28    0.043259 -0.069535  0.068667  -0.023333    0.060201  \n",
       "29    0.124092 -0.019726 -0.016667  -0.056667   -0.043478  \n",
       "30    0.359015 -0.080540 -0.119600   0.050000    0.210702  \n",
       "31   -0.163562 -0.238869 -0.021000  -0.053333   -0.006689  \n",
       "32    0.239027  0.090295 -0.143333  -0.016667    0.183946  \n",
       "33    0.041958 -0.051526  0.029667   0.006667   -0.013378  \n",
       "34    0.142090 -0.110805 -0.072000  -0.070000    0.013378  \n",
       "35   -0.184482 -0.135506  0.080667  -0.076667   -0.220736  \n",
       "36   -0.015984 -0.126358 -0.064400  -0.053333   -0.073579  \n",
       "37   -0.011370  0.023512 -0.008667  -0.036667   -0.006689  \n",
       "38    0.148405  0.012006 -0.136400  -0.050000    0.113712  \n",
       "39   -0.030521 -0.090800  0.027333  -0.010000   -0.050167  \n",
       "40   -0.065829 -0.184800 -0.092800  -0.090000   -0.040134  \n",
       "41   -0.017444 -0.188344  0.016000  -0.003333   -0.083612  \n",
       "42    0.002842 -0.209355 -0.103200  -0.030000    0.010033  \n",
       "43   -0.010399 -0.224700  0.028000  -0.033333    0.060201  \n",
       "44   -0.007045 -0.210692 -0.074800   0.046667   -0.050167  \n",
       "45   -0.014198 -0.213107  0.002000  -0.100000    0.006689  \n",
       "46    0.176508  0.000000 -0.079600   0.026667   -0.023411  \n",
       "47    0.026861  0.035163  0.087333   0.040000    0.033445  \n",
       "48    0.012663 -0.050029 -0.084400  -0.013333   -0.003344  \n",
       "49   -0.026802 -0.146823 -0.086333  -0.156667   -0.006689  \n",
       "50   -0.024629  0.152826 -0.185600  -0.010000   -0.020067  \n",
       "51   -0.010141 -0.421670 -0.024000  -0.293333   -0.016722  \n",
       "52   -0.401304 -0.104631 -0.137200  -0.030000   -0.230769  \n",
       "53   -0.036616  0.052526  0.008667   0.146667   -0.093645  \n",
       "54    0.261762 -0.207354 -0.130000  -0.050000    0.107023  \n",
       "55    0.006399 -0.226701 -0.085333  -0.246667    0.036789  \n",
       "56    0.201564 -0.279017 -0.104000   0.026667   -0.016722  \n",
       "57    0.085886  0.012506 -0.004333   0.143333    0.030100  \n",
       "58   -0.099214 -0.094476  0.050000  -0.030000   -0.117057  \n",
       "59   -0.039293 -0.013007 -0.086667  -0.053333   -0.090301  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>scenario</th>\n",
       "      <th>all</th>\n",
       "      <th>en_affirm</th>\n",
       "      <th>en_negated</th>\n",
       "      <th>conj</th>\n",
       "      <th>disj</th>\n",
       "      <th>de_affirm</th>\n",
       "      <th>de_negated</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>animal_class__affirmative</td>\n",
       "      <td>-0.029397</td>\n",
       "      <td>-0.024309</td>\n",
       "      <td>-0.051784</td>\n",
       "      <td>-0.032516</td>\n",
       "      <td>-0.067000</td>\n",
       "      <td>-0.006667</td>\n",
       "      <td>0.003344</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>animal_class__conjunction</td>\n",
       "      <td>-0.111591</td>\n",
       "      <td>-0.082412</td>\n",
       "      <td>-0.074518</td>\n",
       "      <td>-0.136078</td>\n",
       "      <td>0.078333</td>\n",
       "      <td>-0.066667</td>\n",
       "      <td>-0.130435</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>animal_class__disjunction</td>\n",
       "      <td>0.134387</td>\n",
       "      <td>0.330913</td>\n",
       "      <td>0.001895</td>\n",
       "      <td>0.201351</td>\n",
       "      <td>-0.047600</td>\n",
       "      <td>0.110000</td>\n",
       "      <td>0.026756</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>animal_class__negated</td>\n",
       "      <td>-0.040724</td>\n",
       "      <td>0.077360</td>\n",
       "      <td>-0.026640</td>\n",
       "      <td>-0.029015</td>\n",
       "      <td>0.006333</td>\n",
       "      <td>-0.010000</td>\n",
       "      <td>-0.107023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cities__affirmative</td>\n",
       "      <td>-0.058214</td>\n",
       "      <td>-0.080192</td>\n",
       "      <td>0.127566</td>\n",
       "      <td>-0.091546</td>\n",
       "      <td>-0.031667</td>\n",
       "      <td>-0.083333</td>\n",
       "      <td>0.137124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>cities__conjunction</td>\n",
       "      <td>-0.026342</td>\n",
       "      <td>-0.033786</td>\n",
       "      <td>0.228608</td>\n",
       "      <td>-0.012400</td>\n",
       "      <td>-0.028667</td>\n",
       "      <td>-0.050000</td>\n",
       "      <td>0.110368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>cities__disjunction</td>\n",
       "      <td>-0.012825</td>\n",
       "      <td>-0.004105</td>\n",
       "      <td>-0.087149</td>\n",
       "      <td>0.110805</td>\n",
       "      <td>-0.060800</td>\n",
       "      <td>-0.136667</td>\n",
       "      <td>-0.013378</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>cities__negated</td>\n",
       "      <td>-0.076945</td>\n",
       "      <td>-0.206189</td>\n",
       "      <td>-0.049671</td>\n",
       "      <td>-0.063532</td>\n",
       "      <td>-0.053000</td>\n",
       "      <td>-0.220000</td>\n",
       "      <td>-0.080268</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>element_symb__affirmative</td>\n",
       "      <td>0.118058</td>\n",
       "      <td>0.106676</td>\n",
       "      <td>-0.107989</td>\n",
       "      <td>0.250125</td>\n",
       "      <td>0.000667</td>\n",
       "      <td>0.056667</td>\n",
       "      <td>0.026756</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>element_symb__conjunction</td>\n",
       "      <td>-0.052302</td>\n",
       "      <td>-0.234923</td>\n",
       "      <td>0.023366</td>\n",
       "      <td>-0.024871</td>\n",
       "      <td>0.073333</td>\n",
       "      <td>-0.036667</td>\n",
       "      <td>0.053512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>element_symb__disjunction</td>\n",
       "      <td>-0.112870</td>\n",
       "      <td>-0.207136</td>\n",
       "      <td>-0.150616</td>\n",
       "      <td>-0.234867</td>\n",
       "      <td>-0.120800</td>\n",
       "      <td>-0.040000</td>\n",
       "      <td>-0.127090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>element_symb__negated</td>\n",
       "      <td>0.048535</td>\n",
       "      <td>-0.143353</td>\n",
       "      <td>0.046629</td>\n",
       "      <td>0.017259</td>\n",
       "      <td>0.089000</td>\n",
       "      <td>0.090000</td>\n",
       "      <td>-0.076923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>facts__affirmative</td>\n",
       "      <td>0.090342</td>\n",
       "      <td>0.259401</td>\n",
       "      <td>-0.061257</td>\n",
       "      <td>0.355428</td>\n",
       "      <td>0.037000</td>\n",
       "      <td>0.046667</td>\n",
       "      <td>-0.160535</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>facts__conjunction</td>\n",
       "      <td>-0.125454</td>\n",
       "      <td>-0.292706</td>\n",
       "      <td>-0.010104</td>\n",
       "      <td>0.122070</td>\n",
       "      <td>-0.007667</td>\n",
       "      <td>-0.110000</td>\n",
       "      <td>-0.090301</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>facts__disjunction</td>\n",
       "      <td>-0.090815</td>\n",
       "      <td>-0.071992</td>\n",
       "      <td>-0.023366</td>\n",
       "      <td>-0.046273</td>\n",
       "      <td>-0.061200</td>\n",
       "      <td>-0.063333</td>\n",
       "      <td>-0.003344</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>facts__negated</td>\n",
       "      <td>0.009352</td>\n",
       "      <td>0.119672</td>\n",
       "      <td>-0.015733</td>\n",
       "      <td>-0.001501</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.016667</td>\n",
       "      <td>-0.013378</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>inventors__affirmative</td>\n",
       "      <td>-0.058779</td>\n",
       "      <td>-0.100326</td>\n",
       "      <td>-0.120619</td>\n",
       "      <td>0.014507</td>\n",
       "      <td>-0.028667</td>\n",
       "      <td>-0.026667</td>\n",
       "      <td>-0.090301</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>inventors__conjunction</td>\n",
       "      <td>-0.017255</td>\n",
       "      <td>0.038522</td>\n",
       "      <td>0.005999</td>\n",
       "      <td>-0.210692</td>\n",
       "      <td>-0.035667</td>\n",
       "      <td>-0.010000</td>\n",
       "      <td>-0.026756</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>inventors__disjunction</td>\n",
       "      <td>0.056639</td>\n",
       "      <td>0.087149</td>\n",
       "      <td>0.013578</td>\n",
       "      <td>0.188344</td>\n",
       "      <td>-0.155600</td>\n",
       "      <td>-0.006667</td>\n",
       "      <td>0.023411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>inventors__negated</td>\n",
       "      <td>-0.077373</td>\n",
       "      <td>-0.319545</td>\n",
       "      <td>-0.257878</td>\n",
       "      <td>0.008754</td>\n",
       "      <td>-0.029000</td>\n",
       "      <td>-0.173333</td>\n",
       "      <td>-0.167224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>ambiguous_lie</td>\n",
       "      <td>0.006962</td>\n",
       "      <td>0.008210</td>\n",
       "      <td>0.004105</td>\n",
       "      <td>-0.033517</td>\n",
       "      <td>-0.002667</td>\n",
       "      <td>0.036667</td>\n",
       "      <td>-0.010033</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>ambiguous_truthful_reply</td>\n",
       "      <td>-0.030790</td>\n",
       "      <td>-0.078939</td>\n",
       "      <td>-0.001263</td>\n",
       "      <td>-0.042521</td>\n",
       "      <td>0.013000</td>\n",
       "      <td>-0.090000</td>\n",
       "      <td>0.060201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>honest_reply_despite_incentive_to_lie</td>\n",
       "      <td>-0.001065</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000316</td>\n",
       "      <td>-0.000250</td>\n",
       "      <td>0.001000</td>\n",
       "      <td>-0.003333</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>unambiguous_lie</td>\n",
       "      <td>0.036006</td>\n",
       "      <td>0.106094</td>\n",
       "      <td>0.176508</td>\n",
       "      <td>0.024012</td>\n",
       "      <td>-0.005000</td>\n",
       "      <td>0.110000</td>\n",
       "      <td>0.056856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>unambiguous_truthful_reply</td>\n",
       "      <td>0.023713</td>\n",
       "      <td>-0.004421</td>\n",
       "      <td>0.009788</td>\n",
       "      <td>0.013007</td>\n",
       "      <td>0.004000</td>\n",
       "      <td>-0.030000</td>\n",
       "      <td>0.026756</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>larger_than</td>\n",
       "      <td>0.067821</td>\n",
       "      <td>0.081149</td>\n",
       "      <td>0.071361</td>\n",
       "      <td>0.238369</td>\n",
       "      <td>0.062000</td>\n",
       "      <td>0.070000</td>\n",
       "      <td>0.023411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>smaller_than</td>\n",
       "      <td>-0.047660</td>\n",
       "      <td>-0.033786</td>\n",
       "      <td>-0.149668</td>\n",
       "      <td>0.064282</td>\n",
       "      <td>0.005333</td>\n",
       "      <td>-0.033333</td>\n",
       "      <td>-0.100334</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>sp_en_trans__negated</td>\n",
       "      <td>0.080348</td>\n",
       "      <td>0.285759</td>\n",
       "      <td>0.051902</td>\n",
       "      <td>-0.028014</td>\n",
       "      <td>0.021333</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.036789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>sp_en_trans__affirmative</td>\n",
       "      <td>-0.059942</td>\n",
       "      <td>-0.027373</td>\n",
       "      <td>0.043259</td>\n",
       "      <td>-0.069535</td>\n",
       "      <td>0.068667</td>\n",
       "      <td>-0.023333</td>\n",
       "      <td>0.060201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>sp_en_trans__conjunction</td>\n",
       "      <td>0.024112</td>\n",
       "      <td>0.122513</td>\n",
       "      <td>0.124092</td>\n",
       "      <td>-0.019726</td>\n",
       "      <td>-0.016667</td>\n",
       "      <td>-0.056667</td>\n",
       "      <td>-0.043478</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>sp_en_trans__disjunction</td>\n",
       "      <td>0.000649</td>\n",
       "      <td>0.083991</td>\n",
       "      <td>0.359015</td>\n",
       "      <td>-0.080540</td>\n",
       "      <td>-0.119600</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>0.210702</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>common_claim_true_false</td>\n",
       "      <td>-0.090109</td>\n",
       "      <td>-0.033470</td>\n",
       "      <td>-0.163562</td>\n",
       "      <td>-0.238869</td>\n",
       "      <td>-0.021000</td>\n",
       "      <td>-0.053333</td>\n",
       "      <td>-0.006689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>counterfact_true_false</td>\n",
       "      <td>0.079184</td>\n",
       "      <td>0.031576</td>\n",
       "      <td>0.239027</td>\n",
       "      <td>0.090295</td>\n",
       "      <td>-0.143333</td>\n",
       "      <td>-0.016667</td>\n",
       "      <td>0.183946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>animal_class__affirmative+negated</td>\n",
       "      <td>0.001016</td>\n",
       "      <td>0.013986</td>\n",
       "      <td>0.041958</td>\n",
       "      <td>-0.051526</td>\n",
       "      <td>0.029667</td>\n",
       "      <td>0.006667</td>\n",
       "      <td>-0.013378</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>animal_class__affirmative+disjunction</td>\n",
       "      <td>-0.034150</td>\n",
       "      <td>-0.056610</td>\n",
       "      <td>0.142090</td>\n",
       "      <td>-0.110805</td>\n",
       "      <td>-0.072000</td>\n",
       "      <td>-0.070000</td>\n",
       "      <td>0.013378</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>animal_class__negated+conjunction</td>\n",
       "      <td>-0.119981</td>\n",
       "      <td>-0.081149</td>\n",
       "      <td>-0.184482</td>\n",
       "      <td>-0.135506</td>\n",
       "      <td>0.080667</td>\n",
       "      <td>-0.076667</td>\n",
       "      <td>-0.220736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>animal_class__affirmative+negated+conjunction+...</td>\n",
       "      <td>-0.100587</td>\n",
       "      <td>-0.042957</td>\n",
       "      <td>-0.015984</td>\n",
       "      <td>-0.126358</td>\n",
       "      <td>-0.064400</td>\n",
       "      <td>-0.053333</td>\n",
       "      <td>-0.073579</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>cities__affirmative+negated</td>\n",
       "      <td>-0.033752</td>\n",
       "      <td>-0.000598</td>\n",
       "      <td>-0.011370</td>\n",
       "      <td>0.023512</td>\n",
       "      <td>-0.008667</td>\n",
       "      <td>-0.036667</td>\n",
       "      <td>-0.006689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>cities__affirmative+disjunction</td>\n",
       "      <td>-0.073465</td>\n",
       "      <td>-0.030521</td>\n",
       "      <td>0.148405</td>\n",
       "      <td>0.012006</td>\n",
       "      <td>-0.136400</td>\n",
       "      <td>-0.050000</td>\n",
       "      <td>0.113712</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>cities__negated+conjunction</td>\n",
       "      <td>-0.028234</td>\n",
       "      <td>0.017682</td>\n",
       "      <td>-0.030521</td>\n",
       "      <td>-0.090800</td>\n",
       "      <td>0.027333</td>\n",
       "      <td>-0.010000</td>\n",
       "      <td>-0.050167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>cities__affirmative+negated+conjunction+disjun...</td>\n",
       "      <td>-0.108398</td>\n",
       "      <td>-0.074806</td>\n",
       "      <td>-0.065829</td>\n",
       "      <td>-0.184800</td>\n",
       "      <td>-0.092800</td>\n",
       "      <td>-0.090000</td>\n",
       "      <td>-0.040134</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>element_symb__affirmative+negated</td>\n",
       "      <td>-0.011316</td>\n",
       "      <td>0.124455</td>\n",
       "      <td>-0.017444</td>\n",
       "      <td>-0.188344</td>\n",
       "      <td>0.016000</td>\n",
       "      <td>-0.003333</td>\n",
       "      <td>-0.083612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>element_symb__affirmative+disjunction</td>\n",
       "      <td>-0.043296</td>\n",
       "      <td>0.071453</td>\n",
       "      <td>0.002842</td>\n",
       "      <td>-0.209355</td>\n",
       "      <td>-0.103200</td>\n",
       "      <td>-0.030000</td>\n",
       "      <td>0.010033</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>element_symb__negated+conjunction</td>\n",
       "      <td>-0.059587</td>\n",
       "      <td>-0.274392</td>\n",
       "      <td>-0.010399</td>\n",
       "      <td>-0.224700</td>\n",
       "      <td>0.028000</td>\n",
       "      <td>-0.033333</td>\n",
       "      <td>0.060201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>element_symb__affirmative+negated+conjunction+...</td>\n",
       "      <td>0.009570</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>-0.007045</td>\n",
       "      <td>-0.210692</td>\n",
       "      <td>-0.074800</td>\n",
       "      <td>0.046667</td>\n",
       "      <td>-0.050167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>facts__affirmative+negated</td>\n",
       "      <td>-0.105749</td>\n",
       "      <td>-0.046048</td>\n",
       "      <td>-0.014198</td>\n",
       "      <td>-0.213107</td>\n",
       "      <td>0.002000</td>\n",
       "      <td>-0.100000</td>\n",
       "      <td>0.006689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>facts__affirmative+disjunction</td>\n",
       "      <td>0.042551</td>\n",
       "      <td>0.171911</td>\n",
       "      <td>0.176508</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.079600</td>\n",
       "      <td>0.026667</td>\n",
       "      <td>-0.023411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>facts__negated+conjunction</td>\n",
       "      <td>-0.012567</td>\n",
       "      <td>0.021156</td>\n",
       "      <td>0.026861</td>\n",
       "      <td>0.035163</td>\n",
       "      <td>0.087333</td>\n",
       "      <td>0.040000</td>\n",
       "      <td>0.033445</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>facts__affirmative+negated+conjunction+disjunc...</td>\n",
       "      <td>-0.057324</td>\n",
       "      <td>-0.008058</td>\n",
       "      <td>0.012663</td>\n",
       "      <td>-0.050029</td>\n",
       "      <td>-0.084400</td>\n",
       "      <td>-0.013333</td>\n",
       "      <td>-0.003344</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>inventors__affirmative+negated</td>\n",
       "      <td>-0.085564</td>\n",
       "      <td>-0.105397</td>\n",
       "      <td>-0.026802</td>\n",
       "      <td>-0.146823</td>\n",
       "      <td>-0.086333</td>\n",
       "      <td>-0.156667</td>\n",
       "      <td>-0.006689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>inventors__affirmative+disjunction</td>\n",
       "      <td>-0.034267</td>\n",
       "      <td>-0.094169</td>\n",
       "      <td>-0.024629</td>\n",
       "      <td>0.152826</td>\n",
       "      <td>-0.185600</td>\n",
       "      <td>-0.010000</td>\n",
       "      <td>-0.020067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>inventors__negated+conjunction</td>\n",
       "      <td>-0.167563</td>\n",
       "      <td>-0.398800</td>\n",
       "      <td>-0.010141</td>\n",
       "      <td>-0.421670</td>\n",
       "      <td>-0.024000</td>\n",
       "      <td>-0.293333</td>\n",
       "      <td>-0.016722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>inventors__affirmative+negated+conjunction+dis...</td>\n",
       "      <td>-0.080921</td>\n",
       "      <td>-0.046360</td>\n",
       "      <td>-0.401304</td>\n",
       "      <td>-0.104631</td>\n",
       "      <td>-0.137200</td>\n",
       "      <td>-0.030000</td>\n",
       "      <td>-0.230769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>sp_en_trans__affirmative+negated</td>\n",
       "      <td>0.051704</td>\n",
       "      <td>0.310345</td>\n",
       "      <td>-0.036616</td>\n",
       "      <td>0.052526</td>\n",
       "      <td>0.008667</td>\n",
       "      <td>0.146667</td>\n",
       "      <td>-0.093645</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>sp_en_trans__affirmative+disjunction</td>\n",
       "      <td>-0.103933</td>\n",
       "      <td>0.122645</td>\n",
       "      <td>0.261762</td>\n",
       "      <td>-0.207354</td>\n",
       "      <td>-0.130000</td>\n",
       "      <td>-0.050000</td>\n",
       "      <td>0.107023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>sp_en_trans__negated+conjunction</td>\n",
       "      <td>-0.090314</td>\n",
       "      <td>-0.125039</td>\n",
       "      <td>0.006399</td>\n",
       "      <td>-0.226701</td>\n",
       "      <td>-0.085333</td>\n",
       "      <td>-0.246667</td>\n",
       "      <td>0.036789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>sp_en_trans__affirmative+negated+conjunction+d...</td>\n",
       "      <td>-0.021897</td>\n",
       "      <td>0.019552</td>\n",
       "      <td>0.201564</td>\n",
       "      <td>-0.279017</td>\n",
       "      <td>-0.104000</td>\n",
       "      <td>0.026667</td>\n",
       "      <td>-0.016722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>ambiguous_truthful_reply+unambiguous_lie+unamb...</td>\n",
       "      <td>0.063447</td>\n",
       "      <td>0.302810</td>\n",
       "      <td>0.085886</td>\n",
       "      <td>0.012506</td>\n",
       "      <td>-0.004333</td>\n",
       "      <td>0.143333</td>\n",
       "      <td>0.030100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>animal_class+facts+inventors__affirmative+nega...</td>\n",
       "      <td>-0.032055</td>\n",
       "      <td>-0.000982</td>\n",
       "      <td>-0.099214</td>\n",
       "      <td>-0.094476</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>-0.030000</td>\n",
       "      <td>-0.117057</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>animal_class+facts+inventors__affirmative+nega...</td>\n",
       "      <td>-0.041677</td>\n",
       "      <td>-0.003438</td>\n",
       "      <td>-0.039293</td>\n",
       "      <td>-0.013007</td>\n",
       "      <td>-0.086667</td>\n",
       "      <td>-0.053333</td>\n",
       "      <td>-0.090301</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 47
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:58.939489Z",
     "start_time": "2024-12-07T21:26:58.923432Z"
    }
   },
   "cell_type": "code",
   "source": "lyr18_ttpd_probes_rel_to_baseline_probes_acc_delta_on_unseen_multi_topic_data_groupings_df.describe()",
   "id": "8a2a20a9f5fbf23f",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "             all  en_affirm  en_negated       conj       disj  de_affirm  \\\n",
       "count  60.000000  60.000000   60.000000  60.000000  60.000000  60.000000   \n",
       "mean   -0.027044  -0.004525    0.004781  -0.044980  -0.030094  -0.026000   \n",
       "std     0.065080   0.148862    0.124110   0.141573   0.066574   0.088112   \n",
       "min    -0.167563  -0.398800   -0.401304  -0.421670  -0.185600  -0.293333   \n",
       "25%    -0.077052  -0.079252   -0.041887  -0.135649  -0.080800  -0.058333   \n",
       "50%    -0.032903  -0.004263   -0.008574  -0.033017  -0.022500  -0.030000   \n",
       "75%     0.009406   0.081860    0.044101   0.015195   0.009750   0.019167   \n",
       "max     0.134387   0.330913    0.359015   0.355428   0.089000   0.200000   \n",
       "\n",
       "       de_negated  \n",
       "count   60.000000  \n",
       "mean    -0.015886  \n",
       "std      0.086101  \n",
       "min     -0.230769  \n",
       "25%     -0.077759  \n",
       "50%     -0.008361  \n",
       "75%      0.027592  \n",
       "max      0.210702  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>all</th>\n",
       "      <th>en_affirm</th>\n",
       "      <th>en_negated</th>\n",
       "      <th>conj</th>\n",
       "      <th>disj</th>\n",
       "      <th>de_affirm</th>\n",
       "      <th>de_negated</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>-0.027044</td>\n",
       "      <td>-0.004525</td>\n",
       "      <td>0.004781</td>\n",
       "      <td>-0.044980</td>\n",
       "      <td>-0.030094</td>\n",
       "      <td>-0.026000</td>\n",
       "      <td>-0.015886</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.065080</td>\n",
       "      <td>0.148862</td>\n",
       "      <td>0.124110</td>\n",
       "      <td>0.141573</td>\n",
       "      <td>0.066574</td>\n",
       "      <td>0.088112</td>\n",
       "      <td>0.086101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-0.167563</td>\n",
       "      <td>-0.398800</td>\n",
       "      <td>-0.401304</td>\n",
       "      <td>-0.421670</td>\n",
       "      <td>-0.185600</td>\n",
       "      <td>-0.293333</td>\n",
       "      <td>-0.230769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>-0.077052</td>\n",
       "      <td>-0.079252</td>\n",
       "      <td>-0.041887</td>\n",
       "      <td>-0.135649</td>\n",
       "      <td>-0.080800</td>\n",
       "      <td>-0.058333</td>\n",
       "      <td>-0.077759</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>-0.032903</td>\n",
       "      <td>-0.004263</td>\n",
       "      <td>-0.008574</td>\n",
       "      <td>-0.033017</td>\n",
       "      <td>-0.022500</td>\n",
       "      <td>-0.030000</td>\n",
       "      <td>-0.008361</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.009406</td>\n",
       "      <td>0.081860</td>\n",
       "      <td>0.044101</td>\n",
       "      <td>0.015195</td>\n",
       "      <td>0.009750</td>\n",
       "      <td>0.019167</td>\n",
       "      <td>0.027592</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.134387</td>\n",
       "      <td>0.330913</td>\n",
       "      <td>0.359015</td>\n",
       "      <td>0.355428</td>\n",
       "      <td>0.089000</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.210702</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 48
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:58.956175Z",
     "start_time": "2024-12-07T21:26:58.940494Z"
    }
   },
   "cell_type": "code",
   "source": "lyr18_probes_acc_on_unseen_single_topic_data_groupings_df",
   "id": "ced68d0bbb6315fe",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "                                             scenario  animal class    cities  \\\n",
       "0                           animal_class__affirmative      0.606804  0.792731   \n",
       "1                           animal_class__conjunction      0.687500  0.845187   \n",
       "2                           animal_class__disjunction      0.764009  0.736149   \n",
       "3                               animal_class__negated      0.594146  0.821022   \n",
       "4                                 cities__affirmative      0.633754  0.679188   \n",
       "5                                 cities__conjunction      0.783613  0.910356   \n",
       "6                                 cities__disjunction      0.637955  0.700218   \n",
       "7                                     cities__negated      0.525210  0.587368   \n",
       "8                           element_symb__affirmative      0.714986  0.722593   \n",
       "9                           element_symb__conjunction      0.701681  0.657760   \n",
       "10                          element_symb__disjunction      0.630952  0.583890   \n",
       "11                              element_symb__negated      0.509804  0.519057   \n",
       "12                                 facts__affirmative      0.721289  0.786837   \n",
       "13                                 facts__conjunction      0.581933  0.604519   \n",
       "14                                 facts__disjunction      0.703081  0.689194   \n",
       "15                                     facts__negated      0.537815  0.700786   \n",
       "16                             inventors__affirmative      0.658964  0.904126   \n",
       "17                             inventors__conjunction      0.635854  0.597446   \n",
       "18                             inventors__disjunction      0.509804  0.669155   \n",
       "19                                 inventors__negated      0.475490  0.413949   \n",
       "20                                      ambiguous_lie      0.478291  0.503340   \n",
       "21                           ambiguous_truthful_reply      0.485994  0.477210   \n",
       "22              honest_reply_despite_incentive_to_lie      0.493697  0.498625   \n",
       "23                                    unambiguous_lie      0.525910  0.654028   \n",
       "24                         unambiguous_truthful_reply      0.506303  0.501572   \n",
       "25                                        larger_than      0.692577  0.777210   \n",
       "26                                       smaller_than      0.829832  0.846365   \n",
       "27                               sp_en_trans__negated      0.467787  0.598035   \n",
       "28                           sp_en_trans__affirmative      0.682073  0.842829   \n",
       "29                           sp_en_trans__conjunction      0.698179  0.750295   \n",
       "30                           sp_en_trans__disjunction      0.530812  0.592338   \n",
       "31                            common_claim_true_false      0.528711  0.728291   \n",
       "32                             counterfact_true_false      0.746499  0.866994   \n",
       "33                  animal_class__affirmative+negated      0.585455  0.853635   \n",
       "34              animal_class__affirmative+disjunction      0.717277  0.828684   \n",
       "35                  animal_class__negated+conjunction      0.641361  0.877210   \n",
       "36  animal_class__affirmative+negated+conjunction+...      0.810000  0.878978   \n",
       "37                        cities__affirmative+negated      0.644958  0.577216   \n",
       "38                    cities__affirmative+disjunction      0.680672  0.733678   \n",
       "39                        cities__negated+conjunction      0.761204  0.880725   \n",
       "40  cities__affirmative+negated+conjunction+disjun...      0.770308  0.970000   \n",
       "41                  element_symb__affirmative+negated      0.633053  0.827505   \n",
       "42              element_symb__affirmative+disjunction      0.691176  0.799018   \n",
       "43                  element_symb__negated+conjunction      0.667367  0.756189   \n",
       "44  element_symb__affirmative+negated+conjunction+...      0.712885  0.861493   \n",
       "45                         facts__affirmative+negated      0.620448  0.791749   \n",
       "46                     facts__affirmative+disjunction      0.779412  0.922986   \n",
       "47                         facts__negated+conjunction      0.743697  0.875835   \n",
       "48  facts__affirmative+negated+conjunction+disjunc...      0.725490  0.866994   \n",
       "49                     inventors__affirmative+negated      0.596639  0.745972   \n",
       "50                 inventors__affirmative+disjunction      0.707283  0.900982   \n",
       "51                     inventors__negated+conjunction      0.549720  0.642436   \n",
       "52  inventors__affirmative+negated+conjunction+dis...      0.628151  0.739686   \n",
       "53                   sp_en_trans__affirmative+negated      0.640056  0.816896   \n",
       "54               sp_en_trans__affirmative+disjunction      0.588235  0.707662   \n",
       "55                   sp_en_trans__negated+conjunction      0.525210  0.647544   \n",
       "56  sp_en_trans__affirmative+negated+conjunction+d...      0.629552  0.790570   \n",
       "57  ambiguous_truthful_reply+unambiguous_lie+unamb...      0.589636  0.672299   \n",
       "58  animal_class+facts+inventors__affirmative+nega...      0.568333  0.897642   \n",
       "59  animal_class+facts+inventors__affirmative+nega...      0.725455  0.920629   \n",
       "\n",
       "    element symbols     facts  inventors  spanish english translation  \\\n",
       "0          0.643342  0.600180   0.540795                     0.611726   \n",
       "1          0.626359  0.629446   0.565900                     0.643805   \n",
       "2          0.671196  0.567762   0.581590                     0.676438   \n",
       "3          0.622962  0.674471   0.620816                     0.687500   \n",
       "4          0.569293  0.579919   0.530335                     0.497235   \n",
       "5          0.664402  0.651959   0.584205                     0.683075   \n",
       "6          0.730299  0.568663   0.600941                     0.653208   \n",
       "7          0.537364  0.601081   0.617155                     0.625553   \n",
       "8          0.678849  0.675371   0.601987                     0.739491   \n",
       "9          0.655350  0.667717   0.581067                     0.741704   \n",
       "10         0.567901  0.606033   0.563808                     0.612832   \n",
       "11         0.384137  0.506529   0.650628                     0.590155   \n",
       "12         0.691576  0.604819   0.588912                     0.684735   \n",
       "13         0.701087  0.615921   0.552301                     0.720686   \n",
       "14         0.578125  0.558977   0.563285                     0.547566   \n",
       "15         0.590353  0.560241   0.587343                     0.674226   \n",
       "16         0.669158  0.607834   0.669987                     0.505531   \n",
       "17         0.594429  0.598829   0.546034                     0.517699   \n",
       "18         0.542120  0.562359   0.686969                     0.653208   \n",
       "19         0.487092  0.492571   0.521912                     0.473451   \n",
       "20         0.455163  0.484466   0.535042                     0.454093   \n",
       "21         0.467391  0.510581   0.453452                     0.439712   \n",
       "22         0.512908  0.486718   0.533473                     0.500000   \n",
       "23         0.518342  0.525439   0.546548                     0.498341   \n",
       "24         0.487092  0.515534   0.470711                     0.576881   \n",
       "25         0.778533  0.694732   0.674163                     0.740044   \n",
       "26         0.690217  0.645205   0.602510                     0.621128   \n",
       "27         0.522418  0.499325   0.527197                     0.506878   \n",
       "28         0.708560  0.650608   0.548640                     0.651307   \n",
       "29         0.756114  0.651508   0.695607                     0.542049   \n",
       "30         0.550951  0.545700   0.580544                     0.644495   \n",
       "31         0.697011  0.659163   0.592050                     0.714049   \n",
       "32         0.696332  0.689779   0.658473                     0.716814   \n",
       "33         0.732337  0.717244   0.652197                     0.753319   \n",
       "34         0.672554  0.619541   0.546025                     0.592920   \n",
       "35         0.631114  0.657362   0.578975                     0.684735   \n",
       "36         0.699049  0.701036   0.603556                     0.725664   \n",
       "37         0.745245  0.683926   0.664749                     0.787058   \n",
       "38         0.696332  0.613237   0.540795                     0.578540   \n",
       "39         0.766304  0.712292   0.632845                     0.716261   \n",
       "40         0.737092  0.692931   0.626046                     0.731195   \n",
       "41         0.680909  0.661414   0.654812                     0.790929   \n",
       "42         0.749364  0.644755   0.599372                     0.673673   \n",
       "43         0.553435  0.624043   0.648536                     0.732301   \n",
       "44         0.690000  0.711842   0.675732                     0.838496   \n",
       "45         0.720788  0.526843   0.628138                     0.764381   \n",
       "46         0.663043  0.648276   0.633368                     0.700774   \n",
       "47         0.739810  0.786207   0.687238                     0.816372   \n",
       "48         0.713315  0.707071   0.671025                     0.801991   \n",
       "49         0.627717  0.626745   0.599091                     0.794248   \n",
       "50         0.650815  0.612337   0.685885                     0.522677   \n",
       "51         0.548234  0.521387   0.505964                     0.589049   \n",
       "52         0.762228  0.606033   0.650000                     0.735619   \n",
       "53         0.722147  0.675822   0.642782                     0.725455   \n",
       "54         0.625000  0.552004   0.544979                     0.647799   \n",
       "55         0.696332  0.533994   0.609310                     0.549266   \n",
       "56         0.721467  0.667267   0.640167                     0.760000   \n",
       "57         0.526495  0.570014   0.543933                     0.559735   \n",
       "58         0.686141  0.641068   0.710000                     0.728429   \n",
       "59         0.705163  0.639672   0.564545                     0.782633   \n",
       "\n",
       "    real world scenarios  relative comparison  true false  \n",
       "0               0.688742             0.621970    0.569232  \n",
       "1               0.562914             0.659343    0.577745  \n",
       "2               0.649007             0.780051    0.622041  \n",
       "3               0.562914             0.511616    0.601609  \n",
       "4               0.589404             0.543939    0.569644  \n",
       "5               0.629139             0.842172    0.641676  \n",
       "6               0.543046             0.566162    0.606250  \n",
       "7               0.529801             0.500000    0.553084  \n",
       "8               0.735099             0.758081    0.696957  \n",
       "9               0.582781             0.883586    0.587466  \n",
       "10              0.569536             0.617677    0.564453  \n",
       "11              0.463576             0.586111    0.529000  \n",
       "12              0.761589             0.823485    0.649228  \n",
       "13              0.443709             0.502273    0.564810  \n",
       "14              0.556291             0.532071    0.535838  \n",
       "15              0.569536             0.500253    0.533778  \n",
       "16              0.609272             0.847980    0.633822  \n",
       "17              0.516556             0.722727    0.511067  \n",
       "18              0.463576             0.500000    0.570934  \n",
       "19              0.523179             0.500000    0.462624  \n",
       "20              0.511278             0.500000    0.512495  \n",
       "21              0.523438             0.486616    0.466084  \n",
       "22              0.517241             0.500000    0.498435  \n",
       "23              0.627451             0.518182    0.522382  \n",
       "24              0.468750             0.671212    0.514170  \n",
       "25              0.774834             0.808586    0.689762  \n",
       "26              0.807947             0.789899    0.602708  \n",
       "27              0.443709             0.457576    0.479815  \n",
       "28              0.708609             0.613636    0.617949  \n",
       "29              0.496689             0.578283    0.600209  \n",
       "30              0.509934             0.582071    0.559098  \n",
       "31              0.715232             0.500758    0.657677  \n",
       "32              0.827815             0.941162    0.745169  \n",
       "33              0.695364             0.566919    0.622755  \n",
       "34              0.754967             0.643939    0.583375  \n",
       "35              0.609272             0.640909    0.593124  \n",
       "36              0.668874             0.627273    0.608859  \n",
       "37              0.596026             0.561111    0.613088  \n",
       "38              0.609272             0.607071    0.566678  \n",
       "39              0.668874             0.765404    0.645466  \n",
       "40              0.715232             0.650000    0.605289  \n",
       "41              0.562914             0.592172    0.640523  \n",
       "42              0.748344             0.677778    0.614050  \n",
       "43              0.662252             0.711616    0.624403  \n",
       "44              0.708609             0.703535    0.680370  \n",
       "45              0.715232             0.574242    0.599934  \n",
       "46              0.748344             0.870707    0.671088  \n",
       "47              0.847682             0.678788    0.656368  \n",
       "48              0.821192             0.702778    0.637447  \n",
       "49              0.562914             0.504545    0.609354  \n",
       "50              0.629139             0.868687    0.643461  \n",
       "51              0.761589             0.501010    0.531526  \n",
       "52              0.629139             0.887374    0.623002  \n",
       "53              0.748344             0.571212    0.665596  \n",
       "54              0.688742             0.525000    0.566760  \n",
       "55              0.536424             0.882828    0.523233  \n",
       "56              0.834437             0.621970    0.687675  \n",
       "57              0.833333             0.500758    0.558329  \n",
       "58              0.847682             0.854798    0.697232  \n",
       "59              0.500000             0.916162    0.690402  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>scenario</th>\n",
       "      <th>animal class</th>\n",
       "      <th>cities</th>\n",
       "      <th>element symbols</th>\n",
       "      <th>facts</th>\n",
       "      <th>inventors</th>\n",
       "      <th>spanish english translation</th>\n",
       "      <th>real world scenarios</th>\n",
       "      <th>relative comparison</th>\n",
       "      <th>true false</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>animal_class__affirmative</td>\n",
       "      <td>0.606804</td>\n",
       "      <td>0.792731</td>\n",
       "      <td>0.643342</td>\n",
       "      <td>0.600180</td>\n",
       "      <td>0.540795</td>\n",
       "      <td>0.611726</td>\n",
       "      <td>0.688742</td>\n",
       "      <td>0.621970</td>\n",
       "      <td>0.569232</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>animal_class__conjunction</td>\n",
       "      <td>0.687500</td>\n",
       "      <td>0.845187</td>\n",
       "      <td>0.626359</td>\n",
       "      <td>0.629446</td>\n",
       "      <td>0.565900</td>\n",
       "      <td>0.643805</td>\n",
       "      <td>0.562914</td>\n",
       "      <td>0.659343</td>\n",
       "      <td>0.577745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>animal_class__disjunction</td>\n",
       "      <td>0.764009</td>\n",
       "      <td>0.736149</td>\n",
       "      <td>0.671196</td>\n",
       "      <td>0.567762</td>\n",
       "      <td>0.581590</td>\n",
       "      <td>0.676438</td>\n",
       "      <td>0.649007</td>\n",
       "      <td>0.780051</td>\n",
       "      <td>0.622041</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>animal_class__negated</td>\n",
       "      <td>0.594146</td>\n",
       "      <td>0.821022</td>\n",
       "      <td>0.622962</td>\n",
       "      <td>0.674471</td>\n",
       "      <td>0.620816</td>\n",
       "      <td>0.687500</td>\n",
       "      <td>0.562914</td>\n",
       "      <td>0.511616</td>\n",
       "      <td>0.601609</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cities__affirmative</td>\n",
       "      <td>0.633754</td>\n",
       "      <td>0.679188</td>\n",
       "      <td>0.569293</td>\n",
       "      <td>0.579919</td>\n",
       "      <td>0.530335</td>\n",
       "      <td>0.497235</td>\n",
       "      <td>0.589404</td>\n",
       "      <td>0.543939</td>\n",
       "      <td>0.569644</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>cities__conjunction</td>\n",
       "      <td>0.783613</td>\n",
       "      <td>0.910356</td>\n",
       "      <td>0.664402</td>\n",
       "      <td>0.651959</td>\n",
       "      <td>0.584205</td>\n",
       "      <td>0.683075</td>\n",
       "      <td>0.629139</td>\n",
       "      <td>0.842172</td>\n",
       "      <td>0.641676</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>cities__disjunction</td>\n",
       "      <td>0.637955</td>\n",
       "      <td>0.700218</td>\n",
       "      <td>0.730299</td>\n",
       "      <td>0.568663</td>\n",
       "      <td>0.600941</td>\n",
       "      <td>0.653208</td>\n",
       "      <td>0.543046</td>\n",
       "      <td>0.566162</td>\n",
       "      <td>0.606250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>cities__negated</td>\n",
       "      <td>0.525210</td>\n",
       "      <td>0.587368</td>\n",
       "      <td>0.537364</td>\n",
       "      <td>0.601081</td>\n",
       "      <td>0.617155</td>\n",
       "      <td>0.625553</td>\n",
       "      <td>0.529801</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.553084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>element_symb__affirmative</td>\n",
       "      <td>0.714986</td>\n",
       "      <td>0.722593</td>\n",
       "      <td>0.678849</td>\n",
       "      <td>0.675371</td>\n",
       "      <td>0.601987</td>\n",
       "      <td>0.739491</td>\n",
       "      <td>0.735099</td>\n",
       "      <td>0.758081</td>\n",
       "      <td>0.696957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>element_symb__conjunction</td>\n",
       "      <td>0.701681</td>\n",
       "      <td>0.657760</td>\n",
       "      <td>0.655350</td>\n",
       "      <td>0.667717</td>\n",
       "      <td>0.581067</td>\n",
       "      <td>0.741704</td>\n",
       "      <td>0.582781</td>\n",
       "      <td>0.883586</td>\n",
       "      <td>0.587466</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>element_symb__disjunction</td>\n",
       "      <td>0.630952</td>\n",
       "      <td>0.583890</td>\n",
       "      <td>0.567901</td>\n",
       "      <td>0.606033</td>\n",
       "      <td>0.563808</td>\n",
       "      <td>0.612832</td>\n",
       "      <td>0.569536</td>\n",
       "      <td>0.617677</td>\n",
       "      <td>0.564453</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>element_symb__negated</td>\n",
       "      <td>0.509804</td>\n",
       "      <td>0.519057</td>\n",
       "      <td>0.384137</td>\n",
       "      <td>0.506529</td>\n",
       "      <td>0.650628</td>\n",
       "      <td>0.590155</td>\n",
       "      <td>0.463576</td>\n",
       "      <td>0.586111</td>\n",
       "      <td>0.529000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>facts__affirmative</td>\n",
       "      <td>0.721289</td>\n",
       "      <td>0.786837</td>\n",
       "      <td>0.691576</td>\n",
       "      <td>0.604819</td>\n",
       "      <td>0.588912</td>\n",
       "      <td>0.684735</td>\n",
       "      <td>0.761589</td>\n",
       "      <td>0.823485</td>\n",
       "      <td>0.649228</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>facts__conjunction</td>\n",
       "      <td>0.581933</td>\n",
       "      <td>0.604519</td>\n",
       "      <td>0.701087</td>\n",
       "      <td>0.615921</td>\n",
       "      <td>0.552301</td>\n",
       "      <td>0.720686</td>\n",
       "      <td>0.443709</td>\n",
       "      <td>0.502273</td>\n",
       "      <td>0.564810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>facts__disjunction</td>\n",
       "      <td>0.703081</td>\n",
       "      <td>0.689194</td>\n",
       "      <td>0.578125</td>\n",
       "      <td>0.558977</td>\n",
       "      <td>0.563285</td>\n",
       "      <td>0.547566</td>\n",
       "      <td>0.556291</td>\n",
       "      <td>0.532071</td>\n",
       "      <td>0.535838</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>facts__negated</td>\n",
       "      <td>0.537815</td>\n",
       "      <td>0.700786</td>\n",
       "      <td>0.590353</td>\n",
       "      <td>0.560241</td>\n",
       "      <td>0.587343</td>\n",
       "      <td>0.674226</td>\n",
       "      <td>0.569536</td>\n",
       "      <td>0.500253</td>\n",
       "      <td>0.533778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>inventors__affirmative</td>\n",
       "      <td>0.658964</td>\n",
       "      <td>0.904126</td>\n",
       "      <td>0.669158</td>\n",
       "      <td>0.607834</td>\n",
       "      <td>0.669987</td>\n",
       "      <td>0.505531</td>\n",
       "      <td>0.609272</td>\n",
       "      <td>0.847980</td>\n",
       "      <td>0.633822</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>inventors__conjunction</td>\n",
       "      <td>0.635854</td>\n",
       "      <td>0.597446</td>\n",
       "      <td>0.594429</td>\n",
       "      <td>0.598829</td>\n",
       "      <td>0.546034</td>\n",
       "      <td>0.517699</td>\n",
       "      <td>0.516556</td>\n",
       "      <td>0.722727</td>\n",
       "      <td>0.511067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>inventors__disjunction</td>\n",
       "      <td>0.509804</td>\n",
       "      <td>0.669155</td>\n",
       "      <td>0.542120</td>\n",
       "      <td>0.562359</td>\n",
       "      <td>0.686969</td>\n",
       "      <td>0.653208</td>\n",
       "      <td>0.463576</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.570934</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>inventors__negated</td>\n",
       "      <td>0.475490</td>\n",
       "      <td>0.413949</td>\n",
       "      <td>0.487092</td>\n",
       "      <td>0.492571</td>\n",
       "      <td>0.521912</td>\n",
       "      <td>0.473451</td>\n",
       "      <td>0.523179</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.462624</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>ambiguous_lie</td>\n",
       "      <td>0.478291</td>\n",
       "      <td>0.503340</td>\n",
       "      <td>0.455163</td>\n",
       "      <td>0.484466</td>\n",
       "      <td>0.535042</td>\n",
       "      <td>0.454093</td>\n",
       "      <td>0.511278</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.512495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>ambiguous_truthful_reply</td>\n",
       "      <td>0.485994</td>\n",
       "      <td>0.477210</td>\n",
       "      <td>0.467391</td>\n",
       "      <td>0.510581</td>\n",
       "      <td>0.453452</td>\n",
       "      <td>0.439712</td>\n",
       "      <td>0.523438</td>\n",
       "      <td>0.486616</td>\n",
       "      <td>0.466084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>honest_reply_despite_incentive_to_lie</td>\n",
       "      <td>0.493697</td>\n",
       "      <td>0.498625</td>\n",
       "      <td>0.512908</td>\n",
       "      <td>0.486718</td>\n",
       "      <td>0.533473</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.517241</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.498435</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>unambiguous_lie</td>\n",
       "      <td>0.525910</td>\n",
       "      <td>0.654028</td>\n",
       "      <td>0.518342</td>\n",
       "      <td>0.525439</td>\n",
       "      <td>0.546548</td>\n",
       "      <td>0.498341</td>\n",
       "      <td>0.627451</td>\n",
       "      <td>0.518182</td>\n",
       "      <td>0.522382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>unambiguous_truthful_reply</td>\n",
       "      <td>0.506303</td>\n",
       "      <td>0.501572</td>\n",
       "      <td>0.487092</td>\n",
       "      <td>0.515534</td>\n",
       "      <td>0.470711</td>\n",
       "      <td>0.576881</td>\n",
       "      <td>0.468750</td>\n",
       "      <td>0.671212</td>\n",
       "      <td>0.514170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>larger_than</td>\n",
       "      <td>0.692577</td>\n",
       "      <td>0.777210</td>\n",
       "      <td>0.778533</td>\n",
       "      <td>0.694732</td>\n",
       "      <td>0.674163</td>\n",
       "      <td>0.740044</td>\n",
       "      <td>0.774834</td>\n",
       "      <td>0.808586</td>\n",
       "      <td>0.689762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>smaller_than</td>\n",
       "      <td>0.829832</td>\n",
       "      <td>0.846365</td>\n",
       "      <td>0.690217</td>\n",
       "      <td>0.645205</td>\n",
       "      <td>0.602510</td>\n",
       "      <td>0.621128</td>\n",
       "      <td>0.807947</td>\n",
       "      <td>0.789899</td>\n",
       "      <td>0.602708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>sp_en_trans__negated</td>\n",
       "      <td>0.467787</td>\n",
       "      <td>0.598035</td>\n",
       "      <td>0.522418</td>\n",
       "      <td>0.499325</td>\n",
       "      <td>0.527197</td>\n",
       "      <td>0.506878</td>\n",
       "      <td>0.443709</td>\n",
       "      <td>0.457576</td>\n",
       "      <td>0.479815</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>sp_en_trans__affirmative</td>\n",
       "      <td>0.682073</td>\n",
       "      <td>0.842829</td>\n",
       "      <td>0.708560</td>\n",
       "      <td>0.650608</td>\n",
       "      <td>0.548640</td>\n",
       "      <td>0.651307</td>\n",
       "      <td>0.708609</td>\n",
       "      <td>0.613636</td>\n",
       "      <td>0.617949</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>sp_en_trans__conjunction</td>\n",
       "      <td>0.698179</td>\n",
       "      <td>0.750295</td>\n",
       "      <td>0.756114</td>\n",
       "      <td>0.651508</td>\n",
       "      <td>0.695607</td>\n",
       "      <td>0.542049</td>\n",
       "      <td>0.496689</td>\n",
       "      <td>0.578283</td>\n",
       "      <td>0.600209</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>sp_en_trans__disjunction</td>\n",
       "      <td>0.530812</td>\n",
       "      <td>0.592338</td>\n",
       "      <td>0.550951</td>\n",
       "      <td>0.545700</td>\n",
       "      <td>0.580544</td>\n",
       "      <td>0.644495</td>\n",
       "      <td>0.509934</td>\n",
       "      <td>0.582071</td>\n",
       "      <td>0.559098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>common_claim_true_false</td>\n",
       "      <td>0.528711</td>\n",
       "      <td>0.728291</td>\n",
       "      <td>0.697011</td>\n",
       "      <td>0.659163</td>\n",
       "      <td>0.592050</td>\n",
       "      <td>0.714049</td>\n",
       "      <td>0.715232</td>\n",
       "      <td>0.500758</td>\n",
       "      <td>0.657677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>counterfact_true_false</td>\n",
       "      <td>0.746499</td>\n",
       "      <td>0.866994</td>\n",
       "      <td>0.696332</td>\n",
       "      <td>0.689779</td>\n",
       "      <td>0.658473</td>\n",
       "      <td>0.716814</td>\n",
       "      <td>0.827815</td>\n",
       "      <td>0.941162</td>\n",
       "      <td>0.745169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>animal_class__affirmative+negated</td>\n",
       "      <td>0.585455</td>\n",
       "      <td>0.853635</td>\n",
       "      <td>0.732337</td>\n",
       "      <td>0.717244</td>\n",
       "      <td>0.652197</td>\n",
       "      <td>0.753319</td>\n",
       "      <td>0.695364</td>\n",
       "      <td>0.566919</td>\n",
       "      <td>0.622755</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>animal_class__affirmative+disjunction</td>\n",
       "      <td>0.717277</td>\n",
       "      <td>0.828684</td>\n",
       "      <td>0.672554</td>\n",
       "      <td>0.619541</td>\n",
       "      <td>0.546025</td>\n",
       "      <td>0.592920</td>\n",
       "      <td>0.754967</td>\n",
       "      <td>0.643939</td>\n",
       "      <td>0.583375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>animal_class__negated+conjunction</td>\n",
       "      <td>0.641361</td>\n",
       "      <td>0.877210</td>\n",
       "      <td>0.631114</td>\n",
       "      <td>0.657362</td>\n",
       "      <td>0.578975</td>\n",
       "      <td>0.684735</td>\n",
       "      <td>0.609272</td>\n",
       "      <td>0.640909</td>\n",
       "      <td>0.593124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>animal_class__affirmative+negated+conjunction+...</td>\n",
       "      <td>0.810000</td>\n",
       "      <td>0.878978</td>\n",
       "      <td>0.699049</td>\n",
       "      <td>0.701036</td>\n",
       "      <td>0.603556</td>\n",
       "      <td>0.725664</td>\n",
       "      <td>0.668874</td>\n",
       "      <td>0.627273</td>\n",
       "      <td>0.608859</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>cities__affirmative+negated</td>\n",
       "      <td>0.644958</td>\n",
       "      <td>0.577216</td>\n",
       "      <td>0.745245</td>\n",
       "      <td>0.683926</td>\n",
       "      <td>0.664749</td>\n",
       "      <td>0.787058</td>\n",
       "      <td>0.596026</td>\n",
       "      <td>0.561111</td>\n",
       "      <td>0.613088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>cities__affirmative+disjunction</td>\n",
       "      <td>0.680672</td>\n",
       "      <td>0.733678</td>\n",
       "      <td>0.696332</td>\n",
       "      <td>0.613237</td>\n",
       "      <td>0.540795</td>\n",
       "      <td>0.578540</td>\n",
       "      <td>0.609272</td>\n",
       "      <td>0.607071</td>\n",
       "      <td>0.566678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>cities__negated+conjunction</td>\n",
       "      <td>0.761204</td>\n",
       "      <td>0.880725</td>\n",
       "      <td>0.766304</td>\n",
       "      <td>0.712292</td>\n",
       "      <td>0.632845</td>\n",
       "      <td>0.716261</td>\n",
       "      <td>0.668874</td>\n",
       "      <td>0.765404</td>\n",
       "      <td>0.645466</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>cities__affirmative+negated+conjunction+disjun...</td>\n",
       "      <td>0.770308</td>\n",
       "      <td>0.970000</td>\n",
       "      <td>0.737092</td>\n",
       "      <td>0.692931</td>\n",
       "      <td>0.626046</td>\n",
       "      <td>0.731195</td>\n",
       "      <td>0.715232</td>\n",
       "      <td>0.650000</td>\n",
       "      <td>0.605289</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>element_symb__affirmative+negated</td>\n",
       "      <td>0.633053</td>\n",
       "      <td>0.827505</td>\n",
       "      <td>0.680909</td>\n",
       "      <td>0.661414</td>\n",
       "      <td>0.654812</td>\n",
       "      <td>0.790929</td>\n",
       "      <td>0.562914</td>\n",
       "      <td>0.592172</td>\n",
       "      <td>0.640523</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>element_symb__affirmative+disjunction</td>\n",
       "      <td>0.691176</td>\n",
       "      <td>0.799018</td>\n",
       "      <td>0.749364</td>\n",
       "      <td>0.644755</td>\n",
       "      <td>0.599372</td>\n",
       "      <td>0.673673</td>\n",
       "      <td>0.748344</td>\n",
       "      <td>0.677778</td>\n",
       "      <td>0.614050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>element_symb__negated+conjunction</td>\n",
       "      <td>0.667367</td>\n",
       "      <td>0.756189</td>\n",
       "      <td>0.553435</td>\n",
       "      <td>0.624043</td>\n",
       "      <td>0.648536</td>\n",
       "      <td>0.732301</td>\n",
       "      <td>0.662252</td>\n",
       "      <td>0.711616</td>\n",
       "      <td>0.624403</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>element_symb__affirmative+negated+conjunction+...</td>\n",
       "      <td>0.712885</td>\n",
       "      <td>0.861493</td>\n",
       "      <td>0.690000</td>\n",
       "      <td>0.711842</td>\n",
       "      <td>0.675732</td>\n",
       "      <td>0.838496</td>\n",
       "      <td>0.708609</td>\n",
       "      <td>0.703535</td>\n",
       "      <td>0.680370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>facts__affirmative+negated</td>\n",
       "      <td>0.620448</td>\n",
       "      <td>0.791749</td>\n",
       "      <td>0.720788</td>\n",
       "      <td>0.526843</td>\n",
       "      <td>0.628138</td>\n",
       "      <td>0.764381</td>\n",
       "      <td>0.715232</td>\n",
       "      <td>0.574242</td>\n",
       "      <td>0.599934</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>facts__affirmative+disjunction</td>\n",
       "      <td>0.779412</td>\n",
       "      <td>0.922986</td>\n",
       "      <td>0.663043</td>\n",
       "      <td>0.648276</td>\n",
       "      <td>0.633368</td>\n",
       "      <td>0.700774</td>\n",
       "      <td>0.748344</td>\n",
       "      <td>0.870707</td>\n",
       "      <td>0.671088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>facts__negated+conjunction</td>\n",
       "      <td>0.743697</td>\n",
       "      <td>0.875835</td>\n",
       "      <td>0.739810</td>\n",
       "      <td>0.786207</td>\n",
       "      <td>0.687238</td>\n",
       "      <td>0.816372</td>\n",
       "      <td>0.847682</td>\n",
       "      <td>0.678788</td>\n",
       "      <td>0.656368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>facts__affirmative+negated+conjunction+disjunc...</td>\n",
       "      <td>0.725490</td>\n",
       "      <td>0.866994</td>\n",
       "      <td>0.713315</td>\n",
       "      <td>0.707071</td>\n",
       "      <td>0.671025</td>\n",
       "      <td>0.801991</td>\n",
       "      <td>0.821192</td>\n",
       "      <td>0.702778</td>\n",
       "      <td>0.637447</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>inventors__affirmative+negated</td>\n",
       "      <td>0.596639</td>\n",
       "      <td>0.745972</td>\n",
       "      <td>0.627717</td>\n",
       "      <td>0.626745</td>\n",
       "      <td>0.599091</td>\n",
       "      <td>0.794248</td>\n",
       "      <td>0.562914</td>\n",
       "      <td>0.504545</td>\n",
       "      <td>0.609354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>inventors__affirmative+disjunction</td>\n",
       "      <td>0.707283</td>\n",
       "      <td>0.900982</td>\n",
       "      <td>0.650815</td>\n",
       "      <td>0.612337</td>\n",
       "      <td>0.685885</td>\n",
       "      <td>0.522677</td>\n",
       "      <td>0.629139</td>\n",
       "      <td>0.868687</td>\n",
       "      <td>0.643461</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>inventors__negated+conjunction</td>\n",
       "      <td>0.549720</td>\n",
       "      <td>0.642436</td>\n",
       "      <td>0.548234</td>\n",
       "      <td>0.521387</td>\n",
       "      <td>0.505964</td>\n",
       "      <td>0.589049</td>\n",
       "      <td>0.761589</td>\n",
       "      <td>0.501010</td>\n",
       "      <td>0.531526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>inventors__affirmative+negated+conjunction+dis...</td>\n",
       "      <td>0.628151</td>\n",
       "      <td>0.739686</td>\n",
       "      <td>0.762228</td>\n",
       "      <td>0.606033</td>\n",
       "      <td>0.650000</td>\n",
       "      <td>0.735619</td>\n",
       "      <td>0.629139</td>\n",
       "      <td>0.887374</td>\n",
       "      <td>0.623002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>sp_en_trans__affirmative+negated</td>\n",
       "      <td>0.640056</td>\n",
       "      <td>0.816896</td>\n",
       "      <td>0.722147</td>\n",
       "      <td>0.675822</td>\n",
       "      <td>0.642782</td>\n",
       "      <td>0.725455</td>\n",
       "      <td>0.748344</td>\n",
       "      <td>0.571212</td>\n",
       "      <td>0.665596</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>sp_en_trans__affirmative+disjunction</td>\n",
       "      <td>0.588235</td>\n",
       "      <td>0.707662</td>\n",
       "      <td>0.625000</td>\n",
       "      <td>0.552004</td>\n",
       "      <td>0.544979</td>\n",
       "      <td>0.647799</td>\n",
       "      <td>0.688742</td>\n",
       "      <td>0.525000</td>\n",
       "      <td>0.566760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>sp_en_trans__negated+conjunction</td>\n",
       "      <td>0.525210</td>\n",
       "      <td>0.647544</td>\n",
       "      <td>0.696332</td>\n",
       "      <td>0.533994</td>\n",
       "      <td>0.609310</td>\n",
       "      <td>0.549266</td>\n",
       "      <td>0.536424</td>\n",
       "      <td>0.882828</td>\n",
       "      <td>0.523233</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>sp_en_trans__affirmative+negated+conjunction+d...</td>\n",
       "      <td>0.629552</td>\n",
       "      <td>0.790570</td>\n",
       "      <td>0.721467</td>\n",
       "      <td>0.667267</td>\n",
       "      <td>0.640167</td>\n",
       "      <td>0.760000</td>\n",
       "      <td>0.834437</td>\n",
       "      <td>0.621970</td>\n",
       "      <td>0.687675</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>ambiguous_truthful_reply+unambiguous_lie+unamb...</td>\n",
       "      <td>0.589636</td>\n",
       "      <td>0.672299</td>\n",
       "      <td>0.526495</td>\n",
       "      <td>0.570014</td>\n",
       "      <td>0.543933</td>\n",
       "      <td>0.559735</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>0.500758</td>\n",
       "      <td>0.558329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>animal_class+facts+inventors__affirmative+nega...</td>\n",
       "      <td>0.568333</td>\n",
       "      <td>0.897642</td>\n",
       "      <td>0.686141</td>\n",
       "      <td>0.641068</td>\n",
       "      <td>0.710000</td>\n",
       "      <td>0.728429</td>\n",
       "      <td>0.847682</td>\n",
       "      <td>0.854798</td>\n",
       "      <td>0.697232</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>animal_class+facts+inventors__affirmative+nega...</td>\n",
       "      <td>0.725455</td>\n",
       "      <td>0.920629</td>\n",
       "      <td>0.705163</td>\n",
       "      <td>0.639672</td>\n",
       "      <td>0.564545</td>\n",
       "      <td>0.782633</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.916162</td>\n",
       "      <td>0.690402</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 49
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:58.974611Z",
     "start_time": "2024-12-07T21:26:58.957183Z"
    }
   },
   "cell_type": "code",
   "source": "lyr18_probes_acc_on_unseen_single_topic_data_groupings_df.describe()",
   "id": "6ab4c2752e3f474c",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "       animal class     cities  element symbols      facts  inventors  \\\n",
       "count     60.000000  60.000000        60.000000  60.000000  60.000000   \n",
       "mean       0.636906   0.739048         0.641814   0.614063   0.598574   \n",
       "std        0.094262   0.132850         0.091259   0.068886   0.058142   \n",
       "min        0.467787   0.413949         0.384137   0.484466   0.453452   \n",
       "25%        0.563680   0.652407         0.568945   0.561830   0.548117   \n",
       "50%        0.636905   0.748134         0.670177   0.617731   0.599232   \n",
       "75%        0.708683   0.848183         0.706012   0.662877   0.648902   \n",
       "max        0.829832   0.970000         0.778533   0.786207   0.710000   \n",
       "\n",
       "       spanish english translation  real world scenarios  relative comparison  \\\n",
       "count                    60.000000             60.000000            60.000000   \n",
       "mean                      0.653819              0.636446             0.649268   \n",
       "std                       0.101841              0.115169             0.137284   \n",
       "min                       0.439712              0.443709             0.457576   \n",
       "25%                       0.578125              0.541391             0.523295   \n",
       "50%                       0.673949              0.628295             0.619823   \n",
       "75%                       0.731471              0.720199             0.759912   \n",
       "max                       0.838496              0.847682             0.941162   \n",
       "\n",
       "       true false  \n",
       "count   60.000000  \n",
       "mean     0.596777  \n",
       "std      0.062378  \n",
       "min      0.462624  \n",
       "25%      0.558906  \n",
       "50%      0.602159  \n",
       "75%      0.640811  \n",
       "max      0.745169  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>animal class</th>\n",
       "      <th>cities</th>\n",
       "      <th>element symbols</th>\n",
       "      <th>facts</th>\n",
       "      <th>inventors</th>\n",
       "      <th>spanish english translation</th>\n",
       "      <th>real world scenarios</th>\n",
       "      <th>relative comparison</th>\n",
       "      <th>true false</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.636906</td>\n",
       "      <td>0.739048</td>\n",
       "      <td>0.641814</td>\n",
       "      <td>0.614063</td>\n",
       "      <td>0.598574</td>\n",
       "      <td>0.653819</td>\n",
       "      <td>0.636446</td>\n",
       "      <td>0.649268</td>\n",
       "      <td>0.596777</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.094262</td>\n",
       "      <td>0.132850</td>\n",
       "      <td>0.091259</td>\n",
       "      <td>0.068886</td>\n",
       "      <td>0.058142</td>\n",
       "      <td>0.101841</td>\n",
       "      <td>0.115169</td>\n",
       "      <td>0.137284</td>\n",
       "      <td>0.062378</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.467787</td>\n",
       "      <td>0.413949</td>\n",
       "      <td>0.384137</td>\n",
       "      <td>0.484466</td>\n",
       "      <td>0.453452</td>\n",
       "      <td>0.439712</td>\n",
       "      <td>0.443709</td>\n",
       "      <td>0.457576</td>\n",
       "      <td>0.462624</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.563680</td>\n",
       "      <td>0.652407</td>\n",
       "      <td>0.568945</td>\n",
       "      <td>0.561830</td>\n",
       "      <td>0.548117</td>\n",
       "      <td>0.578125</td>\n",
       "      <td>0.541391</td>\n",
       "      <td>0.523295</td>\n",
       "      <td>0.558906</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.636905</td>\n",
       "      <td>0.748134</td>\n",
       "      <td>0.670177</td>\n",
       "      <td>0.617731</td>\n",
       "      <td>0.599232</td>\n",
       "      <td>0.673949</td>\n",
       "      <td>0.628295</td>\n",
       "      <td>0.619823</td>\n",
       "      <td>0.602159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.708683</td>\n",
       "      <td>0.848183</td>\n",
       "      <td>0.706012</td>\n",
       "      <td>0.662877</td>\n",
       "      <td>0.648902</td>\n",
       "      <td>0.731471</td>\n",
       "      <td>0.720199</td>\n",
       "      <td>0.759912</td>\n",
       "      <td>0.640811</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.829832</td>\n",
       "      <td>0.970000</td>\n",
       "      <td>0.778533</td>\n",
       "      <td>0.786207</td>\n",
       "      <td>0.710000</td>\n",
       "      <td>0.838496</td>\n",
       "      <td>0.847682</td>\n",
       "      <td>0.941162</td>\n",
       "      <td>0.745169</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 50
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:58.992651Z",
     "start_time": "2024-12-07T21:26:58.975634Z"
    }
   },
   "cell_type": "code",
   "source": "lyr18_baseline_linear_probes_acc_on_unseen_single_topic_data_groupings_df",
   "id": "3ab8d9067ae32400",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "                                             scenario  animal class    cities  \\\n",
       "0                           animal_class__affirmative      0.704905  0.834185   \n",
       "1                           animal_class__conjunction      0.705819  0.892731   \n",
       "2                           animal_class__disjunction      0.591595  0.527505   \n",
       "3                               animal_class__negated      0.572785  0.852652   \n",
       "4                                 cities__affirmative      0.636555  0.682248   \n",
       "5                                 cities__conjunction      0.714286  0.710468   \n",
       "6                                 cities__disjunction      0.681373  0.738562   \n",
       "7                                     cities__negated      0.588936  0.682805   \n",
       "8                           element_symb__affirmative      0.648459  0.652849   \n",
       "9                           element_symb__conjunction      0.719188  0.778585   \n",
       "10                          element_symb__disjunction      0.820028  0.883301   \n",
       "11                              element_symb__negated      0.436975  0.586837   \n",
       "12                                 facts__affirmative      0.615546  0.531631   \n",
       "13                                 facts__conjunction      0.629552  0.670334   \n",
       "14                                 facts__disjunction      0.765406  0.740472   \n",
       "15                                     facts__negated      0.543417  0.633399   \n",
       "16                             inventors__affirmative      0.703782  0.866601   \n",
       "17                             inventors__conjunction      0.673669  0.654813   \n",
       "18                             inventors__disjunction      0.619048  0.555206   \n",
       "19                                 inventors__negated      0.560924  0.655599   \n",
       "20                                      ambiguous_lie      0.506303  0.501572   \n",
       "21                           ambiguous_truthful_reply      0.493697  0.498428   \n",
       "22              honest_reply_despite_incentive_to_lie      0.493697  0.498428   \n",
       "23                                    unambiguous_lie      0.506303  0.501572   \n",
       "24                         unambiguous_truthful_reply      0.493697  0.498428   \n",
       "25                                        larger_than      0.715686  0.663458   \n",
       "26                                       smaller_than      0.752101  0.862083   \n",
       "27                               sp_en_trans__negated      0.483894  0.462083   \n",
       "28                           sp_en_trans__affirmative      0.773109  0.822593   \n",
       "29                           sp_en_trans__conjunction      0.724090  0.676228   \n",
       "30                           sp_en_trans__disjunction      0.468487  0.444204   \n",
       "31                            common_claim_true_false      0.784314  0.925737   \n",
       "32                             counterfact_true_false      0.728992  0.759332   \n",
       "33                  animal_class__affirmative+negated      0.680909  0.852259   \n",
       "34              animal_class__affirmative+disjunction      0.869110  0.799607   \n",
       "35                  animal_class__negated+conjunction      0.657068  0.936149   \n",
       "36  animal_class__affirmative+negated+conjunction+...      0.860000  0.945383   \n",
       "37                        cities__affirmative+negated      0.625350  0.542421   \n",
       "38                    cities__affirmative+disjunction      0.691176  0.661603   \n",
       "39                        cities__negated+conjunction      0.791317  0.864981   \n",
       "40  cities__affirmative+negated+conjunction+disjun...      0.885854  0.960000   \n",
       "41                  element_symb__affirmative+negated      0.757703  0.816896   \n",
       "42              element_symb__affirmative+disjunction      0.848739  0.861297   \n",
       "43                  element_symb__negated+conjunction      0.800420  0.925147   \n",
       "44  element_symb__affirmative+negated+conjunction+...      0.878852  0.858939   \n",
       "45                         facts__affirmative+negated      0.770308  0.894303   \n",
       "46                     facts__affirmative+disjunction      0.745798  0.746365   \n",
       "47                         facts__negated+conjunction      0.636555  0.822790   \n",
       "48  facts__affirmative+negated+conjunction+disjunc...      0.754202  0.848723   \n",
       "49                     inventors__affirmative+negated      0.763305  0.878193   \n",
       "50                 inventors__affirmative+disjunction      0.653361  0.864833   \n",
       "51                     inventors__negated+conjunction      0.785714  0.928487   \n",
       "52  inventors__affirmative+negated+conjunction+dis...      0.845238  0.945776   \n",
       "53                   sp_en_trans__affirmative+negated      0.665966  0.655599   \n",
       "54               sp_en_trans__affirmative+disjunction      0.567927  0.654028   \n",
       "55                   sp_en_trans__negated+conjunction      0.766106  0.777407   \n",
       "56  sp_en_trans__affirmative+negated+conjunction+d...      0.636555  0.836542   \n",
       "57  ambiguous_truthful_reply+unambiguous_lie+unamb...      0.495798  0.498428   \n",
       "58  animal_class+facts+inventors__affirmative+nega...      0.566667  0.921611   \n",
       "59  animal_class+facts+inventors__affirmative+nega...      0.729091  0.924361   \n",
       "\n",
       "    element symbols     facts  inventors  spanish english translation  \\\n",
       "0          0.675272  0.618640   0.559100                     0.673119   \n",
       "1          0.693614  0.706439   0.627092                     0.727323   \n",
       "2          0.527174  0.559658   0.548640                     0.529314   \n",
       "3          0.680027  0.611887   0.619770                     0.626659   \n",
       "4          0.611413  0.578118   0.514121                     0.570796   \n",
       "5          0.740489  0.688429   0.610879                     0.713496   \n",
       "6          0.694973  0.569113   0.603033                     0.571350   \n",
       "7          0.753397  0.618640   0.674686                     0.812500   \n",
       "8          0.602644  0.651959   0.506799                     0.617257   \n",
       "9          0.588477  0.648357   0.622385                     0.698009   \n",
       "10         0.808642  0.615489   0.593619                     0.788717   \n",
       "11         0.315708  0.507879   0.631276                     0.510509   \n",
       "12         0.585598  0.557229   0.483264                     0.633296   \n",
       "13         0.703125  0.650203   0.608264                     0.744469   \n",
       "14         0.684103  0.613016   0.553870                     0.591261   \n",
       "15         0.584918  0.551807   0.582636                     0.692478   \n",
       "16         0.747962  0.701036   0.658699                     0.805863   \n",
       "17         0.704484  0.596128   0.558074                     0.625553   \n",
       "18         0.673234  0.555606   0.529745                     0.531527   \n",
       "19         0.540082  0.586673   0.540505                     0.590155   \n",
       "20         0.487092  0.515083   0.466004                     0.498894   \n",
       "21         0.512908  0.484917   0.533996                     0.501106   \n",
       "22         0.512908  0.484917   0.533996                     0.501106   \n",
       "23         0.487092  0.515083   0.466004                     0.498894   \n",
       "24         0.512908  0.484917   0.533996                     0.501106   \n",
       "25         0.548913  0.628546   0.527720                     0.552544   \n",
       "26         0.758832  0.674471   0.612448                     0.728429   \n",
       "27         0.458560  0.458802   0.546548                     0.329436   \n",
       "28         0.648777  0.618640   0.631276                     0.634113   \n",
       "29         0.722826  0.614138   0.618724                     0.532110   \n",
       "30         0.663723  0.521387   0.566423                     0.566514   \n",
       "31         0.679348  0.719946   0.640167                     0.763274   \n",
       "32         0.699049  0.590275   0.682008                     0.665376   \n",
       "33         0.681386  0.716794   0.644351                     0.722345   \n",
       "34         0.595788  0.669968   0.588912                     0.764934   \n",
       "35         0.755435  0.737956   0.703452                     0.806416   \n",
       "36         0.832880  0.751914   0.693515                     0.721792   \n",
       "37         0.785326  0.641603   0.687762                     0.818584   \n",
       "38         0.719429  0.575867   0.586297                     0.592367   \n",
       "39         0.760190  0.695633   0.672594                     0.781527   \n",
       "40         0.897418  0.717244   0.772490                     0.860619   \n",
       "41         0.710000  0.733904   0.643305                     0.855642   \n",
       "42         0.740458  0.684376   0.614540                     0.799226   \n",
       "43         0.601781  0.713192   0.730649                     0.815819   \n",
       "44         0.790000  0.720396   0.707113                     0.888827   \n",
       "45         0.715353  0.656961   0.664749                     0.814159   \n",
       "46         0.749321  0.600862   0.629184                     0.741704   \n",
       "47         0.755435  0.703448   0.657427                     0.821350   \n",
       "48         0.865489  0.767677   0.718619                     0.862832   \n",
       "49         0.677989  0.714543   0.633636                     0.817478   \n",
       "50         0.757473  0.665916   0.572565                     0.688053   \n",
       "51         0.724185  0.689329   0.628231                     0.800885   \n",
       "52         0.816576  0.737956   0.850000                     0.885509   \n",
       "53         0.626359  0.596578   0.689331                     0.716364   \n",
       "54         0.749321  0.564611   0.646967                     0.500000   \n",
       "55         0.788043  0.637551   0.658996                     0.575472   \n",
       "56         0.853261  0.716344   0.663180                     0.960000   \n",
       "57         0.513587  0.499325   0.533996                     0.501106   \n",
       "58         0.761549  0.589316   0.583333                     0.844027   \n",
       "59         0.782609  0.703367   0.680909                     0.841814   \n",
       "\n",
       "    real world scenarios  relative comparison  true false  \n",
       "0               0.596026             0.614899    0.598588  \n",
       "1               0.728477             0.903535    0.693964  \n",
       "2               0.443709             0.500000    0.503131  \n",
       "3               0.569536             0.854040    0.625968  \n",
       "4               0.741722             0.546970    0.648542  \n",
       "5               0.801325             0.938636    0.683007  \n",
       "6               0.642384             0.526263    0.628000  \n",
       "7               0.768212             0.504293    0.629511  \n",
       "8               0.701987             0.571970    0.568930  \n",
       "9               0.596026             0.594697    0.681716  \n",
       "10              0.549669             0.720202    0.654062  \n",
       "11              0.609272             0.502020    0.465892  \n",
       "12              0.476821             0.714646    0.582908  \n",
       "13              0.801325             0.790152    0.701022  \n",
       "14              0.649007             0.742172    0.629044  \n",
       "15              0.549669             0.502778    0.528890  \n",
       "16              0.536424             0.844444    0.701928  \n",
       "17              0.470199             0.571212    0.533696  \n",
       "18              0.569536             0.500000    0.505465  \n",
       "19              0.609272             0.527778    0.522519  \n",
       "20              0.368421             0.500000    0.500000  \n",
       "21              0.476562             0.500000    0.500000  \n",
       "22              0.537931             0.500000    0.500000  \n",
       "23              0.176471             0.500000    0.500000  \n",
       "24              0.302083             0.500000    0.500000  \n",
       "25              0.562914             0.647475    0.646976  \n",
       "26              0.715232             0.757576    0.663975  \n",
       "27              0.403974             0.500000    0.385484  \n",
       "28              0.794702             0.598232    0.708299  \n",
       "29              0.794702             0.500000    0.589114  \n",
       "30              0.496689             0.500253    0.590707  \n",
       "31              0.655629             0.812626    0.708359  \n",
       "32              0.615894             0.834848    0.635056  \n",
       "33              0.761589             0.646465    0.613665  \n",
       "34              0.490066             0.550758    0.635168  \n",
       "35              0.761589             0.870202    0.713709  \n",
       "36              0.576159             0.951263    0.698303  \n",
       "37              0.754967             0.507071    0.666969  \n",
       "38              0.701987             0.530051    0.684489  \n",
       "39              0.814570             0.920202    0.663591  \n",
       "40              0.814570             0.643434    0.726259  \n",
       "41              0.874172             0.923232    0.608722  \n",
       "42              0.549669             0.880808    0.632586  \n",
       "43              0.562914             0.881061    0.650601  \n",
       "44              0.562914             0.903535    0.634179  \n",
       "45              0.867550             0.797727    0.701571  \n",
       "46              0.622517             0.961111    0.621602  \n",
       "47              0.741722             0.815909    0.675262  \n",
       "48              0.821192             0.940404    0.683583  \n",
       "49              0.761589             0.643182    0.684901  \n",
       "50              0.649007             0.701263    0.706487  \n",
       "51              0.841060             0.856818    0.658428  \n",
       "52              0.794702             0.735101    0.700308  \n",
       "53              0.735099             0.500000    0.625144  \n",
       "54              0.668874             0.810101    0.689845  \n",
       "55              0.735099             0.935859    0.608667  \n",
       "56              0.642384             0.849747    0.678640  \n",
       "57              0.791667             0.500000    0.500027  \n",
       "58              0.801325             0.949747    0.722552  \n",
       "59              0.708333             0.928283    0.735171  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>scenario</th>\n",
       "      <th>animal class</th>\n",
       "      <th>cities</th>\n",
       "      <th>element symbols</th>\n",
       "      <th>facts</th>\n",
       "      <th>inventors</th>\n",
       "      <th>spanish english translation</th>\n",
       "      <th>real world scenarios</th>\n",
       "      <th>relative comparison</th>\n",
       "      <th>true false</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>animal_class__affirmative</td>\n",
       "      <td>0.704905</td>\n",
       "      <td>0.834185</td>\n",
       "      <td>0.675272</td>\n",
       "      <td>0.618640</td>\n",
       "      <td>0.559100</td>\n",
       "      <td>0.673119</td>\n",
       "      <td>0.596026</td>\n",
       "      <td>0.614899</td>\n",
       "      <td>0.598588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>animal_class__conjunction</td>\n",
       "      <td>0.705819</td>\n",
       "      <td>0.892731</td>\n",
       "      <td>0.693614</td>\n",
       "      <td>0.706439</td>\n",
       "      <td>0.627092</td>\n",
       "      <td>0.727323</td>\n",
       "      <td>0.728477</td>\n",
       "      <td>0.903535</td>\n",
       "      <td>0.693964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>animal_class__disjunction</td>\n",
       "      <td>0.591595</td>\n",
       "      <td>0.527505</td>\n",
       "      <td>0.527174</td>\n",
       "      <td>0.559658</td>\n",
       "      <td>0.548640</td>\n",
       "      <td>0.529314</td>\n",
       "      <td>0.443709</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.503131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>animal_class__negated</td>\n",
       "      <td>0.572785</td>\n",
       "      <td>0.852652</td>\n",
       "      <td>0.680027</td>\n",
       "      <td>0.611887</td>\n",
       "      <td>0.619770</td>\n",
       "      <td>0.626659</td>\n",
       "      <td>0.569536</td>\n",
       "      <td>0.854040</td>\n",
       "      <td>0.625968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cities__affirmative</td>\n",
       "      <td>0.636555</td>\n",
       "      <td>0.682248</td>\n",
       "      <td>0.611413</td>\n",
       "      <td>0.578118</td>\n",
       "      <td>0.514121</td>\n",
       "      <td>0.570796</td>\n",
       "      <td>0.741722</td>\n",
       "      <td>0.546970</td>\n",
       "      <td>0.648542</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>cities__conjunction</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>0.710468</td>\n",
       "      <td>0.740489</td>\n",
       "      <td>0.688429</td>\n",
       "      <td>0.610879</td>\n",
       "      <td>0.713496</td>\n",
       "      <td>0.801325</td>\n",
       "      <td>0.938636</td>\n",
       "      <td>0.683007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>cities__disjunction</td>\n",
       "      <td>0.681373</td>\n",
       "      <td>0.738562</td>\n",
       "      <td>0.694973</td>\n",
       "      <td>0.569113</td>\n",
       "      <td>0.603033</td>\n",
       "      <td>0.571350</td>\n",
       "      <td>0.642384</td>\n",
       "      <td>0.526263</td>\n",
       "      <td>0.628000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>cities__negated</td>\n",
       "      <td>0.588936</td>\n",
       "      <td>0.682805</td>\n",
       "      <td>0.753397</td>\n",
       "      <td>0.618640</td>\n",
       "      <td>0.674686</td>\n",
       "      <td>0.812500</td>\n",
       "      <td>0.768212</td>\n",
       "      <td>0.504293</td>\n",
       "      <td>0.629511</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>element_symb__affirmative</td>\n",
       "      <td>0.648459</td>\n",
       "      <td>0.652849</td>\n",
       "      <td>0.602644</td>\n",
       "      <td>0.651959</td>\n",
       "      <td>0.506799</td>\n",
       "      <td>0.617257</td>\n",
       "      <td>0.701987</td>\n",
       "      <td>0.571970</td>\n",
       "      <td>0.568930</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>element_symb__conjunction</td>\n",
       "      <td>0.719188</td>\n",
       "      <td>0.778585</td>\n",
       "      <td>0.588477</td>\n",
       "      <td>0.648357</td>\n",
       "      <td>0.622385</td>\n",
       "      <td>0.698009</td>\n",
       "      <td>0.596026</td>\n",
       "      <td>0.594697</td>\n",
       "      <td>0.681716</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>element_symb__disjunction</td>\n",
       "      <td>0.820028</td>\n",
       "      <td>0.883301</td>\n",
       "      <td>0.808642</td>\n",
       "      <td>0.615489</td>\n",
       "      <td>0.593619</td>\n",
       "      <td>0.788717</td>\n",
       "      <td>0.549669</td>\n",
       "      <td>0.720202</td>\n",
       "      <td>0.654062</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>element_symb__negated</td>\n",
       "      <td>0.436975</td>\n",
       "      <td>0.586837</td>\n",
       "      <td>0.315708</td>\n",
       "      <td>0.507879</td>\n",
       "      <td>0.631276</td>\n",
       "      <td>0.510509</td>\n",
       "      <td>0.609272</td>\n",
       "      <td>0.502020</td>\n",
       "      <td>0.465892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>facts__affirmative</td>\n",
       "      <td>0.615546</td>\n",
       "      <td>0.531631</td>\n",
       "      <td>0.585598</td>\n",
       "      <td>0.557229</td>\n",
       "      <td>0.483264</td>\n",
       "      <td>0.633296</td>\n",
       "      <td>0.476821</td>\n",
       "      <td>0.714646</td>\n",
       "      <td>0.582908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>facts__conjunction</td>\n",
       "      <td>0.629552</td>\n",
       "      <td>0.670334</td>\n",
       "      <td>0.703125</td>\n",
       "      <td>0.650203</td>\n",
       "      <td>0.608264</td>\n",
       "      <td>0.744469</td>\n",
       "      <td>0.801325</td>\n",
       "      <td>0.790152</td>\n",
       "      <td>0.701022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>facts__disjunction</td>\n",
       "      <td>0.765406</td>\n",
       "      <td>0.740472</td>\n",
       "      <td>0.684103</td>\n",
       "      <td>0.613016</td>\n",
       "      <td>0.553870</td>\n",
       "      <td>0.591261</td>\n",
       "      <td>0.649007</td>\n",
       "      <td>0.742172</td>\n",
       "      <td>0.629044</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>facts__negated</td>\n",
       "      <td>0.543417</td>\n",
       "      <td>0.633399</td>\n",
       "      <td>0.584918</td>\n",
       "      <td>0.551807</td>\n",
       "      <td>0.582636</td>\n",
       "      <td>0.692478</td>\n",
       "      <td>0.549669</td>\n",
       "      <td>0.502778</td>\n",
       "      <td>0.528890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>inventors__affirmative</td>\n",
       "      <td>0.703782</td>\n",
       "      <td>0.866601</td>\n",
       "      <td>0.747962</td>\n",
       "      <td>0.701036</td>\n",
       "      <td>0.658699</td>\n",
       "      <td>0.805863</td>\n",
       "      <td>0.536424</td>\n",
       "      <td>0.844444</td>\n",
       "      <td>0.701928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>inventors__conjunction</td>\n",
       "      <td>0.673669</td>\n",
       "      <td>0.654813</td>\n",
       "      <td>0.704484</td>\n",
       "      <td>0.596128</td>\n",
       "      <td>0.558074</td>\n",
       "      <td>0.625553</td>\n",
       "      <td>0.470199</td>\n",
       "      <td>0.571212</td>\n",
       "      <td>0.533696</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>inventors__disjunction</td>\n",
       "      <td>0.619048</td>\n",
       "      <td>0.555206</td>\n",
       "      <td>0.673234</td>\n",
       "      <td>0.555606</td>\n",
       "      <td>0.529745</td>\n",
       "      <td>0.531527</td>\n",
       "      <td>0.569536</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.505465</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>inventors__negated</td>\n",
       "      <td>0.560924</td>\n",
       "      <td>0.655599</td>\n",
       "      <td>0.540082</td>\n",
       "      <td>0.586673</td>\n",
       "      <td>0.540505</td>\n",
       "      <td>0.590155</td>\n",
       "      <td>0.609272</td>\n",
       "      <td>0.527778</td>\n",
       "      <td>0.522519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>ambiguous_lie</td>\n",
       "      <td>0.506303</td>\n",
       "      <td>0.501572</td>\n",
       "      <td>0.487092</td>\n",
       "      <td>0.515083</td>\n",
       "      <td>0.466004</td>\n",
       "      <td>0.498894</td>\n",
       "      <td>0.368421</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>ambiguous_truthful_reply</td>\n",
       "      <td>0.493697</td>\n",
       "      <td>0.498428</td>\n",
       "      <td>0.512908</td>\n",
       "      <td>0.484917</td>\n",
       "      <td>0.533996</td>\n",
       "      <td>0.501106</td>\n",
       "      <td>0.476562</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>honest_reply_despite_incentive_to_lie</td>\n",
       "      <td>0.493697</td>\n",
       "      <td>0.498428</td>\n",
       "      <td>0.512908</td>\n",
       "      <td>0.484917</td>\n",
       "      <td>0.533996</td>\n",
       "      <td>0.501106</td>\n",
       "      <td>0.537931</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>unambiguous_lie</td>\n",
       "      <td>0.506303</td>\n",
       "      <td>0.501572</td>\n",
       "      <td>0.487092</td>\n",
       "      <td>0.515083</td>\n",
       "      <td>0.466004</td>\n",
       "      <td>0.498894</td>\n",
       "      <td>0.176471</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>unambiguous_truthful_reply</td>\n",
       "      <td>0.493697</td>\n",
       "      <td>0.498428</td>\n",
       "      <td>0.512908</td>\n",
       "      <td>0.484917</td>\n",
       "      <td>0.533996</td>\n",
       "      <td>0.501106</td>\n",
       "      <td>0.302083</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>larger_than</td>\n",
       "      <td>0.715686</td>\n",
       "      <td>0.663458</td>\n",
       "      <td>0.548913</td>\n",
       "      <td>0.628546</td>\n",
       "      <td>0.527720</td>\n",
       "      <td>0.552544</td>\n",
       "      <td>0.562914</td>\n",
       "      <td>0.647475</td>\n",
       "      <td>0.646976</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>smaller_than</td>\n",
       "      <td>0.752101</td>\n",
       "      <td>0.862083</td>\n",
       "      <td>0.758832</td>\n",
       "      <td>0.674471</td>\n",
       "      <td>0.612448</td>\n",
       "      <td>0.728429</td>\n",
       "      <td>0.715232</td>\n",
       "      <td>0.757576</td>\n",
       "      <td>0.663975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>sp_en_trans__negated</td>\n",
       "      <td>0.483894</td>\n",
       "      <td>0.462083</td>\n",
       "      <td>0.458560</td>\n",
       "      <td>0.458802</td>\n",
       "      <td>0.546548</td>\n",
       "      <td>0.329436</td>\n",
       "      <td>0.403974</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.385484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>sp_en_trans__affirmative</td>\n",
       "      <td>0.773109</td>\n",
       "      <td>0.822593</td>\n",
       "      <td>0.648777</td>\n",
       "      <td>0.618640</td>\n",
       "      <td>0.631276</td>\n",
       "      <td>0.634113</td>\n",
       "      <td>0.794702</td>\n",
       "      <td>0.598232</td>\n",
       "      <td>0.708299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>sp_en_trans__conjunction</td>\n",
       "      <td>0.724090</td>\n",
       "      <td>0.676228</td>\n",
       "      <td>0.722826</td>\n",
       "      <td>0.614138</td>\n",
       "      <td>0.618724</td>\n",
       "      <td>0.532110</td>\n",
       "      <td>0.794702</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.589114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>sp_en_trans__disjunction</td>\n",
       "      <td>0.468487</td>\n",
       "      <td>0.444204</td>\n",
       "      <td>0.663723</td>\n",
       "      <td>0.521387</td>\n",
       "      <td>0.566423</td>\n",
       "      <td>0.566514</td>\n",
       "      <td>0.496689</td>\n",
       "      <td>0.500253</td>\n",
       "      <td>0.590707</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>common_claim_true_false</td>\n",
       "      <td>0.784314</td>\n",
       "      <td>0.925737</td>\n",
       "      <td>0.679348</td>\n",
       "      <td>0.719946</td>\n",
       "      <td>0.640167</td>\n",
       "      <td>0.763274</td>\n",
       "      <td>0.655629</td>\n",
       "      <td>0.812626</td>\n",
       "      <td>0.708359</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>counterfact_true_false</td>\n",
       "      <td>0.728992</td>\n",
       "      <td>0.759332</td>\n",
       "      <td>0.699049</td>\n",
       "      <td>0.590275</td>\n",
       "      <td>0.682008</td>\n",
       "      <td>0.665376</td>\n",
       "      <td>0.615894</td>\n",
       "      <td>0.834848</td>\n",
       "      <td>0.635056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>animal_class__affirmative+negated</td>\n",
       "      <td>0.680909</td>\n",
       "      <td>0.852259</td>\n",
       "      <td>0.681386</td>\n",
       "      <td>0.716794</td>\n",
       "      <td>0.644351</td>\n",
       "      <td>0.722345</td>\n",
       "      <td>0.761589</td>\n",
       "      <td>0.646465</td>\n",
       "      <td>0.613665</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>animal_class__affirmative+disjunction</td>\n",
       "      <td>0.869110</td>\n",
       "      <td>0.799607</td>\n",
       "      <td>0.595788</td>\n",
       "      <td>0.669968</td>\n",
       "      <td>0.588912</td>\n",
       "      <td>0.764934</td>\n",
       "      <td>0.490066</td>\n",
       "      <td>0.550758</td>\n",
       "      <td>0.635168</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>animal_class__negated+conjunction</td>\n",
       "      <td>0.657068</td>\n",
       "      <td>0.936149</td>\n",
       "      <td>0.755435</td>\n",
       "      <td>0.737956</td>\n",
       "      <td>0.703452</td>\n",
       "      <td>0.806416</td>\n",
       "      <td>0.761589</td>\n",
       "      <td>0.870202</td>\n",
       "      <td>0.713709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>animal_class__affirmative+negated+conjunction+...</td>\n",
       "      <td>0.860000</td>\n",
       "      <td>0.945383</td>\n",
       "      <td>0.832880</td>\n",
       "      <td>0.751914</td>\n",
       "      <td>0.693515</td>\n",
       "      <td>0.721792</td>\n",
       "      <td>0.576159</td>\n",
       "      <td>0.951263</td>\n",
       "      <td>0.698303</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>cities__affirmative+negated</td>\n",
       "      <td>0.625350</td>\n",
       "      <td>0.542421</td>\n",
       "      <td>0.785326</td>\n",
       "      <td>0.641603</td>\n",
       "      <td>0.687762</td>\n",
       "      <td>0.818584</td>\n",
       "      <td>0.754967</td>\n",
       "      <td>0.507071</td>\n",
       "      <td>0.666969</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>cities__affirmative+disjunction</td>\n",
       "      <td>0.691176</td>\n",
       "      <td>0.661603</td>\n",
       "      <td>0.719429</td>\n",
       "      <td>0.575867</td>\n",
       "      <td>0.586297</td>\n",
       "      <td>0.592367</td>\n",
       "      <td>0.701987</td>\n",
       "      <td>0.530051</td>\n",
       "      <td>0.684489</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>cities__negated+conjunction</td>\n",
       "      <td>0.791317</td>\n",
       "      <td>0.864981</td>\n",
       "      <td>0.760190</td>\n",
       "      <td>0.695633</td>\n",
       "      <td>0.672594</td>\n",
       "      <td>0.781527</td>\n",
       "      <td>0.814570</td>\n",
       "      <td>0.920202</td>\n",
       "      <td>0.663591</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>cities__affirmative+negated+conjunction+disjun...</td>\n",
       "      <td>0.885854</td>\n",
       "      <td>0.960000</td>\n",
       "      <td>0.897418</td>\n",
       "      <td>0.717244</td>\n",
       "      <td>0.772490</td>\n",
       "      <td>0.860619</td>\n",
       "      <td>0.814570</td>\n",
       "      <td>0.643434</td>\n",
       "      <td>0.726259</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>element_symb__affirmative+negated</td>\n",
       "      <td>0.757703</td>\n",
       "      <td>0.816896</td>\n",
       "      <td>0.710000</td>\n",
       "      <td>0.733904</td>\n",
       "      <td>0.643305</td>\n",
       "      <td>0.855642</td>\n",
       "      <td>0.874172</td>\n",
       "      <td>0.923232</td>\n",
       "      <td>0.608722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>element_symb__affirmative+disjunction</td>\n",
       "      <td>0.848739</td>\n",
       "      <td>0.861297</td>\n",
       "      <td>0.740458</td>\n",
       "      <td>0.684376</td>\n",
       "      <td>0.614540</td>\n",
       "      <td>0.799226</td>\n",
       "      <td>0.549669</td>\n",
       "      <td>0.880808</td>\n",
       "      <td>0.632586</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>element_symb__negated+conjunction</td>\n",
       "      <td>0.800420</td>\n",
       "      <td>0.925147</td>\n",
       "      <td>0.601781</td>\n",
       "      <td>0.713192</td>\n",
       "      <td>0.730649</td>\n",
       "      <td>0.815819</td>\n",
       "      <td>0.562914</td>\n",
       "      <td>0.881061</td>\n",
       "      <td>0.650601</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>element_symb__affirmative+negated+conjunction+...</td>\n",
       "      <td>0.878852</td>\n",
       "      <td>0.858939</td>\n",
       "      <td>0.790000</td>\n",
       "      <td>0.720396</td>\n",
       "      <td>0.707113</td>\n",
       "      <td>0.888827</td>\n",
       "      <td>0.562914</td>\n",
       "      <td>0.903535</td>\n",
       "      <td>0.634179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>facts__affirmative+negated</td>\n",
       "      <td>0.770308</td>\n",
       "      <td>0.894303</td>\n",
       "      <td>0.715353</td>\n",
       "      <td>0.656961</td>\n",
       "      <td>0.664749</td>\n",
       "      <td>0.814159</td>\n",
       "      <td>0.867550</td>\n",
       "      <td>0.797727</td>\n",
       "      <td>0.701571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>facts__affirmative+disjunction</td>\n",
       "      <td>0.745798</td>\n",
       "      <td>0.746365</td>\n",
       "      <td>0.749321</td>\n",
       "      <td>0.600862</td>\n",
       "      <td>0.629184</td>\n",
       "      <td>0.741704</td>\n",
       "      <td>0.622517</td>\n",
       "      <td>0.961111</td>\n",
       "      <td>0.621602</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>facts__negated+conjunction</td>\n",
       "      <td>0.636555</td>\n",
       "      <td>0.822790</td>\n",
       "      <td>0.755435</td>\n",
       "      <td>0.703448</td>\n",
       "      <td>0.657427</td>\n",
       "      <td>0.821350</td>\n",
       "      <td>0.741722</td>\n",
       "      <td>0.815909</td>\n",
       "      <td>0.675262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>facts__affirmative+negated+conjunction+disjunc...</td>\n",
       "      <td>0.754202</td>\n",
       "      <td>0.848723</td>\n",
       "      <td>0.865489</td>\n",
       "      <td>0.767677</td>\n",
       "      <td>0.718619</td>\n",
       "      <td>0.862832</td>\n",
       "      <td>0.821192</td>\n",
       "      <td>0.940404</td>\n",
       "      <td>0.683583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>inventors__affirmative+negated</td>\n",
       "      <td>0.763305</td>\n",
       "      <td>0.878193</td>\n",
       "      <td>0.677989</td>\n",
       "      <td>0.714543</td>\n",
       "      <td>0.633636</td>\n",
       "      <td>0.817478</td>\n",
       "      <td>0.761589</td>\n",
       "      <td>0.643182</td>\n",
       "      <td>0.684901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>inventors__affirmative+disjunction</td>\n",
       "      <td>0.653361</td>\n",
       "      <td>0.864833</td>\n",
       "      <td>0.757473</td>\n",
       "      <td>0.665916</td>\n",
       "      <td>0.572565</td>\n",
       "      <td>0.688053</td>\n",
       "      <td>0.649007</td>\n",
       "      <td>0.701263</td>\n",
       "      <td>0.706487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>inventors__negated+conjunction</td>\n",
       "      <td>0.785714</td>\n",
       "      <td>0.928487</td>\n",
       "      <td>0.724185</td>\n",
       "      <td>0.689329</td>\n",
       "      <td>0.628231</td>\n",
       "      <td>0.800885</td>\n",
       "      <td>0.841060</td>\n",
       "      <td>0.856818</td>\n",
       "      <td>0.658428</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>inventors__affirmative+negated+conjunction+dis...</td>\n",
       "      <td>0.845238</td>\n",
       "      <td>0.945776</td>\n",
       "      <td>0.816576</td>\n",
       "      <td>0.737956</td>\n",
       "      <td>0.850000</td>\n",
       "      <td>0.885509</td>\n",
       "      <td>0.794702</td>\n",
       "      <td>0.735101</td>\n",
       "      <td>0.700308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>sp_en_trans__affirmative+negated</td>\n",
       "      <td>0.665966</td>\n",
       "      <td>0.655599</td>\n",
       "      <td>0.626359</td>\n",
       "      <td>0.596578</td>\n",
       "      <td>0.689331</td>\n",
       "      <td>0.716364</td>\n",
       "      <td>0.735099</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.625144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>sp_en_trans__affirmative+disjunction</td>\n",
       "      <td>0.567927</td>\n",
       "      <td>0.654028</td>\n",
       "      <td>0.749321</td>\n",
       "      <td>0.564611</td>\n",
       "      <td>0.646967</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.668874</td>\n",
       "      <td>0.810101</td>\n",
       "      <td>0.689845</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>sp_en_trans__negated+conjunction</td>\n",
       "      <td>0.766106</td>\n",
       "      <td>0.777407</td>\n",
       "      <td>0.788043</td>\n",
       "      <td>0.637551</td>\n",
       "      <td>0.658996</td>\n",
       "      <td>0.575472</td>\n",
       "      <td>0.735099</td>\n",
       "      <td>0.935859</td>\n",
       "      <td>0.608667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>sp_en_trans__affirmative+negated+conjunction+d...</td>\n",
       "      <td>0.636555</td>\n",
       "      <td>0.836542</td>\n",
       "      <td>0.853261</td>\n",
       "      <td>0.716344</td>\n",
       "      <td>0.663180</td>\n",
       "      <td>0.960000</td>\n",
       "      <td>0.642384</td>\n",
       "      <td>0.849747</td>\n",
       "      <td>0.678640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>ambiguous_truthful_reply+unambiguous_lie+unamb...</td>\n",
       "      <td>0.495798</td>\n",
       "      <td>0.498428</td>\n",
       "      <td>0.513587</td>\n",
       "      <td>0.499325</td>\n",
       "      <td>0.533996</td>\n",
       "      <td>0.501106</td>\n",
       "      <td>0.791667</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.500027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>animal_class+facts+inventors__affirmative+nega...</td>\n",
       "      <td>0.566667</td>\n",
       "      <td>0.921611</td>\n",
       "      <td>0.761549</td>\n",
       "      <td>0.589316</td>\n",
       "      <td>0.583333</td>\n",
       "      <td>0.844027</td>\n",
       "      <td>0.801325</td>\n",
       "      <td>0.949747</td>\n",
       "      <td>0.722552</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>animal_class+facts+inventors__affirmative+nega...</td>\n",
       "      <td>0.729091</td>\n",
       "      <td>0.924361</td>\n",
       "      <td>0.782609</td>\n",
       "      <td>0.703367</td>\n",
       "      <td>0.680909</td>\n",
       "      <td>0.841814</td>\n",
       "      <td>0.708333</td>\n",
       "      <td>0.928283</td>\n",
       "      <td>0.735171</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 51
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:59.010722Z",
     "start_time": "2024-12-07T21:26:58.993649Z"
    }
   },
   "cell_type": "code",
   "source": "lyr18_ttpd_probes_rel_to_baseline_probes_acc_delta_on_unseen_single_topic_data_groupings_df",
   "id": "c7a90c8461a7bb05",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "                                             scenario  animal class    cities  \\\n",
       "0                           animal_class__affirmative     -0.098101 -0.041454   \n",
       "1                           animal_class__conjunction     -0.018319 -0.047544   \n",
       "2                           animal_class__disjunction      0.172414  0.208644   \n",
       "3                               animal_class__negated      0.021361 -0.031631   \n",
       "4                                 cities__affirmative     -0.002801 -0.003061   \n",
       "5                                 cities__conjunction      0.069328  0.199889   \n",
       "6                                 cities__disjunction     -0.043417 -0.038344   \n",
       "7                                     cities__negated     -0.063725 -0.095437   \n",
       "8                           element_symb__affirmative      0.066527  0.069745   \n",
       "9                           element_symb__conjunction     -0.017507 -0.120825   \n",
       "10                          element_symb__disjunction     -0.189076 -0.299411   \n",
       "11                              element_symb__negated      0.072829 -0.067780   \n",
       "12                                 facts__affirmative      0.105742  0.255206   \n",
       "13                                 facts__conjunction     -0.047619 -0.065815   \n",
       "14                                 facts__disjunction     -0.062325 -0.051277   \n",
       "15                                     facts__negated     -0.005602  0.067387   \n",
       "16                             inventors__affirmative     -0.044818  0.037525   \n",
       "17                             inventors__conjunction     -0.037815 -0.057367   \n",
       "18                             inventors__disjunction     -0.109244  0.113949   \n",
       "19                                 inventors__negated     -0.085434 -0.241650   \n",
       "20                                      ambiguous_lie     -0.028011  0.001768   \n",
       "21                           ambiguous_truthful_reply     -0.007703 -0.021218   \n",
       "22              honest_reply_despite_incentive_to_lie      0.000000  0.000196   \n",
       "23                                    unambiguous_lie      0.019608  0.152456   \n",
       "24                         unambiguous_truthful_reply      0.012605  0.003143   \n",
       "25                                        larger_than     -0.023109  0.113752   \n",
       "26                                       smaller_than      0.077731 -0.015717   \n",
       "27                               sp_en_trans__negated     -0.016106  0.135953   \n",
       "28                           sp_en_trans__affirmative     -0.091036  0.020236   \n",
       "29                           sp_en_trans__conjunction     -0.025910  0.074067   \n",
       "30                           sp_en_trans__disjunction      0.062325  0.148134   \n",
       "31                            common_claim_true_false     -0.255602 -0.197446   \n",
       "32                             counterfact_true_false      0.017507  0.107662   \n",
       "33                  animal_class__affirmative+negated     -0.095455  0.001375   \n",
       "34              animal_class__affirmative+disjunction     -0.151832  0.029077   \n",
       "35                  animal_class__negated+conjunction     -0.015707 -0.058939   \n",
       "36  animal_class__affirmative+negated+conjunction+...     -0.050000 -0.066405   \n",
       "37                        cities__affirmative+negated      0.019608  0.034795   \n",
       "38                    cities__affirmative+disjunction     -0.010504  0.072075   \n",
       "39                        cities__negated+conjunction     -0.030112  0.015744   \n",
       "40  cities__affirmative+negated+conjunction+disjun...     -0.115546  0.010000   \n",
       "41                  element_symb__affirmative+negated     -0.124650  0.010609   \n",
       "42              element_symb__affirmative+disjunction     -0.157563 -0.062279   \n",
       "43                  element_symb__negated+conjunction     -0.133053 -0.168959   \n",
       "44  element_symb__affirmative+negated+conjunction+...     -0.165966  0.002554   \n",
       "45                         facts__affirmative+negated     -0.149860 -0.102554   \n",
       "46                     facts__affirmative+disjunction      0.033613  0.176621   \n",
       "47                         facts__negated+conjunction      0.107143  0.053045   \n",
       "48  facts__affirmative+negated+conjunction+disjunc...     -0.028711  0.018271   \n",
       "49                     inventors__affirmative+negated     -0.166667 -0.132220   \n",
       "50                 inventors__affirmative+disjunction      0.053922  0.036149   \n",
       "51                     inventors__negated+conjunction     -0.235994 -0.286051   \n",
       "52  inventors__affirmative+negated+conjunction+dis...     -0.217087 -0.206090   \n",
       "53                   sp_en_trans__affirmative+negated     -0.025910  0.161297   \n",
       "54               sp_en_trans__affirmative+disjunction      0.020308  0.053635   \n",
       "55                   sp_en_trans__negated+conjunction     -0.240896 -0.129862   \n",
       "56  sp_en_trans__affirmative+negated+conjunction+d...     -0.007003 -0.045972   \n",
       "57  ambiguous_truthful_reply+unambiguous_lie+unamb...      0.093838  0.173870   \n",
       "58  animal_class+facts+inventors__affirmative+nega...      0.001667 -0.023969   \n",
       "59  animal_class+facts+inventors__affirmative+nega...     -0.003636 -0.003733   \n",
       "\n",
       "    element symbols     facts  inventors  spanish english translation  \\\n",
       "0         -0.031929 -0.018460  -0.018305                    -0.061394   \n",
       "1         -0.067255 -0.076992  -0.061192                    -0.083518   \n",
       "2          0.144022  0.008104   0.032950                     0.147124   \n",
       "3         -0.057065  0.062584   0.001046                     0.060841   \n",
       "4         -0.042120  0.001801   0.016213                    -0.073562   \n",
       "5         -0.076087 -0.036470  -0.026674                    -0.030420   \n",
       "6          0.035326 -0.000450  -0.002092                     0.081858   \n",
       "7         -0.216033 -0.017560  -0.057531                    -0.186947   \n",
       "8          0.076205  0.023413   0.095188                     0.122235   \n",
       "9          0.066872  0.019361  -0.041318                     0.043695   \n",
       "10        -0.240741 -0.009455  -0.029812                    -0.175885   \n",
       "11         0.068429 -0.001351   0.019351                     0.079646   \n",
       "12         0.105978  0.047590   0.105649                     0.051438   \n",
       "13        -0.002038 -0.034282  -0.055962                    -0.023783   \n",
       "14        -0.105978 -0.054038   0.009414                    -0.043695   \n",
       "15         0.005435  0.008434   0.004707                    -0.018252   \n",
       "16        -0.078804 -0.093201   0.011288                    -0.300332   \n",
       "17        -0.110054  0.002701  -0.012040                    -0.107854   \n",
       "18        -0.131114  0.006754   0.157224                     0.121681   \n",
       "19        -0.052989 -0.094102  -0.018592                    -0.116704   \n",
       "20        -0.031929 -0.030617   0.069038                    -0.044801   \n",
       "21        -0.045516  0.025664  -0.080544                    -0.061394   \n",
       "22         0.000000  0.001801  -0.000523                    -0.001106   \n",
       "23         0.031250  0.010356   0.080544                    -0.000553   \n",
       "24        -0.025815  0.030617  -0.063285                     0.075774   \n",
       "25         0.229620  0.066186   0.146444                     0.187500   \n",
       "26        -0.068614 -0.029266  -0.009937                    -0.107301   \n",
       "27         0.063859  0.040522  -0.019351                     0.177442   \n",
       "28         0.059783  0.031968  -0.082636                     0.017194   \n",
       "29         0.033288  0.037371   0.076883                     0.009939   \n",
       "30        -0.112772  0.024313   0.014121                     0.077982   \n",
       "31         0.017663 -0.060783  -0.048117                    -0.049226   \n",
       "32        -0.002717  0.099505  -0.023536                     0.051438   \n",
       "33         0.050951  0.000450   0.007845                     0.030973   \n",
       "34         0.076766 -0.050428  -0.042887                    -0.172013   \n",
       "35        -0.124321 -0.080594  -0.124477                    -0.121681   \n",
       "36        -0.133832 -0.050878  -0.089958                     0.003872   \n",
       "37        -0.040082  0.042323  -0.023013                    -0.031527   \n",
       "38        -0.023098  0.037371  -0.045502                    -0.013827   \n",
       "39         0.006114  0.016659  -0.039749                    -0.065265   \n",
       "40        -0.160326 -0.024313  -0.146444                    -0.129425   \n",
       "41        -0.029091 -0.072490   0.011506                    -0.064712   \n",
       "42         0.008906 -0.039622  -0.015167                    -0.125553   \n",
       "43        -0.048346 -0.089149  -0.082113                    -0.083518   \n",
       "44        -0.100000 -0.008555  -0.031381                    -0.050332   \n",
       "45         0.005435 -0.130118  -0.036611                    -0.049779   \n",
       "46        -0.086277  0.047414   0.004184                    -0.040929   \n",
       "47        -0.015625  0.082759   0.029812                    -0.004978   \n",
       "48        -0.152174 -0.060606  -0.047594                    -0.060841   \n",
       "49        -0.050272 -0.087798  -0.034545                    -0.023230   \n",
       "50        -0.106658 -0.053579   0.113320                    -0.165376   \n",
       "51        -0.175951 -0.167942  -0.122266                    -0.211836   \n",
       "52        -0.054348 -0.131923  -0.200000                    -0.149889   \n",
       "53         0.095788  0.079244  -0.046548                     0.009091   \n",
       "54        -0.124321 -0.012607  -0.101987                     0.147799   \n",
       "55        -0.091712 -0.103557  -0.049686                    -0.026205   \n",
       "56        -0.131793 -0.049077  -0.023013                    -0.200000   \n",
       "57         0.012908  0.070689   0.009937                     0.058628   \n",
       "58        -0.075408  0.051753   0.126667                    -0.115597   \n",
       "59        -0.077446 -0.063694  -0.116364                    -0.059181   \n",
       "\n",
       "    real world scenarios  relative comparison  true false  \n",
       "0               0.092715             0.007071   -0.029357  \n",
       "1              -0.165563            -0.244192   -0.116219  \n",
       "2               0.205298             0.280051    0.118910  \n",
       "3              -0.006623            -0.342424   -0.024359  \n",
       "4              -0.152318            -0.003030   -0.078898  \n",
       "5              -0.172185            -0.096465   -0.041330  \n",
       "6              -0.099338             0.039899   -0.021750  \n",
       "7              -0.238411            -0.004293   -0.076427  \n",
       "8               0.033113             0.186111    0.128028  \n",
       "9              -0.013245             0.288889   -0.094249  \n",
       "10              0.019868            -0.102525   -0.089608  \n",
       "11             -0.145695             0.084091    0.063108  \n",
       "12              0.284768             0.108838    0.066321  \n",
       "13             -0.357616            -0.287879   -0.136211  \n",
       "14             -0.092715            -0.210101   -0.093206  \n",
       "15              0.019868            -0.002525    0.004888  \n",
       "16              0.072848             0.003535   -0.068106  \n",
       "17              0.046358             0.151515   -0.022629  \n",
       "18             -0.105960             0.000000    0.065469  \n",
       "19             -0.086093            -0.027778   -0.059895  \n",
       "20              0.142857             0.000000    0.012495  \n",
       "21              0.046875            -0.013384   -0.033916  \n",
       "22             -0.020690             0.000000   -0.001565  \n",
       "23              0.450980             0.018182    0.022382  \n",
       "24              0.166667             0.171212    0.014170  \n",
       "25              0.211921             0.161111    0.042786  \n",
       "26              0.092715             0.032323   -0.061268  \n",
       "27              0.039735            -0.042424    0.094332  \n",
       "28             -0.086093             0.015404   -0.090350  \n",
       "29             -0.298013             0.078283    0.011095  \n",
       "30              0.013245             0.081818   -0.031609  \n",
       "31              0.059603            -0.311869   -0.050682  \n",
       "32              0.211921             0.106313    0.110112  \n",
       "33             -0.066225            -0.079545    0.009090  \n",
       "34              0.264901             0.093182   -0.051793  \n",
       "35             -0.152318            -0.229293   -0.120585  \n",
       "36              0.092715            -0.323990   -0.089444  \n",
       "37             -0.158940             0.054040   -0.053880  \n",
       "38             -0.092715             0.077020   -0.117812  \n",
       "39             -0.145695            -0.154798   -0.018125  \n",
       "40             -0.099338             0.006566   -0.120970  \n",
       "41             -0.311258            -0.331061    0.031801  \n",
       "42              0.198675            -0.203030   -0.018537  \n",
       "43              0.099338            -0.169444   -0.026199  \n",
       "44              0.145695            -0.200000    0.046191  \n",
       "45             -0.152318            -0.223485   -0.101637  \n",
       "46              0.125828            -0.090404    0.049486  \n",
       "47              0.105960            -0.137121   -0.018894  \n",
       "48              0.000000            -0.237626   -0.046136  \n",
       "49             -0.198675            -0.138636   -0.075548  \n",
       "50             -0.019868             0.167424   -0.063025  \n",
       "51             -0.079470            -0.355808   -0.126902  \n",
       "52             -0.165563             0.152273   -0.077305  \n",
       "53              0.013245             0.071212    0.040451  \n",
       "54              0.019868            -0.285101   -0.123085  \n",
       "55             -0.198675            -0.053030   -0.085434  \n",
       "56              0.192053            -0.227778    0.009035  \n",
       "57              0.041667             0.000758    0.058302  \n",
       "58              0.046358            -0.094949   -0.025320  \n",
       "59             -0.208333            -0.012121   -0.044769  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>scenario</th>\n",
       "      <th>animal class</th>\n",
       "      <th>cities</th>\n",
       "      <th>element symbols</th>\n",
       "      <th>facts</th>\n",
       "      <th>inventors</th>\n",
       "      <th>spanish english translation</th>\n",
       "      <th>real world scenarios</th>\n",
       "      <th>relative comparison</th>\n",
       "      <th>true false</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>animal_class__affirmative</td>\n",
       "      <td>-0.098101</td>\n",
       "      <td>-0.041454</td>\n",
       "      <td>-0.031929</td>\n",
       "      <td>-0.018460</td>\n",
       "      <td>-0.018305</td>\n",
       "      <td>-0.061394</td>\n",
       "      <td>0.092715</td>\n",
       "      <td>0.007071</td>\n",
       "      <td>-0.029357</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>animal_class__conjunction</td>\n",
       "      <td>-0.018319</td>\n",
       "      <td>-0.047544</td>\n",
       "      <td>-0.067255</td>\n",
       "      <td>-0.076992</td>\n",
       "      <td>-0.061192</td>\n",
       "      <td>-0.083518</td>\n",
       "      <td>-0.165563</td>\n",
       "      <td>-0.244192</td>\n",
       "      <td>-0.116219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>animal_class__disjunction</td>\n",
       "      <td>0.172414</td>\n",
       "      <td>0.208644</td>\n",
       "      <td>0.144022</td>\n",
       "      <td>0.008104</td>\n",
       "      <td>0.032950</td>\n",
       "      <td>0.147124</td>\n",
       "      <td>0.205298</td>\n",
       "      <td>0.280051</td>\n",
       "      <td>0.118910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>animal_class__negated</td>\n",
       "      <td>0.021361</td>\n",
       "      <td>-0.031631</td>\n",
       "      <td>-0.057065</td>\n",
       "      <td>0.062584</td>\n",
       "      <td>0.001046</td>\n",
       "      <td>0.060841</td>\n",
       "      <td>-0.006623</td>\n",
       "      <td>-0.342424</td>\n",
       "      <td>-0.024359</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cities__affirmative</td>\n",
       "      <td>-0.002801</td>\n",
       "      <td>-0.003061</td>\n",
       "      <td>-0.042120</td>\n",
       "      <td>0.001801</td>\n",
       "      <td>0.016213</td>\n",
       "      <td>-0.073562</td>\n",
       "      <td>-0.152318</td>\n",
       "      <td>-0.003030</td>\n",
       "      <td>-0.078898</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>cities__conjunction</td>\n",
       "      <td>0.069328</td>\n",
       "      <td>0.199889</td>\n",
       "      <td>-0.076087</td>\n",
       "      <td>-0.036470</td>\n",
       "      <td>-0.026674</td>\n",
       "      <td>-0.030420</td>\n",
       "      <td>-0.172185</td>\n",
       "      <td>-0.096465</td>\n",
       "      <td>-0.041330</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>cities__disjunction</td>\n",
       "      <td>-0.043417</td>\n",
       "      <td>-0.038344</td>\n",
       "      <td>0.035326</td>\n",
       "      <td>-0.000450</td>\n",
       "      <td>-0.002092</td>\n",
       "      <td>0.081858</td>\n",
       "      <td>-0.099338</td>\n",
       "      <td>0.039899</td>\n",
       "      <td>-0.021750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>cities__negated</td>\n",
       "      <td>-0.063725</td>\n",
       "      <td>-0.095437</td>\n",
       "      <td>-0.216033</td>\n",
       "      <td>-0.017560</td>\n",
       "      <td>-0.057531</td>\n",
       "      <td>-0.186947</td>\n",
       "      <td>-0.238411</td>\n",
       "      <td>-0.004293</td>\n",
       "      <td>-0.076427</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>element_symb__affirmative</td>\n",
       "      <td>0.066527</td>\n",
       "      <td>0.069745</td>\n",
       "      <td>0.076205</td>\n",
       "      <td>0.023413</td>\n",
       "      <td>0.095188</td>\n",
       "      <td>0.122235</td>\n",
       "      <td>0.033113</td>\n",
       "      <td>0.186111</td>\n",
       "      <td>0.128028</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>element_symb__conjunction</td>\n",
       "      <td>-0.017507</td>\n",
       "      <td>-0.120825</td>\n",
       "      <td>0.066872</td>\n",
       "      <td>0.019361</td>\n",
       "      <td>-0.041318</td>\n",
       "      <td>0.043695</td>\n",
       "      <td>-0.013245</td>\n",
       "      <td>0.288889</td>\n",
       "      <td>-0.094249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>element_symb__disjunction</td>\n",
       "      <td>-0.189076</td>\n",
       "      <td>-0.299411</td>\n",
       "      <td>-0.240741</td>\n",
       "      <td>-0.009455</td>\n",
       "      <td>-0.029812</td>\n",
       "      <td>-0.175885</td>\n",
       "      <td>0.019868</td>\n",
       "      <td>-0.102525</td>\n",
       "      <td>-0.089608</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>element_symb__negated</td>\n",
       "      <td>0.072829</td>\n",
       "      <td>-0.067780</td>\n",
       "      <td>0.068429</td>\n",
       "      <td>-0.001351</td>\n",
       "      <td>0.019351</td>\n",
       "      <td>0.079646</td>\n",
       "      <td>-0.145695</td>\n",
       "      <td>0.084091</td>\n",
       "      <td>0.063108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>facts__affirmative</td>\n",
       "      <td>0.105742</td>\n",
       "      <td>0.255206</td>\n",
       "      <td>0.105978</td>\n",
       "      <td>0.047590</td>\n",
       "      <td>0.105649</td>\n",
       "      <td>0.051438</td>\n",
       "      <td>0.284768</td>\n",
       "      <td>0.108838</td>\n",
       "      <td>0.066321</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>facts__conjunction</td>\n",
       "      <td>-0.047619</td>\n",
       "      <td>-0.065815</td>\n",
       "      <td>-0.002038</td>\n",
       "      <td>-0.034282</td>\n",
       "      <td>-0.055962</td>\n",
       "      <td>-0.023783</td>\n",
       "      <td>-0.357616</td>\n",
       "      <td>-0.287879</td>\n",
       "      <td>-0.136211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>facts__disjunction</td>\n",
       "      <td>-0.062325</td>\n",
       "      <td>-0.051277</td>\n",
       "      <td>-0.105978</td>\n",
       "      <td>-0.054038</td>\n",
       "      <td>0.009414</td>\n",
       "      <td>-0.043695</td>\n",
       "      <td>-0.092715</td>\n",
       "      <td>-0.210101</td>\n",
       "      <td>-0.093206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>facts__negated</td>\n",
       "      <td>-0.005602</td>\n",
       "      <td>0.067387</td>\n",
       "      <td>0.005435</td>\n",
       "      <td>0.008434</td>\n",
       "      <td>0.004707</td>\n",
       "      <td>-0.018252</td>\n",
       "      <td>0.019868</td>\n",
       "      <td>-0.002525</td>\n",
       "      <td>0.004888</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>inventors__affirmative</td>\n",
       "      <td>-0.044818</td>\n",
       "      <td>0.037525</td>\n",
       "      <td>-0.078804</td>\n",
       "      <td>-0.093201</td>\n",
       "      <td>0.011288</td>\n",
       "      <td>-0.300332</td>\n",
       "      <td>0.072848</td>\n",
       "      <td>0.003535</td>\n",
       "      <td>-0.068106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>inventors__conjunction</td>\n",
       "      <td>-0.037815</td>\n",
       "      <td>-0.057367</td>\n",
       "      <td>-0.110054</td>\n",
       "      <td>0.002701</td>\n",
       "      <td>-0.012040</td>\n",
       "      <td>-0.107854</td>\n",
       "      <td>0.046358</td>\n",
       "      <td>0.151515</td>\n",
       "      <td>-0.022629</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>inventors__disjunction</td>\n",
       "      <td>-0.109244</td>\n",
       "      <td>0.113949</td>\n",
       "      <td>-0.131114</td>\n",
       "      <td>0.006754</td>\n",
       "      <td>0.157224</td>\n",
       "      <td>0.121681</td>\n",
       "      <td>-0.105960</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.065469</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>inventors__negated</td>\n",
       "      <td>-0.085434</td>\n",
       "      <td>-0.241650</td>\n",
       "      <td>-0.052989</td>\n",
       "      <td>-0.094102</td>\n",
       "      <td>-0.018592</td>\n",
       "      <td>-0.116704</td>\n",
       "      <td>-0.086093</td>\n",
       "      <td>-0.027778</td>\n",
       "      <td>-0.059895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>ambiguous_lie</td>\n",
       "      <td>-0.028011</td>\n",
       "      <td>0.001768</td>\n",
       "      <td>-0.031929</td>\n",
       "      <td>-0.030617</td>\n",
       "      <td>0.069038</td>\n",
       "      <td>-0.044801</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.012495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>ambiguous_truthful_reply</td>\n",
       "      <td>-0.007703</td>\n",
       "      <td>-0.021218</td>\n",
       "      <td>-0.045516</td>\n",
       "      <td>0.025664</td>\n",
       "      <td>-0.080544</td>\n",
       "      <td>-0.061394</td>\n",
       "      <td>0.046875</td>\n",
       "      <td>-0.013384</td>\n",
       "      <td>-0.033916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>honest_reply_despite_incentive_to_lie</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000196</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001801</td>\n",
       "      <td>-0.000523</td>\n",
       "      <td>-0.001106</td>\n",
       "      <td>-0.020690</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.001565</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>unambiguous_lie</td>\n",
       "      <td>0.019608</td>\n",
       "      <td>0.152456</td>\n",
       "      <td>0.031250</td>\n",
       "      <td>0.010356</td>\n",
       "      <td>0.080544</td>\n",
       "      <td>-0.000553</td>\n",
       "      <td>0.450980</td>\n",
       "      <td>0.018182</td>\n",
       "      <td>0.022382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>unambiguous_truthful_reply</td>\n",
       "      <td>0.012605</td>\n",
       "      <td>0.003143</td>\n",
       "      <td>-0.025815</td>\n",
       "      <td>0.030617</td>\n",
       "      <td>-0.063285</td>\n",
       "      <td>0.075774</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.171212</td>\n",
       "      <td>0.014170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>larger_than</td>\n",
       "      <td>-0.023109</td>\n",
       "      <td>0.113752</td>\n",
       "      <td>0.229620</td>\n",
       "      <td>0.066186</td>\n",
       "      <td>0.146444</td>\n",
       "      <td>0.187500</td>\n",
       "      <td>0.211921</td>\n",
       "      <td>0.161111</td>\n",
       "      <td>0.042786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>smaller_than</td>\n",
       "      <td>0.077731</td>\n",
       "      <td>-0.015717</td>\n",
       "      <td>-0.068614</td>\n",
       "      <td>-0.029266</td>\n",
       "      <td>-0.009937</td>\n",
       "      <td>-0.107301</td>\n",
       "      <td>0.092715</td>\n",
       "      <td>0.032323</td>\n",
       "      <td>-0.061268</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>sp_en_trans__negated</td>\n",
       "      <td>-0.016106</td>\n",
       "      <td>0.135953</td>\n",
       "      <td>0.063859</td>\n",
       "      <td>0.040522</td>\n",
       "      <td>-0.019351</td>\n",
       "      <td>0.177442</td>\n",
       "      <td>0.039735</td>\n",
       "      <td>-0.042424</td>\n",
       "      <td>0.094332</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>sp_en_trans__affirmative</td>\n",
       "      <td>-0.091036</td>\n",
       "      <td>0.020236</td>\n",
       "      <td>0.059783</td>\n",
       "      <td>0.031968</td>\n",
       "      <td>-0.082636</td>\n",
       "      <td>0.017194</td>\n",
       "      <td>-0.086093</td>\n",
       "      <td>0.015404</td>\n",
       "      <td>-0.090350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>sp_en_trans__conjunction</td>\n",
       "      <td>-0.025910</td>\n",
       "      <td>0.074067</td>\n",
       "      <td>0.033288</td>\n",
       "      <td>0.037371</td>\n",
       "      <td>0.076883</td>\n",
       "      <td>0.009939</td>\n",
       "      <td>-0.298013</td>\n",
       "      <td>0.078283</td>\n",
       "      <td>0.011095</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>sp_en_trans__disjunction</td>\n",
       "      <td>0.062325</td>\n",
       "      <td>0.148134</td>\n",
       "      <td>-0.112772</td>\n",
       "      <td>0.024313</td>\n",
       "      <td>0.014121</td>\n",
       "      <td>0.077982</td>\n",
       "      <td>0.013245</td>\n",
       "      <td>0.081818</td>\n",
       "      <td>-0.031609</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>common_claim_true_false</td>\n",
       "      <td>-0.255602</td>\n",
       "      <td>-0.197446</td>\n",
       "      <td>0.017663</td>\n",
       "      <td>-0.060783</td>\n",
       "      <td>-0.048117</td>\n",
       "      <td>-0.049226</td>\n",
       "      <td>0.059603</td>\n",
       "      <td>-0.311869</td>\n",
       "      <td>-0.050682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>counterfact_true_false</td>\n",
       "      <td>0.017507</td>\n",
       "      <td>0.107662</td>\n",
       "      <td>-0.002717</td>\n",
       "      <td>0.099505</td>\n",
       "      <td>-0.023536</td>\n",
       "      <td>0.051438</td>\n",
       "      <td>0.211921</td>\n",
       "      <td>0.106313</td>\n",
       "      <td>0.110112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>animal_class__affirmative+negated</td>\n",
       "      <td>-0.095455</td>\n",
       "      <td>0.001375</td>\n",
       "      <td>0.050951</td>\n",
       "      <td>0.000450</td>\n",
       "      <td>0.007845</td>\n",
       "      <td>0.030973</td>\n",
       "      <td>-0.066225</td>\n",
       "      <td>-0.079545</td>\n",
       "      <td>0.009090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>animal_class__affirmative+disjunction</td>\n",
       "      <td>-0.151832</td>\n",
       "      <td>0.029077</td>\n",
       "      <td>0.076766</td>\n",
       "      <td>-0.050428</td>\n",
       "      <td>-0.042887</td>\n",
       "      <td>-0.172013</td>\n",
       "      <td>0.264901</td>\n",
       "      <td>0.093182</td>\n",
       "      <td>-0.051793</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>animal_class__negated+conjunction</td>\n",
       "      <td>-0.015707</td>\n",
       "      <td>-0.058939</td>\n",
       "      <td>-0.124321</td>\n",
       "      <td>-0.080594</td>\n",
       "      <td>-0.124477</td>\n",
       "      <td>-0.121681</td>\n",
       "      <td>-0.152318</td>\n",
       "      <td>-0.229293</td>\n",
       "      <td>-0.120585</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>animal_class__affirmative+negated+conjunction+...</td>\n",
       "      <td>-0.050000</td>\n",
       "      <td>-0.066405</td>\n",
       "      <td>-0.133832</td>\n",
       "      <td>-0.050878</td>\n",
       "      <td>-0.089958</td>\n",
       "      <td>0.003872</td>\n",
       "      <td>0.092715</td>\n",
       "      <td>-0.323990</td>\n",
       "      <td>-0.089444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>cities__affirmative+negated</td>\n",
       "      <td>0.019608</td>\n",
       "      <td>0.034795</td>\n",
       "      <td>-0.040082</td>\n",
       "      <td>0.042323</td>\n",
       "      <td>-0.023013</td>\n",
       "      <td>-0.031527</td>\n",
       "      <td>-0.158940</td>\n",
       "      <td>0.054040</td>\n",
       "      <td>-0.053880</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>cities__affirmative+disjunction</td>\n",
       "      <td>-0.010504</td>\n",
       "      <td>0.072075</td>\n",
       "      <td>-0.023098</td>\n",
       "      <td>0.037371</td>\n",
       "      <td>-0.045502</td>\n",
       "      <td>-0.013827</td>\n",
       "      <td>-0.092715</td>\n",
       "      <td>0.077020</td>\n",
       "      <td>-0.117812</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>cities__negated+conjunction</td>\n",
       "      <td>-0.030112</td>\n",
       "      <td>0.015744</td>\n",
       "      <td>0.006114</td>\n",
       "      <td>0.016659</td>\n",
       "      <td>-0.039749</td>\n",
       "      <td>-0.065265</td>\n",
       "      <td>-0.145695</td>\n",
       "      <td>-0.154798</td>\n",
       "      <td>-0.018125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>cities__affirmative+negated+conjunction+disjun...</td>\n",
       "      <td>-0.115546</td>\n",
       "      <td>0.010000</td>\n",
       "      <td>-0.160326</td>\n",
       "      <td>-0.024313</td>\n",
       "      <td>-0.146444</td>\n",
       "      <td>-0.129425</td>\n",
       "      <td>-0.099338</td>\n",
       "      <td>0.006566</td>\n",
       "      <td>-0.120970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>element_symb__affirmative+negated</td>\n",
       "      <td>-0.124650</td>\n",
       "      <td>0.010609</td>\n",
       "      <td>-0.029091</td>\n",
       "      <td>-0.072490</td>\n",
       "      <td>0.011506</td>\n",
       "      <td>-0.064712</td>\n",
       "      <td>-0.311258</td>\n",
       "      <td>-0.331061</td>\n",
       "      <td>0.031801</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>element_symb__affirmative+disjunction</td>\n",
       "      <td>-0.157563</td>\n",
       "      <td>-0.062279</td>\n",
       "      <td>0.008906</td>\n",
       "      <td>-0.039622</td>\n",
       "      <td>-0.015167</td>\n",
       "      <td>-0.125553</td>\n",
       "      <td>0.198675</td>\n",
       "      <td>-0.203030</td>\n",
       "      <td>-0.018537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>element_symb__negated+conjunction</td>\n",
       "      <td>-0.133053</td>\n",
       "      <td>-0.168959</td>\n",
       "      <td>-0.048346</td>\n",
       "      <td>-0.089149</td>\n",
       "      <td>-0.082113</td>\n",
       "      <td>-0.083518</td>\n",
       "      <td>0.099338</td>\n",
       "      <td>-0.169444</td>\n",
       "      <td>-0.026199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>element_symb__affirmative+negated+conjunction+...</td>\n",
       "      <td>-0.165966</td>\n",
       "      <td>0.002554</td>\n",
       "      <td>-0.100000</td>\n",
       "      <td>-0.008555</td>\n",
       "      <td>-0.031381</td>\n",
       "      <td>-0.050332</td>\n",
       "      <td>0.145695</td>\n",
       "      <td>-0.200000</td>\n",
       "      <td>0.046191</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>facts__affirmative+negated</td>\n",
       "      <td>-0.149860</td>\n",
       "      <td>-0.102554</td>\n",
       "      <td>0.005435</td>\n",
       "      <td>-0.130118</td>\n",
       "      <td>-0.036611</td>\n",
       "      <td>-0.049779</td>\n",
       "      <td>-0.152318</td>\n",
       "      <td>-0.223485</td>\n",
       "      <td>-0.101637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>facts__affirmative+disjunction</td>\n",
       "      <td>0.033613</td>\n",
       "      <td>0.176621</td>\n",
       "      <td>-0.086277</td>\n",
       "      <td>0.047414</td>\n",
       "      <td>0.004184</td>\n",
       "      <td>-0.040929</td>\n",
       "      <td>0.125828</td>\n",
       "      <td>-0.090404</td>\n",
       "      <td>0.049486</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>facts__negated+conjunction</td>\n",
       "      <td>0.107143</td>\n",
       "      <td>0.053045</td>\n",
       "      <td>-0.015625</td>\n",
       "      <td>0.082759</td>\n",
       "      <td>0.029812</td>\n",
       "      <td>-0.004978</td>\n",
       "      <td>0.105960</td>\n",
       "      <td>-0.137121</td>\n",
       "      <td>-0.018894</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>facts__affirmative+negated+conjunction+disjunc...</td>\n",
       "      <td>-0.028711</td>\n",
       "      <td>0.018271</td>\n",
       "      <td>-0.152174</td>\n",
       "      <td>-0.060606</td>\n",
       "      <td>-0.047594</td>\n",
       "      <td>-0.060841</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.237626</td>\n",
       "      <td>-0.046136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>inventors__affirmative+negated</td>\n",
       "      <td>-0.166667</td>\n",
       "      <td>-0.132220</td>\n",
       "      <td>-0.050272</td>\n",
       "      <td>-0.087798</td>\n",
       "      <td>-0.034545</td>\n",
       "      <td>-0.023230</td>\n",
       "      <td>-0.198675</td>\n",
       "      <td>-0.138636</td>\n",
       "      <td>-0.075548</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>inventors__affirmative+disjunction</td>\n",
       "      <td>0.053922</td>\n",
       "      <td>0.036149</td>\n",
       "      <td>-0.106658</td>\n",
       "      <td>-0.053579</td>\n",
       "      <td>0.113320</td>\n",
       "      <td>-0.165376</td>\n",
       "      <td>-0.019868</td>\n",
       "      <td>0.167424</td>\n",
       "      <td>-0.063025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>inventors__negated+conjunction</td>\n",
       "      <td>-0.235994</td>\n",
       "      <td>-0.286051</td>\n",
       "      <td>-0.175951</td>\n",
       "      <td>-0.167942</td>\n",
       "      <td>-0.122266</td>\n",
       "      <td>-0.211836</td>\n",
       "      <td>-0.079470</td>\n",
       "      <td>-0.355808</td>\n",
       "      <td>-0.126902</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>inventors__affirmative+negated+conjunction+dis...</td>\n",
       "      <td>-0.217087</td>\n",
       "      <td>-0.206090</td>\n",
       "      <td>-0.054348</td>\n",
       "      <td>-0.131923</td>\n",
       "      <td>-0.200000</td>\n",
       "      <td>-0.149889</td>\n",
       "      <td>-0.165563</td>\n",
       "      <td>0.152273</td>\n",
       "      <td>-0.077305</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>sp_en_trans__affirmative+negated</td>\n",
       "      <td>-0.025910</td>\n",
       "      <td>0.161297</td>\n",
       "      <td>0.095788</td>\n",
       "      <td>0.079244</td>\n",
       "      <td>-0.046548</td>\n",
       "      <td>0.009091</td>\n",
       "      <td>0.013245</td>\n",
       "      <td>0.071212</td>\n",
       "      <td>0.040451</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>sp_en_trans__affirmative+disjunction</td>\n",
       "      <td>0.020308</td>\n",
       "      <td>0.053635</td>\n",
       "      <td>-0.124321</td>\n",
       "      <td>-0.012607</td>\n",
       "      <td>-0.101987</td>\n",
       "      <td>0.147799</td>\n",
       "      <td>0.019868</td>\n",
       "      <td>-0.285101</td>\n",
       "      <td>-0.123085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>sp_en_trans__negated+conjunction</td>\n",
       "      <td>-0.240896</td>\n",
       "      <td>-0.129862</td>\n",
       "      <td>-0.091712</td>\n",
       "      <td>-0.103557</td>\n",
       "      <td>-0.049686</td>\n",
       "      <td>-0.026205</td>\n",
       "      <td>-0.198675</td>\n",
       "      <td>-0.053030</td>\n",
       "      <td>-0.085434</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>sp_en_trans__affirmative+negated+conjunction+d...</td>\n",
       "      <td>-0.007003</td>\n",
       "      <td>-0.045972</td>\n",
       "      <td>-0.131793</td>\n",
       "      <td>-0.049077</td>\n",
       "      <td>-0.023013</td>\n",
       "      <td>-0.200000</td>\n",
       "      <td>0.192053</td>\n",
       "      <td>-0.227778</td>\n",
       "      <td>0.009035</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>ambiguous_truthful_reply+unambiguous_lie+unamb...</td>\n",
       "      <td>0.093838</td>\n",
       "      <td>0.173870</td>\n",
       "      <td>0.012908</td>\n",
       "      <td>0.070689</td>\n",
       "      <td>0.009937</td>\n",
       "      <td>0.058628</td>\n",
       "      <td>0.041667</td>\n",
       "      <td>0.000758</td>\n",
       "      <td>0.058302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>animal_class+facts+inventors__affirmative+nega...</td>\n",
       "      <td>0.001667</td>\n",
       "      <td>-0.023969</td>\n",
       "      <td>-0.075408</td>\n",
       "      <td>0.051753</td>\n",
       "      <td>0.126667</td>\n",
       "      <td>-0.115597</td>\n",
       "      <td>0.046358</td>\n",
       "      <td>-0.094949</td>\n",
       "      <td>-0.025320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>animal_class+facts+inventors__affirmative+nega...</td>\n",
       "      <td>-0.003636</td>\n",
       "      <td>-0.003733</td>\n",
       "      <td>-0.077446</td>\n",
       "      <td>-0.063694</td>\n",
       "      <td>-0.116364</td>\n",
       "      <td>-0.059181</td>\n",
       "      <td>-0.208333</td>\n",
       "      <td>-0.012121</td>\n",
       "      <td>-0.044769</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 52
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:59.029862Z",
     "start_time": "2024-12-07T21:26:59.011728Z"
    }
   },
   "cell_type": "code",
   "source": "lyr18_ttpd_probes_rel_to_baseline_probes_acc_delta_on_unseen_single_topic_data_groupings_df.describe()",
   "id": "242d5d000d712c07",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "       animal class     cities  element symbols      facts  inventors  \\\n",
       "count     60.000000  60.000000        60.000000  60.000000  60.000000   \n",
       "mean      -0.039523  -0.002070        -0.035101  -0.014271  -0.015457   \n",
       "std        0.092275   0.117435         0.087600   0.058013   0.069418   \n",
       "min       -0.255602  -0.299411        -0.240741  -0.167942  -0.200000   \n",
       "25%       -0.096116  -0.059774        -0.093784  -0.053694  -0.048509   \n",
       "50%       -0.024510   0.001572        -0.041101  -0.004953  -0.021182   \n",
       "75%        0.018032   0.067976         0.014096   0.026902   0.011343   \n",
       "max        0.172414   0.255206         0.229620   0.099505   0.157224   \n",
       "\n",
       "       spanish english translation  real world scenarios  relative comparison  \\\n",
       "count                    60.000000             60.000000            60.000000   \n",
       "mean                     -0.031605             -0.008872            -0.046650   \n",
       "std                       0.100697              0.159499             0.160499   \n",
       "min                      -0.300332             -0.357616            -0.355808   \n",
       "25%                      -0.089463             -0.145695            -0.177083   \n",
       "50%                      -0.036228              0.006623            -0.003662   \n",
       "75%                       0.034154              0.092715             0.072664   \n",
       "max                       0.187500              0.450980             0.288889   \n",
       "\n",
       "       true false  \n",
       "count   60.000000  \n",
       "mean    -0.027143  \n",
       "std      0.066390  \n",
       "min     -0.136211  \n",
       "25%     -0.077704  \n",
       "50%     -0.030483  \n",
       "75%      0.012914  \n",
       "max      0.128028  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>animal class</th>\n",
       "      <th>cities</th>\n",
       "      <th>element symbols</th>\n",
       "      <th>facts</th>\n",
       "      <th>inventors</th>\n",
       "      <th>spanish english translation</th>\n",
       "      <th>real world scenarios</th>\n",
       "      <th>relative comparison</th>\n",
       "      <th>true false</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>-0.039523</td>\n",
       "      <td>-0.002070</td>\n",
       "      <td>-0.035101</td>\n",
       "      <td>-0.014271</td>\n",
       "      <td>-0.015457</td>\n",
       "      <td>-0.031605</td>\n",
       "      <td>-0.008872</td>\n",
       "      <td>-0.046650</td>\n",
       "      <td>-0.027143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.092275</td>\n",
       "      <td>0.117435</td>\n",
       "      <td>0.087600</td>\n",
       "      <td>0.058013</td>\n",
       "      <td>0.069418</td>\n",
       "      <td>0.100697</td>\n",
       "      <td>0.159499</td>\n",
       "      <td>0.160499</td>\n",
       "      <td>0.066390</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-0.255602</td>\n",
       "      <td>-0.299411</td>\n",
       "      <td>-0.240741</td>\n",
       "      <td>-0.167942</td>\n",
       "      <td>-0.200000</td>\n",
       "      <td>-0.300332</td>\n",
       "      <td>-0.357616</td>\n",
       "      <td>-0.355808</td>\n",
       "      <td>-0.136211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>-0.096116</td>\n",
       "      <td>-0.059774</td>\n",
       "      <td>-0.093784</td>\n",
       "      <td>-0.053694</td>\n",
       "      <td>-0.048509</td>\n",
       "      <td>-0.089463</td>\n",
       "      <td>-0.145695</td>\n",
       "      <td>-0.177083</td>\n",
       "      <td>-0.077704</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>-0.024510</td>\n",
       "      <td>0.001572</td>\n",
       "      <td>-0.041101</td>\n",
       "      <td>-0.004953</td>\n",
       "      <td>-0.021182</td>\n",
       "      <td>-0.036228</td>\n",
       "      <td>0.006623</td>\n",
       "      <td>-0.003662</td>\n",
       "      <td>-0.030483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.018032</td>\n",
       "      <td>0.067976</td>\n",
       "      <td>0.014096</td>\n",
       "      <td>0.026902</td>\n",
       "      <td>0.011343</td>\n",
       "      <td>0.034154</td>\n",
       "      <td>0.092715</td>\n",
       "      <td>0.072664</td>\n",
       "      <td>0.012914</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.172414</td>\n",
       "      <td>0.255206</td>\n",
       "      <td>0.229620</td>\n",
       "      <td>0.099505</td>\n",
       "      <td>0.157224</td>\n",
       "      <td>0.187500</td>\n",
       "      <td>0.450980</td>\n",
       "      <td>0.288889</td>\n",
       "      <td>0.128028</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 53
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:59.035289Z",
     "start_time": "2024-12-07T21:26:59.032867Z"
    }
   },
   "cell_type": "code",
   "source": "",
   "id": "497e079d7640ed9a",
   "outputs": [],
   "execution_count": 53
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-07T21:26:59.038689Z",
     "start_time": "2024-12-07T21:26:59.036294Z"
    }
   },
   "cell_type": "code",
   "source": "",
   "id": "76e00371cd03ccaa",
   "outputs": [],
   "execution_count": 53
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}